diff --git a/pyproject.toml b/pyproject.toml index df4d213..d93ae87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-core" -version = "0.1.8" +version = "0.1.9" description = "UiPath Core abstractions" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/src/uipath/core/guardrails/_deterministic_guardrails_service.py b/src/uipath/core/guardrails/_deterministic_guardrails_service.py index 0b88050..27be499 100644 --- a/src/uipath/core/guardrails/_deterministic_guardrails_service.py +++ b/src/uipath/core/guardrails/_deterministic_guardrails_service.py @@ -40,7 +40,7 @@ def evaluate_pre_deterministic_guardrail( if has_output_rule: return GuardrailValidationResult( result=GuardrailValidationResultType.PASSED, - reason="Guardrail contains output-dependent rules that will be evaluated during post-execution", + reason="No rules to apply for input data.", ) return self._evaluate_deterministic_guardrail( input_data=input_data, @@ -66,7 +66,7 @@ def evaluate_post_deterministic_guardrail( if not has_output_rule: return GuardrailValidationResult( result=GuardrailValidationResultType.PASSED, - reason="Guardrail contains only input-dependent rules that were evaluated during pre-execution", + reason="No rules to apply for output data.", ) return self._evaluate_deterministic_guardrail( @@ -117,7 +117,12 @@ def _evaluate_deterministic_guardrail( output_data: dict[str, Any], guardrail: DeterministicGuardrail, ) -> GuardrailValidationResult: - """Evaluate deterministic guardrail rules against input and output data.""" + """Evaluate deterministic guardrail rules against input and output data. + + Validation fails only if ALL guardrail rules are violated. + """ + validated_conditions: list[str] = [] + for rule in guardrail.rules: if isinstance(rule, WordRule): passed, reason = evaluate_word_rule(rule, input_data, output_data) @@ -132,14 +137,25 @@ def _evaluate_deterministic_guardrail( result=GuardrailValidationResultType.VALIDATION_FAILED, reason=f"Unknown rule type: {type(rule)}", ) - - if not passed: + validated_conditions.append(reason) + if passed: return GuardrailValidationResult( - result=GuardrailValidationResultType.VALIDATION_FAILED, - reason=reason or "Rule validation failed", + result=GuardrailValidationResultType.PASSED, + reason=reason, ) + has_always_rule = any( + condition == "Always rule enforced" for condition in validated_conditions + ) + + validated_conditions_str = ", ".join(validated_conditions) + final_reason = ( + "Always rule enforced" + if has_always_rule + else f"Data matched all guardrail conditions: [{validated_conditions_str}]" + ) + return GuardrailValidationResult( - result=GuardrailValidationResultType.PASSED, - reason="All deterministic guardrail rules passed", + result=GuardrailValidationResultType.VALIDATION_FAILED, + reason=final_reason, ) diff --git a/src/uipath/core/guardrails/_evaluators.py b/src/uipath/core/guardrails/_evaluators.py index 111ffa2..502ff7c 100644 --- a/src/uipath/core/guardrails/_evaluators.py +++ b/src/uipath/core/guardrails/_evaluators.py @@ -159,24 +159,44 @@ def get_fields_from_selector( return fields -def format_guardrail_error_message( +def format_guardrail_passed_validation_result_message( field_ref: FieldReference, - operator: str, - expected_value: str | None = None, + operator: str | None, + rule_description: str | None, ) -> str: - """Format a guardrail error message following the standard pattern.""" + """Format a guardrail validation result message following the standard pattern.""" source = "Input" if field_ref.source == FieldSource.INPUT else "Output" - message = f"{source} data didn't match the guardrail condition: [{field_ref.path}] comparing function [{operator}]" - if expected_value and expected_value.strip(): - message += f" [{expected_value.strip()}]" - return message + + if rule_description: + return ( + f"{source} data didn't match the guardrail condition for field " + f"[{field_ref.path}]: {rule_description}" + ) + + return ( + f"{source} data didn't match the guardrail condition: " + f"[{field_ref.path}] comparing function [{operator}]" + ) + + +def get_validated_conditions_description( + field_path: str, + operator: str | None, + rule_description: str | None, +) -> str: + if rule_description: + return rule_description + + return f"[{field_path}] comparing function [{operator}]" def evaluate_word_rule( rule: WordRule, input_data: dict[str, Any], output_data: dict[str, Any] -) -> tuple[bool, str | None]: +) -> tuple[bool, str]: """Evaluate a word rule against input and output data.""" fields = get_fields_from_selector(rule.field_selector, input_data, output_data) + operator = _humanize_guardrail_func(rule.detects_violation) or "violation check" + field_paths = ", ".join({field_ref.path for _, field_ref in fields}) for field_value, field_ref in fields: if field_value is None: @@ -197,22 +217,28 @@ def evaluate_word_rule( # If function raises an exception, treat as failure violation_detected = True - if violation_detected: - operator = ( - _humanize_guardrail_func(rule.detects_violation) or "violation check" + if not violation_detected: + reason = format_guardrail_passed_validation_result_message( + field_ref=field_ref, + operator=operator, + rule_description=rule.rule_description, ) - reason = format_guardrail_error_message(field_ref, operator, None) - return False, reason + return True, reason - return True, "All word rule validations passed" + return False, get_validated_conditions_description( + field_path=field_paths, + operator=operator, + rule_description=rule.rule_description, + ) def evaluate_number_rule( rule: NumberRule, input_data: dict[str, Any], output_data: dict[str, Any] -) -> tuple[bool, str | None]: +) -> tuple[bool, str]: """Evaluate a number rule against input and output data.""" fields = get_fields_from_selector(rule.field_selector, input_data, output_data) - + operator = _humanize_guardrail_func(rule.detects_violation) or "violation check" + field_paths = ", ".join({field_ref.path for _, field_ref in fields}) for field_value, field_ref in fields: if field_value is None: continue @@ -233,24 +259,30 @@ def evaluate_number_rule( # If function raises an exception, treat as failure violation_detected = True - if violation_detected: - operator = ( - _humanize_guardrail_func(rule.detects_violation) or "violation check" + if not violation_detected: + reason = format_guardrail_passed_validation_result_message( + field_ref=field_ref, + operator=operator, + rule_description=rule.rule_description, ) - reason = format_guardrail_error_message(field_ref, operator, None) - return False, reason + return True, reason - return True, "All number rule validations passed" + return False, get_validated_conditions_description( + field_path=field_paths, + operator=operator, + rule_description=rule.rule_description, + ) def evaluate_boolean_rule( rule: BooleanRule, input_data: dict[str, Any], output_data: dict[str, Any], -) -> tuple[bool, str | None]: +) -> tuple[bool, str]: """Evaluate a boolean rule against input and output data.""" fields = get_fields_from_selector(rule.field_selector, input_data, output_data) - + operator = _humanize_guardrail_func(rule.detects_violation) or "violation check" + field_paths = ", ".join({field_ref.path for _, field_ref in fields}) for field_value, field_ref in fields: if field_value is None: continue @@ -270,20 +302,25 @@ def evaluate_boolean_rule( # If function raises an exception, treat as failure violation_detected = True - if violation_detected: - operator = ( - _humanize_guardrail_func(rule.detects_violation) or "violation check" + if not violation_detected: + reason = format_guardrail_passed_validation_result_message( + field_ref=field_ref, + operator=operator, + rule_description=rule.rule_description, ) - reason = format_guardrail_error_message(field_ref, operator, None) - return False, reason + return True, reason - return True, "All boolean rule validations passed" + return False, get_validated_conditions_description( + field_path=field_paths, + operator=operator, + rule_description=rule.rule_description, + ) def evaluate_universal_rule( rule: UniversalRule, output_data: dict[str, Any], -) -> tuple[bool, str | None]: +) -> tuple[bool, str]: """Evaluate a universal rule against input and output data. Universal rules trigger based on the apply_to scope and execution phase: @@ -302,18 +339,18 @@ def evaluate_universal_rule( if rule.apply_to == ApplyTo.INPUT: # INPUT: triggers in pre-execution, does not trigger in post-execution if is_pre_execution: - return False, "Universal rule validation triggered (pre-execution, input)" + return False, "Always rule enforced" else: - return True, "Universal rule validation passed (post-execution, input)" + return True, "No rules to apply for output data" elif rule.apply_to == ApplyTo.OUTPUT: # OUTPUT: does not trigger in pre-execution, triggers in post-execution if is_pre_execution: - return True, "Universal rule validation passed (pre-execution, output)" + return True, "No rules to apply for input data" else: - return False, "Universal rule validation triggered (post-execution, output)" + return False, "Always rule enforced" elif rule.apply_to == ApplyTo.INPUT_AND_OUTPUT: # INPUT_AND_OUTPUT: triggers in both phases - return False, "Universal rule validation triggered (input and output)" + return False, "Always rule enforced" else: return False, f"Unknown apply_to value: {rule.apply_to}" diff --git a/src/uipath/core/guardrails/guardrails.py b/src/uipath/core/guardrails/guardrails.py index 77382cb..54982cb 100644 --- a/src/uipath/core/guardrails/guardrails.py +++ b/src/uipath/core/guardrails/guardrails.py @@ -102,6 +102,11 @@ class WordRule(BaseModel): rule_type: Literal["word"] = Field(alias="$ruleType") field_selector: FieldSelector = Field(alias="fieldSelector") + rule_description: str | None = Field( + default=None, + exclude=True, + description="Human-friendly description of the rule condition.", + ) detects_violation: Callable[[str], bool] = Field( exclude=True, description="Function that returns True if the string violates the rule (validation should fail).", @@ -124,6 +129,11 @@ class NumberRule(BaseModel): rule_type: Literal["number"] = Field(alias="$ruleType") field_selector: FieldSelector = Field(alias="fieldSelector") + rule_description: str | None = Field( + default=None, + exclude=True, + description="Human-friendly description of the rule condition.", + ) detects_violation: Callable[[float], bool] = Field( exclude=True, description="Function that returns True if the number violates the rule (validation should fail).", @@ -137,6 +147,11 @@ class BooleanRule(BaseModel): rule_type: Literal["boolean"] = Field(alias="$ruleType") field_selector: FieldSelector = Field(alias="fieldSelector") + rule_description: str | None = Field( + default=None, + exclude=True, + description="Human-friendly description of the rule condition.", + ) detects_violation: Callable[[bool], bool] = Field( exclude=True, description="Function that returns True if the boolean violates the rule (validation should fail).", diff --git a/tests/guardrails/test_deterministic_guardrails_service.py b/tests/guardrails/test_deterministic_guardrails_service.py index 2a5ed43..1db2da4 100644 --- a/tests/guardrails/test_deterministic_guardrails_service.py +++ b/tests/guardrails/test_deterministic_guardrails_service.py @@ -81,12 +81,9 @@ def test_evaluate_post_deterministic_guardrail_validation_passed( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." - def test_evaluate_post_deterministic_guardrail_validation_failed_age( + def test_evaluate_post_deterministic_guardrail_validation_passes_when_input_data_dont_violates_all_the_rules( self, service: DeterministicGuardrailsService, ) -> None: @@ -146,13 +143,13 @@ def test_evaluate_post_deterministic_guardrail_validation_failed_age( guardrail=deterministic_guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.result == GuardrailValidationResultType.PASSED assert ( result.reason - == "Input data didn't match the guardrail condition: [age] comparing function [(n): n < 21.0]" + == "Input data didn't match the guardrail condition: [isActive] comparing function [(b): b is not True]" ) - def test_evaluate_post_deterministic_guardrail_validation_failed_is_active( + def test_evaluate_post_deterministic_guardrail_validation_passes_when_input_and_output_data_dont_violates_all_the_rules( self, service: DeterministicGuardrailsService, ) -> None: @@ -214,67 +211,22 @@ def test_evaluate_post_deterministic_guardrail_validation_failed_is_active( guardrail=deterministic_guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED - assert ( - result.reason - == "Input data didn't match the guardrail condition: [isActive] comparing function [(b): b is not True]" - ) - - def test_evaluate_post_deterministic_guardrail_matches_regex_positive( - self, - service: DeterministicGuardrailsService, - ) -> None: - """Test deterministic guardrail validation passes when regex matches.""" - deterministic_guardrail = DeterministicGuardrail( - id="test-deterministic-id", - name="Regex Guardrail", - description="Test regex guardrail", - enabled_for_evals=True, - guardrail_type="custom", - selector=GuardrailSelector( - scopes=[GuardrailScope.TOOL], match_names=["test"] - ), - rules=[ - WordRule( - rule_type="word", - field_selector=SpecificFieldsSelector( - selector_type="specific", - fields=[ - FieldReference(path="userName", source=FieldSource.INPUT) - ], - ), - detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)), - ), - ], - ) - - # Input data with userName that matches the regex pattern - input_data = { - "userName": "test123", - } - output_data: dict[str, Any] = {} - - result = service.evaluate_post_deterministic_guardrail( - input_data=input_data, - output_data=output_data, - guardrail=deterministic_guardrail, - ) - assert result.result == GuardrailValidationResultType.PASSED assert ( result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" + == "Input data didn't match the guardrail condition: [age] comparing function [(n): n < 21.0]" ) - def test_evaluate_post_deterministic_guardrail_matches_regex_negative( + def test_evaluate_post_deterministic_guardrail_uses_rule_description( self, service: DeterministicGuardrailsService, ) -> None: - """Test deterministic guardrail validation fails when regex doesn't match.""" + """Ensure rule_description is returned when a rule fails validation.""" + friendly_description = "Username must include 'te' and a digit" deterministic_guardrail = DeterministicGuardrail( - id="test-deterministic-id", - name="Regex Guardrail", - description="Test regex guardrail", + id="test-rule-desc-id", + name="Regex Guardrail With Description", + description="Test regex guardrail with description", enabled_for_evals=True, guardrail_type="custom", selector=GuardrailSelector( @@ -289,18 +241,9 @@ def test_evaluate_post_deterministic_guardrail_matches_regex_negative( FieldReference(path="userName", source=FieldSource.INPUT) ], ), + rule_description=friendly_description, detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)), ), - NumberRule( - rule_type="number", - field_selector=SpecificFieldsSelector( - selector_type="specific", - fields=[ - FieldReference(path="status", source=FieldSource.OUTPUT) - ], - ), - detects_violation=lambda n: n != 200.0, - ), ], ) @@ -308,31 +251,27 @@ def test_evaluate_post_deterministic_guardrail_matches_regex_negative( input_data = { "userName": "test", } - output_data = { - "status": 200, - } - result = service.evaluate_post_deterministic_guardrail( + result = service.evaluate_pre_deterministic_guardrail( input_data=input_data, - output_data=output_data, guardrail=deterministic_guardrail, ) assert result.result == GuardrailValidationResultType.VALIDATION_FAILED assert ( result.reason - == 'Input data didn\'t match the guardrail condition: [userName] comparing function [(s): not bool(re.search(".*te.*3.*", s))]' + == "Data matched all guardrail conditions: [Username must include 'te' and a digit]" ) - def test_evaluate_post_deterministic_guardrail_word_func_positive( + def test_evaluate_post_deterministic_guardrail_passes_validation_when_no_output_rules( self, service: DeterministicGuardrailsService, ) -> None: - """Test deterministic guardrail validation passes when word func returns True.""" + """Test deterministic guardrail validation passes when regex matches.""" deterministic_guardrail = DeterministicGuardrail( id="test-deterministic-id", - name="Word Func Guardrail", - description="Test word func guardrail", + name="Regex Guardrail", + description="Test regex guardrail", enabled_for_evals=True, guardrail_type="custom", selector=GuardrailSelector( @@ -347,14 +286,14 @@ def test_evaluate_post_deterministic_guardrail_word_func_positive( FieldReference(path="userName", source=FieldSource.INPUT) ], ), - detects_violation=lambda s: len(s) <= 5, + detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)), ), ], ) - # Input data with userName that passes the function check + # Input data with userName that matches the regex pattern input_data = { - "userName": "testuser", + "userName": "test123", } output_data: dict[str, Any] = {} @@ -365,20 +304,17 @@ def test_evaluate_post_deterministic_guardrail_word_func_positive( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." - def test_evaluate_post_deterministic_guardrail_word_func_negative( + def test_evaluate_post_deterministic_guardrail_failes_validation_when_data_macthes_rules( self, service: DeterministicGuardrailsService, ) -> None: - """Test deterministic guardrail validation fails when word func returns False.""" + """Test deterministic guardrail validation fails when regex doesn't match.""" deterministic_guardrail = DeterministicGuardrail( id="test-deterministic-id", - name="Word Func Guardrail", - description="Test word func guardrail", + name="Regex Guardrail", + description="Test regex guardrail", enabled_for_evals=True, guardrail_type="custom", selector=GuardrailSelector( @@ -393,7 +329,7 @@ def test_evaluate_post_deterministic_guardrail_word_func_negative( FieldReference(path="userName", source=FieldSource.INPUT) ], ), - detects_violation=lambda s: len(s) <= 5, + detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)), ), NumberRule( rule_type="number", @@ -408,12 +344,12 @@ def test_evaluate_post_deterministic_guardrail_word_func_negative( ], ) - # Input data with userName that fails the function check + # Input data with userName that doesn't match the regex pattern input_data = { "userName": "test", } output_data = { - "status": 200, + "status": 201, } result = service.evaluate_post_deterministic_guardrail( @@ -422,7 +358,14 @@ def test_evaluate_post_deterministic_guardrail_word_func_negative( guardrail=deterministic_guardrail, ) + expected_reason = ( + "Data matched all guardrail conditions: [[userName] comparing function " + '[(s): not bool(re.search(".*te.*3.*", s))], ' + "[status] comparing function [(n): n != 200.0]]" + ) + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == expected_reason def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_violation( self, @@ -467,7 +410,7 @@ def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_v "userName": "andrei", } output_data = { - "status": 200, + "status": 201, } result = service.evaluate_post_deterministic_guardrail( @@ -479,10 +422,11 @@ def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_v assert result.result == GuardrailValidationResultType.VALIDATION_FAILED assert ( result.reason - == 'Input data didn\'t match the guardrail condition: [userName] comparing function [(s): "dre" in s]' + == "Data matched all guardrail conditions: [[userName] comparing function " + '[(s): "dre" in s], [status] comparing function [(n): n != 200.0]]' ) - def test_evaluate_post_deterministic_guardrail_number_func_positive( + def test_evaluate_post_deterministic_guardrail_number_func_passes_when_no_input_rules( self, service: DeterministicGuardrailsService, ) -> None: @@ -521,10 +465,7 @@ def test_evaluate_post_deterministic_guardrail_number_func_positive( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." def test_evaluate_post_deterministic_guardrail_number_func_negative( self, @@ -567,7 +508,7 @@ def test_evaluate_post_deterministic_guardrail_number_func_negative( "age": 70, } output_data = { - "status": 200, + "status": 201, } result = service.evaluate_post_deterministic_guardrail( @@ -578,11 +519,11 @@ def test_evaluate_post_deterministic_guardrail_number_func_negative( assert result.result == GuardrailValidationResultType.VALIDATION_FAILED - def test_should_trigger_policy_pre_execution_only_some_rules_not_met_returns_false( + def test_evaluate_post_execution_pases_when_only_some_rules_not_met( self, service: DeterministicGuardrailsService, ) -> None: - """Test pre-execution guardrail fails when some rules are not met.""" + """Test post-execution guardrail passes when only some rules are not met.""" guardrail = self._create_guardrail_for_pre_execution() input_data = { "userName": "John", @@ -599,7 +540,7 @@ def test_should_trigger_policy_pre_execution_only_some_rules_not_met_returns_fal guardrail=guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.result == GuardrailValidationResultType.PASSED def test_should_ignore_post_execution_guardrail_for_pre_execution_returns_false( self, @@ -649,11 +590,10 @@ def test_should_trigger_policy_post_execution_guardrail_for_pre_execution_return # Pre-execution guardrail should still pass in post-execution assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_with_output_fields_all_conditions_met_returns_true( + def test_should_trigger_policy_post_execution_with_output_fields_when_no_violation_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: - """Test post-execution guardrail passes when all conditions are met.""" guardrail = self._create_guardrail_for_post_execution() input_data = { "userName": "John", @@ -674,11 +614,10 @@ def test_should_trigger_policy_post_execution_with_output_fields_all_conditions_ assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_with_output_fields_input_conditions_not_met_returns_false( + def test_post_execution_with_output_fields_when_only_input_conditions_violated_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: - """Test post-execution guardrail fails when input conditions are not met.""" guardrail = self._create_guardrail_for_post_execution() input_data = { "userName": "John", @@ -697,13 +636,12 @@ def test_should_trigger_policy_post_execution_with_output_fields_input_condition guardrail=guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_with_output_fields_output_conditions_not_met_returns_false( + def test_post_execution_with_input_and_output_fields_output_when_only_output_conditions_violated_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: - """Test post-execution guardrail fails when output conditions are not met.""" guardrail = self._create_guardrail_for_post_execution() input_data = { "userName": "John", @@ -722,13 +660,12 @@ def test_should_trigger_policy_post_execution_with_output_fields_output_conditio guardrail=guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_multiple_rules_all_conditions_must_be_met_returns_true( + def test_post_execution_multiple_rules_when_all_conditions_when_no_condition_is_violated_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: - """Test post-execution guardrail with multiple rules passes when all conditions are met.""" guardrail = self._create_guardrail_with_multiple_rules() input_data = { "userName": "John", @@ -749,7 +686,7 @@ def test_should_trigger_policy_post_execution_multiple_rules_all_conditions_must assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_rule_with_multiple_conditions_all_must_be_met_returns_true( + def test_post_execution_rule_with_multiple_conditions_when_no_condition_is_violated_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -774,7 +711,7 @@ def test_should_trigger_policy_post_execution_rule_with_multiple_conditions_all_ assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_rule_with_multiple_conditions_one_condition_not_met_returns_false( + def test_post_execution_rule_with_multiple_conditions_when_only_some_conditions_are_violated_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -797,9 +734,34 @@ def test_should_trigger_policy_post_execution_rule_with_multiple_conditions_one_ guardrail=guardrail, ) + assert result.result == GuardrailValidationResultType.PASSED + + def test_post_execution_rule_with_multiple_conditions_when_all_condition_are_violated_then_returns_false( + self, + service: DeterministicGuardrailsService, + ) -> None: + """Test guardrail with multiple conditions fails when one condition is not met.""" + guardrail = self._create_guardrail_with_rule_having_multiple_conditions() + input_data = { + "userName": "John", + "age": 15, # < 18 + "isActive": False, # Not True + } + output_data = { + "result": "Success", + "status": 201, # Not 200 + "success": True, + } + + result = service.evaluate_post_deterministic_guardrail( + input_data=input_data, + output_data=output_data, + guardrail=guardrail, + ) + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED - def test_should_trigger_policy_post_execution_with_all_fields_selector_output_schema_has_fields_returns_true( + def test_post_execution_with_all_fields_selector_when_no_field_violates_condition_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -831,7 +793,7 @@ def test_should_trigger_policy_post_execution_with_all_fields_selector_output_sc } output_data = { "result": "Success", - "status": 25, # Matches the rule value + "status": 25, # Doesn't match the rule value "success": True, } @@ -843,7 +805,50 @@ def test_should_trigger_policy_post_execution_with_all_fields_selector_output_sc assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_with_all_fields_selector_empty_output_schema_returns_true( + def test_post_execution_with_all_fields_selector_when_all_fields_violate_condition_then_returns_false( + self, + service: DeterministicGuardrailsService, + ) -> None: + guardrail = DeterministicGuardrail( + id="test-all-fields-id", + name="Guardrail With All Fields Selector", + description="Test all fields selector", + enabled_for_evals=True, + guardrail_type="custom", + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["test"] + ), + rules=[ + NumberRule( + rule_type="number", + field_selector=AllFieldsSelector( + selector_type="all", sources=[FieldSource.OUTPUT] + ), + detects_violation=lambda n: n != 25.0, + ), + ], + ) + + input_data = { + "userName": "John", + "age": 25, + "isActive": True, + } + output_data = { + "result": "Success", + "status": 20, # Matches the rule value + "success": True, + } + + result = service.evaluate_post_deterministic_guardrail( + input_data=input_data, + output_data=output_data, + guardrail=guardrail, + ) + + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + + def test_post_execution_with_all_fields_selector_when_empty_output_schema_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -900,11 +905,10 @@ def test_should_trigger_policy_pre_execution_always_rule_with_input_apply_to_ret guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.VALIDATION_FAILED - ) # Should trigger + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == "Always rule enforced" - def test_should_trigger_policy_pre_execution_always_rule_with_output_apply_to_returns_false( + def test_should_trigger_policy_pre_execution_always_rule_with_output_apply_to_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -923,9 +927,8 @@ def test_should_trigger_policy_pre_execution_always_rule_with_output_apply_to_re guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.PASSED - ) # Should not trigger + assert result.result == GuardrailValidationResultType.PASSED + assert result.reason == "No rules to apply for input data" def test_should_trigger_policy_pre_execution_always_rule_with_input_and_output_apply_to_returns_true( self, @@ -946,9 +949,8 @@ def test_should_trigger_policy_pre_execution_always_rule_with_input_and_output_a guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.VALIDATION_FAILED - ) # Should trigger + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == "Always rule enforced" def test_should_trigger_policy_post_execution_always_rule_with_input_apply_to_returns_false( self, @@ -973,9 +975,8 @@ def test_should_trigger_policy_post_execution_always_rule_with_input_apply_to_re guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.PASSED - ) # Should not trigger + assert result.result == GuardrailValidationResultType.PASSED + assert result.reason == "No rules to apply for output data." def test_should_trigger_policy_post_execution_always_rule_with_output_apply_to_returns_true( self, @@ -1000,9 +1001,8 @@ def test_should_trigger_policy_post_execution_always_rule_with_output_apply_to_r guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.VALIDATION_FAILED - ) # Should trigger + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == "Always rule enforced" def test_should_trigger_policy_post_execution_always_rule_with_input_and_output_apply_to_returns_true( self, @@ -1027,9 +1027,8 @@ def test_should_trigger_policy_post_execution_always_rule_with_input_and_output_ guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.VALIDATION_FAILED - ) # Should trigger + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == "Always rule enforced" # Helper methods to create guardrails @@ -1240,10 +1239,7 @@ def test_evaluate_post_deterministic_guardrail_word_contains_operator_passes( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." def test_evaluate_post_deterministic_guardrail_only_output_rules_passes( self, @@ -1298,7 +1294,10 @@ def test_evaluate_post_deterministic_guardrail_only_output_rules_passes( ) assert result.result == GuardrailValidationResultType.PASSED - assert result.reason == "All deterministic guardrail rules passed" + assert ( + result.reason + == "Output data didn't match the guardrail condition: [status] comparing function [(n): n != 200.0]" + ) def test_evaluate_post_deterministic_guardrail_only_always_rule_fails( self, @@ -1336,6 +1335,7 @@ def test_evaluate_post_deterministic_guardrail_only_always_rule_fails( ) assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == "Always rule enforced" def test_evaluate_post_deterministic_guardrail_only_input_rules_passes( self, @@ -1384,10 +1384,7 @@ def test_evaluate_post_deterministic_guardrail_only_input_rules_passes( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." def test_evaluate_pre_deterministic_guardrail_with_input_and_output_rules_input_true( self, diff --git a/uv.lock b/uv.lock index c6a7e6f..7aeccc3 100644 --- a/uv.lock +++ b/uv.lock @@ -991,7 +991,7 @@ wheels = [ [[package]] name = "uipath-core" -version = "0.1.8" +version = "0.1.9" source = { editable = "." } dependencies = [ { name = "opentelemetry-instrumentation" },