From 370ea2e2fcf0a1d4f9544cf60a31a90ab791fc8e Mon Sep 17 00:00:00 2001 From: Valentina Bojan Date: Tue, 20 Jan 2026 11:46:56 +0200 Subject: [PATCH 1/3] fix: update validation result messages for backwards compatibility with low coded agents UI --- .../_deterministic_guardrails_service.py | 4 +- src/uipath/core/guardrails/_evaluators.py | 10 +-- .../test_deterministic_guardrails_service.py | 61 ++++++------------- 3 files changed, 26 insertions(+), 49 deletions(-) diff --git a/src/uipath/core/guardrails/_deterministic_guardrails_service.py b/src/uipath/core/guardrails/_deterministic_guardrails_service.py index 0b88050..2bfa5b4 100644 --- a/src/uipath/core/guardrails/_deterministic_guardrails_service.py +++ b/src/uipath/core/guardrails/_deterministic_guardrails_service.py @@ -40,7 +40,7 @@ def evaluate_pre_deterministic_guardrail( if has_output_rule: return GuardrailValidationResult( result=GuardrailValidationResultType.PASSED, - reason="Guardrail contains output-dependent rules that will be evaluated during post-execution", + reason="No rules to apply for input data.", ) return self._evaluate_deterministic_guardrail( input_data=input_data, @@ -66,7 +66,7 @@ def evaluate_post_deterministic_guardrail( if not has_output_rule: return GuardrailValidationResult( result=GuardrailValidationResultType.PASSED, - reason="Guardrail contains only input-dependent rules that were evaluated during pre-execution", + reason="No rules to apply for output data.", ) return self._evaluate_deterministic_guardrail( diff --git a/src/uipath/core/guardrails/_evaluators.py b/src/uipath/core/guardrails/_evaluators.py index 111ffa2..562b4c8 100644 --- a/src/uipath/core/guardrails/_evaluators.py +++ b/src/uipath/core/guardrails/_evaluators.py @@ -302,18 +302,18 @@ def evaluate_universal_rule( if rule.apply_to == ApplyTo.INPUT: # INPUT: triggers in pre-execution, does not trigger in post-execution if is_pre_execution: - return False, "Universal rule validation triggered (pre-execution, input)" + return False, "Always rule enforced" else: - return True, "Universal rule validation passed (post-execution, input)" + return True, "No rules to apply for output data" elif rule.apply_to == ApplyTo.OUTPUT: # OUTPUT: does not trigger in pre-execution, triggers in post-execution if is_pre_execution: - return True, "Universal rule validation passed (pre-execution, output)" + return True, "No rules to apply for input data" else: - return False, "Universal rule validation triggered (post-execution, output)" + return False, "Always rule enforced" elif rule.apply_to == ApplyTo.INPUT_AND_OUTPUT: # INPUT_AND_OUTPUT: triggers in both phases - return False, "Universal rule validation triggered (input and output)" + return False, "Always rule enforced" else: return False, f"Unknown apply_to value: {rule.apply_to}" diff --git a/tests/guardrails/test_deterministic_guardrails_service.py b/tests/guardrails/test_deterministic_guardrails_service.py index 2a5ed43..49f6dd3 100644 --- a/tests/guardrails/test_deterministic_guardrails_service.py +++ b/tests/guardrails/test_deterministic_guardrails_service.py @@ -81,10 +81,7 @@ def test_evaluate_post_deterministic_guardrail_validation_passed( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." def test_evaluate_post_deterministic_guardrail_validation_failed_age( self, @@ -261,10 +258,7 @@ def test_evaluate_post_deterministic_guardrail_matches_regex_positive( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." def test_evaluate_post_deterministic_guardrail_matches_regex_negative( self, @@ -365,10 +359,7 @@ def test_evaluate_post_deterministic_guardrail_word_func_positive( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." def test_evaluate_post_deterministic_guardrail_word_func_negative( self, @@ -521,10 +512,7 @@ def test_evaluate_post_deterministic_guardrail_number_func_positive( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." def test_evaluate_post_deterministic_guardrail_number_func_negative( self, @@ -900,9 +888,8 @@ def test_should_trigger_policy_pre_execution_always_rule_with_input_apply_to_ret guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.VALIDATION_FAILED - ) # Should trigger + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == "Always rule enforced" def test_should_trigger_policy_pre_execution_always_rule_with_output_apply_to_returns_false( self, @@ -923,9 +910,8 @@ def test_should_trigger_policy_pre_execution_always_rule_with_output_apply_to_re guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.PASSED - ) # Should not trigger + assert result.result == GuardrailValidationResultType.PASSED + assert result.reason == "All deterministic guardrail rules passed" def test_should_trigger_policy_pre_execution_always_rule_with_input_and_output_apply_to_returns_true( self, @@ -946,9 +932,8 @@ def test_should_trigger_policy_pre_execution_always_rule_with_input_and_output_a guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.VALIDATION_FAILED - ) # Should trigger + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == "Always rule enforced" def test_should_trigger_policy_post_execution_always_rule_with_input_apply_to_returns_false( self, @@ -973,9 +958,8 @@ def test_should_trigger_policy_post_execution_always_rule_with_input_apply_to_re guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.PASSED - ) # Should not trigger + assert result.result == GuardrailValidationResultType.PASSED + assert result.reason == "No rules to apply for output data." def test_should_trigger_policy_post_execution_always_rule_with_output_apply_to_returns_true( self, @@ -1000,9 +984,8 @@ def test_should_trigger_policy_post_execution_always_rule_with_output_apply_to_r guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.VALIDATION_FAILED - ) # Should trigger + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == "Always rule enforced" def test_should_trigger_policy_post_execution_always_rule_with_input_and_output_apply_to_returns_true( self, @@ -1027,9 +1010,8 @@ def test_should_trigger_policy_post_execution_always_rule_with_input_and_output_ guardrail=guardrail, ) - assert ( - result.result == GuardrailValidationResultType.VALIDATION_FAILED - ) # Should trigger + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == "Always rule enforced" # Helper methods to create guardrails @@ -1240,10 +1222,7 @@ def test_evaluate_post_deterministic_guardrail_word_contains_operator_passes( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." def test_evaluate_post_deterministic_guardrail_only_output_rules_passes( self, @@ -1336,6 +1315,7 @@ def test_evaluate_post_deterministic_guardrail_only_always_rule_fails( ) assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == "Always rule enforced" def test_evaluate_post_deterministic_guardrail_only_input_rules_passes( self, @@ -1384,10 +1364,7 @@ def test_evaluate_post_deterministic_guardrail_only_input_rules_passes( ) assert result.result == GuardrailValidationResultType.PASSED - assert ( - result.reason - == "Guardrail contains only input-dependent rules that were evaluated during pre-execution" - ) + assert result.reason == "No rules to apply for output data." def test_evaluate_pre_deterministic_guardrail_with_input_and_output_rules_input_true( self, From a9d58ac2e8281d0fd1d0a449679d8deb44aa8875 Mon Sep 17 00:00:00 2001 From: Valentina Bojan Date: Tue, 20 Jan 2026 15:11:42 +0200 Subject: [PATCH 2/3] fix: return validation_failed only when all rules are violated --- .../_deterministic_guardrails_service.py | 26 +- src/uipath/core/guardrails/_evaluators.py | 106 +++++--- src/uipath/core/guardrails/guardrails.py | 15 ++ .../test_deterministic_guardrails_service.py | 237 +++++++++--------- 4 files changed, 237 insertions(+), 147 deletions(-) diff --git a/src/uipath/core/guardrails/_deterministic_guardrails_service.py b/src/uipath/core/guardrails/_deterministic_guardrails_service.py index 2bfa5b4..2b88461 100644 --- a/src/uipath/core/guardrails/_deterministic_guardrails_service.py +++ b/src/uipath/core/guardrails/_deterministic_guardrails_service.py @@ -118,6 +118,8 @@ def _evaluate_deterministic_guardrail( guardrail: DeterministicGuardrail, ) -> GuardrailValidationResult: """Evaluate deterministic guardrail rules against input and output data.""" + validated_conditions: list[str] = [] + for rule in guardrail.rules: if isinstance(rule, WordRule): passed, reason = evaluate_word_rule(rule, input_data, output_data) @@ -132,14 +134,26 @@ def _evaluate_deterministic_guardrail( result=GuardrailValidationResultType.VALIDATION_FAILED, reason=f"Unknown rule type: {type(rule)}", ) - - if not passed: + validated_conditions.append(reason) + if passed: return GuardrailValidationResult( - result=GuardrailValidationResultType.VALIDATION_FAILED, - reason=reason or "Rule validation failed", + result=GuardrailValidationResultType.PASSED, + reason=reason, ) + has_always_rule = any( + condition == "Always rule enforced" + for condition in validated_conditions + ) + + validated_conditions_str = ", ".join(validated_conditions) + final_reason = ( + "Always rule enforced" + if has_always_rule + else f"Data matched all guardrail conditions: [{validated_conditions_str}]" + ) + return GuardrailValidationResult( - result=GuardrailValidationResultType.PASSED, - reason="All deterministic guardrail rules passed", + result=GuardrailValidationResultType.VALIDATION_FAILED, + reason=final_reason, ) diff --git a/src/uipath/core/guardrails/_evaluators.py b/src/uipath/core/guardrails/_evaluators.py index 562b4c8..82c48b5 100644 --- a/src/uipath/core/guardrails/_evaluators.py +++ b/src/uipath/core/guardrails/_evaluators.py @@ -159,24 +159,47 @@ def get_fields_from_selector( return fields -def format_guardrail_error_message( +def format_guardrail_passed_validation_result_message( field_ref: FieldReference, - operator: str, - expected_value: str | None = None, + operator: str | None, + rule_description: str | None, ) -> str: - """Format a guardrail error message following the standard pattern.""" + """Format a guardrail validation result message following the standard pattern.""" source = "Input" if field_ref.source == FieldSource.INPUT else "Output" - message = f"{source} data didn't match the guardrail condition: [{field_ref.path}] comparing function [{operator}]" - if expected_value and expected_value.strip(): - message += f" [{expected_value.strip()}]" - return message + + if rule_description: + return ( + f"{source} data didn't match the guardrail condition for field " + f"[{field_ref.path}]: {rule_description}" + ) + + return ( + f"{source} data didn't match the guardrail condition: " + f"[{field_ref.path}] comparing function [{operator}]" + ) + +def get_validated_conditions_description( + field_path: str, + operator: str | None, + rule_description: str | None, +) -> str: + if rule_description: + return rule_description + + return f"[{field_path}] comparing function [{operator}]" def evaluate_word_rule( rule: WordRule, input_data: dict[str, Any], output_data: dict[str, Any] -) -> tuple[bool, str | None]: +) -> tuple[bool, str]: """Evaluate a word rule against input and output data.""" fields = get_fields_from_selector(rule.field_selector, input_data, output_data) + operator = ( + _humanize_guardrail_func(rule.detects_violation) or "violation check" + ) + field_paths = ", ".join( + {field_ref.path for _, field_ref in fields} + ) for field_value, field_ref in fields: if field_value is None: @@ -197,14 +220,19 @@ def evaluate_word_rule( # If function raises an exception, treat as failure violation_detected = True - if violation_detected: - operator = ( - _humanize_guardrail_func(rule.detects_violation) or "violation check" + if not violation_detected: + reason = format_guardrail_passed_validation_result_message( + field_ref=field_ref, + operator=operator, + rule_description=rule.rule_description, ) - reason = format_guardrail_error_message(field_ref, operator, None) - return False, reason + return True, reason - return True, "All word rule validations passed" + return False, get_validated_conditions_description( + field_path=field_paths, + operator=operator, + rule_description=rule.rule_description, + ) def evaluate_number_rule( @@ -212,7 +240,12 @@ def evaluate_number_rule( ) -> tuple[bool, str | None]: """Evaluate a number rule against input and output data.""" fields = get_fields_from_selector(rule.field_selector, input_data, output_data) - + operator = ( + _humanize_guardrail_func(rule.detects_violation) or "violation check" + ) + field_paths = ", ".join( + {field_ref.path for _, field_ref in fields} + ) for field_value, field_ref in fields: if field_value is None: continue @@ -233,14 +266,19 @@ def evaluate_number_rule( # If function raises an exception, treat as failure violation_detected = True - if violation_detected: - operator = ( - _humanize_guardrail_func(rule.detects_violation) or "violation check" + if not violation_detected: + reason = format_guardrail_passed_validation_result_message( + field_ref=field_ref, + operator=operator, + rule_description=rule.rule_description, ) - reason = format_guardrail_error_message(field_ref, operator, None) - return False, reason + return True, reason - return True, "All number rule validations passed" + return False, get_validated_conditions_description( + field_path=field_paths, + operator=operator, + rule_description=rule.rule_description, + ) def evaluate_boolean_rule( @@ -250,7 +288,12 @@ def evaluate_boolean_rule( ) -> tuple[bool, str | None]: """Evaluate a boolean rule against input and output data.""" fields = get_fields_from_selector(rule.field_selector, input_data, output_data) - + operator = ( + _humanize_guardrail_func(rule.detects_violation) or "violation check" + ) + field_paths = ", ".join( + {field_ref.path for _, field_ref in fields} + ) for field_value, field_ref in fields: if field_value is None: continue @@ -270,14 +313,19 @@ def evaluate_boolean_rule( # If function raises an exception, treat as failure violation_detected = True - if violation_detected: - operator = ( - _humanize_guardrail_func(rule.detects_violation) or "violation check" + if not violation_detected: + reason = format_guardrail_passed_validation_result_message( + field_ref=field_ref, + operator=operator, + rule_description=rule.rule_description, ) - reason = format_guardrail_error_message(field_ref, operator, None) - return False, reason + return True, reason - return True, "All boolean rule validations passed" + return False, get_validated_conditions_description( + field_path=field_paths, + operator=operator, + rule_description=rule.rule_description, + ) def evaluate_universal_rule( diff --git a/src/uipath/core/guardrails/guardrails.py b/src/uipath/core/guardrails/guardrails.py index 77382cb..54982cb 100644 --- a/src/uipath/core/guardrails/guardrails.py +++ b/src/uipath/core/guardrails/guardrails.py @@ -102,6 +102,11 @@ class WordRule(BaseModel): rule_type: Literal["word"] = Field(alias="$ruleType") field_selector: FieldSelector = Field(alias="fieldSelector") + rule_description: str | None = Field( + default=None, + exclude=True, + description="Human-friendly description of the rule condition.", + ) detects_violation: Callable[[str], bool] = Field( exclude=True, description="Function that returns True if the string violates the rule (validation should fail).", @@ -124,6 +129,11 @@ class NumberRule(BaseModel): rule_type: Literal["number"] = Field(alias="$ruleType") field_selector: FieldSelector = Field(alias="fieldSelector") + rule_description: str | None = Field( + default=None, + exclude=True, + description="Human-friendly description of the rule condition.", + ) detects_violation: Callable[[float], bool] = Field( exclude=True, description="Function that returns True if the number violates the rule (validation should fail).", @@ -137,6 +147,11 @@ class BooleanRule(BaseModel): rule_type: Literal["boolean"] = Field(alias="$ruleType") field_selector: FieldSelector = Field(alias="fieldSelector") + rule_description: str | None = Field( + default=None, + exclude=True, + description="Human-friendly description of the rule condition.", + ) detects_violation: Callable[[bool], bool] = Field( exclude=True, description="Function that returns True if the boolean violates the rule (validation should fail).", diff --git a/tests/guardrails/test_deterministic_guardrails_service.py b/tests/guardrails/test_deterministic_guardrails_service.py index 49f6dd3..35a4757 100644 --- a/tests/guardrails/test_deterministic_guardrails_service.py +++ b/tests/guardrails/test_deterministic_guardrails_service.py @@ -83,7 +83,7 @@ def test_evaluate_post_deterministic_guardrail_validation_passed( assert result.result == GuardrailValidationResultType.PASSED assert result.reason == "No rules to apply for output data." - def test_evaluate_post_deterministic_guardrail_validation_failed_age( + def test_evaluate_post_deterministic_guardrail_validation_passes_when_input_data_dont_violates_all_the_rules( self, service: DeterministicGuardrailsService, ) -> None: @@ -143,13 +143,13 @@ def test_evaluate_post_deterministic_guardrail_validation_failed_age( guardrail=deterministic_guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.result == GuardrailValidationResultType.PASSED assert ( result.reason - == "Input data didn't match the guardrail condition: [age] comparing function [(n): n < 21.0]" + == "Input data didn't match the guardrail condition: [isActive] comparing function [(b): b is not True]" ) - def test_evaluate_post_deterministic_guardrail_validation_failed_is_active( + def test_evaluate_post_deterministic_guardrail_validation_passes_when_input_and_output_data_dont_violates_all_the_rules( self, service: DeterministicGuardrailsService, ) -> None: @@ -211,64 +211,22 @@ def test_evaluate_post_deterministic_guardrail_validation_failed_is_active( guardrail=deterministic_guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.result == GuardrailValidationResultType.PASSED assert ( result.reason - == "Input data didn't match the guardrail condition: [isActive] comparing function [(b): b is not True]" - ) - - def test_evaluate_post_deterministic_guardrail_matches_regex_positive( - self, - service: DeterministicGuardrailsService, - ) -> None: - """Test deterministic guardrail validation passes when regex matches.""" - deterministic_guardrail = DeterministicGuardrail( - id="test-deterministic-id", - name="Regex Guardrail", - description="Test regex guardrail", - enabled_for_evals=True, - guardrail_type="custom", - selector=GuardrailSelector( - scopes=[GuardrailScope.TOOL], match_names=["test"] - ), - rules=[ - WordRule( - rule_type="word", - field_selector=SpecificFieldsSelector( - selector_type="specific", - fields=[ - FieldReference(path="userName", source=FieldSource.INPUT) - ], - ), - detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)), - ), - ], - ) - - # Input data with userName that matches the regex pattern - input_data = { - "userName": "test123", - } - output_data: dict[str, Any] = {} - - result = service.evaluate_post_deterministic_guardrail( - input_data=input_data, - output_data=output_data, - guardrail=deterministic_guardrail, + == "Input data didn't match the guardrail condition: [age] comparing function [(n): n < 21.0]" ) - assert result.result == GuardrailValidationResultType.PASSED - assert result.reason == "No rules to apply for output data." - - def test_evaluate_post_deterministic_guardrail_matches_regex_negative( + def test_evaluate_post_deterministic_guardrail_uses_rule_description( self, service: DeterministicGuardrailsService, ) -> None: - """Test deterministic guardrail validation fails when regex doesn't match.""" + """Ensure rule_description is returned when a rule fails validation.""" + friendly_description = "Username must include 'te' and a digit" deterministic_guardrail = DeterministicGuardrail( - id="test-deterministic-id", - name="Regex Guardrail", - description="Test regex guardrail", + id="test-rule-desc-id", + name="Regex Guardrail With Description", + description="Test regex guardrail with description", enabled_for_evals=True, guardrail_type="custom", selector=GuardrailSelector( @@ -283,18 +241,9 @@ def test_evaluate_post_deterministic_guardrail_matches_regex_negative( FieldReference(path="userName", source=FieldSource.INPUT) ], ), + rule_description=friendly_description, detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)), ), - NumberRule( - rule_type="number", - field_selector=SpecificFieldsSelector( - selector_type="specific", - fields=[ - FieldReference(path="status", source=FieldSource.OUTPUT) - ], - ), - detects_violation=lambda n: n != 200.0, - ), ], ) @@ -302,31 +251,24 @@ def test_evaluate_post_deterministic_guardrail_matches_regex_negative( input_data = { "userName": "test", } - output_data = { - "status": 200, - } - result = service.evaluate_post_deterministic_guardrail( + result = service.evaluate_pre_deterministic_guardrail( input_data=input_data, - output_data=output_data, guardrail=deterministic_guardrail, ) assert result.result == GuardrailValidationResultType.VALIDATION_FAILED - assert ( - result.reason - == 'Input data didn\'t match the guardrail condition: [userName] comparing function [(s): not bool(re.search(".*te.*3.*", s))]' - ) + assert result.reason == "Data matched all guardrail conditions: [Username must include 'te' and a digit]" - def test_evaluate_post_deterministic_guardrail_word_func_positive( + def test_evaluate_post_deterministic_guardrail_passes_validation_when_no_output_rules( self, service: DeterministicGuardrailsService, ) -> None: - """Test deterministic guardrail validation passes when word func returns True.""" + """Test deterministic guardrail validation passes when regex matches.""" deterministic_guardrail = DeterministicGuardrail( id="test-deterministic-id", - name="Word Func Guardrail", - description="Test word func guardrail", + name="Regex Guardrail", + description="Test regex guardrail", enabled_for_evals=True, guardrail_type="custom", selector=GuardrailSelector( @@ -341,14 +283,14 @@ def test_evaluate_post_deterministic_guardrail_word_func_positive( FieldReference(path="userName", source=FieldSource.INPUT) ], ), - detects_violation=lambda s: len(s) <= 5, + detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)), ), ], ) - # Input data with userName that passes the function check + # Input data with userName that matches the regex pattern input_data = { - "userName": "testuser", + "userName": "test123", } output_data: dict[str, Any] = {} @@ -361,15 +303,15 @@ def test_evaluate_post_deterministic_guardrail_word_func_positive( assert result.result == GuardrailValidationResultType.PASSED assert result.reason == "No rules to apply for output data." - def test_evaluate_post_deterministic_guardrail_word_func_negative( + def test_evaluate_post_deterministic_guardrail_failes_validation_when_data_macthes_rules( self, service: DeterministicGuardrailsService, ) -> None: - """Test deterministic guardrail validation fails when word func returns False.""" + """Test deterministic guardrail validation fails when regex doesn't match.""" deterministic_guardrail = DeterministicGuardrail( id="test-deterministic-id", - name="Word Func Guardrail", - description="Test word func guardrail", + name="Regex Guardrail", + description="Test regex guardrail", enabled_for_evals=True, guardrail_type="custom", selector=GuardrailSelector( @@ -384,7 +326,7 @@ def test_evaluate_post_deterministic_guardrail_word_func_negative( FieldReference(path="userName", source=FieldSource.INPUT) ], ), - detects_violation=lambda s: len(s) <= 5, + detects_violation=lambda s: not bool(re.search(".*te.*3.*", s)), ), NumberRule( rule_type="number", @@ -399,12 +341,12 @@ def test_evaluate_post_deterministic_guardrail_word_func_negative( ], ) - # Input data with userName that fails the function check + # Input data with userName that doesn't match the regex pattern input_data = { "userName": "test", } output_data = { - "status": 200, + "status": 201, } result = service.evaluate_post_deterministic_guardrail( @@ -414,6 +356,12 @@ def test_evaluate_post_deterministic_guardrail_word_func_negative( ) assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert ( + result.reason + == "Data matched all guardrail conditions: ['[userName] comparing function " + '[(s): not bool(re.search(".*te.*3.*", s))]', '[status] comparing function ' + "[(n): n != 200.0]']" + ) def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_violation( self, @@ -458,7 +406,7 @@ def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_v "userName": "andrei", } output_data = { - "status": 200, + "status": 201, } result = service.evaluate_post_deterministic_guardrail( @@ -470,10 +418,11 @@ def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_v assert result.result == GuardrailValidationResultType.VALIDATION_FAILED assert ( result.reason - == 'Input data didn\'t match the guardrail condition: [userName] comparing function [(s): "dre" in s]' + == "Data matched all guardrail conditions: [[userName] comparing function " + '[(s): "dre" in s], [status] comparing function [(n): n != 200.0]]' ) - def test_evaluate_post_deterministic_guardrail_number_func_positive( + def test_evaluate_post_deterministic_guardrail_number_func_passes_when_no_input_rules( self, service: DeterministicGuardrailsService, ) -> None: @@ -555,7 +504,7 @@ def test_evaluate_post_deterministic_guardrail_number_func_negative( "age": 70, } output_data = { - "status": 200, + "status": 201, } result = service.evaluate_post_deterministic_guardrail( @@ -566,11 +515,11 @@ def test_evaluate_post_deterministic_guardrail_number_func_negative( assert result.result == GuardrailValidationResultType.VALIDATION_FAILED - def test_should_trigger_policy_pre_execution_only_some_rules_not_met_returns_false( + def test_evaluate_post_execution_pases_when_only_some_rules_not_met( self, service: DeterministicGuardrailsService, ) -> None: - """Test pre-execution guardrail fails when some rules are not met.""" + """Test post-execution guardrail passes when only some rules are not met.""" guardrail = self._create_guardrail_for_pre_execution() input_data = { "userName": "John", @@ -587,7 +536,7 @@ def test_should_trigger_policy_pre_execution_only_some_rules_not_met_returns_fal guardrail=guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.result == GuardrailValidationResultType.PASSED def test_should_ignore_post_execution_guardrail_for_pre_execution_returns_false( self, @@ -637,11 +586,10 @@ def test_should_trigger_policy_post_execution_guardrail_for_pre_execution_return # Pre-execution guardrail should still pass in post-execution assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_with_output_fields_all_conditions_met_returns_true( + def test_should_trigger_policy_post_execution_with_output_fields_when_no_violation_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: - """Test post-execution guardrail passes when all conditions are met.""" guardrail = self._create_guardrail_for_post_execution() input_data = { "userName": "John", @@ -662,11 +610,10 @@ def test_should_trigger_policy_post_execution_with_output_fields_all_conditions_ assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_with_output_fields_input_conditions_not_met_returns_false( + def test_post_execution_with_output_fields_when_only_input_conditions_violated_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: - """Test post-execution guardrail fails when input conditions are not met.""" guardrail = self._create_guardrail_for_post_execution() input_data = { "userName": "John", @@ -685,13 +632,12 @@ def test_should_trigger_policy_post_execution_with_output_fields_input_condition guardrail=guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_with_output_fields_output_conditions_not_met_returns_false( + def test_post_execution_with_input_and_output_fields_output_when_only_output_conditions_violated_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: - """Test post-execution guardrail fails when output conditions are not met.""" guardrail = self._create_guardrail_for_post_execution() input_data = { "userName": "John", @@ -710,13 +656,12 @@ def test_should_trigger_policy_post_execution_with_output_fields_output_conditio guardrail=guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_multiple_rules_all_conditions_must_be_met_returns_true( + def test_post_execution_multiple_rules_when_all_conditions_when_no_condition_is_violated_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: - """Test post-execution guardrail with multiple rules passes when all conditions are met.""" guardrail = self._create_guardrail_with_multiple_rules() input_data = { "userName": "John", @@ -737,7 +682,7 @@ def test_should_trigger_policy_post_execution_multiple_rules_all_conditions_must assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_rule_with_multiple_conditions_all_must_be_met_returns_true( + def test_post_execution_rule_with_multiple_conditions_when_no_condition_is_violated_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -762,7 +707,7 @@ def test_should_trigger_policy_post_execution_rule_with_multiple_conditions_all_ assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_rule_with_multiple_conditions_one_condition_not_met_returns_false( + def test_post_execution_rule_with_multiple_conditions_when_only_some_conditions_are_violated_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -785,9 +730,34 @@ def test_should_trigger_policy_post_execution_rule_with_multiple_conditions_one_ guardrail=guardrail, ) + assert result.result == GuardrailValidationResultType.PASSED + + def test_post_execution_rule_with_multiple_conditions_when_all_condition_are_violated_then_returns_false( + self, + service: DeterministicGuardrailsService, + ) -> None: + """Test guardrail with multiple conditions fails when one condition is not met.""" + guardrail = self._create_guardrail_with_rule_having_multiple_conditions() + input_data = { + "userName": "John", + "age": 15, # < 18 + "isActive": False, # Not True + } + output_data = { + "result": "Success", + "status": 201, # Not 200 + "success": True, + } + + result = service.evaluate_post_deterministic_guardrail( + input_data=input_data, + output_data=output_data, + guardrail=guardrail, + ) + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED - def test_should_trigger_policy_post_execution_with_all_fields_selector_output_schema_has_fields_returns_true( + def test_post_execution_with_all_fields_selector_when_no_field_violates_condition_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -819,7 +789,7 @@ def test_should_trigger_policy_post_execution_with_all_fields_selector_output_sc } output_data = { "result": "Success", - "status": 25, # Matches the rule value + "status": 25, # Doesn't match the rule value "success": True, } @@ -831,7 +801,50 @@ def test_should_trigger_policy_post_execution_with_all_fields_selector_output_sc assert result.result == GuardrailValidationResultType.PASSED - def test_should_trigger_policy_post_execution_with_all_fields_selector_empty_output_schema_returns_true( + def test_post_execution_with_all_fields_selector_when_all_fields_violate_condition_then_returns_false( + self, + service: DeterministicGuardrailsService, + ) -> None: + guardrail = DeterministicGuardrail( + id="test-all-fields-id", + name="Guardrail With All Fields Selector", + description="Test all fields selector", + enabled_for_evals=True, + guardrail_type="custom", + selector=GuardrailSelector( + scopes=[GuardrailScope.TOOL], match_names=["test"] + ), + rules=[ + NumberRule( + rule_type="number", + field_selector=AllFieldsSelector( + selector_type="all", sources=[FieldSource.OUTPUT] + ), + detects_violation=lambda n: n != 25.0, + ), + ], + ) + + input_data = { + "userName": "John", + "age": 25, + "isActive": True, + } + output_data = { + "result": "Success", + "status": 20, # Matches the rule value + "success": True, + } + + result = service.evaluate_post_deterministic_guardrail( + input_data=input_data, + output_data=output_data, + guardrail=guardrail, + ) + + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + + def test_post_execution_with_all_fields_selector_when_empty_output_schema_then_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -891,7 +904,7 @@ def test_should_trigger_policy_pre_execution_always_rule_with_input_apply_to_ret assert result.result == GuardrailValidationResultType.VALIDATION_FAILED assert result.reason == "Always rule enforced" - def test_should_trigger_policy_pre_execution_always_rule_with_output_apply_to_returns_false( + def test_should_trigger_policy_pre_execution_always_rule_with_output_apply_to_returns_true( self, service: DeterministicGuardrailsService, ) -> None: @@ -911,7 +924,7 @@ def test_should_trigger_policy_pre_execution_always_rule_with_output_apply_to_re ) assert result.result == GuardrailValidationResultType.PASSED - assert result.reason == "All deterministic guardrail rules passed" + assert result.reason == "No rules to apply for input data" def test_should_trigger_policy_pre_execution_always_rule_with_input_and_output_apply_to_returns_true( self, @@ -1277,7 +1290,7 @@ def test_evaluate_post_deterministic_guardrail_only_output_rules_passes( ) assert result.result == GuardrailValidationResultType.PASSED - assert result.reason == "All deterministic guardrail rules passed" + assert result.reason == "Output data didn't match the guardrail condition: [status] comparing function [(n): n != 200.0]" def test_evaluate_post_deterministic_guardrail_only_always_rule_fails( self, From 711efc0d0e21d3725406e2e366ffe79d7799ac72 Mon Sep 17 00:00:00 2001 From: Valentina Bojan Date: Tue, 20 Jan 2026 16:05:02 +0200 Subject: [PATCH 3/3] fix: bump version --- pyproject.toml | 2 +- .../_deterministic_guardrails_service.py | 8 +++-- src/uipath/core/guardrails/_evaluators.py | 31 ++++++------------- .../test_deterministic_guardrails_service.py | 25 +++++++++------ uv.lock | 2 +- 5 files changed, 33 insertions(+), 35 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index df4d213..d93ae87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-core" -version = "0.1.8" +version = "0.1.9" description = "UiPath Core abstractions" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/src/uipath/core/guardrails/_deterministic_guardrails_service.py b/src/uipath/core/guardrails/_deterministic_guardrails_service.py index 2b88461..27be499 100644 --- a/src/uipath/core/guardrails/_deterministic_guardrails_service.py +++ b/src/uipath/core/guardrails/_deterministic_guardrails_service.py @@ -117,7 +117,10 @@ def _evaluate_deterministic_guardrail( output_data: dict[str, Any], guardrail: DeterministicGuardrail, ) -> GuardrailValidationResult: - """Evaluate deterministic guardrail rules against input and output data.""" + """Evaluate deterministic guardrail rules against input and output data. + + Validation fails only if ALL guardrail rules are violated. + """ validated_conditions: list[str] = [] for rule in guardrail.rules: @@ -142,8 +145,7 @@ def _evaluate_deterministic_guardrail( ) has_always_rule = any( - condition == "Always rule enforced" - for condition in validated_conditions + condition == "Always rule enforced" for condition in validated_conditions ) validated_conditions_str = ", ".join(validated_conditions) diff --git a/src/uipath/core/guardrails/_evaluators.py b/src/uipath/core/guardrails/_evaluators.py index 82c48b5..502ff7c 100644 --- a/src/uipath/core/guardrails/_evaluators.py +++ b/src/uipath/core/guardrails/_evaluators.py @@ -178,6 +178,7 @@ def format_guardrail_passed_validation_result_message( f"[{field_ref.path}] comparing function [{operator}]" ) + def get_validated_conditions_description( field_path: str, operator: str | None, @@ -194,12 +195,8 @@ def evaluate_word_rule( ) -> tuple[bool, str]: """Evaluate a word rule against input and output data.""" fields = get_fields_from_selector(rule.field_selector, input_data, output_data) - operator = ( - _humanize_guardrail_func(rule.detects_violation) or "violation check" - ) - field_paths = ", ".join( - {field_ref.path for _, field_ref in fields} - ) + operator = _humanize_guardrail_func(rule.detects_violation) or "violation check" + field_paths = ", ".join({field_ref.path for _, field_ref in fields}) for field_value, field_ref in fields: if field_value is None: @@ -237,15 +234,11 @@ def evaluate_word_rule( def evaluate_number_rule( rule: NumberRule, input_data: dict[str, Any], output_data: dict[str, Any] -) -> tuple[bool, str | None]: +) -> tuple[bool, str]: """Evaluate a number rule against input and output data.""" fields = get_fields_from_selector(rule.field_selector, input_data, output_data) - operator = ( - _humanize_guardrail_func(rule.detects_violation) or "violation check" - ) - field_paths = ", ".join( - {field_ref.path for _, field_ref in fields} - ) + operator = _humanize_guardrail_func(rule.detects_violation) or "violation check" + field_paths = ", ".join({field_ref.path for _, field_ref in fields}) for field_value, field_ref in fields: if field_value is None: continue @@ -285,15 +278,11 @@ def evaluate_boolean_rule( rule: BooleanRule, input_data: dict[str, Any], output_data: dict[str, Any], -) -> tuple[bool, str | None]: +) -> tuple[bool, str]: """Evaluate a boolean rule against input and output data.""" fields = get_fields_from_selector(rule.field_selector, input_data, output_data) - operator = ( - _humanize_guardrail_func(rule.detects_violation) or "violation check" - ) - field_paths = ", ".join( - {field_ref.path for _, field_ref in fields} - ) + operator = _humanize_guardrail_func(rule.detects_violation) or "violation check" + field_paths = ", ".join({field_ref.path for _, field_ref in fields}) for field_value, field_ref in fields: if field_value is None: continue @@ -331,7 +320,7 @@ def evaluate_boolean_rule( def evaluate_universal_rule( rule: UniversalRule, output_data: dict[str, Any], -) -> tuple[bool, str | None]: +) -> tuple[bool, str]: """Evaluate a universal rule against input and output data. Universal rules trigger based on the apply_to scope and execution phase: diff --git a/tests/guardrails/test_deterministic_guardrails_service.py b/tests/guardrails/test_deterministic_guardrails_service.py index 35a4757..1db2da4 100644 --- a/tests/guardrails/test_deterministic_guardrails_service.py +++ b/tests/guardrails/test_deterministic_guardrails_service.py @@ -258,7 +258,10 @@ def test_evaluate_post_deterministic_guardrail_uses_rule_description( ) assert result.result == GuardrailValidationResultType.VALIDATION_FAILED - assert result.reason == "Data matched all guardrail conditions: [Username must include 'te' and a digit]" + assert ( + result.reason + == "Data matched all guardrail conditions: [Username must include 'te' and a digit]" + ) def test_evaluate_post_deterministic_guardrail_passes_validation_when_no_output_rules( self, @@ -355,14 +358,15 @@ def test_evaluate_post_deterministic_guardrail_failes_validation_when_data_macth guardrail=deterministic_guardrail, ) - assert result.result == GuardrailValidationResultType.VALIDATION_FAILED - assert ( - result.reason - == "Data matched all guardrail conditions: ['[userName] comparing function " - '[(s): not bool(re.search(".*te.*3.*", s))]', '[status] comparing function ' - "[(n): n != 200.0]']" + expected_reason = ( + "Data matched all guardrail conditions: [[userName] comparing function " + '[(s): not bool(re.search(".*te.*3.*", s))], ' + "[status] comparing function [(n): n != 200.0]]" ) + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.reason == expected_reason + def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_violation( self, service: DeterministicGuardrailsService, @@ -419,7 +423,7 @@ def test_evaluate_post_deterministic_guardrail_word_contains_substring_detects_v assert ( result.reason == "Data matched all guardrail conditions: [[userName] comparing function " - '[(s): "dre" in s], [status] comparing function [(n): n != 200.0]]' + '[(s): "dre" in s], [status] comparing function [(n): n != 200.0]]' ) def test_evaluate_post_deterministic_guardrail_number_func_passes_when_no_input_rules( @@ -1290,7 +1294,10 @@ def test_evaluate_post_deterministic_guardrail_only_output_rules_passes( ) assert result.result == GuardrailValidationResultType.PASSED - assert result.reason == "Output data didn't match the guardrail condition: [status] comparing function [(n): n != 200.0]" + assert ( + result.reason + == "Output data didn't match the guardrail condition: [status] comparing function [(n): n != 200.0]" + ) def test_evaluate_post_deterministic_guardrail_only_always_rule_fails( self, diff --git a/uv.lock b/uv.lock index c6a7e6f..7aeccc3 100644 --- a/uv.lock +++ b/uv.lock @@ -991,7 +991,7 @@ wheels = [ [[package]] name = "uipath-core" -version = "0.1.8" +version = "0.1.9" source = { editable = "." } dependencies = [ { name = "opentelemetry-instrumentation" },