From 7b5e74f1c41ba5b008ce62d6d9ab8e9a0d085581 Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Wed, 24 Jun 2026 13:54:06 -0400 Subject: [PATCH 1/7] Added type_insensitive param for equal/not_equal string part operator. Added tests for this and a negative test for the original operator without the param --- .../check_operators/dataframe_operators.py | 7 +- resources/schema/rule-merged/Operator.json | 6 + resources/schema/rule/Operator.json | 10 +- resources/schema/rule/Operator.md | 6 +- .../test_string_comparison.py | 119 ++++++++++++++++++ 5 files changed, 144 insertions(+), 4 deletions(-) diff --git a/cdisc_rules_engine/check_operators/dataframe_operators.py b/cdisc_rules_engine/check_operators/dataframe_operators.py index ddff41a59..66d2d8fdf 100644 --- a/cdisc_rules_engine/check_operators/dataframe_operators.py +++ b/cdisc_rules_engine/check_operators/dataframe_operators.py @@ -809,6 +809,7 @@ def equals_string_part(self, other_value): column with a regex """ target = other_value.get("target") + type_insensitive = other_value.get("type_insensitive", False) comparator = other_value.get("comparator") regex = other_value.get("regex") value_is_literal: bool = other_value.get("value_is_literal", False) @@ -821,7 +822,11 @@ def equals_string_part(self, other_value): self.value[parsed_id] = parsed_data return self.value.apply( lambda row: self._check_equality( - row, target, parsed_id, value_is_literal=False + row, + target, + parsed_id, + value_is_literal=False, + type_insensitive=type_insensitive, ), axis=1, ) diff --git a/resources/schema/rule-merged/Operator.json b/resources/schema/rule-merged/Operator.json index 1c1f13a64..0f8ca05a8 100644 --- a/resources/schema/rule-merged/Operator.json +++ b/resources/schema/rule-merged/Operator.json @@ -136,6 +136,9 @@ "operator": { "const": "does_not_equal_string_part", "markdownDescription": "\nComplement of `equals_string_part`\n" + }, + "type_insensitive": { + "type": "boolean" } }, "required": ["operator", "value", "regex"], @@ -224,6 +227,9 @@ "operator": { "const": "equals_string_part", "markdownDescription": "\nChecks that the values in the target column equal the result of parsing the value in the comparison column with a regex\n\n> RDOMAIN equals characters 5 and 6 of SUPP dataset name\n\n```yaml\n- name: RDOMAIN\n operator: equals_string_part\n value: dataset_name\n regex: \".{4}(..).*\"\n```\n" + }, + "type_insensitive": { + "type": "boolean" } }, "required": ["operator", "value", "regex"], diff --git a/resources/schema/rule/Operator.json b/resources/schema/rule/Operator.json index 61ed71f2f..021e79f89 100644 --- a/resources/schema/rule/Operator.json +++ b/resources/schema/rule/Operator.json @@ -73,7 +73,10 @@ "type": "object" }, { - "properties": { "operator": { "const": "does_not_equal_string_part" } }, + "properties": { + "operator": { "const": "does_not_equal_string_part" }, + "type_insensitive": { "type": "boolean" } + }, "required": ["operator", "value", "regex"], "type": "object" }, @@ -120,7 +123,10 @@ "type": "object" }, { - "properties": { "operator": { "const": "equals_string_part" } }, + "properties": { + "operator": { "const": "equals_string_part" }, + "type_insensitive": { "type": "boolean" } + }, "required": ["operator", "value", "regex"], "type": "object" }, diff --git a/resources/schema/rule/Operator.md b/resources/schema/rule/Operator.md index 4685cac0f..0a46ea122 100644 --- a/resources/schema/rule/Operator.md +++ b/resources/schema/rule/Operator.md @@ -156,17 +156,21 @@ Text-based operations including regex pattern matching, substring operations, pr ### does_not_equal_string_part -Complement of `equals_string_part` +Complement of `equals_string_part`. Also has the optional parameter 'type_insensitive'. ### equals_string_part Checks that the values in the target column equal the result of parsing the value in the comparison column with a regex +Has optional parameter: + +- 'type_insensitive' when true, both values are converted to strings before comparison to handle type mismatches between string and numeric data. NOTE: all trailing zeroes will be removed in both strings and floats. > RDOMAIN equals characters 5 and 6 of SUPP dataset name ```yaml - name: RDOMAIN operator: equals_string_part + type_insensitive: true value: dataset_name regex: ".{4}(..).*" ``` diff --git a/tests/unit/test_check_operators/test_string_comparison.py b/tests/unit/test_check_operators/test_string_comparison.py index 1ec3277d3..522d6ad5f 100644 --- a/tests/unit/test_check_operators/test_string_comparison.py +++ b/tests/unit/test_check_operators/test_string_comparison.py @@ -22,6 +22,20 @@ DaskDataset, [True, True, False], ), + ( + {"VAR2": ["<40"], "target": [40]}, + "VAR2", + ".(.*)", + PandasDataset, + [False], + ), + ( + {"VAR2": ["<40"], "target": [40]}, + "VAR2", + ".(.*)", + DaskDataset, + [False], + ), ], ) def test_equals_string_part(data, comparator, regex, dataset_type, expected_result): @@ -33,6 +47,111 @@ def test_equals_string_part(data, comparator, regex, dataset_type, expected_resu assert result.equals(df.convert_to_series(expected_result)) +@pytest.mark.parametrize( + "data,comparator,operator,regex,dataset_type,expected_result", + [ + ( + {"VAR2": [">=40", "<=50"], "target": [40, 50]}, + "VAR2", + "equals_string_part", + ".{2}(.*)", + PandasDataset, + [True, True], + ), + ( + {"VAR2": [">=40", "<=50"], "target": [40, 50]}, + "VAR2", + "equals_string_part", + ".{2}(.*)", + DaskDataset, + [True, True], + ), + ( + {"VAR2": [">=42 ", "<=55 "], "target": [40, 50]}, + "VAR2", + "does_not_equal_string_part", + ".{2}(.*)", + PandasDataset, + [True, True], + ), + ( + {"VAR2": [">=42 ", "<=55 "], "target": [40, 50]}, + "VAR2", + "does_not_equal_string_part", + ".{2}(.*)", + DaskDataset, + [True, True], + ), + ( + {"VAR2": [">40", "<50"], "target": [40, 50]}, + "VAR2", + "equals_string_part", + ".(.*)", + DaskDataset, + [True, True], + ), + ( + {"VAR2": [">45", "<52"], "target": [40, 50]}, + "VAR2", + "does_not_equal_string_part", + ".(.*)", + PandasDataset, + [True, True], + ), + ( + {"VAR2": [">45", "<52"], "target": [40, 50]}, + "VAR2", + "does_not_equal_string_part", + ".(.*)", + DaskDataset, + [True, True], + ), + ( + {"VAR2": [">40", "<50"], "target": [40.0, 50.0]}, + "VAR2", + "equals_string_part", + ".(.*)", + PandasDataset, + [True, True], + ), + ( + {"VAR2": [">45", "<52"], "target": [40.0, 50.0]}, + "VAR2", + "does_not_equal_string_part", + ".(.*)", + DaskDataset, + [True, True], + ), + ], +) +def test_equals_string_part_type_insensitive( + data, comparator, operator, regex, dataset_type, expected_result +): + df = dataset_type.from_dict(data) + dataframe_type = DataframeType({"value": df}) + + if operator == "equals_string_part": + result = dataframe_type.equals_string_part( + { + "target": "target", + "comparator": comparator, + "regex": regex, + "type_insensitive": True, + } + ) + else: + result = dataframe_type.does_not_equal_string_part( + { + "target": "target", + "comparator": comparator, + "regex": regex, + "type_insensitive": True, + } + ) + + assert result.equals(df.convert_to_series(expected_result)) + + @pytest.mark.parametrize( "data,comparator,regex,dataset_type,value_is_literal,expected_result", [ From 4d2d10854b46295d458d7f30ea706ae762598e83 Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Wed, 24 Jun 2026 14:50:45 -0400 Subject: [PATCH 2/7] ran the merge schema workflow --- resources/schema/rule-merged/Operator.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/schema/rule-merged/Operator.json b/resources/schema/rule-merged/Operator.json index 0f8ca05a8..d7d57b73a 100644 --- a/resources/schema/rule-merged/Operator.json +++ b/resources/schema/rule-merged/Operator.json @@ -135,7 +135,7 @@ "properties": { "operator": { "const": "does_not_equal_string_part", - "markdownDescription": "\nComplement of `equals_string_part`\n" + "markdownDescription": "\nComplement of `equals_string_part`. Also has the optional parameter 'type_insensitive'.\n" }, "type_insensitive": { "type": "boolean" @@ -226,7 +226,7 @@ "properties": { "operator": { "const": "equals_string_part", - "markdownDescription": "\nChecks that the values in the target column equal the result of parsing the value in the comparison column with a regex\n\n> RDOMAIN equals characters 5 and 6 of SUPP dataset name\n\n```yaml\n- name: RDOMAIN\n operator: equals_string_part\n value: dataset_name\n regex: \".{4}(..).*\"\n```\n" + "markdownDescription": "\nChecks that the values in the target column equal the result of parsing the value in the comparison column with a regex\nHas optional parameter:\n\n- 'type_insensitive' when true, both values are converted to strings before comparison to handle type mismatches between string and numeric data. NOTE: all trailing zeroes will be removed in both strings and floats.\n\n> RDOMAIN equals characters 5 and 6 of SUPP dataset name\n\n```yaml\n- name: RDOMAIN\n operator: equals_string_part\n type_insensitive: true\n value: dataset_name\n regex: \".{4}(..).*\"\n```\n" }, "type_insensitive": { "type": "boolean" From 8f4f8679a466bd7a414f22dfff499ef8a49b15ef Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Fri, 26 Jun 2026 15:57:33 -0400 Subject: [PATCH 3/7] adjusted for pr feedback --- resources/schema/rule/Operator.md | 2 +- .../test_string_comparison.py | 108 ++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/resources/schema/rule/Operator.md b/resources/schema/rule/Operator.md index 0a46ea122..b05e9a22b 100644 --- a/resources/schema/rule/Operator.md +++ b/resources/schema/rule/Operator.md @@ -163,7 +163,7 @@ Complement of `equals_string_part`. Also has the optional parameter 'type_insens Checks that the values in the target column equal the result of parsing the value in the comparison column with a regex Has optional parameter: -- 'type_insensitive' when true, both values are converted to strings before comparison to handle type mismatches between string and numeric data. NOTE: all trailing zeroes will be removed in both strings and floats. +- 'type_insensitive' when true, both values are converted to strings before comparison to handle type mismatches between string and numeric data. NOTE: all leading and trailing zeroes will be removed in both strings and floats. > RDOMAIN equals characters 5 and 6 of SUPP dataset name diff --git a/tests/unit/test_check_operators/test_string_comparison.py b/tests/unit/test_check_operators/test_string_comparison.py index 522d6ad5f..cd0d68d20 100644 --- a/tests/unit/test_check_operators/test_string_comparison.py +++ b/tests/unit/test_check_operators/test_string_comparison.py @@ -46,6 +46,66 @@ def test_equals_string_part(data, comparator, regex, dataset_type, expected_resu ) assert result.equals(df.convert_to_series(expected_result)) +@pytest.mark.parametrize( + "data,comparator,operator,dataset_type,expected_result", + [ + ( + {"VAR2": ["40", "50"], "target": [40, 50]}, + "VAR2", + "equals_string_part", + PandasDataset, + [True, True], + ), + ( + {"VAR2": ["40", "50"], "target": [40, 50]}, + "VAR2", + "equals_string_part", + DaskDataset, + [True, True], + ), + ( + {"VAR2": ["", ""], "target": [0, 0]}, + "VAR2", + "equals_string_part", + PandasDataset, + [True, True], + ), + ( + {"VAR2": ["", ""], "target": [0, 0]}, + "VAR2", + "equals_string_part", + DaskDataset, + [True, True], + ) + ], +) +def test_equals_string_part_no_regex_type_insensitive( + data, comparator, operator, dataset_type, expected_result +): + df = dataset_type.from_dict(data) + dataframe_type = DataframeType({"value": df}) + + if operator == "equals_string_part": + result = dataframe_type.equals_string_part( + { + "target": "target", + "comparator": comparator, + "regex": "(.*)", + "type_insensitive": True, + } + ) + else: + result = dataframe_type.does_not_equal_string_part( + { + "target": "target", + "comparator": comparator, + "regex": "(.*)", + "type_insensitive": True, + } + ) + + assert result.equals(df.convert_to_series(expected_result)) + @pytest.mark.parametrize( "data,comparator,operator,regex,dataset_type,expected_result", @@ -66,6 +126,22 @@ def test_equals_string_part(data, comparator, regex, dataset_type, expected_resu DaskDataset, [True, True], ), + ( + {"VAR2": [">=040", "<=050"], "target": [40, 50]}, + "VAR2", + "equals_string_part", + ".{2}(.*)", + PandasDataset, + [True, True], + ), + ( + {"VAR2": [">=040", "<=050"], "target": [40, 50]}, + "VAR2", + "equals_string_part", + ".{2}(.*)", + DaskDataset, + [True, True], + ), ( {"VAR2": [">=42 ", "<=55 "], "target": [40, 50]}, "VAR2", @@ -122,6 +198,38 @@ def test_equals_string_part(data, comparator, regex, dataset_type, expected_resu DaskDataset, [True, True], ), + ( + {"VAR2": [">40", "<50"], "target": [40, 50]}, + "VAR2", + "does_not_equal_string_part", + ".(.*)", + PandasDataset, + [False, False], + ), + ( + {"VAR2": [">40", "<50"], "target": [40, 50]}, + "VAR2", + "does_not_equal_string_part", + ".(.*)", + DaskDataset, + [False, False], + ), + ( + {"VAR2": ["0040", "0050"], "target": [40, 50]}, + "VAR2", + "equals_string_part", + "(.*)", + PandasDataset, + [True, True], + ), + ( + {"VAR2": ["0040", "0050"], "target": [40, 50]}, + "VAR2", + "equals_string_part", + "(.*)", + DaskDataset, + [True, True], + ), ], ) def test_equals_string_part_type_insensitive( From 1144ef51664de79e160381a19c52647b5ad92bd3 Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Mon, 29 Jun 2026 10:09:20 -0400 Subject: [PATCH 4/7] Adjusted for PR feedback --- .../test_string_comparison.py | 93 +++++++------------ 1 file changed, 32 insertions(+), 61 deletions(-) diff --git a/tests/unit/test_check_operators/test_string_comparison.py b/tests/unit/test_check_operators/test_string_comparison.py index cd0d68d20..647ef08e2 100644 --- a/tests/unit/test_check_operators/test_string_comparison.py +++ b/tests/unit/test_check_operators/test_string_comparison.py @@ -46,67 +46,6 @@ def test_equals_string_part(data, comparator, regex, dataset_type, expected_resu ) assert result.equals(df.convert_to_series(expected_result)) -@pytest.mark.parametrize( - "data,comparator,operator,dataset_type,expected_result", - [ - ( - {"VAR2": ["40", "50"], "target": [40, 50]}, - "VAR2", - "equals_string_part", - PandasDataset, - [True, True], - ), - ( - {"VAR2": ["40", "50"], "target": [40, 50]}, - "VAR2", - "equals_string_part", - DaskDataset, - [True, True], - ), - ( - {"VAR2": ["", ""], "target": [0, 0]}, - "VAR2", - "equals_string_part", - PandasDataset, - [True, True], - ), - ( - {"VAR2": ["", ""], "target": [0, 0]}, - "VAR2", - "equals_string_part", - DaskDataset, - [True, True], - ) - ], -) -def test_equals_string_part_no_regex_type_insensitive( - data, comparator, operator, dataset_type, expected_result -): - df = dataset_type.from_dict(data) - dataframe_type = DataframeType({"value": df}) - - if operator == "equals_string_part": - result = dataframe_type.equals_string_part( - { - "target": "target", - "comparator": comparator, - "regex": "(.*)", - "type_insensitive": True, - } - ) - else: - result = dataframe_type.does_not_equal_string_part( - { - "target": "target", - "comparator": comparator, - "regex": "(.*)", - "type_insensitive": True, - } - ) - - assert result.equals(df.convert_to_series(expected_result)) - - @pytest.mark.parametrize( "data,comparator,operator,regex,dataset_type,expected_result", [ @@ -230,6 +169,38 @@ def test_equals_string_part_no_regex_type_insensitive( DaskDataset, [True, True], ), + ( + {"VAR2": ["40", "50"], "target": [40, 50]}, + "VAR2", + "does_not_equal_string_part", + "^[A-Za-z]+$", + PandasDataset, + [True, True], + ), + ( + {"VAR2": ["40", "50"], "target": [40, 50]}, + "VAR2", + "does_not_equal_string_part", + "^[A-Za-z]+$", + DaskDataset, + [True, True], + ), + ( + {"VAR2": ["", ""], "target": [None, None]}, + "VAR2", + "equals_string_part", + "(.*)", + PandasDataset, + [False, False], + ), + ( + {"VAR2": ["", ""], "target": [None, None]}, + "VAR2", + "equals_string_part", + "(.*)", + DaskDataset, + [False, False], + ) ], ) def test_equals_string_part_type_insensitive( From 5ee41189482bc0ebdbc17a7a2c6f85a075c6d6c8 Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Mon, 29 Jun 2026 10:24:35 -0400 Subject: [PATCH 5/7] Added line for check script failure --- tests/unit/test_check_operators/test_string_comparison.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_check_operators/test_string_comparison.py b/tests/unit/test_check_operators/test_string_comparison.py index 647ef08e2..8ef269d63 100644 --- a/tests/unit/test_check_operators/test_string_comparison.py +++ b/tests/unit/test_check_operators/test_string_comparison.py @@ -46,6 +46,7 @@ def test_equals_string_part(data, comparator, regex, dataset_type, expected_resu ) assert result.equals(df.convert_to_series(expected_result)) + @pytest.mark.parametrize( "data,comparator,operator,regex,dataset_type,expected_result", [ From 75c8ed4bddd3252b9fbebedc19a49513bd6de3e9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 9 Jul 2026 19:14:36 +0000 Subject: [PATCH 6/7] Update merged schema files with markdown descriptions --- resources/schema/rule-merged/Operator.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/schema/rule-merged/Operator.json b/resources/schema/rule-merged/Operator.json index d7d57b73a..e1b988e6a 100644 --- a/resources/schema/rule-merged/Operator.json +++ b/resources/schema/rule-merged/Operator.json @@ -226,7 +226,7 @@ "properties": { "operator": { "const": "equals_string_part", - "markdownDescription": "\nChecks that the values in the target column equal the result of parsing the value in the comparison column with a regex\nHas optional parameter:\n\n- 'type_insensitive' when true, both values are converted to strings before comparison to handle type mismatches between string and numeric data. NOTE: all trailing zeroes will be removed in both strings and floats.\n\n> RDOMAIN equals characters 5 and 6 of SUPP dataset name\n\n```yaml\n- name: RDOMAIN\n operator: equals_string_part\n type_insensitive: true\n value: dataset_name\n regex: \".{4}(..).*\"\n```\n" + "markdownDescription": "\nChecks that the values in the target column equal the result of parsing the value in the comparison column with a regex\nHas optional parameter:\n\n- 'type_insensitive' when true, both values are converted to strings before comparison to handle type mismatches between string and numeric data. NOTE: all leading and trailing zeroes will be removed in both strings and floats.\n\n> RDOMAIN equals characters 5 and 6 of SUPP dataset name\n\n```yaml\n- name: RDOMAIN\n operator: equals_string_part\n type_insensitive: true\n value: dataset_name\n regex: \".{4}(..).*\"\n```\n" }, "type_insensitive": { "type": "boolean" From 61a94de4229f2fec76027f48ecce7f590100bc22 Mon Sep 17 00:00:00 2001 From: Gerry Campion Date: Thu, 9 Jul 2026 16:57:09 -0400 Subject: [PATCH 7/7] black formatting --- tests/unit/test_check_operators/test_string_comparison.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_check_operators/test_string_comparison.py b/tests/unit/test_check_operators/test_string_comparison.py index 8ef269d63..c4a3d7fbc 100644 --- a/tests/unit/test_check_operators/test_string_comparison.py +++ b/tests/unit/test_check_operators/test_string_comparison.py @@ -201,7 +201,7 @@ def test_equals_string_part(data, comparator, regex, dataset_type, expected_resu "(.*)", DaskDataset, [False, False], - ) + ), ], ) def test_equals_string_part_type_insensitive(