From cca24e0d9c5468f540eb3928a53015e9a2483c4f Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Tue, 20 Jan 2026 13:47:53 -0500 Subject: [PATCH 1/4] fix --- .../check_operators/dataframe_operators.py | 134 ++++++++++++------ 1 file changed, 94 insertions(+), 40 deletions(-) diff --git a/cdisc_rules_engine/check_operators/dataframe_operators.py b/cdisc_rules_engine/check_operators/dataframe_operators.py index 011835f65..7aab918df 100644 --- a/cdisc_rules_engine/check_operators/dataframe_operators.py +++ b/cdisc_rules_engine/check_operators/dataframe_operators.py @@ -1258,80 +1258,134 @@ def is_not_unique_set(self, other_value): @type_operator(FIELD_DATAFRAME) def is_not_unique_relationship(self, other_value): """ - Validates one-to-one relationship between - two columns (target and comparator) against a dataset. - One-to-one means that a pair of columns can be duplicated - but its integrity must not be violated: - one value of target always corresponds to - one value of comparator. - Examples: + Validates one-to-one relationship between two columns (target and comparator) + against a dataset. One-to-one means that a pair of columns can be duplicated + but its integrity must not be violated: one value of target always corresponds + to one value of comparator. + + A violation occurs when a NON-NULL value in either column maps to multiple + different values in the other column. Rows are flagged based on their non-null + values that participate in violations. - Valid dataset: - STUDYID STUDYDESC - 1 A - 2 B - 3 C - 1 A - 2 B - - Invalid dataset: - STUDYID STUDYDESC - 1 A - 2 A - 3 C + Examples: + Valid dataset: + STUDYID STUDYDESC + 1 A + 2 B + 3 C + 1 A + 2 B + + Invalid dataset: + STUDYID STUDYDESC + 1 A + 2 A + 3 C """ target = self.replace_prefix(other_value.get("target")) comparator = other_value.get("comparator") + if isinstance(comparator, list): comparator = self.replace_all_prefixes(comparator) else: comparator = self.replace_prefix(comparator) + df_subset = self.value[[target, comparator]].dropna(how="all") df_without_duplicates = df_subset.drop_duplicates() - violated_targets = self._find_relationship_violations( + + violated_targets, violated_comparators = self._find_relationship_violations( df_without_duplicates, target, comparator ) + result = self.value.convert_to_series([False] * len(self.value)) + + # Flag rows where the non-null target value is violated if violated_targets: clean_targets = { v for v in violated_targets if pd.notna(v) and v != "" and v is not None } - has_null_target = any( - pd.isna(v) or v == "" or v is None for v in violated_targets - ) if clean_targets: result = result | self.value[target].isin(clean_targets) - if has_null_target: - result = result | self.value[target].isna() + + # Flag rows where the non-null comparator value is violated + if violated_comparators: + clean_comparators = { + v + for v in violated_comparators + if pd.notna(v) and v != "" and v is not None + } + if clean_comparators: + result = result | self.value[comparator].isin(clean_comparators) + return result def _find_relationship_violations(self, df_without_duplicates, target, comparator): - """Find all target values that violate one-to-one relationship constraints.""" + """Find all values that violate one-to-one relationship constraints. + + Returns two sets: + - violated_targets: non-null target values that map to multiple comparators + - violated_comparators: non-null comparator values that map to multiple targets + + Only non-null values can be in violation. + """ + violated_targets = self._check_target_violations( + df_without_duplicates, target, comparator + ) + violated_comparators = self._check_comparator_violations( + df_without_duplicates, target, comparator + ) + + return violated_targets, violated_comparators + + def _check_target_violations(self, df_without_duplicates, target, comparator): + """Check for non-null target values that map to multiple comparators.""" violated_targets = set() for target_val in df_without_duplicates[target].dropna().unique(): + if target_val == "": + continue + target_rows = df_without_duplicates[ df_without_duplicates[target] == target_val ] - comparator_values = target_rows[comparator] - unique_comparators = set() - for comp_val in comparator_values: - if pd.isna(comp_val) or comp_val == "" or comp_val is None: - unique_comparators.add("NULL_PLACEHOLDER") - else: - unique_comparators.add(comp_val) - if len(unique_comparators) > 1: + + if self._has_multiple_mappings(target_rows[comparator]): violated_targets.add(target_val) + + return violated_targets + + def _check_comparator_violations(self, df_without_duplicates, target, comparator): + """Check for non-null comparator values that map to multiple targets.""" + violated_comparators = set() + for comp_val in df_without_duplicates[comparator].dropna().unique(): if comp_val == "" or pd.isna(comp_val): continue + comp_rows = df_without_duplicates[ df_without_duplicates[comparator] == comp_val ] - target_values = comp_rows[target] - if len(target_values) > 1: - for t_val in target_values: - violated_targets.add(t_val) - return violated_targets + + if self._has_multiple_mappings(comp_rows[target]): + violated_comparators.add(comp_val) + + return violated_comparators + + def _has_multiple_mappings(self, values): + """Check if a series of values contains multiple different values. + + Returns True if there are multiple non-null values, or at least one + non-null value plus null. + """ + unique_values = set() + has_null = False + + for val in values: + if pd.isna(val) or val == "" or val is None: + has_null = True + else: + unique_values.add(val) + + return len(unique_values) > 1 or (len(unique_values) >= 1 and has_null) @log_operator_execution @type_operator(FIELD_DATAFRAME) From 62a26849621ce5db71b186c1eb4aa8b100c17054 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Wed, 21 Jan 2026 16:14:19 -0500 Subject: [PATCH 2/4] current --- .../check_operators/dataframe_operators.py | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/cdisc_rules_engine/check_operators/dataframe_operators.py b/cdisc_rules_engine/check_operators/dataframe_operators.py index 7aab918df..86fc71fc7 100644 --- a/cdisc_rules_engine/check_operators/dataframe_operators.py +++ b/cdisc_rules_engine/check_operators/dataframe_operators.py @@ -1259,28 +1259,11 @@ def is_not_unique_set(self, other_value): def is_not_unique_relationship(self, other_value): """ Validates one-to-one relationship between two columns (target and comparator) - against a dataset. One-to-one means that a pair of columns can be duplicated - but its integrity must not be violated: one value of target always corresponds - to one value of comparator. + within a dataset. One-to-one means that a columns values can be duplicated + but it must always corresponds to one value of comparator. A violation occurs when a NON-NULL value in either column maps to multiple - different values in the other column. Rows are flagged based on their non-null - values that participate in violations. - - Examples: - Valid dataset: - STUDYID STUDYDESC - 1 A - 2 B - 3 C - 1 A - 2 B - - Invalid dataset: - STUDYID STUDYDESC - 1 A - 2 A - 3 C + different values in the other column """ target = self.replace_prefix(other_value.get("target")) comparator = other_value.get("comparator") From 157cb7f306043bc189915e09ce70395707cf8035 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Thu, 22 Jan 2026 11:40:56 -0500 Subject: [PATCH 3/4] update --- .../check_operators/dataframe_operators.py | 65 ++++++------------- 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/cdisc_rules_engine/check_operators/dataframe_operators.py b/cdisc_rules_engine/check_operators/dataframe_operators.py index 86fc71fc7..d5e824d45 100644 --- a/cdisc_rules_engine/check_operators/dataframe_operators.py +++ b/cdisc_rules_engine/check_operators/dataframe_operators.py @@ -1260,10 +1260,10 @@ def is_not_unique_relationship(self, other_value): """ Validates one-to-one relationship between two columns (target and comparator) within a dataset. One-to-one means that a columns values can be duplicated - but it must always corresponds to one value of comparator. + but it must always corresponds to one value of comparator and vice versa. A violation occurs when a NON-NULL value in either column maps to multiple - different values in the other column + different values in the other column. """ target = self.replace_prefix(other_value.get("target")) comparator = other_value.get("comparator") @@ -1275,22 +1275,18 @@ def is_not_unique_relationship(self, other_value): df_subset = self.value[[target, comparator]].dropna(how="all") df_without_duplicates = df_subset.drop_duplicates() - violated_targets, violated_comparators = self._find_relationship_violations( df_without_duplicates, target, comparator ) + # flag violations from target and comparator result = self.value.convert_to_series([False] * len(self.value)) - - # Flag rows where the non-null target value is violated if violated_targets: clean_targets = { v for v in violated_targets if pd.notna(v) and v != "" and v is not None } if clean_targets: result = result | self.value[target].isin(clean_targets) - - # Flag rows where the non-null comparator value is violated if violated_comparators: clean_comparators = { v @@ -1303,59 +1299,36 @@ def is_not_unique_relationship(self, other_value): return result def _find_relationship_violations(self, df_without_duplicates, target, comparator): - """Find all values that violate one-to-one relationship constraints. - + """ + Find all values that violate one-to-one relationship constraints. Returns two sets: - violated_targets: non-null target values that map to multiple comparators - violated_comparators: non-null comparator values that map to multiple targets - - Only non-null values can be in violation. """ - violated_targets = self._check_target_violations( + violated_targets = self._check_column_violations( df_without_duplicates, target, comparator ) - violated_comparators = self._check_comparator_violations( - df_without_duplicates, target, comparator + violated_comparators = self._check_column_violations( + df_without_duplicates, comparator, target ) return violated_targets, violated_comparators - def _check_target_violations(self, df_without_duplicates, target, comparator): - """Check for non-null target values that map to multiple comparators.""" - violated_targets = set() - for target_val in df_without_duplicates[target].dropna().unique(): - if target_val == "": + def _check_column_violations(self, df_without_duplicates, key_column, value_column): + violated_keys = set() + for key_val in df_without_duplicates[key_column].dropna().unique(): + if key_val == "": continue - - target_rows = df_without_duplicates[ - df_without_duplicates[target] == target_val + key_rows = df_without_duplicates[ + df_without_duplicates[key_column] == key_val ] - - if self._has_multiple_mappings(target_rows[comparator]): - violated_targets.add(target_val) - - return violated_targets - - def _check_comparator_violations(self, df_without_duplicates, target, comparator): - """Check for non-null comparator values that map to multiple targets.""" - violated_comparators = set() - - for comp_val in df_without_duplicates[comparator].dropna().unique(): - if comp_val == "" or pd.isna(comp_val): - continue - - comp_rows = df_without_duplicates[ - df_without_duplicates[comparator] == comp_val - ] - - if self._has_multiple_mappings(comp_rows[target]): - violated_comparators.add(comp_val) - - return violated_comparators + if self._has_multiple_mappings(key_rows[value_column]): + violated_keys.add(key_val) + return violated_keys def _has_multiple_mappings(self, values): - """Check if a series of values contains multiple different values. - + """ + Check if a series of values contains multiple different values. Returns True if there are multiple non-null values, or at least one non-null value plus null. """ From 716d53498a38b462d204ca34a16f337b273c37b0 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Tue, 27 Jan 2026 08:54:24 -0500 Subject: [PATCH 4/4] tests --- .../check_operators/dataframe_operators.py | 72 ++++++++-- .../test_relationship_integrity_checks.py | 132 ++++++++++++++++++ 2 files changed, 189 insertions(+), 15 deletions(-) diff --git a/cdisc_rules_engine/check_operators/dataframe_operators.py b/cdisc_rules_engine/check_operators/dataframe_operators.py index 049174b33..8e26e7760 100644 --- a/cdisc_rules_engine/check_operators/dataframe_operators.py +++ b/cdisc_rules_engine/check_operators/dataframe_operators.py @@ -1273,19 +1273,19 @@ def is_not_unique_relationship(self, other_value): Validates one-to-one relationship between two columns (target and comparator) within a dataset. One-to-one means that a columns values can be duplicated but it must always corresponds to one value of comparator and vice versa. - A violation occurs when a NON-NULL value in either column maps to multiple different values in the other column. """ target = self.replace_prefix(other_value.get("target")) comparator = other_value.get("comparator") - if isinstance(comparator, list): comparator = self.replace_all_prefixes(comparator) + columns = [target] + comparator else: comparator = self.replace_prefix(comparator) + columns = [target, comparator] - df_subset = self.value[[target, comparator]].dropna(how="all") + df_subset = self.value[columns].dropna(how="all") df_without_duplicates = df_subset.drop_duplicates() violated_targets, violated_comparators = self._find_relationship_violations( df_without_duplicates, target, comparator @@ -1306,7 +1306,15 @@ def is_not_unique_relationship(self, other_value): if pd.notna(v) and v != "" and v is not None } if clean_comparators: - result = result | self.value[comparator].isin(clean_comparators) + if isinstance(comparator, list): + # For multi-column comparators, match on tuple combinations + for comp_tuple in clean_comparators: + mask = self.value.convert_to_series([True] * len(self.value)) + for i, col in enumerate(comparator): + mask = mask & (self.value[col] == comp_tuple[i]) + result = result | mask + else: + result = result | self.value[comparator].isin(clean_comparators) return result @@ -1323,21 +1331,57 @@ def _find_relationship_violations(self, df_without_duplicates, target, comparato violated_comparators = self._check_column_violations( df_without_duplicates, comparator, target ) - return violated_targets, violated_comparators def _check_column_violations(self, df_without_duplicates, key_column, value_column): violated_keys = set() - for key_val in df_without_duplicates[key_column].dropna().unique(): - if key_val == "": - continue - key_rows = df_without_duplicates[ - df_without_duplicates[key_column] == key_val - ] - if self._has_multiple_mappings(key_rows[value_column]): - violated_keys.add(key_val) + if isinstance(key_column, list): + key_data = df_without_duplicates[key_column] + unique_keys = [tuple(row) for row in key_data.drop_duplicates().values] + else: + unique_keys = df_without_duplicates[key_column].dropna().unique() + for key_val in unique_keys: + if isinstance(key_column, list): + if any(v == "" or pd.isna(v) or v is None for v in key_val): + continue + mask = pd.Series( + [True] * len(df_without_duplicates), + index=df_without_duplicates.index, + ) + for i, col in enumerate(key_column): + mask = mask & (df_without_duplicates[col] == key_val[i]) + key_rows = df_without_duplicates[mask] + else: + if key_val == "": + continue + key_rows = df_without_duplicates[ + df_without_duplicates[key_column] == key_val + ] + + if isinstance(value_column, list): + value_tuples = [tuple(row) for row in key_rows[value_column].values] + if self._has_multiple_mappings_for_tuples(value_tuples): + violated_keys.add(key_val) + else: + if self._has_multiple_mappings(key_rows[value_column]): + violated_keys.add(key_val) return violated_keys + def _has_multiple_mappings_for_tuples(self, value_tuples): + """ + Check if a list of tuples contains multiple different non-null tuples. + Returns True if there are multiple non-null tuples, or at least one + non-null tuple plus a null tuple. + """ + unique_tuples = set() + has_null = False + for val_tuple in value_tuples: + if any(pd.isna(v) or v == "" or v is None for v in val_tuple): + has_null = True + else: + unique_tuples.add(val_tuple) + return len(unique_tuples) > 1 or (len(unique_tuples) >= 1 and has_null) + def _has_multiple_mappings(self, values): """ Check if a series of values contains multiple different values. @@ -1346,13 +1390,11 @@ def _has_multiple_mappings(self, values): """ unique_values = set() has_null = False - for val in values: if pd.isna(val) or val == "" or val is None: has_null = True else: unique_values.add(val) - return len(unique_values) > 1 or (len(unique_values) >= 1 and has_null) @log_operator_execution diff --git a/tests/unit/test_check_operators/test_relationship_integrity_checks.py b/tests/unit/test_check_operators/test_relationship_integrity_checks.py index 943c93b54..df385d635 100644 --- a/tests/unit/test_check_operators/test_relationship_integrity_checks.py +++ b/tests/unit/test_check_operators/test_relationship_integrity_checks.py @@ -207,6 +207,138 @@ def test_has_same_values(data, dataset_type, expected_result): DaskDataset, [True, True, True, False], ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3"], + "VISITNUM": [1, 2, 1, 3], + "target": ["Consulting", None, "Consulting", "Treatment"], + }, + "VISITNUM", + PandasDataset, + [False, False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3"], + "VISITNUM": [1, None, 2, 3], + "target": ["Consulting", "Surgery", "Surgery", "Treatment"], + }, + "VISITNUM", + PandasDataset, + [False, True, True, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [1, None, 1], + "target": ["Consulting", "Consulting", "Consulting"], + }, + "VISITNUM", + PandasDataset, + [True, True, True], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [1, 1, 1], + "target": ["Consulting", None, "Surgery"], + }, + "VISITNUM", + DaskDataset, + [True, True, True], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [1, 2, ""], + "target": ["Consulting", "Surgery", "Treatment"], + }, + "VISITNUM", + PandasDataset, + [False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [1, "", 1], + "target": ["Consulting", "Consulting", "Consulting"], + }, + "VISITNUM", + PandasDataset, + [True, True, True], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3"], + "VISITNUM": [1, None, None, 2], + "target": ["Consulting", None, None, "Surgery"], + }, + "VISITNUM", + DaskDataset, + [False, False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [None, None, None], + "target": [None, None, None], + }, + "VISITNUM", + PandasDataset, + [False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3", "TEST-4"], + "VISITNUM": [1, 1, 2, None, 2], + "target": ["A", "B", "A", "A", "C"], + }, + "VISITNUM", + PandasDataset, + [True, True, True, True, True], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3"], + "VISITNUM": [1, 2, 3, None], + "target": ["A", "B", "C", "D"], + }, + "VISITNUM", + DaskDataset, + [False, False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [None, "", 1], + "target": ["A", "B", "C"], + }, + "VISITNUM", + PandasDataset, + [False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISIT": ["V1", "V2", "V1"], + "VISITNUM": [1, 2, 1], + "target": ["A", "B", "A"], + }, + ["VISIT", "VISITNUM"], + PandasDataset, + [False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3"], + "VISIT": ["V1", "V1", "V2", "V1"], + "VISITNUM": [1, 1, 2, 1], + "target": ["A", "B", "C", "A"], + }, + ["VISIT", "VISITNUM"], + DaskDataset, + [True, True, False, True], + ), ], ) def test_is_not_unique_relationship(data, comparator, dataset_type, expected_result):