diff --git a/cdisc_rules_engine/check_operators/dataframe_operators.py b/cdisc_rules_engine/check_operators/dataframe_operators.py index b86e5572d..8e26e7760 100644 --- a/cdisc_rules_engine/check_operators/dataframe_operators.py +++ b/cdisc_rules_engine/check_operators/dataframe_operators.py @@ -1270,80 +1270,132 @@ def is_not_unique_set(self, other_value): @type_operator(FIELD_DATAFRAME) def is_not_unique_relationship(self, other_value): """ - Validates one-to-one relationship between - two columns (target and comparator) against a dataset. - One-to-one means that a pair of columns can be duplicated - but its integrity must not be violated: - one value of target always corresponds to - one value of comparator. - Examples: - - Valid dataset: - STUDYID STUDYDESC - 1 A - 2 B - 3 C - 1 A - 2 B - - Invalid dataset: - STUDYID STUDYDESC - 1 A - 2 A - 3 C + Validates one-to-one relationship between two columns (target and comparator) + within a dataset. One-to-one means that a columns values can be duplicated + but it must always corresponds to one value of comparator and vice versa. + A violation occurs when a NON-NULL value in either column maps to multiple + different values in the other column. """ target = self.replace_prefix(other_value.get("target")) comparator = other_value.get("comparator") if isinstance(comparator, list): comparator = self.replace_all_prefixes(comparator) + columns = [target] + comparator else: comparator = self.replace_prefix(comparator) - df_subset = self.value[[target, comparator]].dropna(how="all") + columns = [target, comparator] + + df_subset = self.value[columns].dropna(how="all") df_without_duplicates = df_subset.drop_duplicates() - violated_targets = self._find_relationship_violations( + violated_targets, violated_comparators = self._find_relationship_violations( df_without_duplicates, target, comparator ) + + # flag violations from target and comparator result = self.value.convert_to_series([False] * len(self.value)) if violated_targets: clean_targets = { v for v in violated_targets if pd.notna(v) and v != "" and v is not None } - has_null_target = any( - pd.isna(v) or v == "" or v is None for v in violated_targets - ) if clean_targets: result = result | self.value[target].isin(clean_targets) - if has_null_target: - result = result | self.value[target].isna() + if violated_comparators: + clean_comparators = { + v + for v in violated_comparators + if pd.notna(v) and v != "" and v is not None + } + if clean_comparators: + if isinstance(comparator, list): + # For multi-column comparators, match on tuple combinations + for comp_tuple in clean_comparators: + mask = self.value.convert_to_series([True] * len(self.value)) + for i, col in enumerate(comparator): + mask = mask & (self.value[col] == comp_tuple[i]) + result = result | mask + else: + result = result | self.value[comparator].isin(clean_comparators) + return result def _find_relationship_violations(self, df_without_duplicates, target, comparator): - """Find all target values that violate one-to-one relationship constraints.""" - violated_targets = set() - for target_val in df_without_duplicates[target].dropna().unique(): - target_rows = df_without_duplicates[ - df_without_duplicates[target] == target_val - ] - comparator_values = target_rows[comparator] - unique_comparators = set() - for comp_val in comparator_values: - if pd.isna(comp_val) or comp_val == "" or comp_val is None: - unique_comparators.add("NULL_PLACEHOLDER") - else: - unique_comparators.add(comp_val) - if len(unique_comparators) > 1: - violated_targets.add(target_val) - for comp_val in df_without_duplicates[comparator].dropna().unique(): - if comp_val == "" or pd.isna(comp_val): - continue - comp_rows = df_without_duplicates[ - df_without_duplicates[comparator] == comp_val - ] - target_values = comp_rows[target] - if len(target_values) > 1: - for t_val in target_values: - violated_targets.add(t_val) - return violated_targets + """ + Find all values that violate one-to-one relationship constraints. + Returns two sets: + - violated_targets: non-null target values that map to multiple comparators + - violated_comparators: non-null comparator values that map to multiple targets + """ + violated_targets = self._check_column_violations( + df_without_duplicates, target, comparator + ) + violated_comparators = self._check_column_violations( + df_without_duplicates, comparator, target + ) + return violated_targets, violated_comparators + + def _check_column_violations(self, df_without_duplicates, key_column, value_column): + violated_keys = set() + if isinstance(key_column, list): + key_data = df_without_duplicates[key_column] + unique_keys = [tuple(row) for row in key_data.drop_duplicates().values] + else: + unique_keys = df_without_duplicates[key_column].dropna().unique() + for key_val in unique_keys: + if isinstance(key_column, list): + if any(v == "" or pd.isna(v) or v is None for v in key_val): + continue + mask = pd.Series( + [True] * len(df_without_duplicates), + index=df_without_duplicates.index, + ) + for i, col in enumerate(key_column): + mask = mask & (df_without_duplicates[col] == key_val[i]) + key_rows = df_without_duplicates[mask] + else: + if key_val == "": + continue + key_rows = df_without_duplicates[ + df_without_duplicates[key_column] == key_val + ] + + if isinstance(value_column, list): + value_tuples = [tuple(row) for row in key_rows[value_column].values] + if self._has_multiple_mappings_for_tuples(value_tuples): + violated_keys.add(key_val) + else: + if self._has_multiple_mappings(key_rows[value_column]): + violated_keys.add(key_val) + return violated_keys + + def _has_multiple_mappings_for_tuples(self, value_tuples): + """ + Check if a list of tuples contains multiple different non-null tuples. + Returns True if there are multiple non-null tuples, or at least one + non-null tuple plus a null tuple. + """ + unique_tuples = set() + has_null = False + for val_tuple in value_tuples: + if any(pd.isna(v) or v == "" or v is None for v in val_tuple): + has_null = True + else: + unique_tuples.add(val_tuple) + return len(unique_tuples) > 1 or (len(unique_tuples) >= 1 and has_null) + + def _has_multiple_mappings(self, values): + """ + Check if a series of values contains multiple different values. + Returns True if there are multiple non-null values, or at least one + non-null value plus null. + """ + unique_values = set() + has_null = False + for val in values: + if pd.isna(val) or val == "" or val is None: + has_null = True + else: + unique_values.add(val) + return len(unique_values) > 1 or (len(unique_values) >= 1 and has_null) @log_operator_execution @type_operator(FIELD_DATAFRAME) diff --git a/tests/unit/test_check_operators/test_relationship_integrity_checks.py b/tests/unit/test_check_operators/test_relationship_integrity_checks.py index 943c93b54..df385d635 100644 --- a/tests/unit/test_check_operators/test_relationship_integrity_checks.py +++ b/tests/unit/test_check_operators/test_relationship_integrity_checks.py @@ -207,6 +207,138 @@ def test_has_same_values(data, dataset_type, expected_result): DaskDataset, [True, True, True, False], ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3"], + "VISITNUM": [1, 2, 1, 3], + "target": ["Consulting", None, "Consulting", "Treatment"], + }, + "VISITNUM", + PandasDataset, + [False, False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3"], + "VISITNUM": [1, None, 2, 3], + "target": ["Consulting", "Surgery", "Surgery", "Treatment"], + }, + "VISITNUM", + PandasDataset, + [False, True, True, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [1, None, 1], + "target": ["Consulting", "Consulting", "Consulting"], + }, + "VISITNUM", + PandasDataset, + [True, True, True], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [1, 1, 1], + "target": ["Consulting", None, "Surgery"], + }, + "VISITNUM", + DaskDataset, + [True, True, True], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [1, 2, ""], + "target": ["Consulting", "Surgery", "Treatment"], + }, + "VISITNUM", + PandasDataset, + [False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [1, "", 1], + "target": ["Consulting", "Consulting", "Consulting"], + }, + "VISITNUM", + PandasDataset, + [True, True, True], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3"], + "VISITNUM": [1, None, None, 2], + "target": ["Consulting", None, None, "Surgery"], + }, + "VISITNUM", + DaskDataset, + [False, False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [None, None, None], + "target": [None, None, None], + }, + "VISITNUM", + PandasDataset, + [False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3", "TEST-4"], + "VISITNUM": [1, 1, 2, None, 2], + "target": ["A", "B", "A", "A", "C"], + }, + "VISITNUM", + PandasDataset, + [True, True, True, True, True], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3"], + "VISITNUM": [1, 2, 3, None], + "target": ["A", "B", "C", "D"], + }, + "VISITNUM", + DaskDataset, + [False, False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISITNUM": [None, "", 1], + "target": ["A", "B", "C"], + }, + "VISITNUM", + PandasDataset, + [False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2"], + "VISIT": ["V1", "V2", "V1"], + "VISITNUM": [1, 2, 1], + "target": ["A", "B", "A"], + }, + ["VISIT", "VISITNUM"], + PandasDataset, + [False, False, False], + ), + ( + { + "STUDYID": ["TEST", "TEST-1", "TEST-2", "TEST-3"], + "VISIT": ["V1", "V1", "V2", "V1"], + "VISITNUM": [1, 1, 2, 1], + "target": ["A", "B", "C", "A"], + }, + ["VISIT", "VISITNUM"], + DaskDataset, + [True, True, False, True], + ), ], ) def test_is_not_unique_relationship(data, comparator, dataset_type, expected_result):