From 25463a39b61ef71562667991830d39a3ea6e0922 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Thu, 25 Jun 2026 18:36:24 -0400 Subject: [PATCH 01/21] pandas --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 497faca20..7114ff971 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "numpy >=1.26.0", "odmlib >=0.1.4", "openpyxl >=3.1.5", - "pandas >=2.2.0, <3.0.0", + "pandas >=3.0.0", "psutil >=6.1.1", "pyinstaller >=6.11.0", "pympler >=1.1", From 9226b8650b97d9efadb3fdb01f242088913e0687 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Thu, 25 Jun 2026 18:53:04 -0400 Subject: [PATCH 02/21] handle 3.0 none type inferring --- cdisc_rules_engine/operations/day_data_validator.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cdisc_rules_engine/operations/day_data_validator.py b/cdisc_rules_engine/operations/day_data_validator.py index d09893b5a..969303ecd 100644 --- a/cdisc_rules_engine/operations/day_data_validator.py +++ b/cdisc_rules_engine/operations/day_data_validator.py @@ -2,6 +2,7 @@ from cdisc_rules_engine.operations.base_operation import BaseOperation from datetime import datetime import numpy as np +import pandas as pd from cdisc_rules_engine.utilities.sdtm_utilities import tag_source @@ -46,13 +47,17 @@ def parse_timestamp(self, timestamp: str) -> datetime: return dt.date() except TypeError: # Null date time - return None + return pd.NaT except ValueError: # Value is not iso format - return None + return pd.NaT def get_day_difference(self, delta: datetime) -> int: - if delta is None or (isinstance(delta, float) and np.isnan(delta)): + if ( + delta is None + or delta is pd.NaT + or (isinstance(delta, float) and np.isnan(delta)) + ): return "" # Return 1 if the --DTC value is the same as the DY return delta.days if delta.days < 0 else delta.days + 1 From 82b6dca329b287bbb515793b7317c27d0122e786 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 09:19:29 -0400 Subject: [PATCH 03/21] as object --- .../operations/day_data_validator.py | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/cdisc_rules_engine/operations/day_data_validator.py b/cdisc_rules_engine/operations/day_data_validator.py index 969303ecd..ef41d5e90 100644 --- a/cdisc_rules_engine/operations/day_data_validator.py +++ b/cdisc_rules_engine/operations/day_data_validator.py @@ -2,14 +2,15 @@ from cdisc_rules_engine.operations.base_operation import BaseOperation from datetime import datetime import numpy as np -import pandas as pd from cdisc_rules_engine.utilities.sdtm_utilities import tag_source class DayDataValidator(BaseOperation): def _execute_operation(self): - dtc_value = self.evaluation_dataset[self.params.target].map( - self.parse_timestamp + dtc_value = ( + self.evaluation_dataset[self.params.target] + .astype(object) + .map(self.parse_timestamp) ) # Always get RFSTDTC column from DM dataset. dm_datasets = [ @@ -28,17 +29,16 @@ def _execute_operation(self): else: dm_data = self.data_service.get_dataset(dataset_name=dm_datasets[0].name) dm_data = tag_source(dm_data, dm_datasets[0]) - new_dataset = self.evaluation_dataset.merge( dm_data[["USUBJID", "RFSTDTC"]], on="USUBJID", suffixes=("", "_dm") ) rfstdtc_value = "RFSTDTC" if "RFSTDTC_dm" in new_dataset: rfstdtc_value = "RFSTDTC_dm" - delta = (dtc_value - new_dataset[rfstdtc_value].map(self.parse_timestamp)).map( - self.get_day_difference - ) - + delta = ( + dtc_value + - new_dataset[rfstdtc_value].astype(object).map(self.parse_timestamp) + ).map(self.get_day_difference) return self.evaluation_dataset.convert_to_series(delta.replace(np.nan, "")) def parse_timestamp(self, timestamp: str) -> datetime: @@ -47,17 +47,13 @@ def parse_timestamp(self, timestamp: str) -> datetime: return dt.date() except TypeError: # Null date time - return pd.NaT + return None except ValueError: # Value is not iso format - return pd.NaT + return None def get_day_difference(self, delta: datetime) -> int: - if ( - delta is None - or delta is pd.NaT - or (isinstance(delta, float) and np.isnan(delta)) - ): + if delta is None or (isinstance(delta, float) and np.isnan(delta)): return "" # Return 1 if the --DTC value is the same as the DY return delta.days if delta.days < 0 else delta.days + 1 From 9b0cd8dc58e7cd46a853ec48f9a3c192876d3f98 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 09:38:12 -0400 Subject: [PATCH 04/21] try agg --- cdisc_rules_engine/operations/distinct.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index cf42b5d4b..bbb7507dd 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -58,13 +58,16 @@ def get_existing_column_names(group): result = grouped.apply(get_existing_column_names).reset_index() else: + operation_id = self.params.operation_id result = ( result.drop_duplicates( subset=self.params.grouping + [self.params.target] ) - .groupby(self.params.grouping, as_index=False, group_keys=False) - .data[self.params.target] - .apply(list) + .groupby(self.params.grouping, as_index=False, group_keys=False)[ + self.params.target + ] + .agg(list) + .rename(columns={self.params.target: operation_id}) .reset_index() ) return result From 550383c300f3675bed91fbe1898326d5d603d488 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 10:24:05 -0400 Subject: [PATCH 05/21] none --- cdisc_rules_engine/operations/distinct.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index bbb7507dd..bb45f0721 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -32,7 +32,7 @@ def _execute_operation(self): ) data = data.dropna().unique() else: - data = result[self.params.target].unique() + data = result[self.params.target].dropna().unique() if len(data) > 0 and isinstance(data[0], bytes): data = data.astype(str) result = list(data) @@ -58,7 +58,6 @@ def get_existing_column_names(group): result = grouped.apply(get_existing_column_names).reset_index() else: - operation_id = self.params.operation_id result = ( result.drop_duplicates( subset=self.params.grouping + [self.params.target] @@ -66,8 +65,7 @@ def get_existing_column_names(group): .groupby(self.params.grouping, as_index=False, group_keys=False)[ self.params.target ] - .agg(list) - .rename(columns={self.params.target: operation_id}) + .apply(lambda x: list(x.dropna())) .reset_index() ) return result From 077e79fdf2d710e271e09c529627e024b6eb50d2 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 10:40:06 -0400 Subject: [PATCH 06/21] agg --- cdisc_rules_engine/operations/distinct.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index bb45f0721..789b0b9da 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -59,13 +59,12 @@ def get_existing_column_names(group): result = grouped.apply(get_existing_column_names).reset_index() else: result = ( - result.drop_duplicates( - subset=self.params.grouping + [self.params.target] - ) + result.dropna(subset=[self.params.target]) + .drop_duplicates(subset=self.params.grouping + [self.params.target]) .groupby(self.params.grouping, as_index=False, group_keys=False)[ self.params.target ] - .apply(lambda x: list(x.dropna())) + .apply(list) .reset_index() ) return result From 81037e596c3961c60485c3c0251b5bb5ef01aba4 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 10:53:38 -0400 Subject: [PATCH 07/21] distinct --- cdisc_rules_engine/operations/distinct.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 789b0b9da..01591bc39 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -57,16 +57,21 @@ def get_existing_column_names(group): ) result = grouped.apply(get_existing_column_names).reset_index() + elif isinstance(result.data, pd.DataFrame): + grouped = result.data.groupby( + self.params.grouping, as_index=False, group_keys=False + ) + result = grouped[self.params.target].agg(self._unique_values_for_column) else: + grouped = result.data.dropna(subset=[self.params.target]).groupby( + self.params.grouping, as_index=False, group_keys=False + ) result = ( - result.dropna(subset=[self.params.target]) - .drop_duplicates(subset=self.params.grouping + [self.params.target]) - .groupby(self.params.grouping, as_index=False, group_keys=False)[ - self.params.target - ] - .apply(list) - .reset_index() + grouped[self.params.target] + .unique() + .rename({self.params.target: self.params.operation_id}) ) + result = result.apply(list).to_frame().reset_index() return result def _get_referenced_datasets(self): @@ -77,4 +82,4 @@ def _get_referenced_datasets(self): return referenced_datasets def _unique_values_for_column(self, column): - return list(column.unique()) + return pd.Series({self.params.operation_id: list(column.dropna().unique())}) From e35947d36f12ee08b0c634f0e522a72828253e0b Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 10:59:42 -0400 Subject: [PATCH 08/21] remove as_index --- cdisc_rules_engine/operations/distinct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 01591bc39..81058f7a9 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -64,7 +64,7 @@ def get_existing_column_names(group): result = grouped[self.params.target].agg(self._unique_values_for_column) else: grouped = result.data.dropna(subset=[self.params.target]).groupby( - self.params.grouping, as_index=False, group_keys=False + self.params.grouping, group_keys=False ) result = ( grouped[self.params.target] From 9516aad7d36e3f12577e724cbafea12c3dd83b59 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 11:05:20 -0400 Subject: [PATCH 09/21] dask --- cdisc_rules_engine/operations/distinct.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 81058f7a9..391ba5831 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -70,8 +70,10 @@ def get_existing_column_names(group): grouped[self.params.target] .unique() .rename({self.params.target: self.params.operation_id}) + .apply(list, meta=(self.params.operation_id, object)) + .to_frame() + .reset_index() ) - result = result.apply(list).to_frame().reset_index() return result def _get_referenced_datasets(self): From 233b0d13e3a5b54e6d0071d7d0ba55f3e962f38b Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 11:12:01 -0400 Subject: [PATCH 10/21] dask --- cdisc_rules_engine/operations/distinct.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 391ba5831..19669d1ba 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -63,14 +63,14 @@ def get_existing_column_names(group): ) result = grouped[self.params.target].agg(self._unique_values_for_column) else: - grouped = result.data.dropna(subset=[self.params.target]).groupby( - self.params.grouping, group_keys=False - ) + grouped = result.groupby(self.params.grouping, group_keys=False) result = ( - grouped[self.params.target] + grouped.data[self.params.target] # original .unique() .rename({self.params.target: self.params.operation_id}) - .apply(list, meta=(self.params.operation_id, object)) + ) + result = ( + result.apply(list, meta=(self.params.operation_id, object)) .to_frame() .reset_index() ) From f196ff7e3db78e025928aec089565830840a948b Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 11:21:04 -0400 Subject: [PATCH 11/21] dask2 --- cdisc_rules_engine/operations/distinct.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 19669d1ba..20102f832 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -63,17 +63,14 @@ def get_existing_column_names(group): ) result = grouped[self.params.target].agg(self._unique_values_for_column) else: - grouped = result.groupby(self.params.grouping, group_keys=False) result = ( - grouped.data[self.params.target] # original + result.data.dropna(subset=[self.params.target]) + .groupby(self.params.grouping, group_keys=False)[self.params.target] .unique() - .rename({self.params.target: self.params.operation_id}) - ) - result = ( - result.apply(list, meta=(self.params.operation_id, object)) - .to_frame() - .reset_index() ) + result = result.apply( + list, meta=(self.params.target, object) + ).reset_index() return result def _get_referenced_datasets(self): From d823dfcfabb8911985027cd75c11748e24a9d406 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 11:27:04 -0400 Subject: [PATCH 12/21] remove reset_index --- cdisc_rules_engine/operations/distinct.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 20102f832..58e85df3a 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -67,10 +67,8 @@ def get_existing_column_names(group): result.data.dropna(subset=[self.params.target]) .groupby(self.params.grouping, group_keys=False)[self.params.target] .unique() + .apply(list, meta=(self.params.target, object)) ) - result = result.apply( - list, meta=(self.params.target, object) - ).reset_index() return result def _get_referenced_datasets(self): From d5f985aa74ce2d7c95ef02e072c328ade6d04f6b Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 11:31:03 -0400 Subject: [PATCH 13/21] name return --- cdisc_rules_engine/operations/distinct.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 58e85df3a..120b31338 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -68,6 +68,8 @@ def get_existing_column_names(group): .groupby(self.params.grouping, group_keys=False)[self.params.target] .unique() .apply(list, meta=(self.params.target, object)) + .rename_axis(self.params.grouping[0]) + .rename(self.params.target) ) return result From c8bb5387e6b22e51b88a72ce1c8ce59a4aee4cd7 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 11:49:13 -0400 Subject: [PATCH 14/21] replace NaN --- cdisc_rules_engine/utilities/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdisc_rules_engine/utilities/utils.py b/cdisc_rules_engine/utilities/utils.py index 40fe038d7..e8e27def4 100644 --- a/cdisc_rules_engine/utilities/utils.py +++ b/cdisc_rules_engine/utilities/utils.py @@ -324,7 +324,7 @@ def replace_nan_values_in_df(df, columns): if col in df.columns: mask = pd.isna(df[col]) if mask.any(): - df.loc[mask, col] = None + df[col] = df[col].where(~mask, other=None) return df From 4736ea234a658507da46574e9ceb72c7f7320f08 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 12:16:06 -0400 Subject: [PATCH 15/21] refactor --- cdisc_rules_engine/operations/distinct.py | 5 ++--- cdisc_rules_engine/utilities/utils.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 120b31338..7a9d6b894 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -65,11 +65,10 @@ def get_existing_column_names(group): else: result = ( result.data.dropna(subset=[self.params.target]) + .drop_duplicates(subset=self.params.grouping + [self.params.target]) .groupby(self.params.grouping, group_keys=False)[self.params.target] - .unique() .apply(list, meta=(self.params.target, object)) - .rename_axis(self.params.grouping[0]) - .rename(self.params.target) + .reset_index() ) return result diff --git a/cdisc_rules_engine/utilities/utils.py b/cdisc_rules_engine/utilities/utils.py index e8e27def4..40fe038d7 100644 --- a/cdisc_rules_engine/utilities/utils.py +++ b/cdisc_rules_engine/utilities/utils.py @@ -324,7 +324,7 @@ def replace_nan_values_in_df(df, columns): if col in df.columns: mask = pd.isna(df[col]) if mask.any(): - df[col] = df[col].where(~mask, other=None) + df.loc[mask, col] = None return df From 2310ffe4858b2b94700dd3bcc5639c1ce6b8a457 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 13:22:09 -0400 Subject: [PATCH 16/21] revert --- cdisc_rules_engine/operations/base_operation.py | 2 +- cdisc_rules_engine/operations/distinct.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cdisc_rules_engine/operations/base_operation.py b/cdisc_rules_engine/operations/base_operation.py index 40d4561cc..0d0726b36 100644 --- a/cdisc_rules_engine/operations/base_operation.py +++ b/cdisc_rules_engine/operations/base_operation.py @@ -142,7 +142,7 @@ def _handle_grouped_result(self, result): grouping_columns = self._replace_variable_wildcard( grouping_columns, self.params.domain ) - result = result.reset_index() + result = result.reset_index(drop=True) self.evaluation_dataset["_row_order"] = list( range(len(self.evaluation_dataset)) ) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 7a9d6b894..1d17e8df8 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -30,7 +30,7 @@ def _execute_operation(self): ), axis=1, ) - data = data.dropna().unique() + data = data.unique() else: data = result[self.params.target].dropna().unique() if len(data) > 0 and isinstance(data[0], bytes): @@ -64,10 +64,12 @@ def get_existing_column_names(group): result = grouped[self.params.target].agg(self._unique_values_for_column) else: result = ( - result.data.dropna(subset=[self.params.target]) - .drop_duplicates(subset=self.params.grouping + [self.params.target]) - .groupby(self.params.grouping, group_keys=False)[self.params.target] - .apply(list, meta=(self.params.target, object)) + result.drop_duplicates( + subset=self.params.grouping + [self.params.target] + ) + .groupby(self.params.grouping, as_index=False, group_keys=False) + .data[self.params.target] + .apply(list) .reset_index() ) return result From a28c7cda54341bf07fed689857aaa86d66f25690 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 13:30:55 -0400 Subject: [PATCH 17/21] usdm --- cdisc_rules_engine/operations/base_operation.py | 2 +- cdisc_rules_engine/operations/distinct.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cdisc_rules_engine/operations/base_operation.py b/cdisc_rules_engine/operations/base_operation.py index 0d0726b36..40d4561cc 100644 --- a/cdisc_rules_engine/operations/base_operation.py +++ b/cdisc_rules_engine/operations/base_operation.py @@ -142,7 +142,7 @@ def _handle_grouped_result(self, result): grouping_columns = self._replace_variable_wildcard( grouping_columns, self.params.domain ) - result = result.reset_index(drop=True) + result = result.reset_index() self.evaluation_dataset["_row_order"] = list( range(len(self.evaluation_dataset)) ) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 1d17e8df8..51238d25e 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -30,9 +30,9 @@ def _execute_operation(self): ), axis=1, ) - data = data.unique() + data = data.dropna().unique() else: - data = result[self.params.target].dropna().unique() + data = result[self.params.target].unique() if len(data) > 0 and isinstance(data[0], bytes): data = data.astype(str) result = list(data) @@ -57,11 +57,11 @@ def get_existing_column_names(group): ) result = grouped.apply(get_existing_column_names).reset_index() - elif isinstance(result.data, pd.DataFrame): - grouped = result.data.groupby( - self.params.grouping, as_index=False, group_keys=False - ) - result = grouped[self.params.target].agg(self._unique_values_for_column) + # elif isinstance(result.data, pd.DataFrame): + # grouped = result.data.groupby( + # self.params.grouping, as_index=False, group_keys=False + # ) + # result = grouped[self.params.target].agg(self._unique_values_for_column) else: result = ( result.drop_duplicates( @@ -82,4 +82,4 @@ def _get_referenced_datasets(self): return referenced_datasets def _unique_values_for_column(self, column): - return pd.Series({self.params.operation_id: list(column.dropna().unique())}) + return list(column.unique()) From 9bc282b0d9c036a3fd9cdc0c0d37fdcd72705bf6 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 13:48:34 -0400 Subject: [PATCH 18/21] dropna --- cdisc_rules_engine/operations/distinct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 51238d25e..6c8107cef 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -69,7 +69,7 @@ def get_existing_column_names(group): ) .groupby(self.params.grouping, as_index=False, group_keys=False) .data[self.params.target] - .apply(list) + .apply(lambda x: list(x.dropna())) .reset_index() ) return result From 13a110384fb1e00cc8f2fd3dc929c1149c693d95 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 13:56:27 -0400 Subject: [PATCH 19/21] serialize --- cdisc_rules_engine/operations/distinct.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 6c8107cef..152905ad5 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -14,6 +14,10 @@ def _check_column_exists_in_dataset(row, target_col_name, referenced_datasets): return None +def _apply_dropna_list(x): + return list(x.dropna()) + + class Distinct(BaseOperation): def _execute_operation(self): result = self.params.dataframe @@ -69,7 +73,7 @@ def get_existing_column_names(group): ) .groupby(self.params.grouping, as_index=False, group_keys=False) .data[self.params.target] - .apply(lambda x: list(x.dropna())) + .apply(_apply_dropna_list) .reset_index() ) return result From 58b690e2a83a3206313c16e4a8a973b76bf083d6 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Fri, 26 Jun 2026 14:14:21 -0400 Subject: [PATCH 20/21] remove comment --- cdisc_rules_engine/operations/distinct.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index 152905ad5..e8d7ac1ae 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -61,11 +61,6 @@ def get_existing_column_names(group): ) result = grouped.apply(get_existing_column_names).reset_index() - # elif isinstance(result.data, pd.DataFrame): - # grouped = result.data.groupby( - # self.params.grouping, as_index=False, group_keys=False - # ) - # result = grouped[self.params.target].agg(self._unique_values_for_column) else: result = ( result.drop_duplicates( From 66c7892bf7a5e340d7e8eef302ae511ef9fd569c Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Tue, 30 Jun 2026 12:59:54 -0400 Subject: [PATCH 21/21] NaN --- cdisc_rules_engine/operations/distinct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index e8d7ac1ae..e6fd7cfc3 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -36,7 +36,7 @@ def _execute_operation(self): ) data = data.dropna().unique() else: - data = result[self.params.target].unique() + data = result[self.params.target].dropna().unique() if len(data) > 0 and isinstance(data[0], bytes): data = data.astype(str) result = list(data)