diff --git a/cdisc_rules_engine/operations/day_data_validator.py b/cdisc_rules_engine/operations/day_data_validator.py index d09893b5a..ef41d5e90 100644 --- a/cdisc_rules_engine/operations/day_data_validator.py +++ b/cdisc_rules_engine/operations/day_data_validator.py @@ -7,8 +7,10 @@ class DayDataValidator(BaseOperation): def _execute_operation(self): - dtc_value = self.evaluation_dataset[self.params.target].map( - self.parse_timestamp + dtc_value = ( + self.evaluation_dataset[self.params.target] + .astype(object) + .map(self.parse_timestamp) ) # Always get RFSTDTC column from DM dataset. dm_datasets = [ @@ -27,17 +29,16 @@ def _execute_operation(self): else: dm_data = self.data_service.get_dataset(dataset_name=dm_datasets[0].name) dm_data = tag_source(dm_data, dm_datasets[0]) - new_dataset = self.evaluation_dataset.merge( dm_data[["USUBJID", "RFSTDTC"]], on="USUBJID", suffixes=("", "_dm") ) rfstdtc_value = "RFSTDTC" if "RFSTDTC_dm" in new_dataset: rfstdtc_value = "RFSTDTC_dm" - delta = (dtc_value - new_dataset[rfstdtc_value].map(self.parse_timestamp)).map( - self.get_day_difference - ) - + delta = ( + dtc_value + - new_dataset[rfstdtc_value].astype(object).map(self.parse_timestamp) + ).map(self.get_day_difference) return self.evaluation_dataset.convert_to_series(delta.replace(np.nan, "")) def parse_timestamp(self, timestamp: str) -> datetime: diff --git a/cdisc_rules_engine/operations/distinct.py b/cdisc_rules_engine/operations/distinct.py index cf42b5d4b..e6fd7cfc3 100644 --- a/cdisc_rules_engine/operations/distinct.py +++ b/cdisc_rules_engine/operations/distinct.py @@ -14,6 +14,10 @@ def _check_column_exists_in_dataset(row, target_col_name, referenced_datasets): return None +def _apply_dropna_list(x): + return list(x.dropna()) + + class Distinct(BaseOperation): def _execute_operation(self): result = self.params.dataframe @@ -32,7 +36,7 @@ def _execute_operation(self): ) data = data.dropna().unique() else: - data = result[self.params.target].unique() + data = result[self.params.target].dropna().unique() if len(data) > 0 and isinstance(data[0], bytes): data = data.astype(str) result = list(data) @@ -64,7 +68,7 @@ def get_existing_column_names(group): ) .groupby(self.params.grouping, as_index=False, group_keys=False) .data[self.params.target] - .apply(list) + .apply(_apply_dropna_list) .reset_index() ) return result diff --git a/pyproject.toml b/pyproject.toml index 497faca20..7114ff971 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "numpy >=1.26.0", "odmlib >=0.1.4", "openpyxl >=3.1.5", - "pandas >=2.2.0, <3.0.0", + "pandas >=3.0.0", "psutil >=6.1.1", "pyinstaller >=6.11.0", "pympler >=1.1",