pandas

cdisc_rules_engine/operations/day_data_validator.py

@@ -7,8 +7,10 @@
 
 class DayDataValidator(BaseOperation):
     def _execute_operation(self):
-        dtc_value = self.evaluation_dataset[self.params.target].map(
-            self.parse_timestamp
+        dtc_value = (
+            self.evaluation_dataset[self.params.target]
+            .astype(object)
+            .map(self.parse_timestamp)
         )
         # Always get RFSTDTC column from DM dataset.
         dm_datasets = [
@@ -27,17 +29,16 @@ def _execute_operation(self):
         else:
             dm_data = self.data_service.get_dataset(dataset_name=dm_datasets[0].name)
             dm_data = tag_source(dm_data, dm_datasets[0])
-
         new_dataset = self.evaluation_dataset.merge(
             dm_data[["USUBJID", "RFSTDTC"]], on="USUBJID", suffixes=("", "_dm")
         )
         rfstdtc_value = "RFSTDTC"
         if "RFSTDTC_dm" in new_dataset:
             rfstdtc_value = "RFSTDTC_dm"
-        delta = (dtc_value - new_dataset[rfstdtc_value].map(self.parse_timestamp)).map(
-            self.get_day_difference
-        )
-
+        delta = (
+            dtc_value
+            - new_dataset[rfstdtc_value].astype(object).map(self.parse_timestamp)
+        ).map(self.get_day_difference)
         return self.evaluation_dataset.convert_to_series(delta.replace(np.nan, ""))
 
     def parse_timestamp(self, timestamp: str) -> datetime:

cdisc_rules_engine/operations/distinct.py

@@ -14,6 +14,10 @@ def _check_column_exists_in_dataset(row, target_col_name, referenced_datasets):
     return None
 
 
+def _apply_dropna_list(x):
+    return list(x.dropna())
+
+
 class Distinct(BaseOperation):
     def _execute_operation(self):
         result = self.params.dataframe
@@ -32,7 +36,7 @@ def _execute_operation(self):
                 )
                 data = data.dropna().unique()
             else:
-                data = result[self.params.target].unique()
+                data = result[self.params.target].dropna().unique()
             if len(data) > 0 and isinstance(data[0], bytes):
                 data = data.astype(str)
             result = list(data)
@@ -64,7 +68,7 @@ def get_existing_column_names(group):
                     )
                     .groupby(self.params.grouping, as_index=False, group_keys=False)
                     .data[self.params.target]
-                    .apply(list)
+                    .apply(_apply_dropna_list)
                     .reset_index()
                 )
         return result

pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
     "numpy >=1.26.0",
     "odmlib >=0.1.4",
     "openpyxl >=3.1.5",
-    "pandas >=2.2.0, <3.0.0",
+    "pandas >=3.0.0",
     "psutil >=6.1.1",
     "pyinstaller >=6.11.0",
     "pympler >=1.1",