1443 Send49 Rule Blocked Fix

cdisc_rules_engine/dataset_builders/variables_metadata_with_define_and_library_dataset_builder.py

@@ -1,6 +1,7 @@
 from cdisc_rules_engine.dataset_builders.base_dataset_builder import BaseDatasetBuilder
 from typing import List
 from cdisc_rules_engine.models.dataset import DatasetInterface
+import pandas as pd
 
 
 class VariablesMetadataWithDefineAndLibraryDatasetBuilder(BaseDatasetBuilder):
@@ -31,6 +32,14 @@ def build(self):
         define_variable_codelist_coded_values,
         define_variable_codelist_coded_codes,
         define_variable_mandatory,
+        define_vlm_present,
+        define_vlm_item_count,
+        define_vlm_ccodes,
+        define_vlm_has_codelist_any,
+        define_vlm_has_codelist_all,
+        define_vlm_ccode_missing_any,
+        define_vlm_ccode_matches_library_any,
+        define_vlm_ccode_matches_library_all,
         library_variable_name,
         library_variable_label,
         library_variable_data_type,
@@ -95,6 +104,100 @@ def build(self):
             )
         )
 
+        # Third merge: add VLM summary columns
+        define_vlm_records: List[dict] = self.get_define_xml_value_level_metadata()
+        define_vlm_dataset = self.dataset_implementation.from_records(
+            define_vlm_records
+        )
+        define_vlm_df = define_vlm_dataset.data
+        has_vlm = not define_vlm_df.empty
+
+        required_vlm_cols = [
+            "define_variable_name",
+            "define_vlm_ccode",
+            "define_vlm_has_codelist",
+        ]
+
+        # Normalize VLM columns so groupby is safe
+        define_vlm_df = define_vlm_df.reindex(columns=required_vlm_cols)
+        define_vlm_df["define_vlm_ccode"] = define_vlm_df["define_vlm_ccode"].fillna("")
+        define_vlm_df["define_vlm_has_codelist"] = (
+            define_vlm_df["define_vlm_has_codelist"].fillna(False).astype(bool)
+        )
+
+        if has_vlm:
+            vlm_summary = define_vlm_df.groupby(
+                "define_variable_name", as_index=False
+            ).agg(
+                define_vlm_item_count=("define_vlm_ccode", "count"),
+                define_vlm_ccodes=(
+                    "define_vlm_ccode",
+                    lambda x: sorted(set(v for v in x if v != "")),
+                ),
+                define_vlm_has_codelist_any=("define_vlm_has_codelist", "any"),
+                define_vlm_has_codelist_all=("define_vlm_has_codelist", "all"),
+                define_vlm_ccode_missing_any=(
+                    "define_vlm_ccode",
+                    lambda x: (x == "").any(),
+                ),
+            )
+            vlm_summary["define_vlm_present"] = True
+        else:
+            vlm_summary = pd.DataFrame(
+                columns=[
+                    "define_variable_name",
+                    "define_vlm_item_count",
+                    "define_vlm_ccodes",
+                    "define_vlm_has_codelist_any",
+                    "define_vlm_has_codelist_all",
+                    "define_vlm_ccode_missing_any",
+                    "define_vlm_present",
+                ]
+            )
+
+        vlm_summary = vlm_summary.rename(
+            columns={"define_variable_name": "variable_name"}
+        )
+        final_dataframe = final_dataframe.merge(
+            vlm_summary,
+            how="left",
+            on="variable_name",
+        )
+
+        final_dataframe["define_vlm_present"] = final_dataframe[
+            "define_vlm_present"
+        ].fillna(False)
+        final_dataframe["define_vlm_item_count"] = (
+            final_dataframe["define_vlm_item_count"].fillna(0).astype(int)
+        )
+        final_dataframe["define_vlm_ccodes"] = final_dataframe[
+            "define_vlm_ccodes"
+        ].apply(lambda x: x if isinstance(x, list) else [])
+        for col in [
+            "define_vlm_has_codelist_any",
+            "define_vlm_has_codelist_all",
+            "define_vlm_ccode_missing_any",
+        ]:
+            final_dataframe[col] = final_dataframe[col].fillna(False)
+
+        final_dataframe["define_vlm_ccode_matches_library_any"] = final_dataframe.apply(
+            lambda row: (
+                row["library_variable_ccode"] in row["define_vlm_ccodes"]
+                if row["define_vlm_ccodes"]
+                else False
+            ),
+            axis=1,
+        )
+        final_dataframe["define_vlm_ccode_matches_library_all"] = final_dataframe.apply(
+            lambda row: (
+                bool(row["define_vlm_ccodes"])
+                and all(
+                    c == row["library_variable_ccode"] for c in row["define_vlm_ccodes"]
+                )
+            ),
+            axis=1,
+        )
+
         return final_dataframe
 
     def get_variable_null_stats(

resources/schema/rule-merged/MetaVariables.json

@@ -134,6 +134,38 @@
       "const": "define_vlm_ccode",
       "markdownDescription": "\nValueListDef.ItemDef.CodeList.Alias.Name\n"
     },
+    {
+      "const": "define_vlm_ccode_matches_library_all",
+      "markdownDescription": "\nBoolean indicating whether all VLM items' codelist codes match the library standard codelist code for this variable (all non-empty define_vlm_ccodes equal library_variable_ccode)\n"
+    },
+    {
+      "const": "define_vlm_ccode_matches_library_any",
+      "markdownDescription": "\nBoolean indicating whether at least one VLM item's codelist code matches the library standard codelist code for this variable (any define_vlm_ccode equals library_variable_ccode)\n"
+    },
+    {
+      "const": "define_vlm_ccode_missing_any",
+      "markdownDescription": "\nBoolean indicating whether at least one VLM item has an empty or missing codelist code (ValueListDef.ItemDef.CodeList.Alias.Name is empty)\n"
+    },
+    {
+      "const": "define_vlm_ccodes",
+      "markdownDescription": "\nList of distinct non-empty codelist codes from all VLM items for this variable. Derived from ValueListDef.ItemDef.CodeList.Alias.Name entries\n"
+    },
+    {
+      "const": "define_vlm_has_codelist_all",
+      "markdownDescription": "\nBoolean indicating whether all VLM items have a CodeListRef (all ValueListDef.ItemDef.CodeListRef exist)\n"
+    },
+    {
+      "const": "define_vlm_has_codelist_any",
+      "markdownDescription": "\nBoolean indicating whether at least one VLM item has a CodeListRef (ValueListDef.ItemDef.CodeListRef exists)\n"
+    },
+    {
+      "const": "define_vlm_item_count",
+      "markdownDescription": "\nCount of VLM items (ValueListDef.ItemDef entries) for this variable\n"
+    },
+    {
+      "const": "define_vlm_present",
+      "markdownDescription": "\nBoolean indicating whether this variable has one or more VLM (Value Level Metadata) items defined in Define-XML ValueListDef.ItemDef\n"
+    },
     {
       "const": "define_vlm_codelist_coded_values",
       "markdownDescription": "\nValueListDef.ItemDef.CodeList.[CodeListItem/EnumeratedItem].CodedValue\n"

resources/schema/rule-merged/Rule_Type.json

@@ -20,7 +20,7 @@
     {
       "const": "Define Item Metadata Check against Library Metadata",
       "title": "Define xml metadata at variable level and corresponding library variable metadata",
-      "markdownDescription": "\n#### Columns\n\n- `define_variable_name`\n- `define_variable_label`\n- `define_variable_data_type`\n- `define_variable_role`\n- `define_variable_size`\n- `define_variable_ccode`\n- `define_variable_format`\n- `define_variable_allowed_terms`\n- `define_variable_origin_type`\n- `define_variable_is_collected`\n- `define_variable_has_no_data`\n- `define_variable_order_number`\n- `define_variable_has_codelist`\n- `define_variable_codelist_coded_values`\n- `define_variable_codelist_coded_codes`\n- `define_variable_mandatory`\n- `define_variable_has_comment`\n- `define_variable_has_method`\n- `library_variable_name`\n- `library_variable_order_number`\n- `library_variable_label`\n- `library_variable_data_type`\n- `library_variable_role`\n- `library_variable_core`\n- `library_variable_has_codelist`\n- `library_variable_ccode`\n\n#### Rule Macro\n\nChecks variable-level metadata, codelists, and codelist terms from Define-XML against the corresponding standard variable definitions from the CDISC Library.\n"
+      "markdownDescription": "\n#### Columns\n\n- `define_variable_name`\n- `define_variable_label`\n- `define_variable_data_type`\n- `define_variable_role`\n- `define_variable_size`\n- `define_variable_ccode`\n- `define_variable_format`\n- `define_variable_allowed_terms`\n- `define_variable_origin_type`\n- `define_variable_is_collected`\n- `define_variable_has_no_data`\n- `define_variable_order_number`\n- `define_variable_has_codelist`\n- `define_variable_codelist_coded_values`\n- `define_variable_codelist_coded_codes`\n- `define_variable_mandatory`\n- `define_variable_has_comment`\n- `define_variable_has_method`\n- `define_vlm_ccode_matches_library_all`\n- `define_vlm_ccode_matches_library_any`\n- `define_vlm_ccode_missing_any`\n- `define_vlm_ccodes`\n- `define_vlm_has_codelist_all`\n- `define_vlm_has_codelist_any`\n- `define_vlm_item_count`\n- `define_vlm_present`\n- `library_variable_name`\n- `library_variable_order_number`\n- `library_variable_label`\n- `library_variable_data_type`\n- `library_variable_role`\n- `library_variable_core`\n- `library_variable_has_codelist`\n- `library_variable_ccode`\n\n#### Rule Macro\n\nChecks variable-level metadata, codelists, and codelist terms from Define-XML against the corresponding standard variable definitions from the CDISC Library.\n"
     },
     {
       "const": "Domain Presence Check",
@@ -79,7 +79,8 @@
     },
     {
       "const": "Variable Metadata Check against Define XML and Library Metadata",
-      "title": "Combines metadata at the variable level with corresponding define-xml metadata at variable level and corresponding library variable metadata"
+      "title": "Combines metadata at the variable level with corresponding define-xml metadata at variable level and corresponding library variable metadata",
+      "markdownDescription": "\n#### Columns\n\n- `variable_name`\n- `variable_label`\n- `variable_size`\n- `variable_order_number`\n- `variable_data_type`\n- `define_variable_name`\n- `define_variable_label`\n- `define_variable_data_type`\n- `define_variable_is_collected`\n- `define_variable_role`\n- `define_variable_size`\n- `define_variable_ccode`\n- `define_variable_format`\n- `define_variable_allowed_terms`\n- `define_variable_origin_type`\n- `define_variable_has_no_data`\n- `define_variable_order_number`\n- `define_variable_length`\n- `define_variable_has_codelist`\n- `define_variable_codelist_coded_values`\n- `define_variable_codelist_coded_codes`\n- `define_variable_mandatory`\n- `define_variable_has_comment`\n- `define_variable_has_method`\n- `define_vlm_ccode_matches_library_all`\n- `define_vlm_ccode_matches_library_any`\n- `define_vlm_ccode_missing_any`\n- `define_vlm_ccodes`\n- `define_vlm_has_codelist_all`\n- `define_vlm_has_codelist_any`\n- `define_vlm_item_count`\n- `define_vlm_present`\n- `library_variable_name`\n- `library_variable_role`\n- `library_variable_label`\n- `library_variable_core`\n- `library_variable_order_number`\n- `library_variable_data_type`\n- `library_variable_ccode`\n- `variable_has_empty_values`\n\n#### Rule Macro\n\nCombines variable-level metadata from submission dataset contents against both Define-XML variable metadata and CDISC Library standard variable metadata simultaneously.\n"
     },
     {
       "const": "Value Check with Dataset Metadata",

resources/schema/rule/MetaVariables.json

@@ -95,6 +95,30 @@
     {
       "const": "define_vlm_ccode"
     },
+    {
+      "const": "define_vlm_ccode_matches_library_all"
+    },
+    {
+      "const": "define_vlm_ccode_matches_library_any"
+    },
+    {
+      "const": "define_vlm_ccode_missing_any"
+    },
+    {
+      "const": "define_vlm_ccodes"
+    },
+    {
+      "const": "define_vlm_has_codelist_all"
+    },
+    {
+      "const": "define_vlm_has_codelist_any"
+    },
+    {
+      "const": "define_vlm_item_count"
+    },
+    {
+      "const": "define_vlm_present"
+    },
     {
       "const": "define_vlm_codelist_coded_values"
     },

resources/schema/rule/MetaVariables.md

@@ -138,6 +138,38 @@ ValueListDef.ItemDef.CodeList.CodeListItem.Decode.TranslatedText
 
 ValueListDef.ItemDef.CodeList.Alias.Name
 
+## define_vlm_ccode_matches_library_all
+
+Boolean indicating whether all VLM items' codelist codes match the library standard codelist code for this variable (all non-empty define_vlm_ccodes equal library_variable_ccode)
+
+## define_vlm_ccode_matches_library_any
+
+Boolean indicating whether at least one VLM item's codelist code matches the library standard codelist code for this variable (any define_vlm_ccode equals library_variable_ccode)
+
+## define_vlm_ccode_missing_any
+
+Boolean indicating whether at least one VLM item has an empty or missing codelist code (ValueListDef.ItemDef.CodeList.Alias.Name is empty)
+
+## define_vlm_ccodes
+
+List of distinct non-empty codelist codes from all VLM items for this variable. Derived from ValueListDef.ItemDef.CodeList.Alias.Name entries
+
+## define_vlm_has_codelist_all
+
+Boolean indicating whether all VLM items have a CodeListRef (all ValueListDef.ItemDef.CodeListRef exist)
+
+## define_vlm_has_codelist_any
+
+Boolean indicating whether at least one VLM item has a CodeListRef (ValueListDef.ItemDef.CodeListRef exists)
+
+## define_vlm_item_count
+
+Count of VLM items (ValueListDef.ItemDef entries) for this variable
+
+## define_vlm_present
+
+Boolean indicating whether this variable has one or more VLM (Value Level Metadata) items defined in Define-XML ValueListDef.ItemDef
+
 ## define_vlm_codelist_coded_values
 
 ValueListDef.ItemDef.CodeList.[CodeListItem/EnumeratedItem].CodedValue

resources/schema/rule/Rule_Type.md

@@ -313,6 +313,14 @@ all:
 - `define_variable_mandatory`
 - `define_variable_has_comment`
 - `define_variable_has_method`
+- `define_vlm_ccode_matches_library_all`
+- `define_vlm_ccode_matches_library_any`
+- `define_vlm_ccode_missing_any`
+- `define_vlm_ccodes`
+- `define_vlm_has_codelist_all`
+- `define_vlm_has_codelist_any`
+- `define_vlm_item_count`
+- `define_vlm_present`
 - `library_variable_name`
 - `library_variable_order_number`
 - `library_variable_label`
@@ -579,7 +587,7 @@ Combines variable-level metadata from submission dataset contents against the ma
 
 Combines variable-level metadata from submission dataset contents against the corresponding CDISC Library standard variable metadata.
 
-## Variables Metadata Check against Define XML and Library Metadata
+## Variable Metadata Check against Define XML and Library Metadata
 
 #### Columns
 
@@ -607,6 +615,14 @@ Combines variable-level metadata from submission dataset contents against the co
 - `define_variable_mandatory`
 - `define_variable_has_comment`
 - `define_variable_has_method`
+- `define_vlm_ccode_matches_library_all`
+- `define_vlm_ccode_matches_library_any`
+- `define_vlm_ccode_missing_any`
+- `define_vlm_ccodes`
+- `define_vlm_has_codelist_all`
+- `define_vlm_has_codelist_any`
+- `define_vlm_item_count`
+- `define_vlm_present`
 - `library_variable_name`
 - `library_variable_role`
 - `library_variable_label`