From 7891ef4e519fa14a9cf4a5f668eab5dbdd69d7f5 Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Thu, 23 Jul 2026 13:45:21 -0400 Subject: [PATCH 01/12] Add regex_find_replace operation and CoreIssue587 coverage --- cdisc_rules_engine/models/operation_params.py | 4 + .../operations/operations_factory.py | 2 + .../operations/regex_find_replace.py | 98 + .../utilities/rule_processor.py | 4 + resources/schema/rule-merged/CORE-base.json | 142 +- .../schema/rule-merged/CORE-bundled.json | 4205 +++++++++++++++++ resources/schema/rule-merged/Operations.json | 354 +- resources/schema/rule-merged/Operator.json | 487 +- .../rule-merged/Organization_CDISC.json | 112 +- .../rule-merged/Organization_Custom.json | 53 +- .../schema/rule-merged/Organization_FDA.json | 38 +- resources/schema/rule/Operations.json | 37 + .../test_Issues/test_CoreIssue587.py | 118 + tests/resources/CoreIssue587/Rule.yml | 50 + .../boundary_99_missing/Dataset.json | 27 + .../boundary_99_paired/Dataset.json | 34 + .../CoreIssue587/mixed_partial/Dataset.json | 41 + .../multiple_missing/Dataset.json | 34 + .../nonmatching_noise/Dataset.json | 48 + .../CoreIssue587/paired_only/Dataset.json | 34 + .../CoreIssue587/single_missing/Dataset.json | 27 + .../test_regex_find_replace.py | 169 + 22 files changed, 5891 insertions(+), 227 deletions(-) create mode 100644 cdisc_rules_engine/operations/regex_find_replace.py create mode 100644 resources/schema/rule-merged/CORE-bundled.json create mode 100644 tests/QARegressionTests/test_Issues/test_CoreIssue587.py create mode 100644 tests/resources/CoreIssue587/Rule.yml create mode 100644 tests/resources/CoreIssue587/boundary_99_missing/Dataset.json create mode 100644 tests/resources/CoreIssue587/boundary_99_paired/Dataset.json create mode 100644 tests/resources/CoreIssue587/mixed_partial/Dataset.json create mode 100644 tests/resources/CoreIssue587/multiple_missing/Dataset.json create mode 100644 tests/resources/CoreIssue587/nonmatching_noise/Dataset.json create mode 100644 tests/resources/CoreIssue587/paired_only/Dataset.json create mode 100644 tests/resources/CoreIssue587/single_missing/Dataset.json create mode 100644 tests/unit/test_operations/test_regex_find_replace.py diff --git a/cdisc_rules_engine/models/operation_params.py b/cdisc_rules_engine/models/operation_params.py index 5d953ead4..833efce11 100644 --- a/cdisc_rules_engine/models/operation_params.py +++ b/cdisc_rules_engine/models/operation_params.py @@ -55,6 +55,10 @@ class OperationParams: map: List[dict] = None original_target: str = None regex: str = None + find: str = None + replace: str = None + on_no_match: str = "keep_original" + flags: str = "" returntype: str = None source: str = None target: str = None diff --git a/cdisc_rules_engine/operations/operations_factory.py b/cdisc_rules_engine/operations/operations_factory.py index 63db1dfb7..2cd008702 100644 --- a/cdisc_rules_engine/operations/operations_factory.py +++ b/cdisc_rules_engine/operations/operations_factory.py @@ -91,6 +91,7 @@ from cdisc_rules_engine.operations.get_dataset_filtered_variables import ( GetDatasetFilteredVariables, ) +from cdisc_rules_engine.operations.regex_find_replace import RegexFindReplace class OperationsFactory(FactoryInterface): @@ -146,6 +147,7 @@ class OperationsFactory(FactoryInterface): "valid_define_external_dictionary_version": DefineDictionaryVersionValidator, "get_dataset_filtered_variables": GetDatasetFilteredVariables, "get_xhtml_errors": GetXhtmlErrors, + "regex_find_replace": RegexFindReplace, } @classmethod diff --git a/cdisc_rules_engine/operations/regex_find_replace.py b/cdisc_rules_engine/operations/regex_find_replace.py new file mode 100644 index 000000000..f37809b91 --- /dev/null +++ b/cdisc_rules_engine/operations/regex_find_replace.py @@ -0,0 +1,98 @@ +import re +import pandas as pd + +from cdisc_rules_engine.operations.base_operation import BaseOperation +from cdisc_rules_engine.exceptions.custom_exceptions import OperationError + + +class RegexFindReplace(BaseOperation): + _NO_MATCH_POLICIES = {"keep_original", "set_null", "set_empty", "error"} + _FLAG_MAP = { + "i": re.IGNORECASE, + "m": re.MULTILINE, + "s": re.DOTALL, + } + + def _execute_operation(self): + operation_id = self.params.operation_id + target = self.params.target + find = getattr(self.params, "find", None) or getattr(self.params, "regex", None) + replace = getattr(self.params, "replace", None) + on_no_match = getattr(self.params, "on_no_match", "keep_original") + flags_str = getattr(self.params, "flags", "") + + self._validate_required( + operation_id, target, find, replace, on_no_match, flags_str + ) + + if target not in self.evaluation_dataset.columns: + raise OperationError(f"Target column not found: {target}") + + flags = self._parse_flags(flags_str) + pattern = self._compile_pattern(find, flags) + + source = self.evaluation_dataset[target] + transformed = source.map( + lambda value: self._transform_value( + value=value, + pattern=pattern, + replace=replace, + on_no_match=on_no_match, + ) + ) + + return transformed + + def _validate_required( + self, operation_id, target, find, replace, on_no_match, flags_str + ): + if not operation_id: + raise OperationError("regex_find_replace requires id (operation_id)") + if not target: + raise OperationError("regex_find_replace requires name (target)") + if not find: + raise OperationError("regex_find_replace requires find (or regex)") + if replace is None: + raise OperationError("regex_find_replace requires replace") + if on_no_match not in self._NO_MATCH_POLICIES: + raise OperationError( + f"Invalid on_no_match: {on_no_match}. " + f"Must be one of {sorted(self._NO_MATCH_POLICIES)}" + ) + invalid_flags = [f for f in flags_str if f not in self._FLAG_MAP] + if invalid_flags: + raise OperationError( + f"Invalid flags: {''.join(invalid_flags)}. " + f"Allowed flags: {''.join(sorted(self._FLAG_MAP.keys()))}" + ) + + def _parse_flags(self, flags_str): + flags = 0 + for ch in flags_str: + flags |= self._FLAG_MAP[ch] + return flags + + def _compile_pattern(self, find, flags): + try: + return re.compile(find, flags) + except re.error as exc: + raise OperationError(f"Invalid regex pattern '{find}': {exc}") from exc + + def _transform_value(self, value, pattern, replace, on_no_match): + if value is None or (isinstance(value, float) and pd.isna(value)): + return None + + text = str(value) + match = pattern.search(text) + if match: + return pattern.sub(replace, text) + + if on_no_match == "keep_original": + return text + if on_no_match == "set_null": + return None + if on_no_match == "set_empty": + return "" + raise OperationError( + f"No match found for value '{text}' and on_no_match='error'" + ) diff --git a/cdisc_rules_engine/utilities/rule_processor.py b/cdisc_rules_engine/utilities/rule_processor.py index 3f8a5a961..5e90d8a08 100644 --- a/cdisc_rules_engine/utilities/rule_processor.py +++ b/cdisc_rules_engine/utilities/rule_processor.py @@ -417,6 +417,10 @@ def perform_rule_operations( original_target=original_target, subtract=operation.get("subtract"), regex=operation.get("regex"), + find=operation.get("find"), + replace=operation.get("replace"), + on_no_match=operation.get("on_no_match", "keep_original"), + flags=operation.get("flags", ""), returntype=operation.get("returntype"), source=operation.get("source"), standard=standard, diff --git a/resources/schema/rule-merged/CORE-base.json b/resources/schema/rule-merged/CORE-base.json index d8bc945f5..e49252530 100644 --- a/resources/schema/rule-merged/CORE-base.json +++ b/resources/schema/rule-merged/CORE-base.json @@ -9,7 +9,9 @@ "$ref": "#/$defs/CheckItems" } }, - "required": ["all"], + "required": [ + "all" + ], "type": "object" }, { @@ -19,7 +21,9 @@ "$ref": "#/$defs/CheckItems" } }, - "required": ["any"], + "required": [ + "any" + ], "type": "object" }, { @@ -29,7 +33,9 @@ "$ref": "#/$defs/CheckItem" } }, - "required": ["not"], + "required": [ + "not" + ], "type": "object" } ] @@ -128,13 +134,18 @@ "$ref": "#/$defs/Domains" }, "include_split_datasets": { - "enum": [true] + "enum": [ + true + ] } }, "type": "object" }, "JoinType": { - "enum": ["inner", "left"], + "enum": [ + "inner", + "left" + ], "type": "string" }, "LeftRightKeys": { @@ -147,7 +158,10 @@ "$ref": "#/$defs/VariableName" } }, - "required": ["Left", "Right"], + "required": [ + "Left", + "Right" + ], "type": "object" }, "PascalCases": { @@ -232,7 +246,10 @@ "type": "string" } }, - "required": ["Document", "Cited Guidance"], + "required": [ + "Document", + "Cited Guidance" + ], "type": "object" }, "type": "array" @@ -241,10 +258,14 @@ "additionalProperties": false, "anyOf": [ { - "required": ["Logical Expression"] + "required": [ + "Logical Expression" + ] }, { - "required": ["Plain Language Expression"] + "required": [ + "Plain Language Expression" + ] } ], "properties": { @@ -258,18 +279,25 @@ "type": "string" } }, - "required": ["Rule"], + "required": [ + "Rule" + ], "type": "object" }, "Plain Language Expression": { "type": "string" }, "Type": { - "enum": ["Failure", "Success"], + "enum": [ + "Failure", + "Success" + ], "type": "string" } }, - "required": ["Type"], + "required": [ + "Type" + ], "type": "object" }, "Origin": { @@ -283,11 +311,18 @@ "type": "string" }, "Relationship": { - "enum": ["Predecessor", "Related", "Successor"], + "enum": [ + "Predecessor", + "Related", + "Successor" + ], "type": "string" } }, - "required": ["Id", "Relationship"], + "required": [ + "Id", + "Relationship" + ], "type": "object" }, "type": "array" @@ -305,7 +340,9 @@ "type": "string" } }, - "required": ["Id"], + "required": [ + "Id" + ], "type": "object" }, "Validator Rule Message": { @@ -315,7 +352,11 @@ "type": "string" } }, - "required": ["Origin", "Rule Identifier", "Version"], + "required": [ + "Origin", + "Rule Identifier", + "Version" + ], "type": "object" }, "minItems": 1, @@ -328,7 +369,11 @@ "type": "string" } }, - "required": ["Name", "References", "Version"], + "required": [ + "Name", + "References", + "Version" + ], "type": "object" }, "minItems": 1, @@ -351,7 +396,10 @@ "$ref": "Organization_Custom.json" } ], - "required": ["Organization", "Standards"], + "required": [ + "Organization", + "Standards" + ], "type": "object" }, "minItems": 1, @@ -391,10 +439,15 @@ "const": "Published" } }, - "required": ["Id"] + "required": [ + "Id" + ] } ], - "required": ["Status", "Version"], + "required": [ + "Status", + "Version" + ], "type": "object" }, "Description": { @@ -440,7 +493,9 @@ "type": "string" } }, - "required": ["Name"], + "required": [ + "Name" + ], "type": "object" }, "minItems": 1, @@ -466,7 +521,9 @@ "type": "array" } }, - "required": ["Message"], + "required": [ + "Message" + ], "type": "object" }, "Rule Type": { @@ -484,7 +541,9 @@ "$ref": "#/$defs/Classes" } }, - "required": ["Include"], + "required": [ + "Include" + ], "type": "object" }, { @@ -494,7 +553,9 @@ "$ref": "#/$defs/Classes" } }, - "required": ["Exclude"], + "required": [ + "Exclude" + ], "type": "object" } ] @@ -508,7 +569,9 @@ "$ref": "#/$defs/DataStructures" } }, - "required": ["Include"], + "required": [ + "Include" + ], "type": "object" }, { @@ -518,7 +581,9 @@ "$ref": "#/$defs/DataStructures" } }, - "required": ["Exclude"], + "required": [ + "Exclude" + ], "type": "object" } ] @@ -558,10 +623,14 @@ }, "anyOf": [ { - "required": ["Exclude"] + "required": [ + "Exclude" + ] }, { - "required": ["Include"] + "required": [ + "Include" + ] } ], "type": "object" @@ -583,13 +652,20 @@ }, "oneOf": [ { - "required": ["Classes", "Domains"] + "required": [ + "Classes", + "Domains" + ] }, { - "required": ["Data Structures"] + "required": [ + "Data Structures" + ] }, { - "required": ["Entities"] + "required": [ + "Entities" + ] } ], "type": "object" @@ -624,7 +700,9 @@ } }, "then": { - "required": ["Grouping_Variables"] + "required": [ + "Grouping_Variables" + ] }, "type": "object" } diff --git a/resources/schema/rule-merged/CORE-bundled.json b/resources/schema/rule-merged/CORE-bundled.json new file mode 100644 index 000000000..6a6fb0991 --- /dev/null +++ b/resources/schema/rule-merged/CORE-bundled.json @@ -0,0 +1,4205 @@ +{ + "$defs": { + "Boolean": { + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "all": { + "$ref": "#/$defs/CheckItems" + } + }, + "required": [ + "all" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "any": { + "$ref": "#/$defs/CheckItems" + } + }, + "required": [ + "any" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "not": { + "$ref": "#/$defs/CheckItem" + } + }, + "required": [ + "not" + ], + "type": "object" + } + ] + }, + "CheckItem": { + "anyOf": [ + { + "$ref": "#/$defs/Boolean" + }, + { + "$ref": "#/$defs/Operator.json" + } + ] + }, + "CheckItems": { + "items": { + "$ref": "#/$defs/CheckItem" + }, + "type": "array" + }, + "Classes": { + "items": { + "enum": [ + "ALL", + "EVENTS", + "FINDINGS", + "FINDINGS ABOUT", + "INTERVENTIONS", + "RELATIONSHIP", + "SPECIAL PURPOSE", + "STUDY REFERENCE", + "TRIAL DESIGN" + ], + "type": "string" + }, + "type": "array" + }, + "DataStructure": { + "enum": [ + "ADAM OTHER", + "ALL", + "BASIC DATA STRUCTURE", + "OCCURRENCE DATA STRUCTURE", + "SUBJECT LEVEL ANALYSIS DATASET" + ], + "type": "string" + }, + "DataStructures": { + "items": { + "$ref": "#/$defs/DataStructure" + }, + "type": "array" + }, + "Dataset": { + "anyOf": [ + { + "enum": [ + "ALL", + "AP--", + "APRELSUB", + "POOLDEF", + "RELREC", + "RELREF", + "RELSPEC", + "RELSUB", + "SUPP--", + "TUMOR" + ], + "type": "string" + }, + { + "pattern": "^((?!AP)[A-Z]{2}|[A-Z]{4}|SUPP[A-Z]{2}|SUPP[A-Z]{4}|SQAP[A-Z]{4})$", + "type": "string" + } + ] + }, + "Datasets": { + "items": { + "$ref": "#/$defs/Dataset" + }, + "type": "array" + }, + "DomainStructure": { + "additionalProperties": false, + "properties": { + "Exclude": { + "$ref": "#/$defs/Domains" + }, + "Include": { + "$ref": "#/$defs/Domains" + }, + "include_split_datasets": { + "enum": [ + true + ] + } + }, + "type": "object" + }, + "Domains": { + "items": { + "$ref": "#/$defs/Dataset" + }, + "type": "array" + }, + "Executability.json": { + "markdownDescription": "Indicates the extent to which the rule can be automatically executed and validated by the rules engine", + "oneOf": [ + { + "const": "Fully Executable", + "markdownDescription": "\nThe rule can be fully executed and validated automatically by the rules engine.\n" + }, + { + "const": "Not Executable", + "markdownDescription": "\nThe rule cannot be executed automatically by the rules engine.\n" + }, + { + "const": "Partially Executable", + "markdownDescription": "\nThe rule can be partially executed, but may not capture all validation scenarios.\n" + }, + { + "const": "Partially Executable - Possible Overreporting", + "markdownDescription": "\nThe rule can be partially executed but may report more violations than actually exist.\n" + }, + { + "const": "Partially Executable - Possible Underreporting", + "markdownDescription": "\nThe rule can be partially executed but may miss some violations that should be reported.\n" + } + ] + }, + "JoinType": { + "enum": [ + "inner", + "left" + ], + "type": "string" + }, + "LeftRightKeys": { + "additionalProperties": false, + "properties": { + "Left": { + "$ref": "#/$defs/VariableName" + }, + "Right": { + "$ref": "#/$defs/VariableName" + } + }, + "required": [ + "Left", + "Right" + ], + "type": "object" + }, + "MetaVariables.json": { + "anyOf": [ + { + "const": "dataset_label", + "markdownDescription": "\nLabel for the dataset\n" + }, + { + "const": "dataset_location", + "markdownDescription": "\nPath to file\n" + }, + { + "const": "dataset_name", + "markdownDescription": "\nName of the dataset\n" + }, + { + "const": "dataset_size", + "markdownDescription": "\nFile size\n" + }, + { + "const": "define_dataset_class", + "markdownDescription": "\nItemGroupDef.Class.Name\n" + }, + { + "const": "define_dataset_is_non_standard", + "markdownDescription": "\nItemGroupDef.IsNonStandard\n" + }, + { + "const": "define_dataset_key_sequence", + "markdownDescription": "\n[ItemGroupDef/ValueListDef].ItemRef.KeySequence\n" + }, + { + "const": "define_dataset_label", + "markdownDescription": "\nItemGroupDef.Description.TranslatedText\n" + }, + { + "const": "define_dataset_location", + "markdownDescription": "\nItemGroupDef.leaf.href\n" + }, + { + "const": "define_dataset_name", + "markdownDescription": "\nItemGroupDef.Name\n" + }, + { + "const": "define_dataset_structure", + "markdownDescription": "\nItemGroupDef.Structure\n" + }, + { + "const": "define_dataset_variables", + "markdownDescription": "\nList of ItemGroupDef.ItemRef.ItemDef.Name in XML document order (as they appear in ItemRef, no sorting applied)\n" + }, + { + "const": "define_dataset_variable_order", + "markdownDescription": "\nList of ItemGroupDef.ItemRef.ItemDef.Name, sorted by ItemRef OrderNumber when present, otherwise by XML document order\n" + }, + { + "const": "define_variable_allowed_terms", + "markdownDescription": "\nItemGroupDef.ItemDef.CodeList.CodeListItem.Decode.TranslatedText\n" + }, + { + "const": "define_variable_ccode", + "markdownDescription": "\nItemGroupDef.ItemDef.CodeList.Alias.Name\n" + }, + { + "const": "define_variable_codelist_coded_values", + "markdownDescription": "\nItemGroupDef.ItemDef.CodeList.[CodeListItem/EnumeratedItem].CodedValue\n" + }, + { + "const": "define_variable_data_type", + "markdownDescription": "\nItemGroupDef.ItemDef.DataType\n" + }, + { + "const": "define_variable_format", + "markdownDescription": "\n[Not Implemented]\n" + }, + { + "const": "define_variable_has_codelist", + "markdownDescription": "\nItemGroupDef.ItemDef.CodeListRef exists\n" + }, + { + "const": "define_variable_has_comment", + "markdownDescription": "\nItemGroupDef.ItemDef.CommentOID exists\n" + }, + { + "const": "define_variable_has_method", + "markdownDescription": "\nItemGroupDef.ItemRef.MethodOID exists\n" + }, + { + "const": "define_variable_has_no_data", + "markdownDescription": "\nItemGroupDef.ItemRef.HasNoData\n" + }, + { + "const": "define_variable_is_collected", + "markdownDescription": "\nItemGroupDef.ItemDef.Origin.Type = \\\"Collected\\\" (2.1) or \\\"CRF\\\" (2.0)\n" + }, + { + "const": "define_variable_label", + "markdownDescription": "\nItemGroupDef.ItemDef.Description.TranslatedText\n" + }, + { + "const": "define_variable_length", + "markdownDescription": "\nItemGroupDef.ItemDef.Length\n" + }, + { + "const": "define_variable_mandatory", + "markdownDescription": "\nItemGroupDef.ItemRef.Mandatory\n" + }, + { + "const": "define_variable_name", + "markdownDescription": "\nItemGroupDef.ItemDef.Name\n" + }, + { + "const": "define_variable_order_number", + "markdownDescription": "\nItemGroupDef.ItemRef.OrderNumber\n" + }, + { + "const": "define_variable_origin_type", + "markdownDescription": "\nItemGroupDef.ItemDef.Origin.Type\n" + }, + { + "const": "define_variable_role", + "markdownDescription": "\nItemGroupDef.ItemRef.Role\n" + }, + { + "const": "define_variable_size", + "markdownDescription": "\nItemGroupDef.ItemDef.Size\n" + }, + { + "const": "define_vlm_allowed_terms", + "markdownDescription": "\nValueListDef.ItemDef.CodeList.CodeListItem.Decode.TranslatedText\n" + }, + { + "const": "define_vlm_ccode", + "markdownDescription": "\nValueListDef.ItemDef.CodeList.Alias.Name\n" + }, + { + "const": "define_vlm_codelist_coded_values", + "markdownDescription": "\nValueListDef.ItemDef.CodeList.[CodeListItem/EnumeratedItem].CodedValue\n" + }, + { + "const": "define_vlm_data_type", + "markdownDescription": "\nValueListDef.ItemDef.DataType\n" + }, + { + "const": "define_vlm_format", + "markdownDescription": "\n[Not Implemented]\n" + }, + { + "const": "define_vlm_has_codelist", + "markdownDescription": "\nValueListDef.ItemDef.CodeListRef exists\n" + }, + { + "const": "define_vlm_has_comment", + "markdownDescription": "\nValueListDef.ItemDef.CommentOID exists\n" + }, + { + "const": "define_vlm_has_no_data", + "markdownDescription": "\nValueListDef.ItemRef.HasNoData\n" + }, + { + "const": "define_vlm_is_collected", + "markdownDescription": "\nValueListDef.ItemDef.Origin.Type = \\\"Collected\\\" (2.1) or \\\"CRF\\\" (2.0)\n" + }, + { + "const": "define_vlm_label", + "markdownDescription": "\nValueListDef.ItemDef.Description.TranslatedText\n" + }, + { + "const": "define_vlm_length", + "markdownDescription": "\nValueListDef.ItemDef.Length\n" + }, + { + "const": "define_vlm_mandatory", + "markdownDescription": "\nValueListDef.ItemRef.Mandatory\n" + }, + { + "const": "define_vlm_name", + "markdownDescription": "\nValueListDef.ItemDef.Name\n" + }, + { + "const": "define_vlm_order_number", + "markdownDescription": "\nValueListDef.ItemRef.OrderNumber\n" + }, + { + "const": "define_vlm_origin_type", + "markdownDescription": "\nValueListDef.ItemDef.Origin.Type\n" + }, + { + "const": "define_vlm_role", + "markdownDescription": "\nValueListDef.ItemRef.Role\n" + }, + { + "const": "define_vlm_size", + "markdownDescription": "\nValueListDef.ItemDef.Size\n" + }, + { + "const": "filename", + "markdownDescription": "\nName of file\n" + }, + { + "const": "library_variable_core", + "markdownDescription": "\ncore attribute of a variable from the CDISC Library\n" + }, + { + "const": "library_variable_has_codelist", + "markdownDescription": "\nIndicates whether a variable has an associated codelist in the CDISC Library\n" + }, + { + "const": "library_variable_ccode", + "markdownDescription": "\nccode attribute of a variable from the CDISC Library\n" + }, + { + "const": "library_variable_data_type", + "markdownDescription": "\nsimpleDatatype attribute of a variable from the CDISC Library\n" + }, + { + "const": "library_variable_label", + "markdownDescription": "\nlabel attribute of a variable from the CDISC Library\n" + }, + { + "const": "library_variable_name", + "markdownDescription": "\nname attribute of a variable from the CDISC Library\n" + }, + { + "const": "library_variable_order_number", + "markdownDescription": "\nordinal attribute of a variable from the CDISC Library\n" + }, + { + "const": "library_variable_role", + "markdownDescription": "\nrole attribute of a variable from the CDISC Library\n" + }, + { + "const": "row_number", + "markdownDescription": "\n1-based index of record number\n" + }, + { + "const": "variable_data_type", + "markdownDescription": "\nVariable data type (Char or Num)\n" + }, + { + "const": "variable_format", + "markdownDescription": "\nVariable format\n" + }, + { + "const": "variable_has_empty_values", + "markdownDescription": "\nTrue/False value indicating whether a variable has any empty values\n" + }, + { + "const": "variable_is_empty", + "markdownDescription": "\nTrue/False value indicating whether a variable is completely empty\n" + }, + { + "const": "variable_label", + "markdownDescription": "\nVariable long label\n" + }, + { + "const": "variable_max_size", + "markdownDescription": "\nMaximum length of actual data values in the variable\n" + }, + { + "const": "variable_name", + "markdownDescription": "\nVariable short name\n" + }, + { + "const": "variable_order_number", + "markdownDescription": "\nOrder of variable within dataset\n" + }, + { + "const": "variable_size", + "markdownDescription": "\nVariable size\n" + }, + { + "const": "variable_value", + "markdownDescription": "\nValue at `row_number` and `variable_name`\n" + }, + { + "const": "variable_value_length", + "markdownDescription": "\nCalculated length of the value at `row_number` and `variable_name`\n" + } + ] + }, + "OperationResultId": { + "pattern": "^\\$[A-Za-z_][A-Za-z0-9_]*$", + "type": "string" + }, + "Operations.json": { + "additionalProperties": false, + "anyOf": [ + { + "properties": { + "operator": { + "const": "codelist_extensible", + "markdownDescription": "\nReturns a Series indicating whether a specified codelist is extensible. Used in conjunction with codelist_terms to determine if values outside the codelist are acceptable. From the above example, $extensible will contain a bool if the codelist PKUDUG is extensible in all rows of the column.\n\nIf ct_package_type, version, and codelist_code parameters are provided, it will instead attach a new column containing the extensible value for each combination provided in the source dataset.\n\nFor example, given the current dataset:\n\n```\nid \tcodeSystemVersion \t$codelist_code\n1 \t2024-09-27 \tC201264\n2 \t2024-09-27 \tC201265\n3 \t2023-03-29 \tC127262\n```\n\nand the following operation:\n\n```yaml\n- id: $codelist_extensible\n operator: codelist_extensible\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n```\n\nThis will result in the following dataset:\n\n```\nid \tcodeSystemVersion \t$codelist_code \t$codelist_extensible\n1 \t2024-09-27 \tC201264 \tfalse\n2 \t2024-09-27 \tC201265 \tfalse\n3 \t2023-03-29 \tC127262 \ttrue\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "codelist_terms", + "markdownDescription": "\nReturns a list of valid codelist/term values. Used for evaluating whether NCI codes, submission values or NCI preferred terms are valid based on controlled terminology. Expects three parameters: `codelists` which is a list of the codelist submission value(s) to retrieve, `level` which is the level of data (either \"codelist\" or \"term\") at which to return data from, and `returntype` which is the type of values to return: \"code\" for NCI Code(s), \"value\" for submission value(s), or \"pref_term\" for NCI preferred term(s).\n\n```yaml\n- Check:\n - all:\n - name: PPSTRESU\n operator: is_not_contained_by\n value: $terms\n - name: $extensible\n operator: equal_to\n value: true\n- Operations:\n - id: $terms\n operator: codelist_terms\n codelists:\n - PKUDUG\n level: term\n returntype: value\n - id: $extensible\n codelist: PKUDUG\n operator: codelist_extensible\n```\n\nIf `ct_package_type`, `version`, and `codelist_code` parameters are provided, it will instead attach a new column containing the term for each combination provided in the source dataset. If a column name is provided as:\n\n- `term_code`, it will find term information using the term codes in the specified column.\n- `term_value`, it will find term information using the term submission values in the specified column.\n- `term_pref_term`, it will find term information using the term preferred terms in the specified column.\n\nOnly one of `term_code`, `term_value` or `term_pref_term` can be provided. The term information returned will depend on the value of the `returntype` parameter, as described above. If `returntype` is not specified, specifying `term_code` will return the term submission value and specifying either `term_value` or `term_pref_term` will return the term code.\n\nFor example, given the current dataset:\n\n```\nid \tcodeSystemVersion $codelist_code code decode\n1 \t2024-09-27 C201264 C201356 After\n2 \t2024-09-27 C201265 C201352 End to End\n3 \t2023-03-29 C127262 C51282 CLINIC\n```\n\nand the following operations:\n\n```yaml\n- id: $found_term_value\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_code: code\n- id: $found_term_pref_term\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_code: code\n returntype: pref_term\n```\n\nThis will result in the following dataset:\n\n```\n id codeSystemVersion $codelist_code code decode $found_term_value $found_term_pref_term\n 1 2024-09-27 C201264 C201356 After After After Timing Type\n 2 2024-09-27 C201265 C201352 End to End End to End End to End\n 3 2023-03-31 C127262 C51282 CLINIC CLINIC Clinic\n```\n\nConversely, if given the same dataset, and the following operations:\n\n```yaml\n- id: $found_term_code1\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_value: decode\n- id: $found_term_code2\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_pref_term: decode\n```\n\nThis will result in the following dataset:\n\n```\n id codeSystemVersion $codelist_code code decode $found_term_code1 $found_term_code2\n 1 2024-09-27 C201264 C201356 After C201356\n 2 2024-09-27 C201265 C201352 End to End C201352 C201352\n 3 2023-03-31 C127262 C51282 CLINIC C51282 C51282\n```\n\nNote that `$found_term_code2` is:\n\n- `null` for the first record because \"After\" does not match any NCI preferred term in the C201264 codelist.\n- populated for the third record because matching is case-insensitive (i.e., \"CLINIC\" matches \"Clinic\").\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "define_extensible_codelists", + "markdownDescription": "\nReturns a list of valid extensible codelist term's submission values. Used for evaluating whether submission values are valid based on controlled terminology. Expects the parameter codelists which is a list of the codelist submission value(s) to retrieve. If the codelist argument is [\"All\"] will return all extensible terms for the CT in a list.\n\n```yaml\n{\n \"id\": \"$ext_value\",\n \"codelist\": [\"ALL\"],\n \"operator\": \"define_extensible_codelists\",\n}\n```\n" + } + }, + "required": [ + "id", + "operator", + "codelists" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "define_variable_metadata", + "markdownDescription": "\nIf a target variable name is specified, returns the specified metadata in the define for the specified target variable.\n\nInput\n\n```yaml\n- operator: define_variable_metadata\n attribute_name: define_variable_label\n name: LBTESTCD\n id: $LBTESTCD_VARIABLE_LABEL\n```\n\nOutput\n\n```\nLaboratory Test Code\n```\n\nIf no target variable name specified, returns a dictionary containing the specified metadata in the define for all variables.\n\nInput\n\n```yaml\n- operator: define_variable_metadata\n attribute_name: define_variable_label\n id: $VARIABLE_LABEL\n```\n\nOutput\n\n```\n{\n \"STUDYID\": \"Study Identifier\",\n \"USUBJID\": \"Unique Subject Identifier\",\n \"LBTESTCD\": \"Laboratory Test Code\",\n \"...\": \"...\"\n}\n```\n" + } + }, + "required": [ + "id", + "operator", + "attribute_name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "distinct", + "markdownDescription": "\nGet a distinct list of values for the given name.\n\nIf a group list is specified, the distinct value list will be grouped by the variables within group.\nIf a filter object is provided, only values for records that match the filter criteria are included in the distinct values.\nIf `value_is_reference` is set to true, the target column contains the names of other columns, and the operation will check the referenced columns to ensure they exist in the associated dataset before adding them to the distinct list.\n\nIf group is provided, group_aliases may also be provided to assign new grouping variable names so that results grouped by the values in one set of grouping variables can be merged onto a dataset according to the same grouping value(s) stored in different set of grouping variables. When both group and group_aliases are provided, columns are renamed according to corresponding list position (i.e., the 1st column in group is renamed to the 1st column in group_aliases, etc.). If there are more columns listed in group than in group_aliases, only the group columns with corresponding group_aliases columns will be renamed. If there are more columns listed in group_aliases than in group, the extra column names in group_aliases will be ignored. See record_count for an example of the use of group_aliases.\n\n```yaml\nCheck:\n all:\n - name: SSSTRESC\n operator: equal_to\n value: DEAD\n value_is_literal: true\n - name: $ds_dsdecod\n operator: does_not_contain\n value: DEATH\n value_is_literal: true\nOperations:\n - operator: distinct\n domain: DS\n name: DSDECOD\n id: $ds_dsdecod\n group:\n - USUBJID\n filter:\n CAT: \"CATEGORY 1\"\n SCAT: \"SUBCATEGORY A\"\n```\n\n> below, `IDVAR` contains column names, the operation retrieves the value from each column for that row, checks the dataset associated with that column using the CO RDOMAIN. Columns that exist are added to the returns the distinct set.\n\n```yaml\nOperations:\n - domain: CO\n id: $rdomain_variables\n name: IDVAR\n operator: distinct\n value_is_reference: true\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "domain_is_custom", + "markdownDescription": "\nChecks whether the domain is in the set of domains within the provided standard.\n\nInput\n\nTarget Domain: XY\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: domain_is_custom\n id: $domain_is_custom\n```\n\nOutput\n\n```\ntrue\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "related_domain_is_custom", + "markdownDescription": "\nChecks whether the related domain (for example, the parent domain of a SUPP or RELREC dataset) is not present in the set of standard domains for the provided standard and version. This is useful for determining whether relationships point to non-standard or custom domains.\n\nInput\n\nTarget Domain: SUPPEX\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: related_domain_is_custom\n id: $related_domain_is_custom\n```\n\nOutput\n\n```\ntrue\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "domain_label", + "markdownDescription": "\nReturns the label for the domain the operation is executing on within the provided standard.\n\nInput.\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: domain_label\n id: $domain_label\n```\n\nOutput\n\n```\nLaboratory Test Results\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "dy", + "markdownDescription": "\nCalculates the number of days between the DTC and RFSTDTC. The Study Day value is incremented by 1 for each date following RFSTDTC. Dates prior to RFSTDTC are decreased by 1, with the date preceding RFSTDTC designated as Study Day -1 (there is no Study Day 0). All Study Day values are integers. Thus, to calculate Study Day:\n\n- --DY = (date portion of --DTC) - (date portion of RFSTDTC) + 1 if --DTC is on or after RFSTDTC\n- --DY = (date portion of --DTC) - (date portion of RFSTDTC) if --DTC precedes RFSTDTC\n\nThis algorithm should be used across all domains.\n\n```yaml\nCheck:\n all:\n - name: --DY\n operator: non_empty\n - name: --DTC\n operator: is_complete_date\n - name: RFSTDTC\n operator: is_complete_date\n - name: --DY\n operator: not_equal_to\n value: $dy\nOperations:\n - name: --DTC\n operator: dy\n id: $dy\nMatch Datasets:\n - Name: DM\n Keys:\n - USUBJID\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "extract_metadata", + "markdownDescription": "\nReturns the requested dataset level metadata value for the current dataset. Possible name values are:\n\n- dataset_size\n- dataset_location\n- dataset_name\n- dataset_label\n- domain\n- is_ap\n- ap_suffix\n\nExample\n\nInput:\n\nTarget domain: LB\n\n```yaml\n- name: dataset_label\n operator: extract_metadata\n id: $dataset_label\n```\n\nOutput:\n\n```\nLaboratory Test Results\n```\n\nExample: ap_suffix\n\nExtracts the domain suffix (characters 3-4) from AP-related domains. For example, \"FA\" from \"APFA\" DOMAIN value.\n\nInput:\n\nTarget domain: APFA\n\n```yaml\n- name: ap_suffix\n operator: extract_metadata\n id: $ap_suffix\n```\n\nOutput:\n\n```\nFA\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "expected_variables", + "markdownDescription": "\nReturns the expected (\"Core\" = Exp ) variables for the domain in the current standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: expected_variables\n id: $expected_variables\n```\n\nOutput:\n\n```\n[\"LBCAT\", \"LBORRES\", \"LBORRESU\", \"...\"]\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "get_codelist_attributes", + "markdownDescription": "\nFetches controlled terminology attribute values from CT packages based on row-specific CT package and version references. The operation constructs CT package names based on the standard being validated and the values in the `name` and `version` columns (e.g., SDTMIG \u2192 \"sdtmct-{version}\"). When the `name` column contains \"CDISC\" or \"CDISC CT\", it uses the validation run's standard to determine the package prefix and the version found in the cell of the specified column. The operation extracts all codes matching the specified ct_attribute from the package.\n\n**Required Parameters:**\n\n- `ct_attribute`: Attribute to extract - `\"Term CCODE\"`, `\"Codelist CCODE\"`, `\"Term Value\"`, `\"Codelist Value\"`, or `\"Term Preferred Term\"`\n- `name`: Column containing CT reference (e.g., \"TSVCDREF\") - identifies which terminology system is referenced\n- `version`: Column containing CT version (e.g., \"TSVCDVER\")\n\n```yaml\n- id: $VALID_TERM_CODES\n name: TSVCDREF\n operator: get_codelist_attributes\n ct_attribute: Term CCODE\n version: TSVCDVER\n```\n\n**Note:** if using this operator with excel data, you must put the ctpackage versions contained within your data in the library tab for it work properly.\n" + } + }, + "required": [ + "id", + "operator", + "name", + "ct_attribute", + "version" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "get_column_order_from_dataset", + "markdownDescription": "\nReturns list of dataset columns in order\n\n```yaml\nCheck:\n all:\n - name: $column_order_from_dataset\n operator: is_not_ordered_by\n value: $column_order_from_library\nOperations:\n - id: $column_order_from_library\n operator: get_column_order_from_library\n - id: $column_order_from_dataset\n operator: get_column_order_from_dataset\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "get_column_order_from_library", + "markdownDescription": "\nFetches column order for a given domain from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\nOptionally Filters variables based on specified metadata criteria.\n\nRule Type: Variable Metadata Check\n\n```yaml\nCheck:\n all:\n - name: variable_name\n operator: is_not_contained_by\n value: $ig_variables\nOperations:\n - id: $ig_variables\n operator: get_column_order_from_library\n key_name: \"role\" # role, core, etc\n key_value: \"Exp\" # Timing, Req, Exp, Perm, etc\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "get_library_class_domains", + "markdownDescription": "\nReturns the list of domains for a given class from the CDISC Library Implementation Guide. This operation retrieves all domains that belong to a specified class (e.g., \"TRIAL DESIGN\", \"FINDINGS\", \"EVENTS\") based on the current standard and version. The operation uses the standard and version from the validation context as well as the optional `domain_class` parameter which is the name of the class to filter by (e.g., \"TRIAL DESIGN\", \"FINDINGS\", \"EVENTS\", \"INTERVENTIONS). NOTE: Class names are case-sensitive and should match the Library metadata format. If no `domain_class` parameter is provided, the operation returns all domains across all classes in the Implementation Guide:\n\n```yaml\n- operator: get_library_class_domains\n id: $trial_design_domains\n domain_class: \"TRIAL DESIGN\"\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "get_model_column_order", + "markdownDescription": "\nFetches column order for a given model class from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\n\nRule Type: Variable Metadata Check\n\n```yaml\nCheck:\n all:\n - name: variable_name\n operator: is_not_contained_by\n value: $model_variables\nOperations:\n - id: $model_variables\n operator: get_model_column_order\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "get_model_filtered_variables", + "markdownDescription": "\nFetches variable level library model properties filtered by the provided key_name and key_value\n\nExample\n\nInput\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: get_model_filtered_variables\n id: $model_filtered_variables\n key_name: \"role\"\n key_value: \"Timing\"\n```\n\nOutput\n\n```\n[\"VISITNUM\", \"VISIT\", \"VISITDY\", \"TAETORD\", \"...\"]\n```\n" + } + }, + "required": [ + "id", + "operator", + "key_name", + "key_value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "get_parent_model_column_order", + "markdownDescription": "\nFetches column order for a given SUPP's parent model class from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\n\n```yaml\nCheck:\n all:\n - operator: is_not_contained_by\n value: $parent_model_variables\nOperations:\n - id: $parent_model_variables\n operator: get_parent_model_column_order\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "get_dataset_filtered_variables", + "markdownDescription": "\nFilters variables from the dataset based on specified metadata criteria. Returns a list of variable names that exist in the dataset and match the filter criteria.\n\n```yaml\n- operator: get_dataset_filtered_variables\n id: $timing_variables\n key_name: \"role\"\n key_value: \"Timing\"\n```\n" + } + }, + "required": [ + "id", + "operator", + "key_name", + "key_value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "label_referenced_variable_metadata", + "markdownDescription": "\nGenerates a dataframe where each record in the dataframe is the library ig variable metadata corresponding with the variable label found in the column provided in name. The metadata column names are prefixed with the string provided in `id`.\n\nInput\n\nTarget Dataset: SUPPLB\n\nProduct: sdtmig\n\nVersion: 3-4\n\nDataset:\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QLABEL\": [\"Toxicity\", \"Viscosity\", \"Analysis Method\"]\n}\n```\n\nRule:\n\n```yaml\n- operator: label_referenced_variable_metadata\n id: $qlabel_referenced_variable_metadata\n name: \"QLABEL\"\n```\n\nOutput\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QLABEL\": [\"Toxicity\", \"Viscosity\", \"Analysis Method\"],\n \"$qlabel_referenced_variable_metadata_name\": [\"LBTOX\", null, \"LBANMETH\"],\n \"$qlabel_referenced_variable_metadata_role\": [\n \"Variable Qualifier\",\n null,\n \"Record Qualifier\"\n ],\n \"$qlabel_referenced_variable_metadata_ordinal\": [44, null, 38],\n \"$qlabel_referenced_variable_metadata_core\": [\"Req\", \"Req\", \"Req\"],\n \"$qlabel_referenced_variable_metadata_label\": [\"Toxicity\", null, \"Analysis Method\"]\n}\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "map", + "markdownDescription": "\nAllows the creation of a lookup table to take the values from multiple input columns and map them to values in an output column. The map parameter contains a list of objects. Each dictionary contains column names as properties that match the column names in the source dataset and an output property that will be returned as a result.\n\nIf map has a single object and output is the only property specified on that object, this will function as a direct assignment.\n\nFor example, given the following current dataset:\n\n```\nid \tparent_entity \tparent_rel\n1 \tTiming \trelativeToFrom\n2 \tSomething \trelativeToFrom\n3 \tTiming \ttype\n```\n\nand the following operation:\n\n```yaml\nOperations:\n - id: $codelist_code\n operator: map\n map:\n - parent_entity: Timing\n parent_rel: type\n output: C201264\n - parent_entity: Timing\n parent_rel: relativeToFrom\n output: C201265\n```\n\nThis will result in the following dataset:\n\n```\nid \tparent_entity \tparent_rel \t$codelist_code\n1 \tTiming \trelativeToFrom \tC201265\n2 \tSomething \trelativeToFrom \tNone\n3 \tTiming \ttype \tC201264\n```\n\nThe following operation:\n\n```yaml\nOperations:\n - id: $codelist_code\n operator: map\n map:\n - output: C201264\n```\n\nWill result in the following dataset:\n\n```\nid \tparent_entity \tparent_rel \t$codelist_code\n1 \tTiming \trelativeToFrom \tC201264\n2 \tSomething \trelativeToFrom \tC201264\n3 \tTiming \ttype \tC201264\n```\n" + } + }, + "required": [ + "id", + "operator", + "map" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "max", + "markdownDescription": "\nIf no group is provided, returns the max value in name. If group is provided, returns the max value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: \"$max_age\"\n operator: \"greater_than\"\n value: \"MAXAGE\"\nOperations:\n - operator: \"max\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$max_age\"\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "max_date", + "markdownDescription": "\nIf no group is provided, returns the max date value in name. If group is provided, returns the max date value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: USUBJID\n operator: is_contained_by\n value: $ex_usubjid\n - name: RFXENDTC\n operator: not_equal_to\n value: $max_ex_exstdtc\n - name: RFXENDTC\n operator: not_equal_to\n value: $max_ex_exendtc\nOperations:\n - operator: distinct\n domain: EX\n name: USUBJID\n id: $ex_usubjid\n - operator: max_date\n domain: EX\n name: EXSTDTC\n id: $max_ex_exstdtc\n group:\n - USUBJID\n - operator: max_date\n domain: EX\n name: EXENDTC\n id: $max_ex_exendtc\n group:\n - USUBJID\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "mean", + "markdownDescription": "\nExample: AAGE > mean(DM.AGE), where AAGE is a fictitious NSV\n\n```yaml\nCheck:\n all:\n - name: \"AAGE\"\n operator: \"greater_than\"\n value: \"$average_age\"\nOperations:\n - operator: \"mean\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$average_age\"\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "min", + "markdownDescription": "\nIf no group is provided, returns the min value in name. If group is provided, returns the min value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: \"$min_age\"\n operator: \"less_than\"\n value: \"MINAGE\"\nOperations:\n - operator: \"min\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$min_age\"\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "min_date", + "markdownDescription": "\nIf no group is provided, returns the min date value in name. If group is provided, returns the min date value in name, within each unique set of the grouping variables.\n\nExample: RFSTDTC is greater than min AE.AESTDTC for the current USUBJID\n\n```yaml\nCheck:\n all:\n - name: \"RFSTDTC\"\n operator: \"date_greater_than\"\n value: \"$ae_aestdtc\"\nOperations:\n - operator: \"min_date\"\n domain: \"AE\"\n name: \"AESTDTC\"\n id: \"$ae_aestdtc\"\n group:\n - USUBJID\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "minus", + "markdownDescription": "\nComputes set difference: elements in `name` that are not in `subtract`. By default a standard [set difference]() semantics (A \u2216 B) is applied. Optional `order_insensitive` property allows to have the element order to be taken into consideration and only those `name` elements are removed which follow the same order as in `subtract` . Preserves order from the first list. Both `name` and `subtract` must reference other operation results (e.g., `$expected_variables`, `$dataset_variables`). When `subtract` is empty or missing, returns all elements from `name`. Can be computed and added to output variables to display missing elements in error results.\n\n```yaml\nOperations:\n - id: $expected_variables\n operator: expected_variables\n - id: $dataset_variables\n operator: get_column_order_from_dataset\n - id: $expected_minus_dataset\n name: $expected_variables\n operator: minus\n subtract: $dataset_variables\n order_insensitive: false\n```\n" + } + }, + "required": [ + "id", + "operator", + "name", + "subtract" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "name_referenced_variable_metadata", + "markdownDescription": "\nGenerates a dataframe where each record in the dataframe is the library ig variable metadata corresponding with the variable name found in the column provided in name. The metadata column names are prefixed with the string provided in `id`.\n\nInput\n\nTarget Dataset: SUPPLB\n\nProduct: sdtmig\n\nVersion: 3-4\n\nDataset:\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QNAM\": [\"Toxicity\", \"LBVISCOS\", \"Analysis Method\"]\n}\n```\n\nRule:\n\n```yaml\n- operator: name_referenced_variable_metadata\n id: $qnam_referenced_variable_metadata\n name: \"QNAM\"\n```\n\nOutput\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QNAM\": [\"LBTOX\", \"LBVISCOS\", \"LBANMETH\"],\n \"$qnam_referenced_variable_metadata_name\": [\"LBTOX\", null, \"LBANMETH\"],\n \"$qnam_referenced_variable_metadata_role\": [\n \"Variable Qualifier\",\n null,\n \"Record Qualifier\"\n ],\n \"$qnam_referenced_variable_metadata_ordinal\": [44, null, 38],\n \"$qnam_referenced_variable_metadata_core\": [\"Req\", \"Req\", \"Req\"],\n \"$qnam_referenced_variable_metadata_label\": [\"Toxicity\", null, \"Analysis Method\"]\n}\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "permissible_variables", + "markdownDescription": "\nReturns the permissible variables (\"Core\" = Perm ) for a given domain and standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: permissible_variables\n id: $permissible_variables\n```\n\nOutput:\n\n```\n[\"LBGRPID\", \"LBREFID\", \"LBSPID\", \"...\"]\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "record_count", + "markdownDescription": "\nIf no filter or group is provided, returns the number of records in the dataset. If filter is provided, returns the number of records in the dataset that contain the value(s) in the corresponding column(s) provided in the filter. Filter can have a wildcard `&` that when added to the end of the filter value will look for all instances of that prefix (see 4th example below). If group is provided, returns the number of rows matching each unique set of the grouping variables. These can be static column name(s) or can be derived from other operations like get_dataset_filtered_variables.\n\nIf both filter and group are provided, returns the number of records in the dataset that contain the value(s) in the corresponding column(s) provided in the filter that also match each unique set of the grouping variables.\n\n**Wildcard Filtering:** Filter values ending with % will match any records where the column value starts with the specified prefix. For example, RACE% will match RACE1, RACE2, RACE3, etc. This is useful for matching related variables with numeric or alphabetic suffixes.\n\n**Regex Transformation:** If regex is provided along with group, the regex pattern will be applied to transform grouping column values before grouping. The regex is only applied to columns where the pattern matches the data type. For example, using regex `^\\d{4}-\\d{2}-\\d{2}` on a column containing `2022-01-14T08:00` will extract `2022-01-14` for grouping purposes.\n\nIf group is provided, group_aliases may also be provided to assign new grouping variable names so that results grouped by the values in one set of grouping variables can be merged onto a dataset according to the same grouping value(s) stored in different set of grouping variables. When both group and group_aliases are provided, columns are renamed according to corresponding list position (i.e., the 1st column in group is renamed to the 1st column in group_aliases, etc.). If there are more columns listed in group than in group_aliases, only the group columns with corresponding group_aliases columns will be renamed. If there are more columns listed in group_aliases than in group, the extra column names in group_aliases will be ignored.\n\nExample: return the number of records in a dataset.\n\n```yaml\n- operator: record_count\n id: $records_in_dataset\n```\n\nExample: return the number of records where STUDYID = \"CDISC01\" and FLAGVAR = \"Y\".\n\n```yaml\n- operator: record_count\n id: $flagged_cdisc01_records_in_dataset\n filter:\n STUDYID: \"CDISC01\"\n FLAGVAR: \"Y\"\n```\n\nExample: return the number of records grouped by USUBJID and timing variables, extracting only the date portion from datetime values.\n\n```yaml\n- operator: record_count\n id: $records_per_usubjid_date\n group:\n - USUBJID\n - --TESTCD\n - $TIMING_VARIABLES\n regex: \"^\\d{4}-\\d{2}-\\d{2}\"\n```\n\nExample: return the number of records where QNAM starts with \"RACE\" (matches RACE1, RACE2, RACE3, etc.) per USUBJID.\n\n```yaml\n- operator: record_count\n id: $race_records_in_dataset\n filter:\n QNAM: \"RACE&\"\n group:\n - \"USUBJID\"\n```\n\nExample: return the number of records grouped by USUBJID.\n\n```yaml\n- operator: record_count\n id: $records_per_usubjid\n group:\n - USUBJID\n```\n\nExample: return the number of records grouped by USUBJID where FLAGVAR = \"Y\".\n\n```yaml\n- operator: record_count\n id: $flagged_records_per_usubjid\n group:\n - USUBJID\n filter:\n FLAGVAR: \"Y\"\n```\n\nExample: return the number of records grouped by USUBJID and IDVARVAL where QNAM = \"TEST1\" and IDVAR = \"GROUPID\", renaming the IDVARVAL column to GROUPID for subsequent merging.\n\n```yaml\n- operator: record_count\n id: $test1_records_per_usubjid_groupid\n group:\n - USUBJID\n - IDVARVAL\n filter:\n QNAM: \"TEST1\"\n IDVAR: \"GROUPID\"\n group_aliases:\n - USUBJID\n - GROUPID\n```\n\nExample: Group the StudyIdentifier dataset by parent_id and merge the result back to the context dataset StudyVersion using StudyVersion.id == StudyIdentifier.parent_id\n\n```yaml\nScope:\n Entities:\n Include:\n - StudyVersion\nOperations:\n - domain: StudyIdentifier\n filter:\n parent_entity: \"StudyVersion\"\n parent_rel: \"studyIdentifiers\"\n rel_type: \"definition\"\n studyIdentifierScope.organizationType.code: \"C70793\"\n studyIdentifierScope.organizationType.codeSystem: \"http://www.cdisc.org\"\n group:\n - parent_id\n group_aliases:\n - id\n id: $num_sponsor_ids\n operator: record_count\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "required_variables", + "markdownDescription": "\nReturns the required variables ( \"Core\" = Req ) for a given domain and standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: required_variables\n id: $required_variables\n```\n\nOutput:\n\n```\n[\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"LBSEQ\", \"LBTESTCD\", \"LBTEST\"]\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "split_by", + "markdownDescription": "\nSplits a dataset column by a given delimiter\n\n```yaml\nOperations:\n - name: PPSPEC\n delimiter: ;\n id: $ppspec_value\n operator: split_by\n```\n" + } + }, + "required": [ + "id", + "operator", + "delimiter", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "study_domains", + "markdownDescription": "\nReturns a list of the domains in the study\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "dataset_names", + "markdownDescription": "\nReturns a list of the submitted dataset filenames in all uppercase\n\nex. if TS.xpt, AE.xpt, EC.xpt, and SUPPEC.xpt are submitted -> [TS, AE, EC, SUPPEC] will be returned\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "standard_domains", + "markdownDescription": "\nReturns a list of valid SDTM domain names from the standard metadata. This can be used to compare extracted suffixes from DOMAIN values or dataset names.\n\nInput\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: standard_domains\n id: $valid_domain_names\n```\n\nOutput\n\n```\n[\"AE\", \"CM\", \"DM\", \"FA\", \"LB\", \"QS\", ...]\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "valid_codelist_dates", + "markdownDescription": "\nReturns the valid terminology package dates for a given standard.\n\nGiven a list of terminology packages:\n\n```\n[\n \"sdtmct-2023-10-26\",\n \"sdtmct-2023-12-13\",\n \"adamct-2023-12-13\",\n \"cdashct-2023-05-19\"\n]\n```\n\nand standard: sdtmig\n\nthe operation will return:\n\n```\n[\"2023-10-26\", \"2023-12-13\"]\n```\n\nBy default, the standard is as specified when running validation - as the validation runtime parameter and/or as specified in the rule header - and the list of terminology packages is obtained from the current cache. If required, the default standard may be overridden using the optional ct_package_types parameter. For example, given the same list of terminology packages, the following operation:\n\n```yaml\nOperations:\n - operator: valid_codelist_dates\n id: $valid_dates\n ct_package_types:\n - SDTM\n - CDASH\n```\n\nwill return:\n\n```\n[\"2023-05-19\", \"2023-10-26\", \"2023-12-13\"]\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "valid_define_external_dictionary_version", + "markdownDescription": "\nReturns true if the version of an external dictionary provided in the define.xml file matches the version parsed from the dictionary files.\n\nInput:\n\n```yaml\nOperations:\n - operator: valid_define_external_dictionary_version\n id: $is_valid_loinc_version\n external_dictionary_type: loinc\n```\n\nOutput:\n\n```\n[true, true, true, true]\n```\n" + } + }, + "required": [ + "id", + "operator", + "external_dictionary_type" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "valid_external_dictionary_value", + "markdownDescription": "\nReturns true if the target variable contains a valid external dictionary value, otherwise false\n\nCan be case insensitive by setting case_sensitive attribute to false. It is true by default.\n\nInput:\n\n```yaml\nOperations:\n - operator: valid_external_dictionary_value\n name: --DECOD\n id: $is_valid_decod_value\n external_dictionary_type: meddra\n dictionary_term_type: PT\n case_sensitive: false\n```\n\nOutput:\n\n```\n[true, false, false, true]\n```\n" + } + }, + "required": [ + "id", + "operator", + "name", + "external_dictionary_type", + "dictionary_term_type" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "valid_external_dictionary_code", + "markdownDescription": "\nReturns true if the target variable contains a valid external dictionary code, otherwise false\n\nInput:\n\n```yaml\nOperations:\n - operator: valid_external_dictionary_code\n name: --COD\n id: $is_valid_cod_code\n external_dictionary_type: meddra\n dictionary_term_type: PT\n```\n\nOutput:\n\n```\n[true, false, false, true]\n```\n" + } + }, + "required": [ + "id", + "operator", + "name", + "external_dictionary_type", + "dictionary_term_type" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "valid_external_dictionary_code_term_pair", + "markdownDescription": "\nReturns true if the row in the dataset contains a matching pair of code and term, otherwise false\n\nFor this operator, the name parameter should contain the name of the variable containing the code, and the external_dictionary_term_variable parameter should contain the name of the variable containing the term Input:\n\n```yaml\nOperations:\n - operator: valid_external_dictionary_code_term_pair\n name: --COD\n id: $is_valid_loinc_code_term_pair\n external_dictionary_type: loinc\n external_dictionary_term_variable: --DECOD\n```\n\nOutput:\n\n```\n[true, false, false, true]\n```\n" + } + }, + "required": [ + "id", + "operator", + "name", + "external_dictionary_type", + "external_dictionary_term_variable" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "valid_meddra_code_references", + "markdownDescription": "\nDetermines whether the values are valid in the following variables:\n\n- --SOCCD (System Organ Class Code)\n- --HLGTCD (High Level Group Term Code)\n- --HLTCD (High Level Term Code)\n- --PTCD (Preferred Term Code)\n- --LLTCD (Lowest Level Term Code)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_codes\n operator: valid_meddra_code_references\n```\n\nOutput:\n\n```\n[true, false, true, true]\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "valid_meddra_code_term_pairs", + "markdownDescription": "\nDetermines whether the values are valid in the following variable pairs:\n\n- --SOCCD, --SOC (System Organ Class Code and Term)\n- --HLGTCD, --HLGT (High Level Group Term Code and Term)\n- --HLTCD, --HLT (High Level Term Code and Term)\n- --PTCD, --DECOD (Preferred Term Code and Dictionary-Derived Term)\n- --LLTCD, --LLT (Lowest Level Term Code and Term)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_pairs\n operator: valid_meddra_code_term_pairs\n```\n\nOutput:\n\n```\n[true, true, false, true, true]\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "valid_meddra_term_references", + "markdownDescription": "\nDetermines whether the values are valid in the following variables:\n\n- --SOC (System Organ Class)\n- --HLGT (High Level Group Term)\n- --HLT (High Level Term)\n- --DECOD (Dictionary-Derived Term)\n- --LLT (Lowest Level Term)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_terms\n operator: valid_meddra_term_references\n```\n\nOutput:\n\n```\n[true, true, false, true, true]\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "valid_whodrug_references", + "markdownDescription": "\nChecks if a reference to whodrug term in name points to the existing code in Atc Text (INA) file.\n\nInput:\n\n```yaml\nOperations:\n - id: $whodrug_refs_valid\n operator: valid_whodrug_references\n```\n\nOutput:\n\n```\n[true, false, true, true]\n```\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "variable_count", + "markdownDescription": "\nReturns a mapping of variable names to the number of times that variable appears in a domain within the study.\n\nInput\n\n```\n{\n \"AE\": [\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"AETERM\", \"AEENDTC\"],\n \"LB\": [\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"LBTESTCD\", \"LBENDTC\"]\n}\n```\n\nOutput\n\n```\n{\n \"STUDYID\": 2,\n \"DOMAIN\": 2,\n \"USUBJID\": 2,\n \"--TERM\": 1,\n \"--TESTCD\": 1,\n \"--ENDTC\": 2\n}\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "variable_exists", + "markdownDescription": "\nOperation operates only on original submission datasets regardless of rule type. Flags an error if a column exists is in the submission dataset currently being evaluated.\n\nRule Type: Domain Presence Check\n\n```yaml\nCheck:\n all:\n - name: $MIDS_EXISTS\n operator: equal_to\n value: true\n - name: TM\n operator: not_exists\nOperations:\n - id: $MIDS_EXISTS\n name: MIDS\n operator: variable_exists\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "variable_is_null", + "markdownDescription": "\nReturns true if a variable is missing from the dataset or if all values within the variable are null or empty string. This operation first checks if the target variable exists in the dataset, and if it does exist, evaluates whether all its values are null or empty.\nThe operation supports two sources via the `source` parameter:\n\n- **`submission`** : checks against the raw submission dataset\n- **`evaluation`** (default): checks against the evaluation dataset built based on the rule type\n\n```yaml" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "variable_names", + "markdownDescription": "\nReturns the set of variable names from the library for the given standard. This operation extracts all variable names across all domains in the specified standard's library metadata.\n\nInput:\n\nValidation Standard: sdtmig\nValidation Version: 3-4\n\n```yaml\n- operator: variable_names\n id: $all_variables\n```\n\nOutput:\n\n```\n[\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"SUBJID\", \"RFSTDTC\", \"RFENDTC\", \"SITEID\", \"AGE\", \"AGEU\", \"SEX\", \"RACE\", \"ETHNIC\", \"ARMCD\", \"ARM\", \"ACTARMCD\", \"ACTARM\", \"COUNTRY\", \"DMDTC\", \"DMDY\", \"AETERM\", \"AEDECOD\", \"AECAT\", \"AESCAT\", \"AEPRESP\", \"AEBODSYS\", \"AEBDSYCD\", \"AESOC\", \"AESOCCD\", \"AELLT\", \"AELLTCD\", \"AEHLT\", \"AEHLTCD\", \"AEHLGT\", \"AEHLGTCD\", \"AEPTCD\", \"AESTDTC\", \"AEENDTC\", \"AESTDY\", \"AEENDY\", \"AEDUR\", \"AESER\", \"AESEV\", \"AEACN\", \"AEREL\", \"AEOUT\", \"AESCAN\", \"AESCONG\", \"AESDISAB\", \"AESDTH\", \"AESHOSP\", \"AESLIFE\", \"AESOD\", \"AECONTRT\", \"AETOXGR\", \"LBTESTCD\", \"LBTEST\", \"LBCAT\", \"LBSCAT\", \"LBSPEC\", \"LBMETHOD\", \"LBORRES\", \"LBORRESU\", \"LBORNRLO\", \"LBORNRHI\", \"LBSTRESC\", \"LBSTRESN\", \"LBSTRESU\", \"LBSTNRLO\", \"LBSTNRHI\", \"LBNRIND\", \"LBNAM\", \"LBSPEC\", \"LBANTREG\", \"LBFAST\", \"LBDRVFL\", \"LBTOX\", \"LBTOXGR\", \"LBSTDTC\", \"LBENDTC\", \"LBSTDY\", \"LBENDY\", \"LBTPT\", \"LBTPTNUM\", \"LBELTM\", \"LBTPTREF\", \"LBRFTDTC\", \"...\"]\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "variable_value_count", + "markdownDescription": "\nGiven a variable name, returns a mapping of variable values to the number of times that value appears in the variable within all datasets in the study.\n" + } + }, + "required": [ + "id", + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "whodrug_code_hierarchy", + "markdownDescription": "\nDetermines whether the values are valid and in the correct hierarchical structure in the following variables:\n\n- --DECOD\n- --CLAS\n- --CLASCD\n\nInput:\n\n```yaml\nOperations:\n - id: $valid_whodrug_codes\n operator: whodrug_code_hierarchy\n```\n" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "get_xhtml_errors", + "markdownDescription": "\nValidates XHTML fragments in the target column against the specified namespace.\n\n```yaml\nOperations:\n - id: $xhtml_errors\n name: text\n operator: get_xhtml_errors\n namespace: http://www.cdisc.org/ns/usdm/xhtml/v1.0\n```\n\nNote that a local XSD file is required for validation. The file must be stored in the folder indicated by the value of the `LOCAL_XSD_FILE_DIR` default file path and the mapping between the namespace and the local XSD file's `sub-folder/name` must be included in the value of the `LOCAL_XSD_FILE_MAP` default file path.\n" + } + }, + "required": [ + "id", + "operator", + "name", + "namespace" + ], + "type": "object" + }, + { + "properties": { + "find": { + "type": "string" + }, + "flags": { + "pattern": "^[ims]*$", + "type": "string" + }, + "on_no_match": { + "enum": [ + "keep_original", + "set_null", + "set_empty", + "error" + ], + "type": "string" + }, + "operator": { + "const": "regex_find_replace" + }, + "replace": { + "type": "string" + } + }, + "required": [ + "id", + "operator", + "name", + "find", + "replace" + ], + "type": "object" + } + ], + "properties": { + "attribute_name": { + "$ref": "#/$defs/MetaVariables.json" + }, + "case_sensitive": { + "type": "boolean" + }, + "codelist": { + "type": "string" + }, + "codelist_code": { + "type": "string" + }, + "codelists": { + "items": { + "type": "string" + }, + "type": "array" + }, + "ct_attribute": { + "type": "string" + }, + "ct_package_type": { + "enum": [ + "ADAM", + "CDASH", + "COA", + "DDF", + "DEFINE-XML", + "GLOSSARY", + "MRCT", + "PROTOCOL", + "QRS", + "QS-FT", + "SDTM", + "SEND", + "TMF" + ], + "type": "string" + }, + "ct_package_types": { + "items": { + "$ref": "#/$defs/Operations.json/properties/ct_package_type" + }, + "type": "array" + }, + "ct_packages": { + "items": { + "type": "string" + }, + "type": "array" + }, + "ct_version": { + "type": "string" + }, + "delimiter": { + "type": "string" + }, + "dictionary_term_type": { + "enum": [ + "LLT", + "PT", + "HLT", + "HLGT", + "SOC" + ] + }, + "domain": { + "anyOf": [ + { + "$ref": "#/$defs/Dataset" + }, + { + "$ref": "#/$defs/DataStructure" + } + ] + }, + "external_dictionary_type": { + "enum": [ + "meddra" + ] + }, + "filter": { + "type": "object" + }, + "filter_key": { + "type": "string" + }, + "filter_value": { + "type": "string" + }, + "find": { + "type": "string" + }, + "flags": { + "pattern": "^[ims]*$", + "type": "string" + }, + "group": { + "items": { + "$ref": "#/$defs/VariableReference" + }, + "type": "array" + }, + "group_aliases": { + "items": { + "$ref": "#/$defs/VariableReference" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "key_name": { + "enum": [ + "definition", + "examples", + "label", + "name", + "notes", + "ordinal", + "role", + "simpleDatatype", + "variableCcode" + ], + "type": "string" + }, + "key_value": { + "type": "string" + }, + "level": { + "enum": [ + "codelist", + "term" + ], + "type": "string" + }, + "map": { + "items": { + "properties": { + "output": { + "type": "string" + } + }, + "required": [ + "output" + ], + "type": "object" + }, + "type": "array" + }, + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "on_no_match": { + "enum": [ + "keep_original", + "set_null", + "set_empty", + "error" + ], + "type": "string" + }, + "operator": { + "type": "string" + }, + "order_insensitive": { + "type": "boolean" + }, + "regex": { + "type": "string" + }, + "replace": { + "type": "string" + }, + "returntype": { + "enum": [ + "code", + "value", + "pref_term" + ], + "type": "string" + }, + "source": { + "type": "string" + }, + "subtract": { + "type": "string" + }, + "term_code": { + "type": "string" + }, + "term_pref_term": { + "type": "string" + }, + "term_value": { + "type": "string" + }, + "value_is_reference": { + "type": "boolean" + }, + "version": { + "type": "string" + } + }, + "required": [ + "id", + "operator" + ], + "type": "object" + }, + "Operator.json": { + "additionalProperties": false, + "anyOf": [ + { + "properties": { + "operator": { + "const": "additional_columns_empty" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "additional_columns_not_empty" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "contains", + "markdownDescription": "\nWill return True if the value in `value` is contained within the collection/iterable in the target column, or if there's an exact match for non-iterable data.\n\nThe operator checks if every value in a column is a list or set. If yes, it compares row-by-row. If any value is blank or a different type (like a string or number), it compares each value against the entire column instead.\n\nExample:\n\n```yaml\n- name: \"--TOXGR\" # Column containing lists like ['GRADE', 'SEVERITY', 'ONSET']\n operator: \"contains\"\n value: \"GRADE\" # True if 'GRADE' is an element in the list\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "contains_all", + "markdownDescription": "\nTrue if all values in `value` are contained within the variable `name`.\n\n> All of ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment') in ACTARM\n\n```yaml\n- name: \"ACTARM\"\n operator: \"contains_all\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n\nThe operator also supports lists:\n\n```yaml\n- name: \"$spec_codelist\"\n operator: \"contains_all\"\n value: \"$ppspec_value\"\n```\n\nWhere:\n\n| $spec_codelist | $ppspec_value |\n| :-------------------------- | :----------------: |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\", \"CODE2\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE2\", \"CODE3\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\"] |\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "contains_case_insensitive", + "markdownDescription": "\nTrue if the value in `value` is contained within the collection/iterable in the target column, performing case-insensitive comparison.\n\nExample:\n\n```yaml\n- name: \"--TOXGR\" # Column containing lists like ['Grade', 'Severity', 'Onset']\n operator: \"contains_case_insensitive\"\n value: \"grade\" # True if 'Grade'/'GRADE'/'grade' exists in the list\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "date_equal_to", + "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified.\n\nThe `date_component` parameter accepts: `\"year\"`, `\"month\"`, `\"day\"`, `\"hour\"`, `\"minute\"`, `\"second\"`, `\"microsecond\"`, or `\"auto\"`.\n\nWhen `date_component: \"auto\"` is used, the operator automatically detects the precision of both dates and compares at the common (less precise) level.\n\n```yaml\n- name: \"AESTDTC\"\n operator: \"date_equal_to\"\n value: \"RFSTDTC\"\n date_component: \"auto\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "date_greater_than", + "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> Year part of BRTHDTC > 2021\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"date_greater_than\"\n date_component: \"year\"\n value: \"2021\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "date_greater_than_or_equal_to", + "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> Year part of BRTHDTC >= 2021\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"date_greater_than_or_equal_to\"\n date_component: \"year\"\n value: \"2021\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "date_less_than", + "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> AEENDTC < AESTDTC\n\n```yaml\n- name: \"AEENDTC\"\n operator: \"date_less_than\"\n value: \"AESTDTC\"\n```\n\n> SSDTC < all DS.DSSTDTC when SSSTRESC = \"DEAD\"\n\n```yaml\nCheck:\n all:\n - name: \"SSSTRESC\"\n operator: \"equal_to\"\n value: \"DEAD\"\n - name: \"SSDTC\"\n operator: \"date_less_than\"\n value: \"$max_ds_dsstdtc\"\nOperations:\n - operator: \"max_date\"\n domain: \"DS\"\n name: \"DSSTDTC\"\n id: \"$max_ds_dsstdtc\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "date_less_than_or_equal_to", + "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> AEENDTC <= AESTDTC\n\n```yaml\n- name: \"AEENDTC\"\n operator: \"date_less_than_or_equal_to\"\n value: \"AESTDTC\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "date_not_equal_to", + "markdownDescription": "\nComplement of `date_equal_to`\n\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "does_not_contain", + "markdownDescription": "\nComplement of `contains`. Returns True when the value is NOT contained within the target collection.\n\n```yaml\n- name: \"--TOXGR\"\n operator: \"does_not_contain\"\n value: \"GRADE\" # True if 'GRADE' is NOT an element in the list\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "does_not_contain_case_insensitive", + "markdownDescription": "\nComplement of `contains_case_insensitive`. Returns True when the value is NOT contained within the target collection (case-insensitive).\n\nExample:\n\n```yaml\n- name: \"--TOXGR\"\n operator: \"does_not_contain_case_insensitive\"\n value: \"grade\" # True if no case variation of 'grade' exists in the list\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "does_not_equal_string_part", + "markdownDescription": "\nComplement of `equals_string_part`. Also has the optional parameter 'type_insensitive'.\n" + }, + "type_insensitive": { + "type": "boolean" + } + }, + "required": [ + "operator", + "value", + "regex" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "does_not_have_next_corresponding_record", + "markdownDescription": "\nComplement of `has_next_corresponding_record`\n" + } + }, + "required": [ + "operator", + "ordering", + "value", + "within" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "empty", + "markdownDescription": "\nValue presence\n\n> --OCCUR = null\n\n```yaml\n- name: --OCCUR\n operator: empty\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "empty_within_except_last_row", + "markdownDescription": "\n> SEENDTC is not empty when it is not the last record, grouped by USUBJID, sorted by SESTDTC\n\n```yaml\n- name: SEENDTC\n operator: empty_within_except_last_row\n ordering: SESTDTC\n value: USUBJID\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "ends_with", + "markdownDescription": "\nSubstring matching\n\n> DOMAIN ending with 'FOOBAR'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"ends_with\"\n value: \"FOOBAR\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "equal_to", + "markdownDescription": "\nValue comparison. Works for both string and number.\nHas optional parameter:\n\n- 'value_is_reference' when true, the value parameter specifies a column name whose content determines which column to compare against dynamically.\n- 'type_insensitive' when true, both values are converted to strings before comparison to handle type mismatches between string and numeric data. NOTE: all trailing zeroes will be removed in both strings and floats.\n- 'round_values' when true, both the target and value will be rounded to the nearest integer\n\n> --OCCUR = N\n\n```yaml\n- name: --OCCUR\n operator: equal_to\n value: \"N\"\n```\n\n> IDVARVAL = the column specified in the IDVAR column for each row (type insensitive comparison).\n\n```yaml\n- name: IDVARVAL\n operator: equal_to\n value: \"IDVAR\"\n value_is_reference: true\n type_insensitive: true\n```\n\n> --STRESC = --STRESN with rounded values and ignoring the char/num type differences\n> between the two columns\n\n```yaml\n- name: --STRESC\n operator: equal_to\n type_insensitive: true\n value: --STRESN\n round_values: true\n```\n\n> EXDOSE EQ 0\n\n```yaml\n- name: EXDOSE\n operator: equal_to\n value: 0\n```\n" + }, + "round_values": { + "type": "boolean" + }, + "type_insensitive": { + "type": "boolean" + }, + "value_is_reference": { + "type": "boolean" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "equal_to_case_insensitive", + "markdownDescription": "\nCase insensitive `equal_to`. Also has the optional parameters 'value_is_reference', 'round_values' and 'type_insensitive'.\n\n> DSTERM is \"Informed consent obtained\"\n\n```yaml\n- name: DSTERM\n operator: equal_to_case_insensitive\n value: Informed consent obtained\n```\n" + }, + "round_values": { + "type": "boolean" + }, + "type_insensitive": { + "type": "boolean" + }, + "value_is_reference": { + "type": "boolean" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "equals_string_part", + "markdownDescription": "\nChecks that the values in the target column equal the result of parsing the value in the comparison column with a regex\nHas optional parameter:\n\n- 'type_insensitive' when true, both values are converted to strings before comparison to handle type mismatches between string and numeric data. NOTE: all leading and trailing zeroes will be removed in both strings and floats.\n\n> RDOMAIN equals characters 5 and 6 of SUPP dataset name\n\n```yaml\n- name: RDOMAIN\n operator: equals_string_part\n type_insensitive: true\n value: dataset_name\n regex: \".{4}(..).*\"\n```\n" + }, + "type_insensitive": { + "type": "boolean" + } + }, + "required": [ + "operator", + "value", + "regex" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "exists", + "markdownDescription": "\nTrue if the column exists in the current dataframe. (Works for datasets and variables)\n\n> --OCCUR is present in dataset\n\n```yaml\n- name: \"--OCCUR\"\n operator: \"exists\"\n```\n\n> Domain SJ exists\n\n```yaml\nRule Type: Domain Presence Check\nCheck:\n all:\n - name: \"SJ\"\n operator: \"exists\"\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "greater_than", + "markdownDescription": "\nValue comparison\n\n> TSVAL > 0\n\n```yaml\n- name: TSVAL\n operator: greater_than\n value: 0\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "greater_than_or_equal_to", + "markdownDescription": "\nValue comparison\n\n> TSVAL >= 0\n\n```yaml\n- name: TSVAL\n operator: greater_than_or_equal_to\n value: 1\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "has_different_values", + "markdownDescription": "\nComplement of `has_same_values`\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "has_equal_length", + "markdownDescription": "\nLength comparison\n\n> Check whether variable values has equal length of another variable.\n\n```yaml\n- name: SEENDTC\n operator: has_equal_length\n value: SESTDTC\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "has_next_corresponding_record", + "markdownDescription": "\nEnsures that a value of a variable `name` in one record is equal to the value of another variable `value` in the next corresponding record. The rows are grouped by `within` and ordered by `ordering`.\n\n> SEENDTC is equal to the SESTDTC of the next record within a USUBJID. Ordered by SESEQ\n\n```yaml\n- name: SEENDTC\n operator: has_next_corresponding_record\n value: SESTDTC\n within: USUBJID\n ordering: SESEQ\n```\n" + } + }, + "required": [ + "operator", + "ordering", + "value", + "within" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "has_not_equal_length", + "markdownDescription": "\nComplement of `has_equal_length`\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "has_same_values", + "markdownDescription": "\nTrue if all values in `name` are the same\n\n> Condition: MHCAT ^= null\n> Rule: MHCAT ^= the same value for all records\n\n```yaml\nCheck:\n all:\n - name: MHCAT\n operator: non_empty\n - name: MHCAT\n operator: has_same_values\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "invalid_duration", + "markdownDescription": "\nDuration ISO-8601 check, returns True if a duration is not in ISO-8601 format. The negative parameter must be specified to indicate if negative durations are either allowed (True) or disallowed (False)\n\n> DURVAR is invalid (negative durations disallowed)\n\n```yaml\n- name: \"DURVAR\"\n operator: \"invalid_duration\"\n negative: False\n```\n" + } + }, + "required": [ + "operator", + "negative" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "invalid_date", + "markdownDescription": "\nThe operator performs date validation against complete and partial dates with uncertainty in the following order:\n\n1. Attempts to parse using [dateutil.parser.isoparse()](https://dateutil.readthedocs.io/en/stable/parser.html)\n2. If parsing fails and the string contains uncertainty indicators (`/`, `--`, `-:`), validates against an extended ISO 8601 dates regex pattern\n3. If parsing succeeds, dates are still validated against the regex pattern.\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"invalid_date\"\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_complete_date", + "markdownDescription": "\nDate check\n\n> DM.RFSTDTC = complete date\n\n```yaml\n- name: \"RFSTDTC\"\n operator: \"is_complete_date\"\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_contained_by", + "markdownDescription": "\nValue in `name` compared against a list in `value`. The list can have literal values or be a reference to a `$variable`.\n\nThis operator behaves similarly to `contains`. The key distinction: `contains` checks if comparator \u2208 target, while `is_contained_by` checks if target \u2208 comparator.\n\n> ACTARM in ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment')\n\n```yaml\n- name: \"ACTARM\"\n operator: \"is_contained_by\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_contained_by_case_insensitive", + "markdownDescription": "\nValue in `name` case insensitive compared against a list in `value`. The list can have literal values or be a reference to a `$variable`.\n\n> ACTARM in ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment')\n\n```yaml\n- name: \"ACTARM\"\n operator: \"is_contained_by_case_insensitive\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_incomplete_date", + "markdownDescription": "\nComplement of `is_complete_date`\n\nDate check\n\n> DM.RFSTDTC ^= complete date\n\n```yaml\n- name: \"RFSTDTC\"\n operator: \"is_incomplete_date\"\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_not_contained_by", + "markdownDescription": "\nComplement of `is_contained_by`\n\n> ARM not in ('Screen Failure', 'Not Assigned')\n\n```yaml\n- name: \"ARM\"\n operator: \"is_not_contained_by\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_not_contained_by_case_insensitive", + "markdownDescription": "\nComplement of `is_contained_by_case_insensitive`\n\n> ARM not in ('Screen Failure', 'Not Assigned')\n\n```yaml\n- name: \"ARM\"\n operator: \"is_not_contained_by_case_insensitive\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_not_ordered_by", + "markdownDescription": "\nComplement of `is_ordered_by`\n" + } + }, + "required": [ + "operator", + "order" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_not_ordered_set" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_not_unique_relationship", + "markdownDescription": "\nComplement of `is_unique_relationship`\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_not_unique_set", + "markdownDescription": "\nComplement of `is_unique_set`.\n\n> --SEQ is not unique within DOMAIN, USUBJID, and --TESTCD\n\n```yaml\n- name: \"--SEQ\"\n operator: is_not_unique_set\n value:\n - \"DOMAIN\"\n - \"USUBJID\"\n - \"--TESTCD\"\n```\n\n> `name` can be a variable containing a list of columns and `value` does not need to be present\n\n```yaml\nRule Type: Dataset Contents Check against Define XML\nCheck:\n all:\n - name: define_dataset_key_sequence # contains list of dataset key columns\n operator: is_not_unique_set\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_ordered_by", + "markdownDescription": "\nTrue if the dataset rows are ordered by the values within `name`, given the ordering specified by `order`\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_by\n order: asc\n```\n" + } + }, + "required": [ + "operator", + "order" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_ordered_set", + "markdownDescription": "\nTrue if the dataset rows are in ascending order of the values within `name`, grouped by the values within `value`. Value can either be a single column or multiple.\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_set\n value: USUBJID\n```\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_set\n value:\n - USUBJID\n - \"--TESTCD\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_unique_relationship", + "markdownDescription": "\nRelationship Integrity Check looking for a 1-1 relationship between name and value. Ensures uniqueness of both name and value.\n\n> AETERM and AEDECOD has a 1-to-1 relationship\n\n```yaml\n- name: AETERM\n operator: is_unique_relationship\n value: AEDECOD\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_inconsistent_across_dataset", + "markdownDescription": "\nChecks if a variable maintains consistent values within groups defined by one or more grouping variables. Groups records by specified value(s) and validates that the target variable maintains the same value within each unique combination of grouping variables. When inconsistency is detected within a group, the operator attempts to identify a majority value. If one value appears more frequently than all others, only the minority records (those not matching the majority value) are flagged. If no single majority exists \u2014 i.e., two or more values are tied for the highest frequency \u2014 all records in that group are flagged.\n\nSingle grouping variable - true if the values of BGSTRESU differ within USUBJID:\n\nIf a regex parameter is provided, it is applied to the values of the target variable before the consistency check. The first capture group of the regex is used as the normalized value for comparison. This can be useful when only part of the value should be considered during comparison (for example, comparing only the date portion of a datetime value).\n\n- regex is optional.\n- The pattern must include at least one capture group(or whole regex will be wrapped to capture group).\n- Only the first capture group is used for comparison.\n- If the pattern does not match a value, the original value is used.\n\n```yaml\n- name: \"BGSTRESU\"\n operator: is_inconsistent_across_dataset\n value: \"USUBJID\"\n```\n\nMultiple grouping variables - true if the values of --STRESU differ within each combination of --TESTCD, --CAT, --SCAT, --SPEC, and --METHOD:\n\n```yaml\n- name: \"--STRESU\"\n operator: is_inconsistent_across_dataset\n value:\n - \"--TESTCD\"\n - \"--CAT\"\n - \"--SCAT\"\n - \"--SPEC\"\n - \"--METHOD\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_unique_set", + "markdownDescription": "\nRelationship Integrity Check\n\n> --SEQ is unique within DOMAIN, USUBJID, and --TESTCD\n\n```yaml\n- name: \"--SEQ\"\n operator: is_unique_set\n value:\n - \"DOMAIN\"\n - \"USUBJID\"\n - \"--TESTCD\"\n```\n\n> `name` can be a variable containing a list of columns and `value` does not need to be present\n\n> The `regex` parameter allows you to extract portions of values using a regex pattern before checking uniqueness.\n\n> Compare date only (YYYY-MM-DD) for uniqueness\n\n```yaml\n- name: \"--REPNUM\"\n operator: is_not_unique_set\n value:\n - \"USUBJID\"\n - \"--TESTCD\"\n - \"$TIMING_VARIABLES\"\n regex: '^\\d{4}-\\d{2}-\\d{2}'\n```\n\n> Compare by first N characters of a string\n\n```yaml\n- name: \"ITEM_ID\"\n operator: is_not_unique_set\n value:\n - \"USUBJID\"\n - \"CATEGORY\"\n regex: \"^.{2}\"\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "less_than", + "markdownDescription": "\nValue comparison\n\n> TSVAL < 1\n\n```yaml\n- name: TSVAL\n operator: less_than\n value: 1\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "less_than_or_equal_to", + "markdownDescription": "\nValue comparison\n\n> TSVAL <= 1\n\n```yaml\n- name: TSVAL\n operator: less_than_or_equal_to\n value: 1\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "longer_than", + "markdownDescription": "\nLength comparison\n\n> SETCD value length > 8\n\n```yaml\n- name: \"SETCD\"\n operator: \"longer_than\"\n value: 8\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "longer_than_or_equal_to", + "markdownDescription": "\nLength comparison\n\n> TSVAL value length >= 201\n\n```yaml\n- name: \"TSVAL\"\n operator: \"longer_than_or_equal_to\"\n value: 201\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "matches_regex", + "markdownDescription": "\nRegular Expression value matching\n\n- Determine if each string starts with a match of a regular expression. Refer to this pandas documentation: https://pandas.pydata.org/docs/reference/api/pandas.Series.str.match.html\n- To \"search\" for a regex within the entire text, prefix the regex with `.*` and do not use anchors `^` , `$`\n- To do a \"fullmatch\" of a regex with the entire text, suffix the regex with an anchor `$` and do not prefix the regex with `.*`\n- For syntax guide, refer to this Python documentation: [Regular Expression HOWTO](https://docs.python.org/3/howto/regex.html).\n- Suggestion for an on-line regular expression logic. tester: https://regex101.com, choose the Python dialect.\n- For regex token visualization, try https://www.debuggex.com.\n\n> --DOSTXT value is non-numeric\n\n```yaml\n- name: --DOSTXT\n operator: matches_regex\n value: ^\\d*\\.?\\d*$\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "non_empty", + "markdownDescription": "\nComplement of `empty`\n\n> --OCCUR ^= null\n\n```yaml\n- name: --OCCUR\n operator: non_empty\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "non_empty_within_except_last_row", + "markdownDescription": "\nComplement of `empty_within_except_last_row`\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "not_contains_all", + "markdownDescription": "\nComplement of `contains_all`\n\n> All of ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment') not in ACTARM\n\n```yaml\n- name: \"ACTARM\"\n operator: \"not_contains_all\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n\nThe operator also supports lists:\n\n```yaml\n- name: \"$spec_codelist\"\n operator: \"not_contains_all\"\n value: \"$ppspec_value\"\n```\n\nWhere:\n\n| $spec_codelist | $ppspec_value |\n| :-------------------------- | :----------------: |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\", \"CODE2\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE2\", \"CODE3\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\"] |\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "not_equal_to", + "markdownDescription": "\nComplement of `equal_to`. Also has the optional parameters 'value_is_reference', 'round_values' and 'type_insensitive'.\n\n> --OCCUR ^= Y\n\n```yaml\n- name: --OCCUR\n operator: not_equal_to\n value: \"Y\"\n```\n" + }, + "round_values": { + "type": "boolean" + }, + "type_insensitive": { + "type": "boolean" + }, + "value_is_reference": { + "type": "boolean" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "not_equal_to_case_insensitive", + "markdownDescription": "\nComplement of `equal_to_case_insensitive`. Also has the optional parameters 'value_is_reference', 'round_values' and 'type_insensitive'.\n" + }, + "round_values": { + "type": "boolean" + }, + "type_insensitive": { + "type": "boolean" + }, + "value_is_reference": { + "type": "boolean" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "not_exists", + "markdownDescription": "\nComplement of `exists`\n\n> AEOCCUR not present in dataset\n\n```yaml\n- name: \"AEOCCUR\"\n operator: \"not_exists\"\n```\n\n> Domain SJ does not exist\n\n```yaml\nRule Type: Domain Presence Check\nCheck:\n all:\n - name: \"SJ\"\n operator: \"not_exists\"\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "not_matches_regex", + "markdownDescription": "\nComplement of `matches_regex`\n\n> --TESTCD <= 8 chars and contains only letters, numbers, and underscores and can not start with a number\n\n```yaml\n- name: --TESTCD\n operator: not_matches_regex\n value: ^[A-Z_][A-Z0-9_]{0,7}$\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "not_prefix_matches_regex", + "markdownDescription": "\nComplement of `prefix_matches_regex`\n" + } + }, + "required": [ + "operator", + "prefix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "not_present_on_multiple_rows_within", + "markdownDescription": "\nComplement of `present_on_multiple_rows_within`\n\n```yaml\n- operator: \"not_present_on_multiple_rows_within\"\n name: \"RELID\"\n value: 4 (optional)\n within: \"USUBJID\"\n```\n" + } + }, + "required": [ + "operator", + "within" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "not_suffix_matches_regex", + "markdownDescription": "\nComplement of `suffix_matches_regex`\n\n> QNAM does not end with numbers\n\n```yaml\n- name: \"QNAM\"\n operator: \"not_suffix_matches_regex\"\n suffix: 2\n value: \"\\d\\d\"\n```\n" + } + }, + "required": [ + "operator", + "suffix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "prefix_is_contained_by", + "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match one of the strings in the list in `value`\n\n> Check if a variable's domain identifier exists in the study\n\n```yaml\n- name: variable_name\n operator: prefix_is_contained_by\n prefix: 2\n value: $study_domains\n```\n" + } + }, + "required": [ + "operator", + "prefix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "prefix_equal_to", + "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match the string in `value`\n\n```yaml\n- name: dataset_name\n operator: prefix_equal_to\n prefix: 2\n value: DOMAIN\n```\n" + } + }, + "required": [ + "operator", + "prefix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "prefix_is_not_contained_by", + "markdownDescription": "\nComplement of `prefix_is_contained_by`\n" + } + }, + "required": [ + "operator", + "prefix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "prefix_matches_regex", + "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match a regular expression in `value`\n\n```yaml\n- name: DOMAIN\n operator: prefix_matches_regex\n prefix: 2\n value: (AP|ap)\n```\n" + } + }, + "required": [ + "operator", + "prefix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "prefix_not_equal_to", + "markdownDescription": "\nComplement of `prefix_equal_to`\n" + } + }, + "required": [ + "operator", + "prefix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "present_on_multiple_rows_within", + "markdownDescription": "\nTrue if the same value of `name` is present on multiple rows, grouped by `within`. A maximum allowed number of occurrences can be specified in the value attribute. In this instance the value: 4 means that an error will be flagged if the same value appears more than 4 times within a USUBJID. By default the operator will flag any time a value appears more than once.\n\n```yaml\n- operator: \"present_on_multiple_rows_within\"\n name: \"RELID\"\n value: 4 (optional)\n within: \"USUBJID\"\n```\n" + } + }, + "required": [ + "operator", + "within" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "shares_at_least_one_element_with", + "markdownDescription": "\nWill raise an issue if at least one of the values in `name` is the same as one of the values in `value`. See [shares_no_elements_with](#shares_no_elements_with).\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "shares_exactly_one_element_with", + "markdownDescription": "\nWill raise an issue if exactly one of the values in `name` is the same as one of the values in `value`. See [shares_no_elements_with](#shares_no_elements_with).\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "shares_no_elements_with", + "markdownDescription": "\nWill raise an issue if the values in `name` do not share any of the values in `value`\n\n> Check if $dataset_variables shares no elements with $timing_variables\n\n```yaml\nRule Type: Dataset Metadata Check # One record per dataset\nCheck:\n - all:\n name: $dataset_variables\n operator: shares_no_elements_with\n value: $timing_variables\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "shorter_than", + "markdownDescription": "\nLength comparison\n\n> SETCD value length < 9\n\n```yaml\n- name: \"SETCD\"\n operator: \"shorter_than\"\n value: 9\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "shorter_than_or_equal_to", + "markdownDescription": "\nLength comparison\n\n> TSVAL value length <= 200\n\n```yaml\n- name: \"TSVAL\"\n operator: \"shorter_than_or_equal_to\"\n value: 201\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "split_parts_have_equal_length", + "markdownDescription": "\nSplits a string by a separator and checks if both parts have equal length. Generic operator for validating paired data formats where both parts must have the same level of detail or precision.\n\nParameters:\n\n- `separator`: The delimiter to split on (default: \"/\")\n\n> Check that string parts separated by a delimiter have equal length\n\n```yaml\n- name: --DTC\n operator: split_parts_have_equal_length\n separator: \"/\"\n```\n\nUse cases:\n\n- **Date/time intervals**: `2003-12-15T10:00/2003-12-15T10:30` \u2192 True (both 16 characters)\n- **Date ranges**: `2003-12-01/2003-12-10` \u2192 True (both 10 characters)\n- **Version ranges**: `1.2.3/2.0.0` \u2192 True (both 5 characters)\n- **Product codes**: `ABC-123/XYZ-789` \u2192 True (both 7 characters)\n\nInvalid example:\n\n- `2003-12-15T10:00/2003-12-15T10:30:15` \u2192 False (16 vs 19 characters - different precision)\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "split_parts_have_unequal_length", + "markdownDescription": "\nComplement of `split_parts_have_equal_length`. Returns True when parts have unequal lengths (indicates a violation).\n\n```yaml\n- name: --DTC\n operator: split_parts_have_unequal_length\n separator: \"/\"\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "starts_with", + "markdownDescription": "\nSubstring matching\n\n> DOMAIN beginning with 'AP'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"starts_with\"\n value: \"AP\"\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "suffix_equal_to", + "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match the string in `value`\n\n```yaml\n- name: dataset_name\n operator: suffix_equal_to\n prefix: 2\n value: DOMAIN\n```\n" + } + }, + "required": [ + "operator", + "suffix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "suffix_is_contained_by", + "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match one of the strings in the list in `value`\n\n> Check if a supp's parent domain exists in the study\n\n```yaml\n- name: dataset_name\n operator: suffix_is_contained_by\n prefix: 2\n value: $study_domains\n```\n" + } + }, + "required": [ + "operator", + "suffix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "suffix_is_not_contained_by", + "markdownDescription": "\nComplement of `suffix_is_contained_by`\n" + } + }, + "required": [ + "operator", + "suffix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "suffix_matches_regex", + "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match a regular expression in `value`\n\n> QNAM ends with numbers\n\n```yaml\n- name: \"QNAM\"\n operator: \"suffix_matches_regex\"\n suffix: 2\n value: \"\\d\\d\"\n```\n" + } + }, + "required": [ + "operator", + "suffix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "suffix_not_equal_to", + "markdownDescription": "\nComplement of `suffix_equal_to`\n" + } + }, + "required": [ + "operator", + "suffix", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "target_is_not_sorted_by", + "markdownDescription": "\nComplement of `target_is_sorted_by`\n" + } + }, + "required": [ + "operator", + "value", + "within" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "target_is_sorted_by", + "markdownDescription": "\nTrue if the values in name are ordered according to the values specified by value\nin ascending/descending order, grouped by the values in within. Each value entry\nrequires a variable name, a sort_order of asc or desc, and an optional\nnull_position of first or last (defaults to last) which controls where null/empty\ncomparator values are placed in the expected ordering. Within accepts either a\nsingle column or an ordered list of columns. Columns can be either number or Char\nDates in ISO8601 YYYY-MM-DD format. Date value(s) with different precisions that\noverlap (e.g. 2005-10, 2005-10-3 and 2005-10-08) are all flagged as not sorted as\ntheir order cannot be inferred.\n\nOptionally supports a `regex` parameter that extracts a portion of the target\nvalue for sorting. The regex must contain at least one capturing group. The first\ncaptured group is extracted and converted to numeric if possible, allowing proper\nsorting of sequence numbers (e.g., \"MIDS1\", \"MIDS2\", ..., \"MIDS10\" with regex\n`.*?(\\\\d+)$`). This is particularly useful for variables that end with sequence\nnumbers that may or may not be zero-padded.\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n within:\n - USUBJID\n - MIDSTYPE\n operator: target_is_sorted_by\n value:\n - name: --STDTC\n sort_order: asc\n null_position: last\n```\n\nExample with regex for extracting sequence numbers:\n\n```yaml\nCheck:\n all:\n - name: MIDS\n operator: target_is_sorted_by\n regex: \".*?(\\\\d+)$\" # Extract trailing digits, convert to numeric\n value:\n - name: SMSTDTC\n sort_order: asc\n within:\n - USUBJID\n - MIDSTYPE\n```\n" + } + }, + "required": [ + "operator", + "value", + "within" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "value_does_not_have_multiple_references", + "markdownDescription": "\nComplement of `value_has_multiple_references`\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "value_has_multiple_references", + "markdownDescription": "\nTrue if the value in `name` has more than one count in the dictionary defined in `value`\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "inconsistent_enumerated_columns", + "markdownDescription": "\nChecks for inconsistencies in enumerated columns of a DataFrame. Starting with the smallest/largest enumeration of the given variable, returns True if VARIABLE(N+1) is populated but VARIABLE(N) is not populated. Repeats for all variables belonging to the enumeration. Note that the initial variable will not have an index (VARIABLE) and the next enumerated variable has index 1 (VARIABLE1).\n\nex: Check if there are inconsistencies in the TSVAL columns (TSVAL, TSVAL1, TSVAL2, etc.)\n\n```yaml\nCheck:\n all:\n - name: \"TSVAL\"\n operator: \"inconsistent_enumerated_columns\"\n```\n" + } + }, + "required": [ + "operator", + "name" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_ordered_subset_of", + "markdownDescription": "\nChecks if elements in the target list appear in the same relative order in the comparator list.\n\n> Check if dataset column order is a correctly ordered subset of library column order\n\n```yaml\n- name: $column_order_from_dataset\n operator: is_ordered_subset_of\n value: $column_order_from_library\n```\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_not_ordered_subset_of", + "markdownDescription": "\nComplement of `is_ordered_subset_of`\n" + } + }, + "required": [ + "operator", + "value" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_title_case", + "markdownDescription": "\nValidates that variable labels follow proper title case formatting rules using the titlecase PyPi library. Title case capitalizes the first word and all major words, while keeping articles (a, an, the), conjunctions (and, but, or), and prepositions (in, of, for) in lowercase unless they are the first word. \nNOTE: The titlecase library may produce false positives or false negatives in syntactic edge cases (e.g. hyphenated words, slash-separated terms, uncommon prepositions).\n\n> Check that AELABEL values are in proper title case\n\n```yaml\n- name: AELABEL\n operator: is_title_case\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "is_not_title_case", + "markdownDescription": "\nComplement of `is_title_case`. Returns True when values are NOT in proper title case.\n\n> Flag AELABEL values that violate title case rules\n\n```yaml\n- name: AELABEL\n operator: is_not_title_case\n```\n" + } + }, + "required": [ + "operator" + ], + "type": "object" + } + ], + "properties": { + "codelistcheck": { + "enum": [ + "code", + "value" + ], + "type": "string" + }, + "codelistlevel": { + "enum": [ + "term", + "codelist" + ], + "type": "string" + }, + "comparator": { + "type": [ + "number", + "string" + ] + }, + "context": { + "type": "string" + }, + "date_component": { + "enum": [ + "year", + "month", + "day", + "hour", + "minute", + "second", + "microsecond", + "auto" + ], + "type": "string" + }, + "metadata": { + "$ref": "#/$defs/Operations.json/properties/id" + }, + "name": { + "anyOf": [ + { + "$ref": "#/$defs/Operations.json/properties/id" + }, + { + "$ref": "#/$defs/MetaVariables.json" + }, + { + "$ref": "#/$defs/VariableName" + } + ] + }, + "negative": { + "type": "boolean" + }, + "operator": { + "type": "string" + }, + "order": { + "enum": [ + "asc", + "dsc" + ], + "type": "string" + }, + "ordering": { + "$ref": "#/$defs/VariableName" + }, + "prefix": { + "type": "integer" + }, + "regex": { + "type": "string" + }, + "round_values": { + "type": "boolean" + }, + "separator": { + "type": "string" + }, + "suffix": { + "type": "integer" + }, + "type_insensitive": { + "type": "boolean" + }, + "value": { + "oneOf": [ + { + "type": [ + "boolean", + "number", + "string" + ] + }, + { + "items": { + "type": [ + "number" + ] + }, + "type": "array" + }, + { + "items": { + "type": [ + "string" + ] + }, + "type": "array" + }, + { + "items": { + "properties": { + "name": { + "$ref": "#/$defs/Operator.json/properties/name" + }, + "null_position": { + "enum": [ + "first", + "last" + ], + "type": "string" + }, + "order": { + "$ref": "#/$defs/Operator.json/properties/order" + } + }, + "type": "object" + }, + "type": "array" + } + ] + }, + "value_is_literal": { + "const": true + }, + "value_is_reference": { + "type": "boolean" + }, + "within": { + "oneOf": [ + { + "$ref": "#/$defs/VariableName" + }, + { + "items": { + "$ref": "#/$defs/VariableName" + }, + "minItems": 1, + "type": "array" + } + ] + } + }, + "required": [ + "operator" + ], + "type": "object" + }, + "Organization_CDISC.json": { + "properties": { + "Organization": { + "const": "CDISC" + }, + "Standards": { + "items": { + "oneOf": [ + { + "properties": { + "Name": { + "const": "ADaMIG" + }, + "References": { + "items": { + "properties": { + "Criteria": { + "properties": { + "Type": { + "const": "Failure" + } + }, + "required": [ + "Type" + ], + "type": "object" + }, + "Origin": { + "const": "ADaM Conformance Rules" + }, + "Rule Identifier": { + "properties": { + "Id": { + "pattern": "^\\d{1,3}(\\.\\d{1,2})?$", + "type": "string" + } + }, + "type": "object" + }, + "Version": { + "enum": [ + "5.0" + ] + } + }, + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Version": { + "enum": [ + "1.0", + "1.1", + "1.2", + "1.3" + ] + } + }, + "type": "object" + }, + { + "properties": { + "Name": { + "const": "SDTMIG" + }, + "References": { + "items": { + "properties": { + "Criteria": { + "properties": { + "Type": { + "const": "Success" + } + }, + "required": [ + "Type" + ], + "type": "object" + }, + "Origin": { + "const": "SDTM and SDTMIG Conformance Rules" + }, + "Rule Identifier": { + "properties": { + "Id": { + "pattern": "^CG\\d{4}$", + "type": "string" + }, + "Version": { + "enum": [ + "1", + "2", + "3" + ] + } + }, + "type": "object" + }, + "Version": { + "enum": [ + "2.0" + ] + } + }, + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Version": { + "enum": [ + "3.2", + "3.3", + "3.4" + ] + } + }, + "type": "object" + }, + { + "properties": { + "Name": { + "const": "SENDIG" + }, + "References": { + "items": { + "properties": { + "Criteria": { + "properties": { + "Type": { + "const": "Success" + } + }, + "required": [ + "Type" + ], + "type": "object" + }, + "Origin": { + "const": "SEND Conformance Rules" + }, + "Rule Identifier": { + "properties": { + "Id": { + "pattern": "^SEND\\d{1,3}(\\.\\d+)?$", + "type": "string" + } + }, + "type": "object" + }, + "Version": { + "enum": [ + "5.0" + ] + } + }, + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Version": { + "enum": [ + "3.0", + "3.1", + "3.1.1" + ] + } + }, + "type": "object" + }, + { + "properties": { + "Name": { + "const": "SENDIG-DART" + }, + "References": { + "items": { + "properties": { + "Criteria": { + "properties": { + "Type": { + "const": "Success" + } + }, + "required": [ + "Type" + ], + "type": "object" + }, + "Origin": { + "const": "SEND Conformance Rules" + }, + "Rule Identifier": { + "properties": { + "Id": { + "pattern": "^SEND\\d{1,3}(\\.\\d+)?$", + "type": "string" + } + }, + "type": "object" + }, + "Version": { + "enum": [ + "5.0" + ] + } + }, + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Version": { + "enum": [ + "1.1", + "1.2" + ] + } + }, + "type": "object" + }, + { + "properties": { + "Name": { + "const": "SENDIG-GENETOX" + }, + "References": { + "items": { + "properties": { + "Criteria": { + "properties": { + "Type": { + "const": "Success" + } + }, + "required": [ + "Type" + ], + "type": "object" + }, + "Origin": { + "const": "SEND Conformance Rules" + }, + "Rule Identifier": { + "properties": { + "Id": { + "pattern": "^SEND\\d{1,3}(\\.\\d+)?$", + "type": "string" + } + }, + "type": "object" + }, + "Version": { + "enum": [ + "5.0" + ] + } + }, + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Version": { + "enum": [ + "1.0" + ] + } + }, + "type": "object" + }, + { + "properties": { + "Name": { + "const": "TIG" + }, + "References": { + "items": { + "properties": { + "Criteria": { + "properties": { + "Type": { + "const": "Success" + } + }, + "required": [ + "Type" + ], + "type": "object" + }, + "Origin": { + "const": "TIG Conformance Rules" + }, + "Rule Identifier": { + "properties": { + "Id": { + "pattern": "^TIG\\d{4}[a-z]?$", + "type": "string" + } + }, + "type": "object" + }, + "Version": { + "enum": [ + "1.0" + ] + } + }, + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Substandard": { + "enum": [ + "SDTM", + "SEND", + "ADaM", + "CDASH" + ] + }, + "Version": { + "enum": [ + "1.0" + ] + } + }, + "required": [ + "Name", + "Version", + "Substandard" + ], + "type": "object" + }, + { + "properties": { + "Name": { + "const": "USDM" + }, + "References": { + "items": { + "properties": { + "Origin": { + "const": "USDM Conformance Rules" + }, + "Rule Identifier": { + "properties": { + "Id": { + "pattern": "^DDF\\d{5}$", + "type": "string" + }, + "Version": { + "enum": [ + "1" + ] + } + }, + "type": "object" + }, + "Version": { + "enum": [ + "1.0" + ] + } + }, + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Version": { + "enum": [ + "3.0", + "4.0" + ] + } + }, + "type": "object" + } + ], + "type": "object" + }, + "minItems": 1, + "type": "array" + } + }, + "type": "object" + }, + "Organization_Custom.json": { + "$comment": "Version: 1.0.0, Last Updated: 2025-03-09", + "description": "Schema for defining custom organizations and their rule categorization structure", + "properties": { + "Category": { + "additionalProperties": true, + "description": "Custom categorization for rule governance", + "properties": { + "CompanyRuleLibrary": { + "description": "Whether the rule is part of a company rule library", + "type": "boolean" + }, + "Keywords": { + "description": "Custom keywords for further categorization", + "items": { + "type": "string" + }, + "type": "array" + }, + "OutputType": { + "description": "Output type of the rule validation result", + "enum": [ + "Check", + "Listing" + ], + "type": "string" + }, + "Purpose": { + "description": "Specific purpose of the rule, e.g., 'RAW data validation', 'External data validation'", + "type": "string" + }, + "Sponsors": { + "description": "List of sponsors the rule applies to", + "items": { + "type": "string" + }, + "type": "array" + }, + "TherapeuticAreas": { + "description": "List of therapeutic areas the rule applies to", + "items": { + "type": "string" + }, + "type": "array" + }, + "Trials": { + "description": "List of trials the rule applies to", + "items": { + "type": "string" + }, + "type": "array" + }, + "Vendors": { + "description": "List of vendors the rule applies to", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "Organization": { + "description": "Name of your custom organization", + "not": { + "enum": [ + "CDISC", + "FDA" + ] + }, + "type": "string" + }, + "Standards": { + "items": { + "properties": { + "Name": { + "description": "Name of the standard", + "type": "string" + }, + "References": { + "items": { + "properties": { + "Criteria": { + "anyOf": [ + { + "required": [ + "Logical Expression" + ] + }, + { + "required": [ + "Plain Language Expression" + ] + } + ], + "properties": { + "Logical Expression": { + "properties": { + "Condition": { + "type": "string" + }, + "Rule": { + "type": "string" + } + }, + "required": [ + "Rule" + ], + "type": "object" + }, + "Plain Language Expression": { + "type": "string" + }, + "Type": { + "enum": [ + "Failure", + "Success" + ], + "type": "string" + } + }, + "required": [ + "Type" + ], + "type": "object" + }, + "Origin": { + "description": "Origin of the rule (e.g., 'Custom Conformance Rules')", + "type": "string" + }, + "Rule Identifier": { + "properties": { + "Id": { + "description": "Custom rule identifier pattern", + "type": "string" + }, + "Version": { + "description": "Version of the rule", + "type": "string" + } + }, + "required": [ + "Id" + ], + "type": "object" + }, + "Version": { + "description": "Version of the references", + "type": "string" + } + }, + "required": [ + "Origin", + "Rule Identifier", + "Version" + ], + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Substandard": { + "description": "Optional substandard name", + "type": "string" + }, + "Version": { + "description": "Version of the standard", + "type": "string" + } + }, + "required": [ + "Name", + "References", + "Version" + ], + "type": "object" + }, + "minItems": 1, + "type": "array" + } + }, + "required": [ + "Organization", + "Standards", + "Category" + ], + "title": "Custom Organization Schema", + "type": "object" + }, + "Organization_FDA.json": { + "properties": { + "Organization": { + "const": "FDA" + }, + "Standards": { + "items": { + "oneOf": [ + { + "properties": { + "Name": { + "const": "SDTMIG" + }, + "Version": { + "enum": [ + "3.2", + "3.3", + "3.4" + ] + } + }, + "type": "object" + }, + { + "properties": { + "Name": { + "const": "SENDIG" + }, + "Version": { + "enum": [ + "3.0", + "3.1", + "3.1.1" + ] + } + }, + "type": "object" + }, + { + "properties": { + "Name": { + "const": "SENDIG-AR" + }, + "Version": { + "enum": [ + "1.0" + ] + } + }, + "type": "object" + }, + { + "properties": { + "Name": { + "const": "SENDIG-DART" + }, + "Version": { + "enum": [ + "1.1", + "1.2" + ] + } + }, + "type": "object" + }, + { + "properties": { + "Name": { + "const": "SENDIG-GENETOX" + }, + "Version": { + "enum": [ + "1.0" + ] + } + }, + "type": "object" + } + ], + "properties": { + "References": { + "items": { + "properties": { + "Citations": { + "items": { + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "Cited Guidance": { + "type": "string" + }, + "Document": { + "const": "CDISC" + } + } + }, + { + "additionalProperties": false, + "properties": { + "Cited Guidance": { + "type": "string" + }, + "Document": { + "const": "FDA" + }, + "Section": { + "pattern": "^(FDAB\\d{3}|TRC\\d{4}[A-Z]?|\\d{4}[A-Z])$", + "type": "string" + } + } + } + ], + "required": [ + "Document", + "Cited Guidance" + ], + "type": "object" + }, + "type": "array" + }, + "Criteria": { + "properties": { + "Type": { + "const": "Success" + } + }, + "required": [ + "Type" + ], + "type": "object" + }, + "Origin": { + "const": "FDA Business Rules" + }, + "Rule Identifier": { + "properties": { + "Id": { + "pattern": "^(FB|CT|SD|SE)\\d{4}[A-Z]?$|^TRC.*$", + "type": "string" + } + }, + "type": "object" + }, + "Version": { + "enum": [ + "1.5" + ] + } + }, + "type": "object" + }, + "minItems": 1, + "type": "array" + } + }, + "type": "object" + }, + "minItems": 1, + "type": "array" + } + }, + "type": "object" + }, + "PascalCases": { + "items": { + "pattern": "^[A-Z](([a-z]+[A-Z]?)*)$", + "type": "string" + }, + "minItems": 1, + "type": "array" + }, + "Rule_Type.json": { + "anyOf": [ + { + "const": "Dataset Contents Check against Define XML", + "markdownDescription": "\n#### Columns\n\nColumns are the columns within the original dataset along with the following columns:\n\n- `dataset_label`\n- `dataset_location`\n- `dataset_name`\n- `dataset_size`\n- `dataset_domain`\n- `define_dataset_class`\n- `define_dataset_domain`\n- `define_dataset_has_no_data`\n- `define_dataset_is_non_standard`\n- `define_dataset_key_sequence`\n- `define_dataset_label`\n- `define_dataset_location`\n- `define_dataset_name`\n- `define_dataset_structure`\n- `define_dataset_variables`\n", + "title": "Content data at record level, dataset metadata at dataset level, and define xml metadata at dataset level" + }, + { + "const": "Dataset Metadata Check", + "markdownDescription": "\n#### Columns\n\n- `dataset_label`\n- `dataset_location`\n- `dataset_name`\n- `dataset_size`\n\n#### Rule Macro\n\nPairs dataset metadata sourced from the submission contents.\n\n#### Example\n\n```yaml\n- name: dataset_name\n operator: longer_than\n value: 6\n```\n", + "title": "Content metadata at dataset level" + }, + { + "const": "Dataset Metadata Check against Define XML", + "markdownDescription": "\nReturns a dataset where each dataset is a row in the new dataset. The define xml dataset metadata is attached to each row.\n\n#### Columns\n\n- `dataset_size`\n- `dataset_location`\n- `dataset_name`\n- `dataset_label`\n- `dataset_domain`\n- `dataset_columns`\n- `define_dataset_class`\n- `define_dataset_domain`\n- `define_dataset_has_no_data`\n- `define_dataset_is_non_standard`\n- `define_dataset_key_sequence`\n- `define_dataset_label`\n- `define_dataset_location`\n- `define_dataset_name`\n- `define_dataset_structure`\n- `define_dataset_variables`\n- `define_dataset_variable_order`\n\n#### Rule Macro\n\nAllows comparing dataset metadata against define xml dataset metadata.\n\n#### Example\n\n```yaml\nall:\n - name: dataset_name\n operator: not_equal_to\n value: define_dataset_name\n```\n\nOr\n\n```yaml\nany:\n - name: dataset_name\n operator: empty\n - name: define_dataset_name\n operator: empty\n```\n", + "title": "Content metadata at dataset level and define xml metadata at dataset level" + }, + { + "const": "Define Item Metadata Check against Library Metadata", + "markdownDescription": "\n#### Columns\n\n- `define_variable_name`\n- `define_variable_label`\n- `define_variable_data_type`\n- `define_variable_role`\n- `define_variable_size`\n- `define_variable_ccode`\n- `define_variable_format`\n- `define_variable_allowed_terms`\n- `define_variable_origin_type`\n- `define_variable_is_collected`\n- `define_variable_has_no_data`\n- `define_variable_order_number`\n- `define_variable_has_codelist`\n- `define_variable_codelist_coded_values`\n- `define_variable_codelist_coded_codes`\n- `define_variable_mandatory`\n- `define_variable_has_comment`\n- `define_variable_has_method`\n- `library_variable_name`\n- `library_variable_order_number`\n- `library_variable_label`\n- `library_variable_data_type`\n- `library_variable_role`\n- `library_variable_core`\n- `library_variable_has_codelist`\n- `library_variable_ccode`\n\n#### Rule Macro\n\nChecks variable-level metadata, codelists, and codelist terms from Define-XML against the corresponding standard variable definitions from the CDISC Library.\n", + "title": "Define xml metadata at variable level and corresponding library variable metadata" + }, + { + "const": "Domain Presence Check", + "markdownDescription": "\n#### Columns\n\nSingle row contains a column for each domain and the value of that column is the domain's file name\n\n| AE | EC |\n| ------ | ------ |\n| ae.xpt | ec.xpt |\n\n#### Rule Macro\n\nChecks which dataset files are physically present in the submission contents. Each column represents one domain that exists in the submission, holding the filename as its value.\n\n#### Example\n\n```yaml\nall:\n - name: PP\n operator: exists\n - name: PC\n operator: not_exists\n```\n", + "title": "Content domain presence at study level" + }, + { + "const": "Domain Presence Check against Define XML", + "markdownDescription": "\n#### Columns\n\nOne row per dataset defined in Define-XML:\n\n- `domain` - The domain if the dataset exists, null otherwise\n- `filename` - The file name if dataset exists, null otherwise\n- `define_dataset_name`\n- `define_dataset_label`\n- `define_dataset_location`\n- `define_dataset_domain`\n- `define_dataset_class`\n- `define_dataset_structure`\n- `define_dataset_is_non_standard`\n- `define_dataset_has_no_data`\n- `define_dataset_key_sequence`\n- `define_dataset_variables`\n\n#### Rule Macro\n\nReconciles dataset file presence in the submission against Define-XML dataset declarations. Each row represents one dataset defined in Define-XML, with the corresponding submission filename joined if the file exists.\n\n#### Example\n\nCheck if SE domain is defined in Define-XML without HasNoData=\"Yes\" but the dataset file doesn't exist:\n\n```yaml\nall:\n - name: define_dataset_name\n operator: equal_to\n value: \"SE\"\n - name: define_dataset_has_no_data\n operator: equal_to\n value: False\n - name: filename\n operator: not_exists\n```\n", + "title": "Content domain presence at study level with define xml metadata at dataset level" + }, + { + "const": "JSON Schema Check", + "markdownDescription": "\n#### Columns\n\n- `json_path`\n- `error_attribute`\n- `error_value`\n- `validator`\n- `validator_value`\n- `message`\n- `dataset`\n- `id`\n- `_path`\n", + "title": "Apply JSON schema validation to a JSON file" + }, + { + "const": "JSONata", + "markdownDescription": "\nApply a JSONata query to a JSON file. [JSONata documentation](https://docs.jsonata.org)\n\n### Example\n\n#### Rule\n\n```yaml\nCheck: |\n **.$filter($, $myutils.equals).{\"row\":_path, \"A\":A, \"B\":B}\nCore:\n Id: JSONATA Test\n Status: Draft\nOutcome:\n Message: \"A equals B\"\n Output Variables:\n - row\n - A\n - B\nRule Type: JSONata\nScope:\n Entities:\n Include:\n - ALL\nSensitivity: Record\n```\n\n#### Custom user function contained in external file \"equals.jsonata\"\n\n\\* Note that in the CLI, you can pass a variable name and directory of such files using `-jcf` or `--jsonata-custom-functions`. The engine's built-in JSONata functions are accessible from the `$utils` variable (see [JSONata Functions](JSONata_Functions.md)). For example to load two more directories containing functions into `$myutils` and `$yourutils`, add the options:\n`-jcf myutils path/to/myutils -jcf yourutils path/to/yourutils`\n\n```yaml\n{\n \"equals\": function($v){ $v.A=$v.B }\n}\n```\n\n#### JSON Data\n\n```json\n{\n \"A\": \"same value 1\",\n \"B\": \"same value 1\",\n \"C\": {\n \"A\": \"different value 1\",\n \"B\": \"different value 2\",\n \"C\": { \"A\": \"same value 2\", \"B\": \"same value 2\" }\n }\n}\n```\n\n#### Result\n\n```json\n[\n {\n \"executionStatus\": \"success\",\n \"dataset\": \"\",\n \"domain\": \"\",\n \"variables\": [\"A\", \"B\", \"row\"],\n \"message\": \"A equals B\",\n \"errors\": [\n {\n \"value\": { \"row\": \"\", \"A\": \"same value 1\", \"B\": \"same value 1\" },\n \"dataset\": \"\",\n \"row\": \"\"\n },\n {\n \"value\": {\n \"row\": \"/C/C\",\n \"A\": \"same value 2\",\n \"B\": \"same value 2\"\n },\n \"dataset\": \"\",\n \"row\": \"/C/C\"\n }\n ]\n }\n]\n```\n\n### Preprocessing\n\nWhen the JSONata Rule Type is used, the input JSON file will be preprocessed to assign a `_path` attribute to each node in the JSON tree. The syntax for this path value will use the [JSON Pointer](https://datatracker.ietf.org/doc/html/rfc6901) syntax. This `_path` attribute can be referenced throughout the JSONata query.\n\n### Output Variables and Report column mapping\n\nYou can use `Outcome.Output Variables` to specify which properties to display from the result JSON. The following result property names will map to the column names in the Excel output report.\n\nMapping of Result property names to Report Issue Details Column Names:\n\n| JSONata Result Name | JSON report property | Excel Column |\n| ------------------- | -------------------- | ------------ |\n| dataset | dataset | Dataset |\n| row | row | Record |\n| SEQ | SEQ | Sequence |\n| USUBJID | USUBJID | USUBJID |\n| entity | entity | Entity |\n| instance_id | instance_id | Instance ID |\n| path | path | Path |\n\n### Scope\n\nA JSONata rule will always run once for the entire JSON file, regardless of the Scope. The `Entity` determination must come from the rule's JSONata result property.\n", + "title": "Apply a JSONata query to a JSON file" + }, + { + "const": "Record Data", + "markdownDescription": "\n#### Columns\n\nColumns are the columns within the original dataset\n\n#### Rule Macro\n\nChecks record-level data values sourced directly from submission dataset contents.\n\n#### Example\n\n```yaml\nall:\n - name: --SCAT\n operator: non_empty\n - name: --SCAT\n operator: equal_to\n value: --CAT\n```\n", + "title": "Content data at record level. Most common Rule Type" + }, + { + "const": "Value Check against Define XML Variable", + "markdownDescription": "\n#### Columns\n\n- `row_number`\n- `variable_name`\n- `variable_value`\n- `define_variable_name`\n- `define_variable_label`\n- `define_variable_data_type`\n- `define_variable_`...\n\n#### Rule Macro\n\nChecks individual cell values from submission dataset contents pivoted to a long format, with the matching Define-XML variable-level metadata joined to each row.\n\n#### Example\n\n```yaml\nall:\n - name: define_variable_ccode\n operator: empty\n - name: variable_value\n operator: non_empty\n - name: define_variable_has_codelist\n operator: equal_to\n value: true\n - name: variable_value\n operator: is_not_contained_by\n value: define_variable_codelist_coded_values\n```\n", + "title": "Content data at value level and define xml metadata at variable level" + }, + { + "const": "Value Check against Define XML VLM", + "markdownDescription": "\n#### Columns\n\n- `row_number`\n- `variable_name`\n- `variable_value`\n- `define_vlm_name`\n- `define_vlm_label`\n- `define_vlm_data_type`\n- `define_vlm_is_collected`\n- `define_vlm_role`\n- `define_vlm_size`\n- `define_vlm_ccode`\n- `define_vlm_format`\n- `define_vlm_allowed_terms`\n- `define_vlm_origin_type`\n- `define_vlm_has_no_data`\n- `define_vlm_order_number`\n- `define_vlm_length`\n- `define_vlm_has_codelist`\n- `define_vlm_codelist_coded_values`\n- `define_vlm_mandatory`\n- `define_variable_name`\n- `type_check`\n- `length_check`\n- `variable_value_length`\n\n#### Rule Macro\n\nChecks individual cell values from submission dataset contents pivoted to a long format, with the matching Define-XML Value Level Metadata (VLM) joined to each row. Only rows where VLM exists for that variable are produced \u2014 records without matching VLM are excluded.\n\n#### Example\n\n```yaml\nall:\n - name: define_vlm_ccode\n operator: empty\n - name: variable_value\n operator: non_empty\n - name: define_vlm_has_codelist\n operator: equal_to\n value: true\n - name: variable_value\n operator: is_not_contained_by\n value: define_vlm_codelist_coded_values\n```\n\n```yaml\nall:\n - name: variable_value\n operator: empty\n - name: define_vlm_mandatory\n operator: equal_to\n value: Yes\n```\n", + "title": "Content data at value level and define xml metadata at value level" + }, + { + "const": "Value Check against Library Metadata", + "markdownDescription": "\n#### Columns\n\n- `row_number`\n- `variable_name`\n- `variable_value`\n- `library_variable__links`\n- `library_variable_core`\n- `library_variable_description`\n- `library_variable_label`\n- `library_variable_name`\n- `library_variable_ordinal`\n- `library_variable_role`\n- `library_variable_simpleDatatype`\n- `library_variable_ccode`\n- `library_variable_has_codelist`\n- `library_variable_valueList`\n- `library_variable_definition`\n- `library_variable_notes`\n- `library_variable_variableCcode`\n- `library_variable_examples`\n- `library_variable_usageRestrictions`\n- `library_variable_describedValueDomain`\n\n#### Rule Macro\n\nChecks individual cell values from submission dataset contents pivoted to a long format, with the matching Library Metadata joined to each row.\n\n#### Example\n\n```yaml\nall:\n - name: variable_value\n operator: non_empty\n - name: library_variable_has_codelist\n operator: equal_to\n value: true\n```\n", + "title": "Content data at value level and Library Metadata at value level" + }, + { + "const": "Variable Metadata Check", + "markdownDescription": "\n#### Columns\n\n- `variable_name`\n- `variable_order_number`\n- `variable_label`\n- `variable_size`\n- `variable_data_type`\n- `variable_format`\n- `variable_max_size` (if needed by the rule)\n\n#### Rule Macro\n\nChecks variable-level metadata sourced from the submission dataset contents.\n\n#### Example\n\n```yaml\n- name: variable_label\n operator: longer_than\n value: 40\n```\n", + "title": "Content metadata at variable level" + }, + { + "const": "Variable Metadata Check against Define XML", + "markdownDescription": "\n#### Columns\n\n- `variable_name`\n- `variable_order_number`\n- `variable_label`\n- `variable_size`\n- `variable_data_type`\n- `define_variable_name`\n- `define_variable_label`\n- `define_variable_data_type`\n- `define_variable_is_collected`\n- `define_variable_role`\n- `define_variable_size`\n- `define_variable_ccode`\n- `define_variable_format`\n- `define_variable_allowed_terms`\n- `define_variable_origin_type`\n- `define_variable_has_no_data`\n- `define_variable_order_number`\n- `define_variable_length`\n- `define_variable_has_codelist`\n- `define_variable_codelist_coded_values`\n- `define_variable_codelist_coded_codes`\n- `define_variable_mandatory`\n- `define_variable_has_comment`\n- `define_variable_has_method`\n\n#### Rule Macro\n\nCombines variable-level metadata from submission dataset contents against the matching variable definitions in Define-XML.\n\n#### Example\n\n```yaml\n- name: variable_name\n operator: not_equal_to\n value: define_variable_name\n```\n", + "title": "Content metadata at variable level and define xml metadata at variable level" + }, + { + "const": "Variable Metadata Check against Library Metadata", + "markdownDescription": "\n#### Columns\n\n- `variable_name`\n- `variable_order_number`\n- `variable_label`\n- `variable_size`\n- `variable_data_type`\n- `variable_format`\n- `variable_has_empty_values`\n- `library_variable_name`\n- `library_variable_role`\n- `library_variable_label`\n- `library_variable_core`\n- `library_variable_order_number`\n- `library_variable_data_type`\n- `library_variable_ccode`\n\n#### Rule Macro\n\nCombines variable-level metadata from submission dataset contents against the corresponding CDISC Library standard variable metadata.\n", + "title": "Content metadata at the variable level and the corresponding library metadata" + }, + { + "const": "Variable Metadata Check against Define XML and Library Metadata", + "title": "Combines metadata at the variable level with corresponding define-xml metadata at variable level and corresponding library variable metadata" + }, + { + "const": "Value Check with Dataset Metadata", + "markdownDescription": "\n#### Columns\n\n- `row_number`\n- `variable_name`\n- `variable_value`\n- `dataset_label`\n- `dataset_location`\n- `dataset_name`\n- `dataset_size`\n\n#### Rule Macro\n\nChecks individual cell values from submission dataset contents pivoted to a long format, with dataset-level metadata attached to each row.\n\n#### Example\n\n```yaml\nall:\n - name: variable_name\n operator: starts_with\n value: \"DM\"\n - name: dataset_name\n operator: not_equal_to\n value: \"DM\"\n```\n", + "title": "Content data at value level and dataset metadata at dataset level" + }, + { + "const": "Value Check with Variable Metadata", + "markdownDescription": "\n#### Columns\n\n- `row_number`\n- `variable_name`\n- `variable_value`\n- `variable_order_number`\n- `variable_label`\n- `variable_size`\n- `variable_data_type`\n- `variable_format`\n- `variable_value_length`\n\n#### Rule Macro\n\nChecks individual cell values from submission dataset contents pivoted to a long format, with variable-level metadata from the submission dataset attached to each row.\n\n#### Example\n\n```yaml\nall:\n - name: variable_data_type\n operator: equal_to\n value: char\n - name: variable_value\n operator: longer_than\n value: 200\n```\n", + "title": "Content data at value level and variable metadata at variable level" + } + ], + "markdownDescription": "Determines how the primary dataset should be built before the rule is applied" + }, + "Sensitivity.json": { + "anyOf": [ + { + "const": "Dataset", + "markdownDescription": "\nReport one result per dataset generated by the `Rule Type`, where a dataset or record within the dataset matches the rule failure criteria\n" + }, + { + "const": "Record", + "markdownDescription": "\nReport one result per record within the dataset generated by the `Rule Type`, where the record matches the rule failure criteria\n" + }, + { + "const": "Group", + "markdownDescription": "\nReports one issue per each logical group of records within the dataset generated by the `Rule Type`, where the group matches the rule failure criteria. Records are organized into logical groups based on matching values for the `Grouping_Variables` specified in the rule (e.g., records with the same `SETCD` value form one group). For each group that violates the rule, the engine reports a single error using the first record from that group that matches the failure criteria. Groups are defined by the required variable `Grouping_Variables` specified in the rule.\n\n### Example\n\n```yaml\nSensitivity: Group\nGrouping Variables:\n - SETCD\n```\n" + }, + { + "const": "Study", + "markdownDescription": "\nWill report once per submitted data study. Lends itself to Domain Presence Check rule types as these are cross-study checks that do not involve within-dataset checks. Reports one result per failing row in the dataset generated by the rule type, collapsed to a single study-level result.\n" + } + ], + "markdownDescription": "Determines what level of granularity issues should be generated within the report" + }, + "Subclasses": { + "items": { + "enum": [ + "ADVERSE EVENT", + "ALL", + "MEDICAL DEVICE TIME-TO-EVENT", + "NON-COMPARTMENTAL ANALYSIS", + "POPULATION PHARMACOKINETIC ANALYSIS", + "TIME-TO-EVENT" + ], + "type": "string" + }, + "type": "array" + }, + "VariableName": { + "pattern": "^(--[A-Z0-9]{1,6}|[A-Z][A-Z0-9]{0,7})$", + "type": "string" + }, + "VariableReference": { + "anyOf": [ + { + "$ref": "#/$defs/VariableName" + }, + { + "$ref": "#/$defs/OperationResultId" + } + ], + "description": "Can reference either a dataset variable name or an operation result" + }, + "metadata": { + "changelog": [ + { + "changes": [ + "Added Category object structure with standard properties", + "Implemented support for Sponsors, Vendors, TherapeuticAreas arrays", + "Added support for X, Y and Z enum", + "Enabled extensibility with additionalProperties: true" + ], + "date": "2025-03-09", + "description": "Initial release of Custom Organization Schema", + "version": "1.0.0" + } + ], + "maintainer": { + "email": "standards@yourorganization.com", + "name": "Your Organization Name" + }, + "releaseDate": "2025-03-09", + "schemaVersion": "1.0.0" + } + }, + "$id": "https://raw.githubusercontent.com/cdisc-org/cdisc-rules-engine/refs/heads/main/resources/schema/rule-merged/CORE-base.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "if": { + "properties": { + "Sensitivity": { + "const": "Group" + } + } + }, + "markdownDescription": "Validation schema CDISC Rules 1.0", + "properties": { + "Authorities": { + "items": { + "additionalProperties": false, + "oneOf": [ + { + "$ref": "#/$defs/Organization_CDISC.json" + }, + { + "$ref": "#/$defs/Organization_FDA.json" + }, + { + "$ref": "#/$defs/Organization_Custom.json" + } + ], + "properties": { + "Category": { + "additionalProperties": true, + "description": "Custom categorization for rule governance defined in Organization_Custom.json", + "type": "object" + }, + "Organization": { + "type": "string" + }, + "Standards": { + "items": { + "additionalProperties": false, + "properties": { + "Name": { + "type": "string" + }, + "References": { + "items": { + "additionalProperties": false, + "properties": { + "Citations": { + "items": { + "additionalProperties": false, + "properties": { + "Cited Guidance": { + "type": "string" + }, + "Document": { + "type": "string" + }, + "Item": { + "type": "string" + }, + "Section": { + "type": "string" + } + }, + "required": [ + "Document", + "Cited Guidance" + ], + "type": "object" + }, + "type": "array" + }, + "Criteria": { + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "Logical Expression" + ] + }, + { + "required": [ + "Plain Language Expression" + ] + } + ], + "properties": { + "Logical Expression": { + "additionalProperties": false, + "properties": { + "Condition": { + "type": "string" + }, + "Rule": { + "type": "string" + } + }, + "required": [ + "Rule" + ], + "type": "object" + }, + "Plain Language Expression": { + "type": "string" + }, + "Type": { + "enum": [ + "Failure", + "Success" + ], + "type": "string" + } + }, + "required": [ + "Type" + ], + "type": "object" + }, + "Origin": { + "type": "string" + }, + "Related Rules": { + "items": { + "additionalProperties": false, + "properties": { + "Id": { + "type": "string" + }, + "Relationship": { + "enum": [ + "Predecessor", + "Related", + "Successor" + ], + "type": "string" + } + }, + "required": [ + "Id", + "Relationship" + ], + "type": "object" + }, + "type": "array" + }, + "Release Notes": { + "type": "string" + }, + "Rule Identifier": { + "additionalProperties": false, + "properties": { + "Id": { + "type": "string" + }, + "Version": { + "type": "string" + } + }, + "required": [ + "Id" + ], + "type": "object" + }, + "Validator Rule Message": { + "type": "string" + }, + "Version": { + "type": "string" + } + }, + "required": [ + "Origin", + "Rule Identifier", + "Version" + ], + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Substandard": { + "type": "string" + }, + "Version": { + "type": "string" + } + }, + "required": [ + "Name", + "References", + "Version" + ], + "type": "object" + }, + "minItems": 1, + "type": "array" + } + }, + "required": [ + "Organization", + "Standards" + ], + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Check": { + "anyOf": [ + { + "$ref": "#/$defs/Boolean" + }, + { + "type": "string" + } + ] + }, + "Core": { + "oneOf": [ + { + "properties": { + "Status": { + "const": "Draft" + } + } + }, + { + "properties": { + "Id": { + "pattern": "^CORE-\\d{6}$", + "type": "string" + }, + "Status": { + "const": "Published" + } + }, + "required": [ + "Id" + ] + } + ], + "properties": { + "Version": { + "const": "1" + } + }, + "required": [ + "Status", + "Version" + ], + "type": "object" + }, + "Description": { + "type": "string" + }, + "Executability": { + "$ref": "#/$defs/Executability.json" + }, + "Grouping_Variables": { + "items": { + "$ref": "#/$defs/VariableReference" + }, + "markdownDescription": "Variables to group by when using Group sensitivity. Required when Sensitivity is set to Group.", + "type": "array" + }, + "Match Datasets": { + "items": { + "additionalProperties": false, + "properties": { + "Child": { + "const": true + }, + "Join Type": { + "$ref": "#/$defs/JoinType" + }, + "Keys": { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/VariableName" + }, + { + "$ref": "#/$defs/LeftRightKeys" + } + ] + }, + "type": "array" + }, + "Name": { + "anyOf": [ + { + "$ref": "#/$defs/Dataset" + }, + { + "$ref": "#/$defs/DataStructure" + } + ] + }, + "Wildcard": { + "type": "string" + } + }, + "required": [ + "Name" + ], + "type": "object" + }, + "minItems": 1, + "type": "array" + }, + "Operations": { + "items": { + "$ref": "#/$defs/Operations.json" + }, + "minItems": 1, + "type": "array" + }, + "Outcome": { + "additionalProperties": false, + "properties": { + "Message": { + "type": "string" + }, + "Output Variables": { + "items": { + "$ref": "#/$defs/Operator.json/properties/name" + }, + "type": "array" + } + }, + "required": [ + "Message" + ], + "type": "object" + }, + "Rule Type": { + "$ref": "#/$defs/Rule_Type.json" + }, + "Scope": { + "additionalProperties": false, + "oneOf": [ + { + "required": [ + "Classes", + "Domains" + ] + }, + { + "required": [ + "Data Structures" + ] + }, + { + "required": [ + "Entities" + ] + } + ], + "properties": { + "Classes": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "Include": { + "$ref": "#/$defs/Classes" + } + }, + "required": [ + "Include" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "Exclude": { + "$ref": "#/$defs/Classes" + } + }, + "required": [ + "Exclude" + ], + "type": "object" + } + ] + }, + "Data Structures": { + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "Include": { + "$ref": "#/$defs/DataStructures" + } + }, + "required": [ + "Include" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "Exclude": { + "$ref": "#/$defs/DataStructures" + } + }, + "required": [ + "Exclude" + ], + "type": "object" + } + ] + }, + "Dataset or Domain or Item Group": { + "$ref": "#/$defs/DomainStructure" + }, + "Datasets": { + "additionalProperties": false, + "properties": { + "Exclude": { + "$ref": "#/$defs/Datasets" + }, + "Include": { + "$ref": "#/$defs/Datasets" + } + }, + "type": "object" + }, + "Domains": { + "$ref": "#/$defs/DomainStructure" + }, + "Entities": { + "anyOf": [ + { + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "Exclude" + ] + }, + { + "required": [ + "Include" + ] + } + ], + "properties": { + "Exclude": { + "$ref": "#/$defs/PascalCases" + }, + "Include": { + "$ref": "#/$defs/PascalCases" + } + }, + "type": "object" + } + ] + }, + "Subclasses": { + "additionalProperties": false, + "properties": { + "Exclude": { + "$ref": "#/$defs/Subclasses" + }, + "Include": { + "$ref": "#/$defs/Subclasses" + } + }, + "type": "object" + }, + "Use Case": { + "type": "string" + } + }, + "type": "object" + }, + "Sensitivity": { + "$ref": "#/$defs/Sensitivity.json" + } + }, + "required": [ + "Authorities", + "Check", + "Core", + "Description", + "Outcome", + "Executability", + "Rule Type", + "Scope", + "Sensitivity" + ], + "then": { + "required": [ + "Grouping_Variables" + ] + }, + "type": "object" +} diff --git a/resources/schema/rule-merged/Operations.json b/resources/schema/rule-merged/Operations.json index 6bceb8ebc..ce0152e9b 100644 --- a/resources/schema/rule-merged/Operations.json +++ b/resources/schema/rule-merged/Operations.json @@ -10,7 +10,10 @@ "markdownDescription": "\nReturns a Series indicating whether a specified codelist is extensible. Used in conjunction with codelist_terms to determine if values outside the codelist are acceptable. From the above example, $extensible will contain a bool if the codelist PKUDUG is extensible in all rows of the column.\n\nIf ct_package_type, version, and codelist_code parameters are provided, it will instead attach a new column containing the extensible value for each combination provided in the source dataset.\n\nFor example, given the current dataset:\n\n```\nid \tcodeSystemVersion \t$codelist_code\n1 \t2024-09-27 \tC201264\n2 \t2024-09-27 \tC201265\n3 \t2023-03-29 \tC127262\n```\n\nand the following operation:\n\n```yaml\n- id: $codelist_extensible\n operator: codelist_extensible\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n```\n\nThis will result in the following dataset:\n\n```\nid \tcodeSystemVersion \t$codelist_code \t$codelist_extensible\n1 \t2024-09-27 \tC201264 \tfalse\n2 \t2024-09-27 \tC201265 \tfalse\n3 \t2023-03-29 \tC127262 \ttrue\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -20,7 +23,10 @@ "markdownDescription": "\nReturns a list of valid codelist/term values. Used for evaluating whether NCI codes, submission values or NCI preferred terms are valid based on controlled terminology. Expects three parameters: `codelists` which is a list of the codelist submission value(s) to retrieve, `level` which is the level of data (either \"codelist\" or \"term\") at which to return data from, and `returntype` which is the type of values to return: \"code\" for NCI Code(s), \"value\" for submission value(s), or \"pref_term\" for NCI preferred term(s).\n\n```yaml\n- Check:\n - all:\n - name: PPSTRESU\n operator: is_not_contained_by\n value: $terms\n - name: $extensible\n operator: equal_to\n value: true\n- Operations:\n - id: $terms\n operator: codelist_terms\n codelists:\n - PKUDUG\n level: term\n returntype: value\n - id: $extensible\n codelist: PKUDUG\n operator: codelist_extensible\n```\n\nIf `ct_package_type`, `version`, and `codelist_code` parameters are provided, it will instead attach a new column containing the term for each combination provided in the source dataset. If a column name is provided as:\n\n- `term_code`, it will find term information using the term codes in the specified column.\n- `term_value`, it will find term information using the term submission values in the specified column.\n- `term_pref_term`, it will find term information using the term preferred terms in the specified column.\n\nOnly one of `term_code`, `term_value` or `term_pref_term` can be provided. The term information returned will depend on the value of the `returntype` parameter, as described above. If `returntype` is not specified, specifying `term_code` will return the term submission value and specifying either `term_value` or `term_pref_term` will return the term code.\n\nFor example, given the current dataset:\n\n```\nid \tcodeSystemVersion $codelist_code code decode\n1 \t2024-09-27 C201264 C201356 After\n2 \t2024-09-27 C201265 C201352 End to End\n3 \t2023-03-29 C127262 C51282 CLINIC\n```\n\nand the following operations:\n\n```yaml\n- id: $found_term_value\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_code: code\n- id: $found_term_pref_term\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_code: code\n returntype: pref_term\n```\n\nThis will result in the following dataset:\n\n```\n id codeSystemVersion $codelist_code code decode $found_term_value $found_term_pref_term\n 1 2024-09-27 C201264 C201356 After After After Timing Type\n 2 2024-09-27 C201265 C201352 End to End End to End End to End\n 3 2023-03-31 C127262 C51282 CLINIC CLINIC Clinic\n```\n\nConversely, if given the same dataset, and the following operations:\n\n```yaml\n- id: $found_term_code1\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_value: decode\n- id: $found_term_code2\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_pref_term: decode\n```\n\nThis will result in the following dataset:\n\n```\n id codeSystemVersion $codelist_code code decode $found_term_code1 $found_term_code2\n 1 2024-09-27 C201264 C201356 After C201356\n 2 2024-09-27 C201265 C201352 End to End C201352 C201352\n 3 2023-03-31 C127262 C51282 CLINIC C51282 C51282\n```\n\nNote that `$found_term_code2` is:\n\n- `null` for the first record because \"After\" does not match any NCI preferred term in the C201264 codelist.\n- populated for the third record because matching is case-insensitive (i.e., \"CLINIC\" matches \"Clinic\").\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -30,7 +36,11 @@ "markdownDescription": "\nReturns a list of valid extensible codelist term's submission values. Used for evaluating whether submission values are valid based on controlled terminology. Expects the parameter codelists which is a list of the codelist submission value(s) to retrieve. If the codelist argument is [\"All\"] will return all extensible terms for the CT in a list.\n\n```yaml\n{\n \"id\": \"$ext_value\",\n \"codelist\": [\"ALL\"],\n \"operator\": \"define_extensible_codelists\",\n}\n```\n" } }, - "required": ["id", "operator", "codelists"], + "required": [ + "id", + "operator", + "codelists" + ], "type": "object" }, { @@ -40,7 +50,11 @@ "markdownDescription": "\nIf a target variable name is specified, returns the specified metadata in the define for the specified target variable.\n\nInput\n\n```yaml\n- operator: define_variable_metadata\n attribute_name: define_variable_label\n name: LBTESTCD\n id: $LBTESTCD_VARIABLE_LABEL\n```\n\nOutput\n\n```\nLaboratory Test Code\n```\n\nIf no target variable name specified, returns a dictionary containing the specified metadata in the define for all variables.\n\nInput\n\n```yaml\n- operator: define_variable_metadata\n attribute_name: define_variable_label\n id: $VARIABLE_LABEL\n```\n\nOutput\n\n```\n{\n \"STUDYID\": \"Study Identifier\",\n \"USUBJID\": \"Unique Subject Identifier\",\n \"LBTESTCD\": \"Laboratory Test Code\",\n \"...\": \"...\"\n}\n```\n" } }, - "required": ["id", "operator", "attribute_name"], + "required": [ + "id", + "operator", + "attribute_name" + ], "type": "object" }, { @@ -50,7 +64,11 @@ "markdownDescription": "\nGet a distinct list of values for the given name.\n\nIf a group list is specified, the distinct value list will be grouped by the variables within group.\nIf a filter object is provided, only values for records that match the filter criteria are included in the distinct values.\nIf `value_is_reference` is set to true, the target column contains the names of other columns, and the operation will check the referenced columns to ensure they exist in the associated dataset before adding them to the distinct list.\n\nIf group is provided, group_aliases may also be provided to assign new grouping variable names so that results grouped by the values in one set of grouping variables can be merged onto a dataset according to the same grouping value(s) stored in different set of grouping variables. When both group and group_aliases are provided, columns are renamed according to corresponding list position (i.e., the 1st column in group is renamed to the 1st column in group_aliases, etc.). If there are more columns listed in group than in group_aliases, only the group columns with corresponding group_aliases columns will be renamed. If there are more columns listed in group_aliases than in group, the extra column names in group_aliases will be ignored. See record_count for an example of the use of group_aliases.\n\n```yaml\nCheck:\n all:\n - name: SSSTRESC\n operator: equal_to\n value: DEAD\n value_is_literal: true\n - name: $ds_dsdecod\n operator: does_not_contain\n value: DEATH\n value_is_literal: true\nOperations:\n - operator: distinct\n domain: DS\n name: DSDECOD\n id: $ds_dsdecod\n group:\n - USUBJID\n filter:\n CAT: \"CATEGORY 1\"\n SCAT: \"SUBCATEGORY A\"\n```\n\n> below, `IDVAR` contains column names, the operation retrieves the value from each column for that row, checks the dataset associated with that column using the CO RDOMAIN. Columns that exist are added to the returns the distinct set.\n\n```yaml\nOperations:\n - domain: CO\n id: $rdomain_variables\n name: IDVAR\n operator: distinct\n value_is_reference: true\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -60,7 +78,10 @@ "markdownDescription": "\nChecks whether the domain is in the set of domains within the provided standard.\n\nInput\n\nTarget Domain: XY\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: domain_is_custom\n id: $domain_is_custom\n```\n\nOutput\n\n```\ntrue\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -70,7 +91,10 @@ "markdownDescription": "\nChecks whether the related domain (for example, the parent domain of a SUPP or RELREC dataset) is not present in the set of standard domains for the provided standard and version. This is useful for determining whether relationships point to non-standard or custom domains.\n\nInput\n\nTarget Domain: SUPPEX\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: related_domain_is_custom\n id: $related_domain_is_custom\n```\n\nOutput\n\n```\ntrue\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -80,7 +104,10 @@ "markdownDescription": "\nReturns the label for the domain the operation is executing on within the provided standard.\n\nInput.\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: domain_label\n id: $domain_label\n```\n\nOutput\n\n```\nLaboratory Test Results\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -90,7 +117,11 @@ "markdownDescription": "\nCalculates the number of days between the DTC and RFSTDTC. The Study Day value is incremented by 1 for each date following RFSTDTC. Dates prior to RFSTDTC are decreased by 1, with the date preceding RFSTDTC designated as Study Day -1 (there is no Study Day 0). All Study Day values are integers. Thus, to calculate Study Day:\n\n- --DY = (date portion of --DTC) - (date portion of RFSTDTC) + 1 if --DTC is on or after RFSTDTC\n- --DY = (date portion of --DTC) - (date portion of RFSTDTC) if --DTC precedes RFSTDTC\n\nThis algorithm should be used across all domains.\n\n```yaml\nCheck:\n all:\n - name: --DY\n operator: non_empty\n - name: --DTC\n operator: is_complete_date\n - name: RFSTDTC\n operator: is_complete_date\n - name: --DY\n operator: not_equal_to\n value: $dy\nOperations:\n - name: --DTC\n operator: dy\n id: $dy\nMatch Datasets:\n - Name: DM\n Keys:\n - USUBJID\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -100,7 +131,11 @@ "markdownDescription": "\nReturns the requested dataset level metadata value for the current dataset. Possible name values are:\n\n- dataset_size\n- dataset_location\n- dataset_name\n- dataset_label\n- domain\n- is_ap\n- ap_suffix\n\nExample\n\nInput:\n\nTarget domain: LB\n\n```yaml\n- name: dataset_label\n operator: extract_metadata\n id: $dataset_label\n```\n\nOutput:\n\n```\nLaboratory Test Results\n```\n\nExample: ap_suffix\n\nExtracts the domain suffix (characters 3-4) from AP-related domains. For example, \"FA\" from \"APFA\" DOMAIN value.\n\nInput:\n\nTarget domain: APFA\n\n```yaml\n- name: ap_suffix\n operator: extract_metadata\n id: $ap_suffix\n```\n\nOutput:\n\n```\nFA\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -110,7 +145,10 @@ "markdownDescription": "\nReturns the expected (\"Core\" = Exp ) variables for the domain in the current standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: expected_variables\n id: $expected_variables\n```\n\nOutput:\n\n```\n[\"LBCAT\", \"LBORRES\", \"LBORRESU\", \"...\"]\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -120,7 +158,13 @@ "markdownDescription": "\nFetches controlled terminology attribute values from CT packages based on row-specific CT package and version references. The operation constructs CT package names based on the standard being validated and the values in the `name` and `version` columns (e.g., SDTMIG \u2192 \"sdtmct-{version}\"). When the `name` column contains \"CDISC\" or \"CDISC CT\", it uses the validation run's standard to determine the package prefix and the version found in the cell of the specified column. The operation extracts all codes matching the specified ct_attribute from the package.\n\n**Required Parameters:**\n\n- `ct_attribute`: Attribute to extract - `\"Term CCODE\"`, `\"Codelist CCODE\"`, `\"Term Value\"`, `\"Codelist Value\"`, or `\"Term Preferred Term\"`\n- `name`: Column containing CT reference (e.g., \"TSVCDREF\") - identifies which terminology system is referenced\n- `version`: Column containing CT version (e.g., \"TSVCDVER\")\n\n```yaml\n- id: $VALID_TERM_CODES\n name: TSVCDREF\n operator: get_codelist_attributes\n ct_attribute: Term CCODE\n version: TSVCDVER\n```\n\n**Note:** if using this operator with excel data, you must put the ctpackage versions contained within your data in the library tab for it work properly.\n" } }, - "required": ["id", "operator", "name", "ct_attribute", "version"], + "required": [ + "id", + "operator", + "name", + "ct_attribute", + "version" + ], "type": "object" }, { @@ -130,7 +174,10 @@ "markdownDescription": "\nReturns list of dataset columns in order\n\n```yaml\nCheck:\n all:\n - name: $column_order_from_dataset\n operator: is_not_ordered_by\n value: $column_order_from_library\nOperations:\n - id: $column_order_from_library\n operator: get_column_order_from_library\n - id: $column_order_from_dataset\n operator: get_column_order_from_dataset\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -140,7 +187,10 @@ "markdownDescription": "\nFetches column order for a given domain from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\nOptionally Filters variables based on specified metadata criteria.\n\nRule Type: Variable Metadata Check\n\n```yaml\nCheck:\n all:\n - name: variable_name\n operator: is_not_contained_by\n value: $ig_variables\nOperations:\n - id: $ig_variables\n operator: get_column_order_from_library\n key_name: \"role\" # role, core, etc\n key_value: \"Exp\" # Timing, Req, Exp, Perm, etc\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -150,7 +200,10 @@ "markdownDescription": "\nReturns the list of domains for a given class from the CDISC Library Implementation Guide. This operation retrieves all domains that belong to a specified class (e.g., \"TRIAL DESIGN\", \"FINDINGS\", \"EVENTS\") based on the current standard and version. The operation uses the standard and version from the validation context as well as the optional `domain_class` parameter which is the name of the class to filter by (e.g., \"TRIAL DESIGN\", \"FINDINGS\", \"EVENTS\", \"INTERVENTIONS). NOTE: Class names are case-sensitive and should match the Library metadata format. If no `domain_class` parameter is provided, the operation returns all domains across all classes in the Implementation Guide:\n\n```yaml\n- operator: get_library_class_domains\n id: $trial_design_domains\n domain_class: \"TRIAL DESIGN\"\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -160,7 +213,10 @@ "markdownDescription": "\nFetches column order for a given model class from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\n\nRule Type: Variable Metadata Check\n\n```yaml\nCheck:\n all:\n - name: variable_name\n operator: is_not_contained_by\n value: $model_variables\nOperations:\n - id: $model_variables\n operator: get_model_column_order\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -170,7 +226,12 @@ "markdownDescription": "\nFetches variable level library model properties filtered by the provided key_name and key_value\n\nExample\n\nInput\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: get_model_filtered_variables\n id: $model_filtered_variables\n key_name: \"role\"\n key_value: \"Timing\"\n```\n\nOutput\n\n```\n[\"VISITNUM\", \"VISIT\", \"VISITDY\", \"TAETORD\", \"...\"]\n```\n" } }, - "required": ["id", "operator", "key_name", "key_value"], + "required": [ + "id", + "operator", + "key_name", + "key_value" + ], "type": "object" }, { @@ -180,7 +241,10 @@ "markdownDescription": "\nFetches column order for a given SUPP's parent model class from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\n\n```yaml\nCheck:\n all:\n - operator: is_not_contained_by\n value: $parent_model_variables\nOperations:\n - id: $parent_model_variables\n operator: get_parent_model_column_order\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -190,7 +254,12 @@ "markdownDescription": "\nFilters variables from the dataset based on specified metadata criteria. Returns a list of variable names that exist in the dataset and match the filter criteria.\n\n```yaml\n- operator: get_dataset_filtered_variables\n id: $timing_variables\n key_name: \"role\"\n key_value: \"Timing\"\n```\n" } }, - "required": ["id", "operator", "key_name", "key_value"], + "required": [ + "id", + "operator", + "key_name", + "key_value" + ], "type": "object" }, { @@ -200,7 +269,11 @@ "markdownDescription": "\nGenerates a dataframe where each record in the dataframe is the library ig variable metadata corresponding with the variable label found in the column provided in name. The metadata column names are prefixed with the string provided in `id`.\n\nInput\n\nTarget Dataset: SUPPLB\n\nProduct: sdtmig\n\nVersion: 3-4\n\nDataset:\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QLABEL\": [\"Toxicity\", \"Viscosity\", \"Analysis Method\"]\n}\n```\n\nRule:\n\n```yaml\n- operator: label_referenced_variable_metadata\n id: $qlabel_referenced_variable_metadata\n name: \"QLABEL\"\n```\n\nOutput\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QLABEL\": [\"Toxicity\", \"Viscosity\", \"Analysis Method\"],\n \"$qlabel_referenced_variable_metadata_name\": [\"LBTOX\", null, \"LBANMETH\"],\n \"$qlabel_referenced_variable_metadata_role\": [\n \"Variable Qualifier\",\n null,\n \"Record Qualifier\"\n ],\n \"$qlabel_referenced_variable_metadata_ordinal\": [44, null, 38],\n \"$qlabel_referenced_variable_metadata_core\": [\"Req\", \"Req\", \"Req\"],\n \"$qlabel_referenced_variable_metadata_label\": [\"Toxicity\", null, \"Analysis Method\"]\n}\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -210,7 +283,11 @@ "markdownDescription": "\nAllows the creation of a lookup table to take the values from multiple input columns and map them to values in an output column. The map parameter contains a list of objects. Each dictionary contains column names as properties that match the column names in the source dataset and an output property that will be returned as a result.\n\nIf map has a single object and output is the only property specified on that object, this will function as a direct assignment.\n\nFor example, given the following current dataset:\n\n```\nid \tparent_entity \tparent_rel\n1 \tTiming \trelativeToFrom\n2 \tSomething \trelativeToFrom\n3 \tTiming \ttype\n```\n\nand the following operation:\n\n```yaml\nOperations:\n - id: $codelist_code\n operator: map\n map:\n - parent_entity: Timing\n parent_rel: type\n output: C201264\n - parent_entity: Timing\n parent_rel: relativeToFrom\n output: C201265\n```\n\nThis will result in the following dataset:\n\n```\nid \tparent_entity \tparent_rel \t$codelist_code\n1 \tTiming \trelativeToFrom \tC201265\n2 \tSomething \trelativeToFrom \tNone\n3 \tTiming \ttype \tC201264\n```\n\nThe following operation:\n\n```yaml\nOperations:\n - id: $codelist_code\n operator: map\n map:\n - output: C201264\n```\n\nWill result in the following dataset:\n\n```\nid \tparent_entity \tparent_rel \t$codelist_code\n1 \tTiming \trelativeToFrom \tC201264\n2 \tSomething \trelativeToFrom \tC201264\n3 \tTiming \ttype \tC201264\n```\n" } }, - "required": ["id", "operator", "map"], + "required": [ + "id", + "operator", + "map" + ], "type": "object" }, { @@ -220,7 +297,11 @@ "markdownDescription": "\nIf no group is provided, returns the max value in name. If group is provided, returns the max value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: \"$max_age\"\n operator: \"greater_than\"\n value: \"MAXAGE\"\nOperations:\n - operator: \"max\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$max_age\"\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -230,7 +311,11 @@ "markdownDescription": "\nIf no group is provided, returns the max date value in name. If group is provided, returns the max date value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: USUBJID\n operator: is_contained_by\n value: $ex_usubjid\n - name: RFXENDTC\n operator: not_equal_to\n value: $max_ex_exstdtc\n - name: RFXENDTC\n operator: not_equal_to\n value: $max_ex_exendtc\nOperations:\n - operator: distinct\n domain: EX\n name: USUBJID\n id: $ex_usubjid\n - operator: max_date\n domain: EX\n name: EXSTDTC\n id: $max_ex_exstdtc\n group:\n - USUBJID\n - operator: max_date\n domain: EX\n name: EXENDTC\n id: $max_ex_exendtc\n group:\n - USUBJID\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -240,7 +325,11 @@ "markdownDescription": "\nExample: AAGE > mean(DM.AGE), where AAGE is a fictitious NSV\n\n```yaml\nCheck:\n all:\n - name: \"AAGE\"\n operator: \"greater_than\"\n value: \"$average_age\"\nOperations:\n - operator: \"mean\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$average_age\"\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -250,7 +339,11 @@ "markdownDescription": "\nIf no group is provided, returns the min value in name. If group is provided, returns the min value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: \"$min_age\"\n operator: \"less_than\"\n value: \"MINAGE\"\nOperations:\n - operator: \"min\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$min_age\"\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -260,7 +353,11 @@ "markdownDescription": "\nIf no group is provided, returns the min date value in name. If group is provided, returns the min date value in name, within each unique set of the grouping variables.\n\nExample: RFSTDTC is greater than min AE.AESTDTC for the current USUBJID\n\n```yaml\nCheck:\n all:\n - name: \"RFSTDTC\"\n operator: \"date_greater_than\"\n value: \"$ae_aestdtc\"\nOperations:\n - operator: \"min_date\"\n domain: \"AE\"\n name: \"AESTDTC\"\n id: \"$ae_aestdtc\"\n group:\n - USUBJID\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -270,7 +367,12 @@ "markdownDescription": "\nComputes set difference: elements in `name` that are not in `subtract`. By default a standard [set difference]() semantics (A \u2216 B) is applied. Optional `order_insensitive` property allows to have the element order to be taken into consideration and only those `name` elements are removed which follow the same order as in `subtract` . Preserves order from the first list. Both `name` and `subtract` must reference other operation results (e.g., `$expected_variables`, `$dataset_variables`). When `subtract` is empty or missing, returns all elements from `name`. Can be computed and added to output variables to display missing elements in error results.\n\n```yaml\nOperations:\n - id: $expected_variables\n operator: expected_variables\n - id: $dataset_variables\n operator: get_column_order_from_dataset\n - id: $expected_minus_dataset\n name: $expected_variables\n operator: minus\n subtract: $dataset_variables\n order_insensitive: false\n```\n" } }, - "required": ["id", "operator", "name", "subtract"], + "required": [ + "id", + "operator", + "name", + "subtract" + ], "type": "object" }, { @@ -280,7 +382,11 @@ "markdownDescription": "\nGenerates a dataframe where each record in the dataframe is the library ig variable metadata corresponding with the variable name found in the column provided in name. The metadata column names are prefixed with the string provided in `id`.\n\nInput\n\nTarget Dataset: SUPPLB\n\nProduct: sdtmig\n\nVersion: 3-4\n\nDataset:\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QNAM\": [\"Toxicity\", \"LBVISCOS\", \"Analysis Method\"]\n}\n```\n\nRule:\n\n```yaml\n- operator: name_referenced_variable_metadata\n id: $qnam_referenced_variable_metadata\n name: \"QNAM\"\n```\n\nOutput\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QNAM\": [\"LBTOX\", \"LBVISCOS\", \"LBANMETH\"],\n \"$qnam_referenced_variable_metadata_name\": [\"LBTOX\", null, \"LBANMETH\"],\n \"$qnam_referenced_variable_metadata_role\": [\n \"Variable Qualifier\",\n null,\n \"Record Qualifier\"\n ],\n \"$qnam_referenced_variable_metadata_ordinal\": [44, null, 38],\n \"$qnam_referenced_variable_metadata_core\": [\"Req\", \"Req\", \"Req\"],\n \"$qnam_referenced_variable_metadata_label\": [\"Toxicity\", null, \"Analysis Method\"]\n}\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -290,7 +396,10 @@ "markdownDescription": "\nReturns the permissible variables (\"Core\" = Perm ) for a given domain and standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: permissible_variables\n id: $permissible_variables\n```\n\nOutput:\n\n```\n[\"LBGRPID\", \"LBREFID\", \"LBSPID\", \"...\"]\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -300,7 +409,10 @@ "markdownDescription": "\nIf no filter or group is provided, returns the number of records in the dataset. If filter is provided, returns the number of records in the dataset that contain the value(s) in the corresponding column(s) provided in the filter. Filter can have a wildcard `&` that when added to the end of the filter value will look for all instances of that prefix (see 4th example below). If group is provided, returns the number of rows matching each unique set of the grouping variables. These can be static column name(s) or can be derived from other operations like get_dataset_filtered_variables.\n\nIf both filter and group are provided, returns the number of records in the dataset that contain the value(s) in the corresponding column(s) provided in the filter that also match each unique set of the grouping variables.\n\n**Wildcard Filtering:** Filter values ending with % will match any records where the column value starts with the specified prefix. For example, RACE% will match RACE1, RACE2, RACE3, etc. This is useful for matching related variables with numeric or alphabetic suffixes.\n\n**Regex Transformation:** If regex is provided along with group, the regex pattern will be applied to transform grouping column values before grouping. The regex is only applied to columns where the pattern matches the data type. For example, using regex `^\\d{4}-\\d{2}-\\d{2}` on a column containing `2022-01-14T08:00` will extract `2022-01-14` for grouping purposes.\n\nIf group is provided, group_aliases may also be provided to assign new grouping variable names so that results grouped by the values in one set of grouping variables can be merged onto a dataset according to the same grouping value(s) stored in different set of grouping variables. When both group and group_aliases are provided, columns are renamed according to corresponding list position (i.e., the 1st column in group is renamed to the 1st column in group_aliases, etc.). If there are more columns listed in group than in group_aliases, only the group columns with corresponding group_aliases columns will be renamed. If there are more columns listed in group_aliases than in group, the extra column names in group_aliases will be ignored.\n\nExample: return the number of records in a dataset.\n\n```yaml\n- operator: record_count\n id: $records_in_dataset\n```\n\nExample: return the number of records where STUDYID = \"CDISC01\" and FLAGVAR = \"Y\".\n\n```yaml\n- operator: record_count\n id: $flagged_cdisc01_records_in_dataset\n filter:\n STUDYID: \"CDISC01\"\n FLAGVAR: \"Y\"\n```\n\nExample: return the number of records grouped by USUBJID and timing variables, extracting only the date portion from datetime values.\n\n```yaml\n- operator: record_count\n id: $records_per_usubjid_date\n group:\n - USUBJID\n - --TESTCD\n - $TIMING_VARIABLES\n regex: \"^\\d{4}-\\d{2}-\\d{2}\"\n```\n\nExample: return the number of records where QNAM starts with \"RACE\" (matches RACE1, RACE2, RACE3, etc.) per USUBJID.\n\n```yaml\n- operator: record_count\n id: $race_records_in_dataset\n filter:\n QNAM: \"RACE&\"\n group:\n - \"USUBJID\"\n```\n\nExample: return the number of records grouped by USUBJID.\n\n```yaml\n- operator: record_count\n id: $records_per_usubjid\n group:\n - USUBJID\n```\n\nExample: return the number of records grouped by USUBJID where FLAGVAR = \"Y\".\n\n```yaml\n- operator: record_count\n id: $flagged_records_per_usubjid\n group:\n - USUBJID\n filter:\n FLAGVAR: \"Y\"\n```\n\nExample: return the number of records grouped by USUBJID and IDVARVAL where QNAM = \"TEST1\" and IDVAR = \"GROUPID\", renaming the IDVARVAL column to GROUPID for subsequent merging.\n\n```yaml\n- operator: record_count\n id: $test1_records_per_usubjid_groupid\n group:\n - USUBJID\n - IDVARVAL\n filter:\n QNAM: \"TEST1\"\n IDVAR: \"GROUPID\"\n group_aliases:\n - USUBJID\n - GROUPID\n```\n\nExample: Group the StudyIdentifier dataset by parent_id and merge the result back to the context dataset StudyVersion using StudyVersion.id == StudyIdentifier.parent_id\n\n```yaml\nScope:\n Entities:\n Include:\n - StudyVersion\nOperations:\n - domain: StudyIdentifier\n filter:\n parent_entity: \"StudyVersion\"\n parent_rel: \"studyIdentifiers\"\n rel_type: \"definition\"\n studyIdentifierScope.organizationType.code: \"C70793\"\n studyIdentifierScope.organizationType.codeSystem: \"http://www.cdisc.org\"\n group:\n - parent_id\n group_aliases:\n - id\n id: $num_sponsor_ids\n operator: record_count\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -310,7 +422,10 @@ "markdownDescription": "\nReturns the required variables ( \"Core\" = Req ) for a given domain and standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: required_variables\n id: $required_variables\n```\n\nOutput:\n\n```\n[\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"LBSEQ\", \"LBTESTCD\", \"LBTEST\"]\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -320,7 +435,12 @@ "markdownDescription": "\nSplits a dataset column by a given delimiter\n\n```yaml\nOperations:\n - name: PPSPEC\n delimiter: ;\n id: $ppspec_value\n operator: split_by\n```\n" } }, - "required": ["id", "operator", "delimiter", "name"], + "required": [ + "id", + "operator", + "delimiter", + "name" + ], "type": "object" }, { @@ -330,7 +450,10 @@ "markdownDescription": "\nReturns a list of the domains in the study\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -340,7 +463,10 @@ "markdownDescription": "\nReturns a list of the submitted dataset filenames in all uppercase\n\nex. if TS.xpt, AE.xpt, EC.xpt, and SUPPEC.xpt are submitted -> [TS, AE, EC, SUPPEC] will be returned\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -350,7 +476,10 @@ "markdownDescription": "\nReturns a list of valid SDTM domain names from the standard metadata. This can be used to compare extracted suffixes from DOMAIN values or dataset names.\n\nInput\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: standard_domains\n id: $valid_domain_names\n```\n\nOutput\n\n```\n[\"AE\", \"CM\", \"DM\", \"FA\", \"LB\", \"QS\", ...]\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -360,7 +489,10 @@ "markdownDescription": "\nReturns the valid terminology package dates for a given standard.\n\nGiven a list of terminology packages:\n\n```\n[\n \"sdtmct-2023-10-26\",\n \"sdtmct-2023-12-13\",\n \"adamct-2023-12-13\",\n \"cdashct-2023-05-19\"\n]\n```\n\nand standard: sdtmig\n\nthe operation will return:\n\n```\n[\"2023-10-26\", \"2023-12-13\"]\n```\n\nBy default, the standard is as specified when running validation - as the validation runtime parameter and/or as specified in the rule header - and the list of terminology packages is obtained from the current cache. If required, the default standard may be overridden using the optional ct_package_types parameter. For example, given the same list of terminology packages, the following operation:\n\n```yaml\nOperations:\n - operator: valid_codelist_dates\n id: $valid_dates\n ct_package_types:\n - SDTM\n - CDASH\n```\n\nwill return:\n\n```\n[\"2023-05-19\", \"2023-10-26\", \"2023-12-13\"]\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -370,7 +502,11 @@ "markdownDescription": "\nReturns true if the version of an external dictionary provided in the define.xml file matches the version parsed from the dictionary files.\n\nInput:\n\n```yaml\nOperations:\n - operator: valid_define_external_dictionary_version\n id: $is_valid_loinc_version\n external_dictionary_type: loinc\n```\n\nOutput:\n\n```\n[true, true, true, true]\n```\n" } }, - "required": ["id", "operator", "external_dictionary_type"], + "required": [ + "id", + "operator", + "external_dictionary_type" + ], "type": "object" }, { @@ -428,7 +564,10 @@ "markdownDescription": "\nDetermines whether the values are valid in the following variables:\n\n- --SOCCD (System Organ Class Code)\n- --HLGTCD (High Level Group Term Code)\n- --HLTCD (High Level Term Code)\n- --PTCD (Preferred Term Code)\n- --LLTCD (Lowest Level Term Code)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_codes\n operator: valid_meddra_code_references\n```\n\nOutput:\n\n```\n[true, false, true, true]\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -438,7 +577,10 @@ "markdownDescription": "\nDetermines whether the values are valid in the following variable pairs:\n\n- --SOCCD, --SOC (System Organ Class Code and Term)\n- --HLGTCD, --HLGT (High Level Group Term Code and Term)\n- --HLTCD, --HLT (High Level Term Code and Term)\n- --PTCD, --DECOD (Preferred Term Code and Dictionary-Derived Term)\n- --LLTCD, --LLT (Lowest Level Term Code and Term)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_pairs\n operator: valid_meddra_code_term_pairs\n```\n\nOutput:\n\n```\n[true, true, false, true, true]\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -448,7 +590,10 @@ "markdownDescription": "\nDetermines whether the values are valid in the following variables:\n\n- --SOC (System Organ Class)\n- --HLGT (High Level Group Term)\n- --HLT (High Level Term)\n- --DECOD (Dictionary-Derived Term)\n- --LLT (Lowest Level Term)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_terms\n operator: valid_meddra_term_references\n```\n\nOutput:\n\n```\n[true, true, false, true, true]\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -458,7 +603,11 @@ "markdownDescription": "\nChecks if a reference to whodrug term in name points to the existing code in Atc Text (INA) file.\n\nInput:\n\n```yaml\nOperations:\n - id: $whodrug_refs_valid\n operator: valid_whodrug_references\n```\n\nOutput:\n\n```\n[true, false, true, true]\n```\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -468,7 +617,10 @@ "markdownDescription": "\nReturns a mapping of variable names to the number of times that variable appears in a domain within the study.\n\nInput\n\n```\n{\n \"AE\": [\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"AETERM\", \"AEENDTC\"],\n \"LB\": [\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"LBTESTCD\", \"LBENDTC\"]\n}\n```\n\nOutput\n\n```\n{\n \"STUDYID\": 2,\n \"DOMAIN\": 2,\n \"USUBJID\": 2,\n \"--TERM\": 1,\n \"--TESTCD\": 1,\n \"--ENDTC\": 2\n}\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -478,7 +630,10 @@ "markdownDescription": "\nOperation operates only on original submission datasets regardless of rule type. Flags an error if a column exists is in the submission dataset currently being evaluated.\n\nRule Type: Domain Presence Check\n\n```yaml\nCheck:\n all:\n - name: $MIDS_EXISTS\n operator: equal_to\n value: true\n - name: TM\n operator: not_exists\nOperations:\n - id: $MIDS_EXISTS\n name: MIDS\n operator: variable_exists\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -488,7 +643,10 @@ "markdownDescription": "\nReturns true if a variable is missing from the dataset or if all values within the variable are null or empty string. This operation first checks if the target variable exists in the dataset, and if it does exist, evaluates whether all its values are null or empty.\nThe operation supports two sources via the `source` parameter:\n\n- **`submission`** : checks against the raw submission dataset\n- **`evaluation`** (default): checks against the evaluation dataset built based on the rule type\n\n```yaml" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -498,7 +656,10 @@ "markdownDescription": "\nReturns the set of variable names from the library for the given standard. This operation extracts all variable names across all domains in the specified standard's library metadata.\n\nInput:\n\nValidation Standard: sdtmig\nValidation Version: 3-4\n\n```yaml\n- operator: variable_names\n id: $all_variables\n```\n\nOutput:\n\n```\n[\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"SUBJID\", \"RFSTDTC\", \"RFENDTC\", \"SITEID\", \"AGE\", \"AGEU\", \"SEX\", \"RACE\", \"ETHNIC\", \"ARMCD\", \"ARM\", \"ACTARMCD\", \"ACTARM\", \"COUNTRY\", \"DMDTC\", \"DMDY\", \"AETERM\", \"AEDECOD\", \"AECAT\", \"AESCAT\", \"AEPRESP\", \"AEBODSYS\", \"AEBDSYCD\", \"AESOC\", \"AESOCCD\", \"AELLT\", \"AELLTCD\", \"AEHLT\", \"AEHLTCD\", \"AEHLGT\", \"AEHLGTCD\", \"AEPTCD\", \"AESTDTC\", \"AEENDTC\", \"AESTDY\", \"AEENDY\", \"AEDUR\", \"AESER\", \"AESEV\", \"AEACN\", \"AEREL\", \"AEOUT\", \"AESCAN\", \"AESCONG\", \"AESDISAB\", \"AESDTH\", \"AESHOSP\", \"AESLIFE\", \"AESOD\", \"AECONTRT\", \"AETOXGR\", \"LBTESTCD\", \"LBTEST\", \"LBCAT\", \"LBSCAT\", \"LBSPEC\", \"LBMETHOD\", \"LBORRES\", \"LBORRESU\", \"LBORNRLO\", \"LBORNRHI\", \"LBSTRESC\", \"LBSTRESN\", \"LBSTRESU\", \"LBSTNRLO\", \"LBSTNRHI\", \"LBNRIND\", \"LBNAM\", \"LBSPEC\", \"LBANTREG\", \"LBFAST\", \"LBDRVFL\", \"LBTOX\", \"LBTOXGR\", \"LBSTDTC\", \"LBENDTC\", \"LBSTDY\", \"LBENDY\", \"LBTPT\", \"LBTPTNUM\", \"LBELTM\", \"LBTPTREF\", \"LBRFTDTC\", \"...\"]\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -508,7 +669,11 @@ "markdownDescription": "\nGiven a variable name, returns a mapping of variable values to the number of times that value appears in the variable within all datasets in the study.\n" } }, - "required": ["id", "operator", "name"], + "required": [ + "id", + "operator", + "name" + ], "type": "object" }, { @@ -518,7 +683,10 @@ "markdownDescription": "\nDetermines whether the values are valid and in the correct hierarchical structure in the following variables:\n\n- --DECOD\n- --CLAS\n- --CLASCD\n\nInput:\n\n```yaml\nOperations:\n - id: $valid_whodrug_codes\n operator: whodrug_code_hierarchy\n```\n" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" }, { @@ -528,7 +696,46 @@ "markdownDescription": "\nValidates XHTML fragments in the target column against the specified namespace.\n\n```yaml\nOperations:\n - id: $xhtml_errors\n name: text\n operator: get_xhtml_errors\n namespace: http://www.cdisc.org/ns/usdm/xhtml/v1.0\n```\n\nNote that a local XSD file is required for validation. The file must be stored in the folder indicated by the value of the `LOCAL_XSD_FILE_DIR` default file path and the mapping between the namespace and the local XSD file's `sub-folder/name` must be included in the value of the `LOCAL_XSD_FILE_MAP` default file path.\n" } }, - "required": ["id", "operator", "name", "namespace"], + "required": [ + "id", + "operator", + "name", + "namespace" + ], + "type": "object" + }, + { + "properties": { + "operator": { + "const": "regex_find_replace" + }, + "find": { + "type": "string" + }, + "replace": { + "type": "string" + }, + "on_no_match": { + "type": "string", + "enum": [ + "keep_original", + "set_null", + "set_empty", + "error" + ] + }, + "flags": { + "type": "string", + "pattern": "^[ims]*$" + } + }, + "required": [ + "id", + "operator", + "name", + "find", + "replace" + ], "type": "object" } ], @@ -591,7 +798,13 @@ "type": "string" }, "dictionary_term_type": { - "enum": ["LLT", "PT", "HLT", "HLGT", "SOC"] + "enum": [ + "LLT", + "PT", + "HLT", + "HLGT", + "SOC" + ] }, "domain": { "anyOf": [ @@ -604,7 +817,9 @@ ] }, "external_dictionary_type": { - "enum": ["meddra"] + "enum": [ + "meddra" + ] }, "filter": { "type": "object" @@ -649,7 +864,10 @@ }, "level": { "type": "string", - "enum": ["codelist", "term"] + "enum": [ + "codelist", + "term" + ] }, "map": { "type": "array", @@ -660,7 +878,9 @@ "type": "string" } }, - "required": ["output"] + "required": [ + "output" + ] } }, "name": { @@ -675,9 +895,32 @@ "regex": { "type": "string" }, + "find": { + "type": "string" + }, + "replace": { + "type": "string" + }, + "on_no_match": { + "type": "string", + "enum": [ + "keep_original", + "set_null", + "set_empty", + "error" + ] + }, + "flags": { + "type": "string", + "pattern": "^[ims]*$" + }, "returntype": { "type": "string", - "enum": ["code", "value", "pref_term"] + "enum": [ + "code", + "value", + "pref_term" + ] }, "source": { "type": "string" @@ -704,6 +947,9 @@ "type": "string" } }, - "required": ["id", "operator"], + "required": [ + "id", + "operator" + ], "type": "object" } diff --git a/resources/schema/rule-merged/Operator.json b/resources/schema/rule-merged/Operator.json index e1b988e6a..f3a269a4d 100644 --- a/resources/schema/rule-merged/Operator.json +++ b/resources/schema/rule-merged/Operator.json @@ -9,7 +9,9 @@ "const": "additional_columns_empty" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -18,7 +20,9 @@ "const": "additional_columns_not_empty" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -28,7 +32,10 @@ "markdownDescription": "\nWill return True if the value in `value` is contained within the collection/iterable in the target column, or if there's an exact match for non-iterable data.\n\nThe operator checks if every value in a column is a list or set. If yes, it compares row-by-row. If any value is blank or a different type (like a string or number), it compares each value against the entire column instead.\n\nExample:\n\n```yaml\n- name: \"--TOXGR\" # Column containing lists like ['GRADE', 'SEVERITY', 'ONSET']\n operator: \"contains\"\n value: \"GRADE\" # True if 'GRADE' is an element in the list\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -38,7 +45,9 @@ "markdownDescription": "\nTrue if all values in `value` are contained within the variable `name`.\n\n> All of ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment') in ACTARM\n\n```yaml\n- name: \"ACTARM\"\n operator: \"contains_all\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n\nThe operator also supports lists:\n\n```yaml\n- name: \"$spec_codelist\"\n operator: \"contains_all\"\n value: \"$ppspec_value\"\n```\n\nWhere:\n\n| $spec_codelist | $ppspec_value |\n| :-------------------------- | :----------------: |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\", \"CODE2\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE2\", \"CODE3\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\"] |\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -48,7 +57,10 @@ "markdownDescription": "\nTrue if the value in `value` is contained within the collection/iterable in the target column, performing case-insensitive comparison.\n\nExample:\n\n```yaml\n- name: \"--TOXGR\" # Column containing lists like ['Grade', 'Severity', 'Onset']\n operator: \"contains_case_insensitive\"\n value: \"grade\" # True if 'Grade'/'GRADE'/'grade' exists in the list\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -58,7 +70,10 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified.\n\nThe `date_component` parameter accepts: `\"year\"`, `\"month\"`, `\"day\"`, `\"hour\"`, `\"minute\"`, `\"second\"`, `\"microsecond\"`, or `\"auto\"`.\n\nWhen `date_component: \"auto\"` is used, the operator automatically detects the precision of both dates and compares at the common (less precise) level.\n\n```yaml\n- name: \"AESTDTC\"\n operator: \"date_equal_to\"\n value: \"RFSTDTC\"\n date_component: \"auto\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -68,7 +83,10 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> Year part of BRTHDTC > 2021\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"date_greater_than\"\n date_component: \"year\"\n value: \"2021\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -78,7 +96,10 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> Year part of BRTHDTC >= 2021\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"date_greater_than_or_equal_to\"\n date_component: \"year\"\n value: \"2021\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -88,7 +109,10 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> AEENDTC < AESTDTC\n\n```yaml\n- name: \"AEENDTC\"\n operator: \"date_less_than\"\n value: \"AESTDTC\"\n```\n\n> SSDTC < all DS.DSSTDTC when SSSTRESC = \"DEAD\"\n\n```yaml\nCheck:\n all:\n - name: \"SSSTRESC\"\n operator: \"equal_to\"\n value: \"DEAD\"\n - name: \"SSDTC\"\n operator: \"date_less_than\"\n value: \"$max_ds_dsstdtc\"\nOperations:\n - operator: \"max_date\"\n domain: \"DS\"\n name: \"DSSTDTC\"\n id: \"$max_ds_dsstdtc\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -98,7 +122,10 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> AEENDTC <= AESTDTC\n\n```yaml\n- name: \"AEENDTC\"\n operator: \"date_less_than_or_equal_to\"\n value: \"AESTDTC\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -108,7 +135,10 @@ "markdownDescription": "\nComplement of `date_equal_to`\n\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -118,7 +148,10 @@ "markdownDescription": "\nComplement of `contains`. Returns True when the value is NOT contained within the target collection.\n\n```yaml\n- name: \"--TOXGR\"\n operator: \"does_not_contain\"\n value: \"GRADE\" # True if 'GRADE' is NOT an element in the list\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -128,7 +161,10 @@ "markdownDescription": "\nComplement of `contains_case_insensitive`. Returns True when the value is NOT contained within the target collection (case-insensitive).\n\nExample:\n\n```yaml\n- name: \"--TOXGR\"\n operator: \"does_not_contain_case_insensitive\"\n value: \"grade\" # True if no case variation of 'grade' exists in the list\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -141,7 +177,11 @@ "type": "boolean" } }, - "required": ["operator", "value", "regex"], + "required": [ + "operator", + "value", + "regex" + ], "type": "object" }, { @@ -151,7 +191,12 @@ "markdownDescription": "\nComplement of `has_next_corresponding_record`\n" } }, - "required": ["operator", "ordering", "value", "within"], + "required": [ + "operator", + "ordering", + "value", + "within" + ], "type": "object" }, { @@ -161,7 +206,9 @@ "markdownDescription": "\nValue presence\n\n> --OCCUR = null\n\n```yaml\n- name: --OCCUR\n operator: empty\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -171,7 +218,10 @@ "markdownDescription": "\n> SEENDTC is not empty when it is not the last record, grouped by USUBJID, sorted by SESTDTC\n\n```yaml\n- name: SEENDTC\n operator: empty_within_except_last_row\n ordering: SESTDTC\n value: USUBJID\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -181,7 +231,10 @@ "markdownDescription": "\nSubstring matching\n\n> DOMAIN ending with 'FOOBAR'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"ends_with\"\n value: \"FOOBAR\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -200,7 +253,10 @@ "type": "boolean" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -219,7 +275,10 @@ "type": "boolean" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -232,7 +291,11 @@ "type": "boolean" } }, - "required": ["operator", "value", "regex"], + "required": [ + "operator", + "value", + "regex" + ], "type": "object" }, { @@ -242,7 +305,9 @@ "markdownDescription": "\nTrue if the column exists in the current dataframe. (Works for datasets and variables)\n\n> --OCCUR is present in dataset\n\n```yaml\n- name: \"--OCCUR\"\n operator: \"exists\"\n```\n\n> Domain SJ exists\n\n```yaml\nRule Type: Domain Presence Check\nCheck:\n all:\n - name: \"SJ\"\n operator: \"exists\"\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -252,7 +317,10 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL > 0\n\n```yaml\n- name: TSVAL\n operator: greater_than\n value: 0\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -262,7 +330,10 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL >= 0\n\n```yaml\n- name: TSVAL\n operator: greater_than_or_equal_to\n value: 1\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -272,7 +343,9 @@ "markdownDescription": "\nComplement of `has_same_values`\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -282,7 +355,9 @@ "markdownDescription": "\nLength comparison\n\n> Check whether variable values has equal length of another variable.\n\n```yaml\n- name: SEENDTC\n operator: has_equal_length\n value: SESTDTC\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -292,7 +367,12 @@ "markdownDescription": "\nEnsures that a value of a variable `name` in one record is equal to the value of another variable `value` in the next corresponding record. The rows are grouped by `within` and ordered by `ordering`.\n\n> SEENDTC is equal to the SESTDTC of the next record within a USUBJID. Ordered by SESEQ\n\n```yaml\n- name: SEENDTC\n operator: has_next_corresponding_record\n value: SESTDTC\n within: USUBJID\n ordering: SESEQ\n```\n" } }, - "required": ["operator", "ordering", "value", "within"], + "required": [ + "operator", + "ordering", + "value", + "within" + ], "type": "object" }, { @@ -302,7 +382,9 @@ "markdownDescription": "\nComplement of `has_equal_length`\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -312,7 +394,9 @@ "markdownDescription": "\nTrue if all values in `name` are the same\n\n> Condition: MHCAT ^= null\n> Rule: MHCAT ^= the same value for all records\n\n```yaml\nCheck:\n all:\n - name: MHCAT\n operator: non_empty\n - name: MHCAT\n operator: has_same_values\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -322,7 +406,10 @@ "markdownDescription": "\nDuration ISO-8601 check, returns True if a duration is not in ISO-8601 format. The negative parameter must be specified to indicate if negative durations are either allowed (True) or disallowed (False)\n\n> DURVAR is invalid (negative durations disallowed)\n\n```yaml\n- name: \"DURVAR\"\n operator: \"invalid_duration\"\n negative: False\n```\n" } }, - "required": ["operator", "negative"], + "required": [ + "operator", + "negative" + ], "type": "object" }, { @@ -332,7 +419,9 @@ "markdownDescription": "\nThe operator performs date validation against complete and partial dates with uncertainty in the following order:\n\n1. Attempts to parse using [dateutil.parser.isoparse()](https://dateutil.readthedocs.io/en/stable/parser.html)\n2. If parsing fails and the string contains uncertainty indicators (`/`, `--`, `-:`), validates against an extended ISO 8601 dates regex pattern\n3. If parsing succeeds, dates are still validated against the regex pattern.\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"invalid_date\"\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -342,7 +431,9 @@ "markdownDescription": "\nDate check\n\n> DM.RFSTDTC = complete date\n\n```yaml\n- name: \"RFSTDTC\"\n operator: \"is_complete_date\"\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -352,7 +443,10 @@ "markdownDescription": "\nValue in `name` compared against a list in `value`. The list can have literal values or be a reference to a `$variable`.\n\nThis operator behaves similarly to `contains`. The key distinction: `contains` checks if comparator \u2208 target, while `is_contained_by` checks if target \u2208 comparator.\n\n> ACTARM in ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment')\n\n```yaml\n- name: \"ACTARM\"\n operator: \"is_contained_by\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -362,7 +456,10 @@ "markdownDescription": "\nValue in `name` case insensitive compared against a list in `value`. The list can have literal values or be a reference to a `$variable`.\n\n> ACTARM in ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment')\n\n```yaml\n- name: \"ACTARM\"\n operator: \"is_contained_by_case_insensitive\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -372,7 +469,9 @@ "markdownDescription": "\nComplement of `is_complete_date`\n\nDate check\n\n> DM.RFSTDTC ^= complete date\n\n```yaml\n- name: \"RFSTDTC\"\n operator: \"is_incomplete_date\"\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -382,7 +481,10 @@ "markdownDescription": "\nComplement of `is_contained_by`\n\n> ARM not in ('Screen Failure', 'Not Assigned')\n\n```yaml\n- name: \"ARM\"\n operator: \"is_not_contained_by\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -392,7 +494,10 @@ "markdownDescription": "\nComplement of `is_contained_by_case_insensitive`\n\n> ARM not in ('Screen Failure', 'Not Assigned')\n\n```yaml\n- name: \"ARM\"\n operator: \"is_not_contained_by_case_insensitive\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -402,7 +507,10 @@ "markdownDescription": "\nComplement of `is_ordered_by`\n" } }, - "required": ["operator", "order"], + "required": [ + "operator", + "order" + ], "type": "object" }, { @@ -411,7 +519,10 @@ "const": "is_not_ordered_set" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -421,7 +532,10 @@ "markdownDescription": "\nComplement of `is_unique_relationship`\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -431,7 +545,9 @@ "markdownDescription": "\nComplement of `is_unique_set`.\n\n> --SEQ is not unique within DOMAIN, USUBJID, and --TESTCD\n\n```yaml\n- name: \"--SEQ\"\n operator: is_not_unique_set\n value:\n - \"DOMAIN\"\n - \"USUBJID\"\n - \"--TESTCD\"\n```\n\n> `name` can be a variable containing a list of columns and `value` does not need to be present\n\n```yaml\nRule Type: Dataset Contents Check against Define XML\nCheck:\n all:\n - name: define_dataset_key_sequence # contains list of dataset key columns\n operator: is_not_unique_set\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -441,7 +557,10 @@ "markdownDescription": "\nTrue if the dataset rows are ordered by the values within `name`, given the ordering specified by `order`\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_by\n order: asc\n```\n" } }, - "required": ["operator", "order"], + "required": [ + "operator", + "order" + ], "type": "object" }, { @@ -451,7 +570,10 @@ "markdownDescription": "\nTrue if the dataset rows are in ascending order of the values within `name`, grouped by the values within `value`. Value can either be a single column or multiple.\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_set\n value: USUBJID\n```\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_set\n value:\n - USUBJID\n - \"--TESTCD\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -461,7 +583,10 @@ "markdownDescription": "\nRelationship Integrity Check looking for a 1-1 relationship between name and value. Ensures uniqueness of both name and value.\n\n> AETERM and AEDECOD has a 1-to-1 relationship\n\n```yaml\n- name: AETERM\n operator: is_unique_relationship\n value: AEDECOD\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -471,7 +596,10 @@ "markdownDescription": "\nChecks if a variable maintains consistent values within groups defined by one or more grouping variables. Groups records by specified value(s) and validates that the target variable maintains the same value within each unique combination of grouping variables. When inconsistency is detected within a group, the operator attempts to identify a majority value. If one value appears more frequently than all others, only the minority records (those not matching the majority value) are flagged. If no single majority exists \u2014 i.e., two or more values are tied for the highest frequency \u2014 all records in that group are flagged.\n\nSingle grouping variable - true if the values of BGSTRESU differ within USUBJID:\n\nIf a regex parameter is provided, it is applied to the values of the target variable before the consistency check. The first capture group of the regex is used as the normalized value for comparison. This can be useful when only part of the value should be considered during comparison (for example, comparing only the date portion of a datetime value).\n\n- regex is optional.\n- The pattern must include at least one capture group(or whole regex will be wrapped to capture group).\n- Only the first capture group is used for comparison.\n- If the pattern does not match a value, the original value is used.\n\n```yaml\n- name: \"BGSTRESU\"\n operator: is_inconsistent_across_dataset\n value: \"USUBJID\"\n```\n\nMultiple grouping variables - true if the values of --STRESU differ within each combination of --TESTCD, --CAT, --SCAT, --SPEC, and --METHOD:\n\n```yaml\n- name: \"--STRESU\"\n operator: is_inconsistent_across_dataset\n value:\n - \"--TESTCD\"\n - \"--CAT\"\n - \"--SCAT\"\n - \"--SPEC\"\n - \"--METHOD\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -481,7 +609,9 @@ "markdownDescription": "\nRelationship Integrity Check\n\n> --SEQ is unique within DOMAIN, USUBJID, and --TESTCD\n\n```yaml\n- name: \"--SEQ\"\n operator: is_unique_set\n value:\n - \"DOMAIN\"\n - \"USUBJID\"\n - \"--TESTCD\"\n```\n\n> `name` can be a variable containing a list of columns and `value` does not need to be present\n\n> The `regex` parameter allows you to extract portions of values using a regex pattern before checking uniqueness.\n\n> Compare date only (YYYY-MM-DD) for uniqueness\n\n```yaml\n- name: \"--REPNUM\"\n operator: is_not_unique_set\n value:\n - \"USUBJID\"\n - \"--TESTCD\"\n - \"$TIMING_VARIABLES\"\n regex: '^\\d{4}-\\d{2}-\\d{2}'\n```\n\n> Compare by first N characters of a string\n\n```yaml\n- name: \"ITEM_ID\"\n operator: is_not_unique_set\n value:\n - \"USUBJID\"\n - \"CATEGORY\"\n regex: \"^.{2}\"\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -491,7 +621,10 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL < 1\n\n```yaml\n- name: TSVAL\n operator: less_than\n value: 1\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -501,7 +634,10 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL <= 1\n\n```yaml\n- name: TSVAL\n operator: less_than_or_equal_to\n value: 1\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -511,7 +647,10 @@ "markdownDescription": "\nLength comparison\n\n> SETCD value length > 8\n\n```yaml\n- name: \"SETCD\"\n operator: \"longer_than\"\n value: 8\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -521,7 +660,10 @@ "markdownDescription": "\nLength comparison\n\n> TSVAL value length >= 201\n\n```yaml\n- name: \"TSVAL\"\n operator: \"longer_than_or_equal_to\"\n value: 201\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -531,7 +673,10 @@ "markdownDescription": "\nRegular Expression value matching\n\n- Determine if each string starts with a match of a regular expression. Refer to this pandas documentation: https://pandas.pydata.org/docs/reference/api/pandas.Series.str.match.html\n- To \"search\" for a regex within the entire text, prefix the regex with `.*` and do not use anchors `^` , `$`\n- To do a \"fullmatch\" of a regex with the entire text, suffix the regex with an anchor `$` and do not prefix the regex with `.*`\n- For syntax guide, refer to this Python documentation: [Regular Expression HOWTO](https://docs.python.org/3/howto/regex.html).\n- Suggestion for an on-line regular expression logic. tester: https://regex101.com, choose the Python dialect.\n- For regex token visualization, try https://www.debuggex.com.\n\n> --DOSTXT value is non-numeric\n\n```yaml\n- name: --DOSTXT\n operator: matches_regex\n value: ^\\d*\\.?\\d*$\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -541,7 +686,9 @@ "markdownDescription": "\nComplement of `empty`\n\n> --OCCUR ^= null\n\n```yaml\n- name: --OCCUR\n operator: non_empty\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -551,7 +698,10 @@ "markdownDescription": "\nComplement of `empty_within_except_last_row`\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -561,7 +711,9 @@ "markdownDescription": "\nComplement of `contains_all`\n\n> All of ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment') not in ACTARM\n\n```yaml\n- name: \"ACTARM\"\n operator: \"not_contains_all\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n\nThe operator also supports lists:\n\n```yaml\n- name: \"$spec_codelist\"\n operator: \"not_contains_all\"\n value: \"$ppspec_value\"\n```\n\nWhere:\n\n| $spec_codelist | $ppspec_value |\n| :-------------------------- | :----------------: |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\", \"CODE2\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE2\", \"CODE3\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\"] |\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -580,7 +732,10 @@ "type": "boolean" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -599,7 +754,10 @@ "type": "boolean" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -609,7 +767,9 @@ "markdownDescription": "\nComplement of `exists`\n\n> AEOCCUR not present in dataset\n\n```yaml\n- name: \"AEOCCUR\"\n operator: \"not_exists\"\n```\n\n> Domain SJ does not exist\n\n```yaml\nRule Type: Domain Presence Check\nCheck:\n all:\n - name: \"SJ\"\n operator: \"not_exists\"\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -619,7 +779,10 @@ "markdownDescription": "\nComplement of `matches_regex`\n\n> --TESTCD <= 8 chars and contains only letters, numbers, and underscores and can not start with a number\n\n```yaml\n- name: --TESTCD\n operator: not_matches_regex\n value: ^[A-Z_][A-Z0-9_]{0,7}$\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -629,7 +792,11 @@ "markdownDescription": "\nComplement of `prefix_matches_regex`\n" } }, - "required": ["operator", "prefix", "value"], + "required": [ + "operator", + "prefix", + "value" + ], "type": "object" }, { @@ -639,7 +806,10 @@ "markdownDescription": "\nComplement of `present_on_multiple_rows_within`\n\n```yaml\n- operator: \"not_present_on_multiple_rows_within\"\n name: \"RELID\"\n value: 4 (optional)\n within: \"USUBJID\"\n```\n" } }, - "required": ["operator", "within"], + "required": [ + "operator", + "within" + ], "type": "object" }, { @@ -649,7 +819,11 @@ "markdownDescription": "\nComplement of `suffix_matches_regex`\n\n> QNAM does not end with numbers\n\n```yaml\n- name: \"QNAM\"\n operator: \"not_suffix_matches_regex\"\n suffix: 2\n value: \"\\d\\d\"\n```\n" } }, - "required": ["operator", "suffix", "value"], + "required": [ + "operator", + "suffix", + "value" + ], "type": "object" }, { @@ -659,7 +833,11 @@ "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match one of the strings in the list in `value`\n\n> Check if a variable's domain identifier exists in the study\n\n```yaml\n- name: variable_name\n operator: prefix_is_contained_by\n prefix: 2\n value: $study_domains\n```\n" } }, - "required": ["operator", "prefix", "value"], + "required": [ + "operator", + "prefix", + "value" + ], "type": "object" }, { @@ -669,7 +847,11 @@ "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match the string in `value`\n\n```yaml\n- name: dataset_name\n operator: prefix_equal_to\n prefix: 2\n value: DOMAIN\n```\n" } }, - "required": ["operator", "prefix", "value"], + "required": [ + "operator", + "prefix", + "value" + ], "type": "object" }, { @@ -679,7 +861,11 @@ "markdownDescription": "\nComplement of `prefix_is_contained_by`\n" } }, - "required": ["operator", "prefix", "value"], + "required": [ + "operator", + "prefix", + "value" + ], "type": "object" }, { @@ -689,7 +875,11 @@ "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match a regular expression in `value`\n\n```yaml\n- name: DOMAIN\n operator: prefix_matches_regex\n prefix: 2\n value: (AP|ap)\n```\n" } }, - "required": ["operator", "prefix", "value"], + "required": [ + "operator", + "prefix", + "value" + ], "type": "object" }, { @@ -699,7 +889,11 @@ "markdownDescription": "\nComplement of `prefix_equal_to`\n" } }, - "required": ["operator", "prefix", "value"], + "required": [ + "operator", + "prefix", + "value" + ], "type": "object" }, { @@ -709,7 +903,10 @@ "markdownDescription": "\nTrue if the same value of `name` is present on multiple rows, grouped by `within`. A maximum allowed number of occurrences can be specified in the value attribute. In this instance the value: 4 means that an error will be flagged if the same value appears more than 4 times within a USUBJID. By default the operator will flag any time a value appears more than once.\n\n```yaml\n- operator: \"present_on_multiple_rows_within\"\n name: \"RELID\"\n value: 4 (optional)\n within: \"USUBJID\"\n```\n" } }, - "required": ["operator", "within"], + "required": [ + "operator", + "within" + ], "type": "object" }, { @@ -719,7 +916,10 @@ "markdownDescription": "\nWill raise an issue if at least one of the values in `name` is the same as one of the values in `value`. See [shares_no_elements_with](#shares_no_elements_with).\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -729,7 +929,10 @@ "markdownDescription": "\nWill raise an issue if exactly one of the values in `name` is the same as one of the values in `value`. See [shares_no_elements_with](#shares_no_elements_with).\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -739,7 +942,10 @@ "markdownDescription": "\nWill raise an issue if the values in `name` do not share any of the values in `value`\n\n> Check if $dataset_variables shares no elements with $timing_variables\n\n```yaml\nRule Type: Dataset Metadata Check # One record per dataset\nCheck:\n - all:\n name: $dataset_variables\n operator: shares_no_elements_with\n value: $timing_variables\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -749,7 +955,10 @@ "markdownDescription": "\nLength comparison\n\n> SETCD value length < 9\n\n```yaml\n- name: \"SETCD\"\n operator: \"shorter_than\"\n value: 9\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -759,7 +968,10 @@ "markdownDescription": "\nLength comparison\n\n> TSVAL value length <= 200\n\n```yaml\n- name: \"TSVAL\"\n operator: \"shorter_than_or_equal_to\"\n value: 201\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -769,7 +981,9 @@ "markdownDescription": "\nSplits a string by a separator and checks if both parts have equal length. Generic operator for validating paired data formats where both parts must have the same level of detail or precision.\n\nParameters:\n\n- `separator`: The delimiter to split on (default: \"/\")\n\n> Check that string parts separated by a delimiter have equal length\n\n```yaml\n- name: --DTC\n operator: split_parts_have_equal_length\n separator: \"/\"\n```\n\nUse cases:\n\n- **Date/time intervals**: `2003-12-15T10:00/2003-12-15T10:30` \u2192 True (both 16 characters)\n- **Date ranges**: `2003-12-01/2003-12-10` \u2192 True (both 10 characters)\n- **Version ranges**: `1.2.3/2.0.0` \u2192 True (both 5 characters)\n- **Product codes**: `ABC-123/XYZ-789` \u2192 True (both 7 characters)\n\nInvalid example:\n\n- `2003-12-15T10:00/2003-12-15T10:30:15` \u2192 False (16 vs 19 characters - different precision)\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -779,7 +993,9 @@ "markdownDescription": "\nComplement of `split_parts_have_equal_length`. Returns True when parts have unequal lengths (indicates a violation).\n\n```yaml\n- name: --DTC\n operator: split_parts_have_unequal_length\n separator: \"/\"\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -789,7 +1005,10 @@ "markdownDescription": "\nSubstring matching\n\n> DOMAIN beginning with 'AP'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"starts_with\"\n value: \"AP\"\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -799,7 +1018,11 @@ "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match the string in `value`\n\n```yaml\n- name: dataset_name\n operator: suffix_equal_to\n prefix: 2\n value: DOMAIN\n```\n" } }, - "required": ["operator", "suffix", "value"], + "required": [ + "operator", + "suffix", + "value" + ], "type": "object" }, { @@ -809,7 +1032,11 @@ "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match one of the strings in the list in `value`\n\n> Check if a supp's parent domain exists in the study\n\n```yaml\n- name: dataset_name\n operator: suffix_is_contained_by\n prefix: 2\n value: $study_domains\n```\n" } }, - "required": ["operator", "suffix", "value"], + "required": [ + "operator", + "suffix", + "value" + ], "type": "object" }, { @@ -819,7 +1046,11 @@ "markdownDescription": "\nComplement of `suffix_is_contained_by`\n" } }, - "required": ["operator", "suffix", "value"], + "required": [ + "operator", + "suffix", + "value" + ], "type": "object" }, { @@ -829,7 +1060,11 @@ "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match a regular expression in `value`\n\n> QNAM ends with numbers\n\n```yaml\n- name: \"QNAM\"\n operator: \"suffix_matches_regex\"\n suffix: 2\n value: \"\\d\\d\"\n```\n" } }, - "required": ["operator", "suffix", "value"], + "required": [ + "operator", + "suffix", + "value" + ], "type": "object" }, { @@ -839,7 +1074,11 @@ "markdownDescription": "\nComplement of `suffix_equal_to`\n" } }, - "required": ["operator", "suffix", "value"], + "required": [ + "operator", + "suffix", + "value" + ], "type": "object" }, { @@ -849,7 +1088,11 @@ "markdownDescription": "\nComplement of `target_is_sorted_by`\n" } }, - "required": ["operator", "value", "within"], + "required": [ + "operator", + "value", + "within" + ], "type": "object" }, { @@ -859,7 +1102,11 @@ "markdownDescription": "\nTrue if the values in name are ordered according to the values specified by value\nin ascending/descending order, grouped by the values in within. Each value entry\nrequires a variable name, a sort_order of asc or desc, and an optional\nnull_position of first or last (defaults to last) which controls where null/empty\ncomparator values are placed in the expected ordering. Within accepts either a\nsingle column or an ordered list of columns. Columns can be either number or Char\nDates in ISO8601 YYYY-MM-DD format. Date value(s) with different precisions that\noverlap (e.g. 2005-10, 2005-10-3 and 2005-10-08) are all flagged as not sorted as\ntheir order cannot be inferred.\n\nOptionally supports a `regex` parameter that extracts a portion of the target\nvalue for sorting. The regex must contain at least one capturing group. The first\ncaptured group is extracted and converted to numeric if possible, allowing proper\nsorting of sequence numbers (e.g., \"MIDS1\", \"MIDS2\", ..., \"MIDS10\" with regex\n`.*?(\\\\d+)$`). This is particularly useful for variables that end with sequence\nnumbers that may or may not be zero-padded.\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n within:\n - USUBJID\n - MIDSTYPE\n operator: target_is_sorted_by\n value:\n - name: --STDTC\n sort_order: asc\n null_position: last\n```\n\nExample with regex for extracting sequence numbers:\n\n```yaml\nCheck:\n all:\n - name: MIDS\n operator: target_is_sorted_by\n regex: \".*?(\\\\d+)$\" # Extract trailing digits, convert to numeric\n value:\n - name: SMSTDTC\n sort_order: asc\n within:\n - USUBJID\n - MIDSTYPE\n```\n" } }, - "required": ["operator", "value", "within"], + "required": [ + "operator", + "value", + "within" + ], "type": "object" }, { @@ -869,7 +1116,10 @@ "markdownDescription": "\nComplement of `value_has_multiple_references`\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -879,7 +1129,10 @@ "markdownDescription": "\nTrue if the value in `name` has more than one count in the dictionary defined in `value`\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -889,7 +1142,10 @@ "markdownDescription": "\nChecks for inconsistencies in enumerated columns of a DataFrame. Starting with the smallest/largest enumeration of the given variable, returns True if VARIABLE(N+1) is populated but VARIABLE(N) is not populated. Repeats for all variables belonging to the enumeration. Note that the initial variable will not have an index (VARIABLE) and the next enumerated variable has index 1 (VARIABLE1).\n\nex: Check if there are inconsistencies in the TSVAL columns (TSVAL, TSVAL1, TSVAL2, etc.)\n\n```yaml\nCheck:\n all:\n - name: \"TSVAL\"\n operator: \"inconsistent_enumerated_columns\"\n```\n" } }, - "required": ["operator", "name"], + "required": [ + "operator", + "name" + ], "type": "object" }, { @@ -899,7 +1155,10 @@ "markdownDescription": "\nChecks if elements in the target list appear in the same relative order in the comparator list.\n\n> Check if dataset column order is a correctly ordered subset of library column order\n\n```yaml\n- name: $column_order_from_dataset\n operator: is_ordered_subset_of\n value: $column_order_from_library\n```\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -909,7 +1168,10 @@ "markdownDescription": "\nComplement of `is_ordered_subset_of`\n" } }, - "required": ["operator", "value"], + "required": [ + "operator", + "value" + ], "type": "object" }, { @@ -919,7 +1181,9 @@ "markdownDescription": "\nValidates that variable labels follow proper title case formatting rules using the titlecase PyPi library. Title case capitalizes the first word and all major words, while keeping articles (a, an, the), conjunctions (and, but, or), and prepositions (in, of, for) in lowercase unless they are the first word. \nNOTE: The titlecase library may produce false positives or false negatives in syntactic edge cases (e.g. hyphenated words, slash-separated terms, uncommon prepositions).\n\n> Check that AELABEL values are in proper title case\n\n```yaml\n- name: AELABEL\n operator: is_title_case\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" }, { @@ -929,13 +1193,18 @@ "markdownDescription": "\nComplement of `is_title_case`. Returns True when values are NOT in proper title case.\n\n> Flag AELABEL values that violate title case rules\n\n```yaml\n- name: AELABEL\n operator: is_not_title_case\n```\n" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" } ], "properties": { "comparator": { - "type": ["number", "string"] + "type": [ + "number", + "string" + ] }, "context": { "type": "string" @@ -973,18 +1242,27 @@ "type": "boolean" }, "codelistcheck": { - "enum": ["code", "value"], + "enum": [ + "code", + "value" + ], "type": "string" }, "codelistlevel": { - "enum": ["term", "codelist"], + "enum": [ + "term", + "codelist" + ], "type": "string" }, "operator": { "type": "string" }, "order": { - "enum": ["asc", "dsc"], + "enum": [ + "asc", + "dsc" + ], "type": "string" }, "ordering": { @@ -1002,17 +1280,25 @@ "value": { "oneOf": [ { - "type": ["boolean", "number", "string"] + "type": [ + "boolean", + "number", + "string" + ] }, { "items": { - "type": ["number"] + "type": [ + "number" + ] }, "type": "array" }, { "items": { - "type": ["string"] + "type": [ + "string" + ] }, "type": "array" }, @@ -1023,7 +1309,10 @@ "$ref": "Operator.json#/properties/name" }, "null_position": { - "enum": ["first", "last"], + "enum": [ + "first", + "last" + ], "type": "string" }, "order": { @@ -1066,6 +1355,8 @@ "type": "string" } }, - "required": ["operator"], + "required": [ + "operator" + ], "type": "object" } diff --git a/resources/schema/rule-merged/Organization_CDISC.json b/resources/schema/rule-merged/Organization_CDISC.json index db1041921..9aaef8a76 100644 --- a/resources/schema/rule-merged/Organization_CDISC.json +++ b/resources/schema/rule-merged/Organization_CDISC.json @@ -22,7 +22,9 @@ "const": "Failure" } }, - "required": ["Type"], + "required": [ + "Type" + ], "type": "object" }, "Origin": { @@ -38,7 +40,9 @@ "type": "object" }, "Version": { - "enum": ["5.0"] + "enum": [ + "5.0" + ] } }, "type": "object" @@ -47,7 +51,12 @@ "type": "array" }, "Version": { - "enum": ["1.0", "1.1", "1.2", "1.3"] + "enum": [ + "1.0", + "1.1", + "1.2", + "1.3" + ] } }, "type": "object" @@ -66,7 +75,9 @@ "const": "Success" } }, - "required": ["Type"], + "required": [ + "Type" + ], "type": "object" }, "Origin": { @@ -79,13 +90,19 @@ "type": "string" }, "Version": { - "enum": ["1", "2", "3"] + "enum": [ + "1", + "2", + "3" + ] } }, "type": "object" }, "Version": { - "enum": ["2.0"] + "enum": [ + "2.0" + ] } }, "type": "object" @@ -94,7 +111,11 @@ "type": "array" }, "Version": { - "enum": ["3.2", "3.3", "3.4"] + "enum": [ + "3.2", + "3.3", + "3.4" + ] } }, "type": "object" @@ -113,7 +134,9 @@ "const": "Success" } }, - "required": ["Type"], + "required": [ + "Type" + ], "type": "object" }, "Origin": { @@ -129,7 +152,9 @@ "type": "object" }, "Version": { - "enum": ["5.0"] + "enum": [ + "5.0" + ] } }, "type": "object" @@ -138,7 +163,11 @@ "type": "array" }, "Version": { - "enum": ["3.0", "3.1", "3.1.1"] + "enum": [ + "3.0", + "3.1", + "3.1.1" + ] } }, "type": "object" @@ -157,7 +186,9 @@ "const": "Success" } }, - "required": ["Type"], + "required": [ + "Type" + ], "type": "object" }, "Origin": { @@ -173,7 +204,9 @@ "type": "object" }, "Version": { - "enum": ["5.0"] + "enum": [ + "5.0" + ] } }, "type": "object" @@ -182,7 +215,10 @@ "type": "array" }, "Version": { - "enum": ["1.1", "1.2"] + "enum": [ + "1.1", + "1.2" + ] } }, "type": "object" @@ -201,7 +237,9 @@ "const": "Success" } }, - "required": ["Type"], + "required": [ + "Type" + ], "type": "object" }, "Origin": { @@ -217,7 +255,9 @@ "type": "object" }, "Version": { - "enum": ["5.0"] + "enum": [ + "5.0" + ] } }, "type": "object" @@ -226,7 +266,9 @@ "type": "array" }, "Version": { - "enum": ["1.0"] + "enum": [ + "1.0" + ] } }, "type": "object" @@ -245,7 +287,9 @@ "const": "Success" } }, - "required": ["Type"], + "required": [ + "Type" + ], "type": "object" }, "Origin": { @@ -261,7 +305,9 @@ "type": "object" }, "Version": { - "enum": ["1.0"] + "enum": [ + "1.0" + ] } }, "type": "object" @@ -270,13 +316,24 @@ "type": "array" }, "Version": { - "enum": ["1.0"] + "enum": [ + "1.0" + ] }, "Substandard": { - "enum": ["SDTM", "SEND", "ADaM", "CDASH"] + "enum": [ + "SDTM", + "SEND", + "ADaM", + "CDASH" + ] } }, - "required": ["Name", "Version", "Substandard"], + "required": [ + "Name", + "Version", + "Substandard" + ], "type": "object" }, { @@ -297,13 +354,17 @@ "type": "string" }, "Version": { - "enum": ["1"] + "enum": [ + "1" + ] } }, "type": "object" }, "Version": { - "enum": ["1.0"] + "enum": [ + "1.0" + ] } }, "type": "object" @@ -312,7 +373,10 @@ "type": "array" }, "Version": { - "enum": ["3.0", "4.0"] + "enum": [ + "3.0", + "4.0" + ] } }, "type": "object" diff --git a/resources/schema/rule-merged/Organization_Custom.json b/resources/schema/rule-merged/Organization_Custom.json index bf5bd7276..c5f591394 100644 --- a/resources/schema/rule-merged/Organization_Custom.json +++ b/resources/schema/rule-merged/Organization_Custom.json @@ -9,7 +9,10 @@ "type": "string", "description": "Name of your custom organization", "not": { - "enum": ["CDISC", "FDA"] + "enum": [ + "CDISC", + "FDA" + ] } }, "Standards": { @@ -45,7 +48,9 @@ "description": "Version of the rule" } }, - "required": ["Id"], + "required": [ + "Id" + ], "type": "object" }, "Version": { @@ -55,7 +60,10 @@ "Criteria": { "properties": { "Type": { - "enum": ["Failure", "Success"], + "enum": [ + "Failure", + "Success" + ], "type": "string" }, "Plain Language Expression": { @@ -70,30 +78,46 @@ "type": "string" } }, - "required": ["Rule"], + "required": [ + "Rule" + ], "type": "object" } }, - "required": ["Type"], + "required": [ + "Type" + ], "anyOf": [ { - "required": ["Logical Expression"] + "required": [ + "Logical Expression" + ] }, { - "required": ["Plain Language Expression"] + "required": [ + "Plain Language Expression" + ] } ], "type": "object" } }, - "required": ["Origin", "Rule Identifier", "Version"], + "required": [ + "Origin", + "Rule Identifier", + "Version" + ], "type": "object" }, "minItems": 1, "type": "array" } }, - "required": ["Name", "References", "Version"], + "required": [ + "Name", + "References", + "Version" + ], "type": "object" }, "minItems": 1, @@ -141,7 +165,10 @@ }, "OutputType": { "type": "string", - "enum": ["Check", "Listing"], + "enum": [ + "Check", + "Listing" + ], "description": "Output type of the rule validation result" }, "Keywords": { @@ -155,7 +182,11 @@ "additionalProperties": true } }, - "required": ["Organization", "Standards", "Category"], + "required": [ + "Organization", + "Standards", + "Category" + ], "type": "object", "$defs": { "metadata": { diff --git a/resources/schema/rule-merged/Organization_FDA.json b/resources/schema/rule-merged/Organization_FDA.json index b0f7de783..94af54bc4 100644 --- a/resources/schema/rule-merged/Organization_FDA.json +++ b/resources/schema/rule-merged/Organization_FDA.json @@ -41,7 +41,10 @@ } } ], - "required": ["Document", "Cited Guidance"], + "required": [ + "Document", + "Cited Guidance" + ], "type": "object" }, "type": "array" @@ -52,7 +55,9 @@ "const": "Success" } }, - "required": ["Type"], + "required": [ + "Type" + ], "type": "object" }, "Origin": { @@ -68,7 +73,9 @@ "type": "object" }, "Version": { - "enum": ["1.5"] + "enum": [ + "1.5" + ] } }, "type": "object" @@ -84,7 +91,11 @@ "const": "SDTMIG" }, "Version": { - "enum": ["3.2", "3.3", "3.4"] + "enum": [ + "3.2", + "3.3", + "3.4" + ] } }, "type": "object" @@ -95,7 +106,11 @@ "const": "SENDIG" }, "Version": { - "enum": ["3.0", "3.1", "3.1.1"] + "enum": [ + "3.0", + "3.1", + "3.1.1" + ] } }, "type": "object" @@ -106,7 +121,9 @@ "const": "SENDIG-AR" }, "Version": { - "enum": ["1.0"] + "enum": [ + "1.0" + ] } }, "type": "object" @@ -117,7 +134,10 @@ "const": "SENDIG-DART" }, "Version": { - "enum": ["1.1", "1.2"] + "enum": [ + "1.1", + "1.2" + ] } }, "type": "object" @@ -128,7 +148,9 @@ "const": "SENDIG-GENETOX" }, "Version": { - "enum": ["1.0"] + "enum": [ + "1.0" + ] } }, "type": "object" diff --git a/resources/schema/rule/Operations.json b/resources/schema/rule/Operations.json index e375fa0af..f2ec6ca76 100644 --- a/resources/schema/rule/Operations.json +++ b/resources/schema/rule/Operations.json @@ -477,6 +477,29 @@ }, "required": ["id", "operator", "name", "namespace"], "type": "object" + }, + { + "properties": { + "operator": { + "const": "regex_find_replace" + }, + "find": { + "type": "string" + }, + "replace": { + "type": "string" + }, + "on_no_match": { + "type": "string", + "enum": ["keep_original", "set_null", "set_empty", "error"] + }, + "flags": { + "type": "string", + "pattern": "^[ims]*$" + } + }, + "required": ["id", "operator", "name", "find", "replace"], + "type": "object" } ], "properties": { @@ -622,6 +645,20 @@ "regex": { "type": "string" }, + "find": { + "type": "string" + }, + "replace": { + "type": "string" + }, + "on_no_match": { + "type": "string", + "enum": ["keep_original", "set_null", "set_empty", "error"] + }, + "flags": { + "type": "string", + "pattern": "^[ims]*$" + }, "returntype": { "type": "string", "enum": ["code", "value", "pref_term"] diff --git a/tests/QARegressionTests/test_Issues/test_CoreIssue587.py b/tests/QARegressionTests/test_Issues/test_CoreIssue587.py new file mode 100644 index 000000000..b1f3f41f7 --- /dev/null +++ b/tests/QARegressionTests/test_Issues/test_CoreIssue587.py @@ -0,0 +1,118 @@ +import json +import os +import subprocess + +import pytest +from conftest import get_python_executable + + +def _latest_json_report(before_files: set[str]) -> str: + files = set(os.listdir()) + new_json = sorted( + [ + f + for f in (files - before_files) + if f.startswith("CORE-Report-") and f.endswith(".json") + ] + ) + if new_json: + return new_json[-1] + # Fallback if report file already existed from prior runs + all_json = sorted( + [f for f in files if f.startswith("CORE-Report-") and f.endswith(".json")] + ) + assert all_json, "No CORE JSON report was produced" + return all_json[-1] + + +def _issue_row_text(issue_row: dict) -> str: + # Make a resilient searchable blob regardless of key naming + return " | ".join(str(v) for v in issue_row.values() if v is not None) + + +@pytest.mark.regression +@pytest.mark.parametrize( + "case_folder,expected_missing_trtxxa", + [ + ("paired_only", []), + ("single_missing", ["TRT01A"]), + ("mixed_partial", ["TRT01A"]), + ("multiple_missing", ["TRT01A", "TRT02A"]), + ("boundary_99_missing", ["TRT99A"]), + ("boundary_99_paired", []), + ("nonmatching_noise", []), + ], +) +def test_coreissue587_adam64(case_folder, expected_missing_trtxxa): + # Expected folder structure: + # tests/resources/CoreIssue587//Dataset.json + # tests/resources/CoreIssue587//Rule.yml + + case_root = os.path.join("tests", "resources", "CoreIssue587", case_folder) + dataset_path = os.path.join(case_root, "Dataset.json") + rule_path = os.path.join("tests", "resources", "CoreIssue587", "Rule.yml") + + assert os.path.exists(dataset_path), f"Missing dataset file: {dataset_path}" + assert os.path.exists(rule_path), f"Missing rule file: {rule_path}" + + command = [ + f"{get_python_executable()}", + "-m", + "core", + "validate", + "-s", + "adamig", + "-v", + "1-3", + "-dp", + dataset_path, + "-lr", + rule_path, + "-ps", + "1", + "-of", + "json", + ] + + before_files = set(os.listdir()) + subprocess.run(command, check=True) + + json_report_path = _latest_json_report(before_files) + try: + with open(json_report_path, encoding="utf-8") as f: + json_report = json.load(f) + + assert { + "Conformance_Details", + "Dataset_Details", + "Issue_Summary", + "Issue_Details", + "Rules_Report", + }.issubset(json_report.keys()) + + issue_details = json_report.get("Issue_Details", []) + rules_report = json_report.get("Rules_Report", []) + assert rules_report, "Rules_Report should contain at least one row" + + # Core count expectation for this rule + assert len(issue_details) == len(expected_missing_trtxxa), ( + f"Expected {len(expected_missing_trtxxa)} issues, got {len(issue_details)}. " + f"Issues: {issue_details}" + ) + + # Status expectation + expected_status = "ISSUE REPORTED" if expected_missing_trtxxa else "SUCCESS" + assert rules_report[0].get("status") == expected_status + + # Optional stronger assertion: + # ensure each expected missing TRTxxA token appears in at least one issue row payload + if expected_missing_trtxxa: + issue_payload = "\n".join(_issue_row_text(row) for row in issue_details) + for token in expected_missing_trtxxa: + assert ( + token in issue_payload + ), f"Expected token {token} not found in issue payload:\n{issue_payload}" + + finally: + if os.path.exists(json_report_path): + os.remove(json_report_path) diff --git a/tests/resources/CoreIssue587/Rule.yml b/tests/resources/CoreIssue587/Rule.yml new file mode 100644 index 000000000..f1a1aeed8 --- /dev/null +++ b/tests/resources/CoreIssue587/Rule.yml @@ -0,0 +1,50 @@ +Authorities: + - Organization: CDISC + Standards: + - Name: ADaMIG + References: + - Citations: + - Cited Guidance: "" + Document: "" + Origin: ADaM Conformance Rules + Rule Identifier: + Id: '41' + Version: "1" + Version: "5.0" + Version: "1.0" +Check: + all: + - name: variable_name + operator: matches_regex + value: ^TRT[0-9]{2}AN$ + - name: $generated_variable_name + operator: is_not_contained_by + value: variable_name +Core: + Id: CORE-000587 + Status: Draft + Version: "1" +Description: If TRTxxAN exists in ADSL, TRTxxA must also exist. +Executability: Fully Executable +Operations: + - id: $generated_variable_name + operator: regex_find_replace + name: variable_name + find: ^TRT([0-9]{2})AN$ + replace: TRT\1A + on_no_match: keep_original + flags: "" +Outcome: + Message: TRTxxAN is present but matching TRTxxA is missing. + Output Variables: + - variable_name + - $generated_variable_name +Rule Type: Variable Metadata Check +Scope: + Classes: + Include: + - "ALL" + Domains: + Include: + - "ADSL" +Sensitivity: Record diff --git a/tests/resources/CoreIssue587/boundary_99_missing/Dataset.json b/tests/resources/CoreIssue587/boundary_99_missing/Dataset.json new file mode 100644 index 000000000..c2f423031 --- /dev/null +++ b/tests/resources/CoreIssue587/boundary_99_missing/Dataset.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "filename": "adsl.xpt", + "label": "Subject-Level Analysis Dataset", + "domain": "ADSL", + "variables": [ + { + "name": "USUBJID", + "label": "Unique Subject Identifier", + "type": "char", + "length": 12 + }, + { + "name": "TRT99AN", + "label": "Actual Treatment for Period 99 (N)", + "type": "num", + "length": 8 + } + ], + "records": { + "USUBJID": ["SUBJ001"], + "TRT99AN": [1] + } + } + ] +} \ No newline at end of file diff --git a/tests/resources/CoreIssue587/boundary_99_paired/Dataset.json b/tests/resources/CoreIssue587/boundary_99_paired/Dataset.json new file mode 100644 index 000000000..96311a935 --- /dev/null +++ b/tests/resources/CoreIssue587/boundary_99_paired/Dataset.json @@ -0,0 +1,34 @@ +{ + "datasets": [ + { + "filename": "adsl.xpt", + "label": "Subject-Level Analysis Dataset", + "domain": "ADSL", + "variables": [ + { + "name": "USUBJID", + "label": "Unique Subject Identifier", + "type": "char", + "length": 12 + }, + { + "name": "TRT99AN", + "label": "Actual Treatment for Period 99 (N)", + "type": "num", + "length": 8 + }, + { + "name": "TRT99A", + "label": "Actual Treatment for Period 99", + "type": "char", + "length": 20 + } + ], + "records": { + "USUBJID": ["SUBJ001"], + "TRT99AN": [1], + "TRT99A": ["Active"] + } + } + ] +} \ No newline at end of file diff --git a/tests/resources/CoreIssue587/mixed_partial/Dataset.json b/tests/resources/CoreIssue587/mixed_partial/Dataset.json new file mode 100644 index 000000000..87f221f3c --- /dev/null +++ b/tests/resources/CoreIssue587/mixed_partial/Dataset.json @@ -0,0 +1,41 @@ +{ + "datasets": [ + { + "filename": "adsl.xpt", + "label": "Subject-Level Analysis Dataset", + "domain": "ADSL", + "variables": [ + { + "name": "USUBJID", + "label": "Unique Subject Identifier", + "type": "char", + "length": 12 + }, + { + "name": "TRT01AN", + "label": "Actual Treatment for Period 01 (N)", + "type": "num", + "length": 8 + }, + { + "name": "TRT02AN", + "label": "Actual Treatment for Period 02 (N)", + "type": "num", + "length": 8 + }, + { + "name": "TRT02A", + "label": "Actual Treatment for Period 02", + "type": "char", + "length": 20 + } + ], + "records": { + "USUBJID": ["SUBJ001"], + "TRT01AN": [1], + "TRT02AN": [2], + "TRT02A": ["Active"] + } + } + ] +} \ No newline at end of file diff --git a/tests/resources/CoreIssue587/multiple_missing/Dataset.json b/tests/resources/CoreIssue587/multiple_missing/Dataset.json new file mode 100644 index 000000000..effe676e7 --- /dev/null +++ b/tests/resources/CoreIssue587/multiple_missing/Dataset.json @@ -0,0 +1,34 @@ +{ + "datasets": [ + { + "filename": "adsl.xpt", + "label": "Subject-Level Analysis Dataset", + "domain": "ADSL", + "variables": [ + { + "name": "USUBJID", + "label": "Unique Subject Identifier", + "type": "char", + "length": 12 + }, + { + "name": "TRT01AN", + "label": "Actual Treatment for Period 01 (N)", + "type": "num", + "length": 8 + }, + { + "name": "TRT02AN", + "label": "Actual Treatment for Period 02 (N)", + "type": "num", + "length": 8 + } + ], + "records": { + "USUBJID": ["SUBJ001"], + "TRT01AN": [1], + "TRT02AN": [2] + } + } + ] +} \ No newline at end of file diff --git a/tests/resources/CoreIssue587/nonmatching_noise/Dataset.json b/tests/resources/CoreIssue587/nonmatching_noise/Dataset.json new file mode 100644 index 000000000..916a93d11 --- /dev/null +++ b/tests/resources/CoreIssue587/nonmatching_noise/Dataset.json @@ -0,0 +1,48 @@ +{ + "datasets": [ + { + "filename": "adsl.xpt", + "label": "Subject-Level Analysis Dataset", + "domain": "ADSL", + "variables": [ + { + "name": "USUBJID", + "label": "Unique Subject Identifier", + "type": "char", + "length": 12 + }, + { + "name": "TRTA", + "label": "Actual Treatment", + "type": "char", + "length": 20 + }, + { + "name": "TRTAA", + "label": "Nonmatching Treatment Variable", + "type": "char", + "length": 20 + }, + { + "name": "TRTXXAN", + "label": "Alphabetic Placeholder Treatment Variable", + "type": "char", + "length": 20 + }, + { + "name": "ARMCD", + "label": "Planned Arm Code", + "type": "char", + "length": 20 + } + ], + "records": { + "USUBJID": ["SUBJ001"], + "TRTA": ["Placebo"], + "TRTAA": ["Placebo"], + "TRTXXAN": ["Placebo"], + "ARMCD": ["PBO"] + } + } + ] +} \ No newline at end of file diff --git a/tests/resources/CoreIssue587/paired_only/Dataset.json b/tests/resources/CoreIssue587/paired_only/Dataset.json new file mode 100644 index 000000000..ecd8d6c4a --- /dev/null +++ b/tests/resources/CoreIssue587/paired_only/Dataset.json @@ -0,0 +1,34 @@ +{ + "datasets": [ + { + "filename": "adsl.xpt", + "label": "Subject-Level Analysis Dataset", + "domain": "ADSL", + "variables": [ + { + "name": "USUBJID", + "label": "Unique Subject Identifier", + "type": "char", + "length": 12 + }, + { + "name": "TRT01AN", + "label": "Actual Treatment for Period 01 (N)", + "type": "num", + "length": 8 + }, + { + "name": "TRT01A", + "label": "Actual Treatment for Period 01", + "type": "char", + "length": 20 + } + ], + "records": { + "USUBJID": ["SUBJ001"], + "TRT01AN": [1], + "TRT01A": ["Placebo"] + } + } + ] +} \ No newline at end of file diff --git a/tests/resources/CoreIssue587/single_missing/Dataset.json b/tests/resources/CoreIssue587/single_missing/Dataset.json new file mode 100644 index 000000000..f90c987f8 --- /dev/null +++ b/tests/resources/CoreIssue587/single_missing/Dataset.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "filename": "adsl.xpt", + "label": "Subject-Level Analysis Dataset", + "domain": "ADSL", + "variables": [ + { + "name": "USUBJID", + "label": "Unique Subject Identifier", + "type": "char", + "length": 12 + }, + { + "name": "TRT01AN", + "label": "Actual Treatment for Period 01 (N)", + "type": "num", + "length": 8 + } + ], + "records": { + "USUBJID": ["SUBJ001"], + "TRT01AN": [1] + } + } + ] +} \ No newline at end of file diff --git a/tests/unit/test_operations/test_regex_find_replace.py b/tests/unit/test_operations/test_regex_find_replace.py new file mode 100644 index 000000000..699bd86d1 --- /dev/null +++ b/tests/unit/test_operations/test_regex_find_replace.py @@ -0,0 +1,169 @@ +from unittest.mock import MagicMock + +import pytest + +from cdisc_rules_engine.models.dataset.dask_dataset import DaskDataset +from cdisc_rules_engine.models.dataset.pandas_dataset import PandasDataset +from cdisc_rules_engine.models.operation_params import OperationParams + +# Update this import if you choose a different class/module name. +from cdisc_rules_engine.operations.regex_find_replace import RegexFindReplace + + +@pytest.mark.parametrize( + "data,find,replace,on_no_match,flags,expected_generated", + [ + ( + {"variable_name": ["TRT01AN", "TRT99AN"]}, + r"^TRT([0-9]{2})AN$", + r"TRT\1A", + "keep_original", + "", + ["TRT01A", "TRT99A"], + ), + ( + {"variable_name": ["TRT1AN", "TRT001AN", "ABCDE"]}, + r"^TRT([0-9]{2})AN$", + r"TRT\1A", + "keep_original", + "", + ["TRT1AN", "TRT001AN", "ABCDE"], + ), + ( + {"variable_name": ["TRT1AN", "TRT02AN", "ABCDE"]}, + r"^TRT([0-9]{2})AN$", + r"TRT\1A", + "set_null", + "", + [None, "TRT02A", None], + ), + ( + {"variable_name": ["TRT1AN", "TRT02AN", "ABCDE"]}, + r"^TRT([0-9]{2})AN$", + r"TRT\1A", + "set_empty", + "", + ["", "TRT02A", ""], + ), + ( + {"variable_name": [None, "TRT03AN", ""]}, + r"^TRT([0-9]{2})AN$", + r"TRT\1A", + "keep_original", + "", + [None, "TRT03A", ""], + ), + ( + {"variable_name": ["trt04an", "TRT04AN"]}, + r"^TRT([0-9]{2})AN$", + r"TRT\1A", + "keep_original", + "i", + ["TRT04A", "TRT04A"], + ), + ], +) +@pytest.mark.parametrize("dataset_type", [PandasDataset, DaskDataset]) +def test_regex_find_replace_matrix( + operation_params: OperationParams, + dataset_type, + data, + find, + replace, + on_no_match, + flags, + expected_generated, +): + eval_dataset = dataset_type.from_dict(data) + + operation_params.operation_name = "regex_find_replace" + operation_params.operation_id = "$generated_variable_name" + operation_params.target = "variable_name" + + # These are expected to be mapped in rule_processor into OperationParams. + # Assign directly here for unit testing the operation class. + operation_params.find = find + operation_params.replace = replace + operation_params.on_no_match = on_no_match + operation_params.flags = flags + + operation = RegexFindReplace( + operation_params, + eval_dataset, + MagicMock(), + MagicMock(), + ) + result = operation.execute() + + assert operation_params.operation_id in result + assert result[operation_params.operation_id].equals( + eval_dataset.convert_to_series(expected_generated) + ) + + +@pytest.mark.parametrize("dataset_type", [PandasDataset, DaskDataset]) +def test_regex_find_replace_no_match_error_matrix( + operation_params: OperationParams, + dataset_type, +): + eval_dataset = dataset_type.from_dict({"variable_name": ["TRT1AN"]}) + + operation_params.operation_name = "regex_find_replace" + operation_params.operation_id = "$generated_variable_name" + operation_params.target = "variable_name" + operation_params.find = r"^TRT([0-9]{2})AN$" + operation_params.replace = r"TRT\1A" + operation_params.on_no_match = "error" + operation_params.flags = "" + + operation = RegexFindReplace( + operation_params, + eval_dataset, + MagicMock(), + MagicMock(), + ) + with pytest.raises(Exception, match="no match|No match|on_no_match"): + operation.execute() + + +@pytest.mark.parametrize( + "overrides,expected_error_match", + [ + ({"operation_id": None}, "id|operation_id"), + ({"target": None}, "name|target"), + ({"find": None}, "find|regex"), + ({"replace": None}, "replace"), + ({"find": r"^TRT([0-9]{2}AN$"}, "regex|pattern"), + ({"on_no_match": "bad_value"}, "on_no_match"), + ({"flags": "z"}, "flags"), + ({"target": "missing_column"}, "missing_column|target"), + ], +) +@pytest.mark.parametrize("dataset_type", [PandasDataset, DaskDataset]) +def test_regex_find_replace_validation_matrix( + operation_params: OperationParams, + dataset_type, + overrides, + expected_error_match, +): + eval_dataset = dataset_type.from_dict({"variable_name": ["TRT01AN"]}) + + operation_params.operation_name = "regex_find_replace" + operation_params.operation_id = "$generated_variable_name" + operation_params.target = "variable_name" + operation_params.find = r"^TRT([0-9]{2})AN$" + operation_params.replace = r"TRT\1A" + operation_params.on_no_match = "keep_original" + operation_params.flags = "" + + for key, value in overrides.items(): + setattr(operation_params, key, value) + + operation = RegexFindReplace( + operation_params, + eval_dataset, + MagicMock(), + MagicMock(), + ) + with pytest.raises(Exception, match=expected_error_match): + operation.execute() From edaea85f4c9c82a5e9a8b83c628c00244fff1d25 Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Thu, 23 Jul 2026 13:53:40 -0400 Subject: [PATCH 02/12] Update merged rule schema files --- resources/schema/rule-merged/CORE-base.json | 142 +- .../schema/rule-merged/CORE-bundled.json | 1155 ++++------------- resources/schema/rule-merged/Operations.json | 323 +---- resources/schema/rule-merged/Operator.json | 487 ++----- .../rule-merged/Organization_CDISC.json | 112 +- .../rule-merged/Organization_Custom.json | 53 +- .../schema/rule-merged/Organization_FDA.json | 38 +- 7 files changed, 460 insertions(+), 1850 deletions(-) diff --git a/resources/schema/rule-merged/CORE-base.json b/resources/schema/rule-merged/CORE-base.json index e49252530..d8bc945f5 100644 --- a/resources/schema/rule-merged/CORE-base.json +++ b/resources/schema/rule-merged/CORE-base.json @@ -9,9 +9,7 @@ "$ref": "#/$defs/CheckItems" } }, - "required": [ - "all" - ], + "required": ["all"], "type": "object" }, { @@ -21,9 +19,7 @@ "$ref": "#/$defs/CheckItems" } }, - "required": [ - "any" - ], + "required": ["any"], "type": "object" }, { @@ -33,9 +29,7 @@ "$ref": "#/$defs/CheckItem" } }, - "required": [ - "not" - ], + "required": ["not"], "type": "object" } ] @@ -134,18 +128,13 @@ "$ref": "#/$defs/Domains" }, "include_split_datasets": { - "enum": [ - true - ] + "enum": [true] } }, "type": "object" }, "JoinType": { - "enum": [ - "inner", - "left" - ], + "enum": ["inner", "left"], "type": "string" }, "LeftRightKeys": { @@ -158,10 +147,7 @@ "$ref": "#/$defs/VariableName" } }, - "required": [ - "Left", - "Right" - ], + "required": ["Left", "Right"], "type": "object" }, "PascalCases": { @@ -246,10 +232,7 @@ "type": "string" } }, - "required": [ - "Document", - "Cited Guidance" - ], + "required": ["Document", "Cited Guidance"], "type": "object" }, "type": "array" @@ -258,14 +241,10 @@ "additionalProperties": false, "anyOf": [ { - "required": [ - "Logical Expression" - ] + "required": ["Logical Expression"] }, { - "required": [ - "Plain Language Expression" - ] + "required": ["Plain Language Expression"] } ], "properties": { @@ -279,25 +258,18 @@ "type": "string" } }, - "required": [ - "Rule" - ], + "required": ["Rule"], "type": "object" }, "Plain Language Expression": { "type": "string" }, "Type": { - "enum": [ - "Failure", - "Success" - ], + "enum": ["Failure", "Success"], "type": "string" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -311,18 +283,11 @@ "type": "string" }, "Relationship": { - "enum": [ - "Predecessor", - "Related", - "Successor" - ], + "enum": ["Predecessor", "Related", "Successor"], "type": "string" } }, - "required": [ - "Id", - "Relationship" - ], + "required": ["Id", "Relationship"], "type": "object" }, "type": "array" @@ -340,9 +305,7 @@ "type": "string" } }, - "required": [ - "Id" - ], + "required": ["Id"], "type": "object" }, "Validator Rule Message": { @@ -352,11 +315,7 @@ "type": "string" } }, - "required": [ - "Origin", - "Rule Identifier", - "Version" - ], + "required": ["Origin", "Rule Identifier", "Version"], "type": "object" }, "minItems": 1, @@ -369,11 +328,7 @@ "type": "string" } }, - "required": [ - "Name", - "References", - "Version" - ], + "required": ["Name", "References", "Version"], "type": "object" }, "minItems": 1, @@ -396,10 +351,7 @@ "$ref": "Organization_Custom.json" } ], - "required": [ - "Organization", - "Standards" - ], + "required": ["Organization", "Standards"], "type": "object" }, "minItems": 1, @@ -439,15 +391,10 @@ "const": "Published" } }, - "required": [ - "Id" - ] + "required": ["Id"] } ], - "required": [ - "Status", - "Version" - ], + "required": ["Status", "Version"], "type": "object" }, "Description": { @@ -493,9 +440,7 @@ "type": "string" } }, - "required": [ - "Name" - ], + "required": ["Name"], "type": "object" }, "minItems": 1, @@ -521,9 +466,7 @@ "type": "array" } }, - "required": [ - "Message" - ], + "required": ["Message"], "type": "object" }, "Rule Type": { @@ -541,9 +484,7 @@ "$ref": "#/$defs/Classes" } }, - "required": [ - "Include" - ], + "required": ["Include"], "type": "object" }, { @@ -553,9 +494,7 @@ "$ref": "#/$defs/Classes" } }, - "required": [ - "Exclude" - ], + "required": ["Exclude"], "type": "object" } ] @@ -569,9 +508,7 @@ "$ref": "#/$defs/DataStructures" } }, - "required": [ - "Include" - ], + "required": ["Include"], "type": "object" }, { @@ -581,9 +518,7 @@ "$ref": "#/$defs/DataStructures" } }, - "required": [ - "Exclude" - ], + "required": ["Exclude"], "type": "object" } ] @@ -623,14 +558,10 @@ }, "anyOf": [ { - "required": [ - "Exclude" - ] + "required": ["Exclude"] }, { - "required": [ - "Include" - ] + "required": ["Include"] } ], "type": "object" @@ -652,20 +583,13 @@ }, "oneOf": [ { - "required": [ - "Classes", - "Domains" - ] + "required": ["Classes", "Domains"] }, { - "required": [ - "Data Structures" - ] + "required": ["Data Structures"] }, { - "required": [ - "Entities" - ] + "required": ["Entities"] } ], "type": "object" @@ -700,9 +624,7 @@ } }, "then": { - "required": [ - "Grouping_Variables" - ] + "required": ["Grouping_Variables"] }, "type": "object" } diff --git a/resources/schema/rule-merged/CORE-bundled.json b/resources/schema/rule-merged/CORE-bundled.json index 6a6fb0991..4f1ac8868 100644 --- a/resources/schema/rule-merged/CORE-bundled.json +++ b/resources/schema/rule-merged/CORE-bundled.json @@ -9,9 +9,7 @@ "$ref": "#/$defs/CheckItems" } }, - "required": [ - "all" - ], + "required": ["all"], "type": "object" }, { @@ -21,9 +19,7 @@ "$ref": "#/$defs/CheckItems" } }, - "required": [ - "any" - ], + "required": ["any"], "type": "object" }, { @@ -33,9 +29,7 @@ "$ref": "#/$defs/CheckItem" } }, - "required": [ - "not" - ], + "required": ["not"], "type": "object" } ] @@ -128,9 +122,7 @@ "$ref": "#/$defs/Domains" }, "include_split_datasets": { - "enum": [ - true - ] + "enum": [true] } }, "type": "object" @@ -167,10 +159,7 @@ ] }, "JoinType": { - "enum": [ - "inner", - "left" - ], + "enum": ["inner", "left"], "type": "string" }, "LeftRightKeys": { @@ -183,10 +172,7 @@ "$ref": "#/$defs/VariableName" } }, - "required": [ - "Left", - "Right" - ], + "required": ["Left", "Right"], "type": "object" }, "MetaVariables.json": { @@ -483,10 +469,7 @@ "markdownDescription": "\nReturns a Series indicating whether a specified codelist is extensible. Used in conjunction with codelist_terms to determine if values outside the codelist are acceptable. From the above example, $extensible will contain a bool if the codelist PKUDUG is extensible in all rows of the column.\n\nIf ct_package_type, version, and codelist_code parameters are provided, it will instead attach a new column containing the extensible value for each combination provided in the source dataset.\n\nFor example, given the current dataset:\n\n```\nid \tcodeSystemVersion \t$codelist_code\n1 \t2024-09-27 \tC201264\n2 \t2024-09-27 \tC201265\n3 \t2023-03-29 \tC127262\n```\n\nand the following operation:\n\n```yaml\n- id: $codelist_extensible\n operator: codelist_extensible\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n```\n\nThis will result in the following dataset:\n\n```\nid \tcodeSystemVersion \t$codelist_code \t$codelist_extensible\n1 \t2024-09-27 \tC201264 \tfalse\n2 \t2024-09-27 \tC201265 \tfalse\n3 \t2023-03-29 \tC127262 \ttrue\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -496,10 +479,7 @@ "markdownDescription": "\nReturns a list of valid codelist/term values. Used for evaluating whether NCI codes, submission values or NCI preferred terms are valid based on controlled terminology. Expects three parameters: `codelists` which is a list of the codelist submission value(s) to retrieve, `level` which is the level of data (either \"codelist\" or \"term\") at which to return data from, and `returntype` which is the type of values to return: \"code\" for NCI Code(s), \"value\" for submission value(s), or \"pref_term\" for NCI preferred term(s).\n\n```yaml\n- Check:\n - all:\n - name: PPSTRESU\n operator: is_not_contained_by\n value: $terms\n - name: $extensible\n operator: equal_to\n value: true\n- Operations:\n - id: $terms\n operator: codelist_terms\n codelists:\n - PKUDUG\n level: term\n returntype: value\n - id: $extensible\n codelist: PKUDUG\n operator: codelist_extensible\n```\n\nIf `ct_package_type`, `version`, and `codelist_code` parameters are provided, it will instead attach a new column containing the term for each combination provided in the source dataset. If a column name is provided as:\n\n- `term_code`, it will find term information using the term codes in the specified column.\n- `term_value`, it will find term information using the term submission values in the specified column.\n- `term_pref_term`, it will find term information using the term preferred terms in the specified column.\n\nOnly one of `term_code`, `term_value` or `term_pref_term` can be provided. The term information returned will depend on the value of the `returntype` parameter, as described above. If `returntype` is not specified, specifying `term_code` will return the term submission value and specifying either `term_value` or `term_pref_term` will return the term code.\n\nFor example, given the current dataset:\n\n```\nid \tcodeSystemVersion $codelist_code code decode\n1 \t2024-09-27 C201264 C201356 After\n2 \t2024-09-27 C201265 C201352 End to End\n3 \t2023-03-29 C127262 C51282 CLINIC\n```\n\nand the following operations:\n\n```yaml\n- id: $found_term_value\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_code: code\n- id: $found_term_pref_term\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_code: code\n returntype: pref_term\n```\n\nThis will result in the following dataset:\n\n```\n id codeSystemVersion $codelist_code code decode $found_term_value $found_term_pref_term\n 1 2024-09-27 C201264 C201356 After After After Timing Type\n 2 2024-09-27 C201265 C201352 End to End End to End End to End\n 3 2023-03-31 C127262 C51282 CLINIC CLINIC Clinic\n```\n\nConversely, if given the same dataset, and the following operations:\n\n```yaml\n- id: $found_term_code1\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_value: decode\n- id: $found_term_code2\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_pref_term: decode\n```\n\nThis will result in the following dataset:\n\n```\n id codeSystemVersion $codelist_code code decode $found_term_code1 $found_term_code2\n 1 2024-09-27 C201264 C201356 After C201356\n 2 2024-09-27 C201265 C201352 End to End C201352 C201352\n 3 2023-03-31 C127262 C51282 CLINIC C51282 C51282\n```\n\nNote that `$found_term_code2` is:\n\n- `null` for the first record because \"After\" does not match any NCI preferred term in the C201264 codelist.\n- populated for the third record because matching is case-insensitive (i.e., \"CLINIC\" matches \"Clinic\").\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -509,11 +489,7 @@ "markdownDescription": "\nReturns a list of valid extensible codelist term's submission values. Used for evaluating whether submission values are valid based on controlled terminology. Expects the parameter codelists which is a list of the codelist submission value(s) to retrieve. If the codelist argument is [\"All\"] will return all extensible terms for the CT in a list.\n\n```yaml\n{\n \"id\": \"$ext_value\",\n \"codelist\": [\"ALL\"],\n \"operator\": \"define_extensible_codelists\",\n}\n```\n" } }, - "required": [ - "id", - "operator", - "codelists" - ], + "required": ["id", "operator", "codelists"], "type": "object" }, { @@ -523,11 +499,7 @@ "markdownDescription": "\nIf a target variable name is specified, returns the specified metadata in the define for the specified target variable.\n\nInput\n\n```yaml\n- operator: define_variable_metadata\n attribute_name: define_variable_label\n name: LBTESTCD\n id: $LBTESTCD_VARIABLE_LABEL\n```\n\nOutput\n\n```\nLaboratory Test Code\n```\n\nIf no target variable name specified, returns a dictionary containing the specified metadata in the define for all variables.\n\nInput\n\n```yaml\n- operator: define_variable_metadata\n attribute_name: define_variable_label\n id: $VARIABLE_LABEL\n```\n\nOutput\n\n```\n{\n \"STUDYID\": \"Study Identifier\",\n \"USUBJID\": \"Unique Subject Identifier\",\n \"LBTESTCD\": \"Laboratory Test Code\",\n \"...\": \"...\"\n}\n```\n" } }, - "required": [ - "id", - "operator", - "attribute_name" - ], + "required": ["id", "operator", "attribute_name"], "type": "object" }, { @@ -537,11 +509,7 @@ "markdownDescription": "\nGet a distinct list of values for the given name.\n\nIf a group list is specified, the distinct value list will be grouped by the variables within group.\nIf a filter object is provided, only values for records that match the filter criteria are included in the distinct values.\nIf `value_is_reference` is set to true, the target column contains the names of other columns, and the operation will check the referenced columns to ensure they exist in the associated dataset before adding them to the distinct list.\n\nIf group is provided, group_aliases may also be provided to assign new grouping variable names so that results grouped by the values in one set of grouping variables can be merged onto a dataset according to the same grouping value(s) stored in different set of grouping variables. When both group and group_aliases are provided, columns are renamed according to corresponding list position (i.e., the 1st column in group is renamed to the 1st column in group_aliases, etc.). If there are more columns listed in group than in group_aliases, only the group columns with corresponding group_aliases columns will be renamed. If there are more columns listed in group_aliases than in group, the extra column names in group_aliases will be ignored. See record_count for an example of the use of group_aliases.\n\n```yaml\nCheck:\n all:\n - name: SSSTRESC\n operator: equal_to\n value: DEAD\n value_is_literal: true\n - name: $ds_dsdecod\n operator: does_not_contain\n value: DEATH\n value_is_literal: true\nOperations:\n - operator: distinct\n domain: DS\n name: DSDECOD\n id: $ds_dsdecod\n group:\n - USUBJID\n filter:\n CAT: \"CATEGORY 1\"\n SCAT: \"SUBCATEGORY A\"\n```\n\n> below, `IDVAR` contains column names, the operation retrieves the value from each column for that row, checks the dataset associated with that column using the CO RDOMAIN. Columns that exist are added to the returns the distinct set.\n\n```yaml\nOperations:\n - domain: CO\n id: $rdomain_variables\n name: IDVAR\n operator: distinct\n value_is_reference: true\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -551,10 +519,7 @@ "markdownDescription": "\nChecks whether the domain is in the set of domains within the provided standard.\n\nInput\n\nTarget Domain: XY\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: domain_is_custom\n id: $domain_is_custom\n```\n\nOutput\n\n```\ntrue\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -564,10 +529,7 @@ "markdownDescription": "\nChecks whether the related domain (for example, the parent domain of a SUPP or RELREC dataset) is not present in the set of standard domains for the provided standard and version. This is useful for determining whether relationships point to non-standard or custom domains.\n\nInput\n\nTarget Domain: SUPPEX\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: related_domain_is_custom\n id: $related_domain_is_custom\n```\n\nOutput\n\n```\ntrue\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -577,10 +539,7 @@ "markdownDescription": "\nReturns the label for the domain the operation is executing on within the provided standard.\n\nInput.\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: domain_label\n id: $domain_label\n```\n\nOutput\n\n```\nLaboratory Test Results\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -590,11 +549,7 @@ "markdownDescription": "\nCalculates the number of days between the DTC and RFSTDTC. The Study Day value is incremented by 1 for each date following RFSTDTC. Dates prior to RFSTDTC are decreased by 1, with the date preceding RFSTDTC designated as Study Day -1 (there is no Study Day 0). All Study Day values are integers. Thus, to calculate Study Day:\n\n- --DY = (date portion of --DTC) - (date portion of RFSTDTC) + 1 if --DTC is on or after RFSTDTC\n- --DY = (date portion of --DTC) - (date portion of RFSTDTC) if --DTC precedes RFSTDTC\n\nThis algorithm should be used across all domains.\n\n```yaml\nCheck:\n all:\n - name: --DY\n operator: non_empty\n - name: --DTC\n operator: is_complete_date\n - name: RFSTDTC\n operator: is_complete_date\n - name: --DY\n operator: not_equal_to\n value: $dy\nOperations:\n - name: --DTC\n operator: dy\n id: $dy\nMatch Datasets:\n - Name: DM\n Keys:\n - USUBJID\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -604,11 +559,7 @@ "markdownDescription": "\nReturns the requested dataset level metadata value for the current dataset. Possible name values are:\n\n- dataset_size\n- dataset_location\n- dataset_name\n- dataset_label\n- domain\n- is_ap\n- ap_suffix\n\nExample\n\nInput:\n\nTarget domain: LB\n\n```yaml\n- name: dataset_label\n operator: extract_metadata\n id: $dataset_label\n```\n\nOutput:\n\n```\nLaboratory Test Results\n```\n\nExample: ap_suffix\n\nExtracts the domain suffix (characters 3-4) from AP-related domains. For example, \"FA\" from \"APFA\" DOMAIN value.\n\nInput:\n\nTarget domain: APFA\n\n```yaml\n- name: ap_suffix\n operator: extract_metadata\n id: $ap_suffix\n```\n\nOutput:\n\n```\nFA\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -618,10 +569,7 @@ "markdownDescription": "\nReturns the expected (\"Core\" = Exp ) variables for the domain in the current standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: expected_variables\n id: $expected_variables\n```\n\nOutput:\n\n```\n[\"LBCAT\", \"LBORRES\", \"LBORRESU\", \"...\"]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -631,13 +579,7 @@ "markdownDescription": "\nFetches controlled terminology attribute values from CT packages based on row-specific CT package and version references. The operation constructs CT package names based on the standard being validated and the values in the `name` and `version` columns (e.g., SDTMIG \u2192 \"sdtmct-{version}\"). When the `name` column contains \"CDISC\" or \"CDISC CT\", it uses the validation run's standard to determine the package prefix and the version found in the cell of the specified column. The operation extracts all codes matching the specified ct_attribute from the package.\n\n**Required Parameters:**\n\n- `ct_attribute`: Attribute to extract - `\"Term CCODE\"`, `\"Codelist CCODE\"`, `\"Term Value\"`, `\"Codelist Value\"`, or `\"Term Preferred Term\"`\n- `name`: Column containing CT reference (e.g., \"TSVCDREF\") - identifies which terminology system is referenced\n- `version`: Column containing CT version (e.g., \"TSVCDVER\")\n\n```yaml\n- id: $VALID_TERM_CODES\n name: TSVCDREF\n operator: get_codelist_attributes\n ct_attribute: Term CCODE\n version: TSVCDVER\n```\n\n**Note:** if using this operator with excel data, you must put the ctpackage versions contained within your data in the library tab for it work properly.\n" } }, - "required": [ - "id", - "operator", - "name", - "ct_attribute", - "version" - ], + "required": ["id", "operator", "name", "ct_attribute", "version"], "type": "object" }, { @@ -647,10 +589,7 @@ "markdownDescription": "\nReturns list of dataset columns in order\n\n```yaml\nCheck:\n all:\n - name: $column_order_from_dataset\n operator: is_not_ordered_by\n value: $column_order_from_library\nOperations:\n - id: $column_order_from_library\n operator: get_column_order_from_library\n - id: $column_order_from_dataset\n operator: get_column_order_from_dataset\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -660,10 +599,7 @@ "markdownDescription": "\nFetches column order for a given domain from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\nOptionally Filters variables based on specified metadata criteria.\n\nRule Type: Variable Metadata Check\n\n```yaml\nCheck:\n all:\n - name: variable_name\n operator: is_not_contained_by\n value: $ig_variables\nOperations:\n - id: $ig_variables\n operator: get_column_order_from_library\n key_name: \"role\" # role, core, etc\n key_value: \"Exp\" # Timing, Req, Exp, Perm, etc\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -673,10 +609,7 @@ "markdownDescription": "\nReturns the list of domains for a given class from the CDISC Library Implementation Guide. This operation retrieves all domains that belong to a specified class (e.g., \"TRIAL DESIGN\", \"FINDINGS\", \"EVENTS\") based on the current standard and version. The operation uses the standard and version from the validation context as well as the optional `domain_class` parameter which is the name of the class to filter by (e.g., \"TRIAL DESIGN\", \"FINDINGS\", \"EVENTS\", \"INTERVENTIONS). NOTE: Class names are case-sensitive and should match the Library metadata format. If no `domain_class` parameter is provided, the operation returns all domains across all classes in the Implementation Guide:\n\n```yaml\n- operator: get_library_class_domains\n id: $trial_design_domains\n domain_class: \"TRIAL DESIGN\"\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -686,10 +619,7 @@ "markdownDescription": "\nFetches column order for a given model class from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\n\nRule Type: Variable Metadata Check\n\n```yaml\nCheck:\n all:\n - name: variable_name\n operator: is_not_contained_by\n value: $model_variables\nOperations:\n - id: $model_variables\n operator: get_model_column_order\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -699,12 +629,7 @@ "markdownDescription": "\nFetches variable level library model properties filtered by the provided key_name and key_value\n\nExample\n\nInput\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: get_model_filtered_variables\n id: $model_filtered_variables\n key_name: \"role\"\n key_value: \"Timing\"\n```\n\nOutput\n\n```\n[\"VISITNUM\", \"VISIT\", \"VISITDY\", \"TAETORD\", \"...\"]\n```\n" } }, - "required": [ - "id", - "operator", - "key_name", - "key_value" - ], + "required": ["id", "operator", "key_name", "key_value"], "type": "object" }, { @@ -714,10 +639,7 @@ "markdownDescription": "\nFetches column order for a given SUPP's parent model class from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\n\n```yaml\nCheck:\n all:\n - operator: is_not_contained_by\n value: $parent_model_variables\nOperations:\n - id: $parent_model_variables\n operator: get_parent_model_column_order\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -727,12 +649,7 @@ "markdownDescription": "\nFilters variables from the dataset based on specified metadata criteria. Returns a list of variable names that exist in the dataset and match the filter criteria.\n\n```yaml\n- operator: get_dataset_filtered_variables\n id: $timing_variables\n key_name: \"role\"\n key_value: \"Timing\"\n```\n" } }, - "required": [ - "id", - "operator", - "key_name", - "key_value" - ], + "required": ["id", "operator", "key_name", "key_value"], "type": "object" }, { @@ -742,11 +659,7 @@ "markdownDescription": "\nGenerates a dataframe where each record in the dataframe is the library ig variable metadata corresponding with the variable label found in the column provided in name. The metadata column names are prefixed with the string provided in `id`.\n\nInput\n\nTarget Dataset: SUPPLB\n\nProduct: sdtmig\n\nVersion: 3-4\n\nDataset:\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QLABEL\": [\"Toxicity\", \"Viscosity\", \"Analysis Method\"]\n}\n```\n\nRule:\n\n```yaml\n- operator: label_referenced_variable_metadata\n id: $qlabel_referenced_variable_metadata\n name: \"QLABEL\"\n```\n\nOutput\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QLABEL\": [\"Toxicity\", \"Viscosity\", \"Analysis Method\"],\n \"$qlabel_referenced_variable_metadata_name\": [\"LBTOX\", null, \"LBANMETH\"],\n \"$qlabel_referenced_variable_metadata_role\": [\n \"Variable Qualifier\",\n null,\n \"Record Qualifier\"\n ],\n \"$qlabel_referenced_variable_metadata_ordinal\": [44, null, 38],\n \"$qlabel_referenced_variable_metadata_core\": [\"Req\", \"Req\", \"Req\"],\n \"$qlabel_referenced_variable_metadata_label\": [\"Toxicity\", null, \"Analysis Method\"]\n}\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -756,11 +669,7 @@ "markdownDescription": "\nAllows the creation of a lookup table to take the values from multiple input columns and map them to values in an output column. The map parameter contains a list of objects. Each dictionary contains column names as properties that match the column names in the source dataset and an output property that will be returned as a result.\n\nIf map has a single object and output is the only property specified on that object, this will function as a direct assignment.\n\nFor example, given the following current dataset:\n\n```\nid \tparent_entity \tparent_rel\n1 \tTiming \trelativeToFrom\n2 \tSomething \trelativeToFrom\n3 \tTiming \ttype\n```\n\nand the following operation:\n\n```yaml\nOperations:\n - id: $codelist_code\n operator: map\n map:\n - parent_entity: Timing\n parent_rel: type\n output: C201264\n - parent_entity: Timing\n parent_rel: relativeToFrom\n output: C201265\n```\n\nThis will result in the following dataset:\n\n```\nid \tparent_entity \tparent_rel \t$codelist_code\n1 \tTiming \trelativeToFrom \tC201265\n2 \tSomething \trelativeToFrom \tNone\n3 \tTiming \ttype \tC201264\n```\n\nThe following operation:\n\n```yaml\nOperations:\n - id: $codelist_code\n operator: map\n map:\n - output: C201264\n```\n\nWill result in the following dataset:\n\n```\nid \tparent_entity \tparent_rel \t$codelist_code\n1 \tTiming \trelativeToFrom \tC201264\n2 \tSomething \trelativeToFrom \tC201264\n3 \tTiming \ttype \tC201264\n```\n" } }, - "required": [ - "id", - "operator", - "map" - ], + "required": ["id", "operator", "map"], "type": "object" }, { @@ -770,11 +679,7 @@ "markdownDescription": "\nIf no group is provided, returns the max value in name. If group is provided, returns the max value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: \"$max_age\"\n operator: \"greater_than\"\n value: \"MAXAGE\"\nOperations:\n - operator: \"max\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$max_age\"\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -784,11 +689,7 @@ "markdownDescription": "\nIf no group is provided, returns the max date value in name. If group is provided, returns the max date value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: USUBJID\n operator: is_contained_by\n value: $ex_usubjid\n - name: RFXENDTC\n operator: not_equal_to\n value: $max_ex_exstdtc\n - name: RFXENDTC\n operator: not_equal_to\n value: $max_ex_exendtc\nOperations:\n - operator: distinct\n domain: EX\n name: USUBJID\n id: $ex_usubjid\n - operator: max_date\n domain: EX\n name: EXSTDTC\n id: $max_ex_exstdtc\n group:\n - USUBJID\n - operator: max_date\n domain: EX\n name: EXENDTC\n id: $max_ex_exendtc\n group:\n - USUBJID\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -798,11 +699,7 @@ "markdownDescription": "\nExample: AAGE > mean(DM.AGE), where AAGE is a fictitious NSV\n\n```yaml\nCheck:\n all:\n - name: \"AAGE\"\n operator: \"greater_than\"\n value: \"$average_age\"\nOperations:\n - operator: \"mean\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$average_age\"\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -812,11 +709,7 @@ "markdownDescription": "\nIf no group is provided, returns the min value in name. If group is provided, returns the min value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: \"$min_age\"\n operator: \"less_than\"\n value: \"MINAGE\"\nOperations:\n - operator: \"min\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$min_age\"\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -826,11 +719,7 @@ "markdownDescription": "\nIf no group is provided, returns the min date value in name. If group is provided, returns the min date value in name, within each unique set of the grouping variables.\n\nExample: RFSTDTC is greater than min AE.AESTDTC for the current USUBJID\n\n```yaml\nCheck:\n all:\n - name: \"RFSTDTC\"\n operator: \"date_greater_than\"\n value: \"$ae_aestdtc\"\nOperations:\n - operator: \"min_date\"\n domain: \"AE\"\n name: \"AESTDTC\"\n id: \"$ae_aestdtc\"\n group:\n - USUBJID\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -840,12 +729,7 @@ "markdownDescription": "\nComputes set difference: elements in `name` that are not in `subtract`. By default a standard [set difference]() semantics (A \u2216 B) is applied. Optional `order_insensitive` property allows to have the element order to be taken into consideration and only those `name` elements are removed which follow the same order as in `subtract` . Preserves order from the first list. Both `name` and `subtract` must reference other operation results (e.g., `$expected_variables`, `$dataset_variables`). When `subtract` is empty or missing, returns all elements from `name`. Can be computed and added to output variables to display missing elements in error results.\n\n```yaml\nOperations:\n - id: $expected_variables\n operator: expected_variables\n - id: $dataset_variables\n operator: get_column_order_from_dataset\n - id: $expected_minus_dataset\n name: $expected_variables\n operator: minus\n subtract: $dataset_variables\n order_insensitive: false\n```\n" } }, - "required": [ - "id", - "operator", - "name", - "subtract" - ], + "required": ["id", "operator", "name", "subtract"], "type": "object" }, { @@ -855,11 +739,7 @@ "markdownDescription": "\nGenerates a dataframe where each record in the dataframe is the library ig variable metadata corresponding with the variable name found in the column provided in name. The metadata column names are prefixed with the string provided in `id`.\n\nInput\n\nTarget Dataset: SUPPLB\n\nProduct: sdtmig\n\nVersion: 3-4\n\nDataset:\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QNAM\": [\"Toxicity\", \"LBVISCOS\", \"Analysis Method\"]\n}\n```\n\nRule:\n\n```yaml\n- operator: name_referenced_variable_metadata\n id: $qnam_referenced_variable_metadata\n name: \"QNAM\"\n```\n\nOutput\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QNAM\": [\"LBTOX\", \"LBVISCOS\", \"LBANMETH\"],\n \"$qnam_referenced_variable_metadata_name\": [\"LBTOX\", null, \"LBANMETH\"],\n \"$qnam_referenced_variable_metadata_role\": [\n \"Variable Qualifier\",\n null,\n \"Record Qualifier\"\n ],\n \"$qnam_referenced_variable_metadata_ordinal\": [44, null, 38],\n \"$qnam_referenced_variable_metadata_core\": [\"Req\", \"Req\", \"Req\"],\n \"$qnam_referenced_variable_metadata_label\": [\"Toxicity\", null, \"Analysis Method\"]\n}\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -869,10 +749,7 @@ "markdownDescription": "\nReturns the permissible variables (\"Core\" = Perm ) for a given domain and standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: permissible_variables\n id: $permissible_variables\n```\n\nOutput:\n\n```\n[\"LBGRPID\", \"LBREFID\", \"LBSPID\", \"...\"]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -882,10 +759,7 @@ "markdownDescription": "\nIf no filter or group is provided, returns the number of records in the dataset. If filter is provided, returns the number of records in the dataset that contain the value(s) in the corresponding column(s) provided in the filter. Filter can have a wildcard `&` that when added to the end of the filter value will look for all instances of that prefix (see 4th example below). If group is provided, returns the number of rows matching each unique set of the grouping variables. These can be static column name(s) or can be derived from other operations like get_dataset_filtered_variables.\n\nIf both filter and group are provided, returns the number of records in the dataset that contain the value(s) in the corresponding column(s) provided in the filter that also match each unique set of the grouping variables.\n\n**Wildcard Filtering:** Filter values ending with % will match any records where the column value starts with the specified prefix. For example, RACE% will match RACE1, RACE2, RACE3, etc. This is useful for matching related variables with numeric or alphabetic suffixes.\n\n**Regex Transformation:** If regex is provided along with group, the regex pattern will be applied to transform grouping column values before grouping. The regex is only applied to columns where the pattern matches the data type. For example, using regex `^\\d{4}-\\d{2}-\\d{2}` on a column containing `2022-01-14T08:00` will extract `2022-01-14` for grouping purposes.\n\nIf group is provided, group_aliases may also be provided to assign new grouping variable names so that results grouped by the values in one set of grouping variables can be merged onto a dataset according to the same grouping value(s) stored in different set of grouping variables. When both group and group_aliases are provided, columns are renamed according to corresponding list position (i.e., the 1st column in group is renamed to the 1st column in group_aliases, etc.). If there are more columns listed in group than in group_aliases, only the group columns with corresponding group_aliases columns will be renamed. If there are more columns listed in group_aliases than in group, the extra column names in group_aliases will be ignored.\n\nExample: return the number of records in a dataset.\n\n```yaml\n- operator: record_count\n id: $records_in_dataset\n```\n\nExample: return the number of records where STUDYID = \"CDISC01\" and FLAGVAR = \"Y\".\n\n```yaml\n- operator: record_count\n id: $flagged_cdisc01_records_in_dataset\n filter:\n STUDYID: \"CDISC01\"\n FLAGVAR: \"Y\"\n```\n\nExample: return the number of records grouped by USUBJID and timing variables, extracting only the date portion from datetime values.\n\n```yaml\n- operator: record_count\n id: $records_per_usubjid_date\n group:\n - USUBJID\n - --TESTCD\n - $TIMING_VARIABLES\n regex: \"^\\d{4}-\\d{2}-\\d{2}\"\n```\n\nExample: return the number of records where QNAM starts with \"RACE\" (matches RACE1, RACE2, RACE3, etc.) per USUBJID.\n\n```yaml\n- operator: record_count\n id: $race_records_in_dataset\n filter:\n QNAM: \"RACE&\"\n group:\n - \"USUBJID\"\n```\n\nExample: return the number of records grouped by USUBJID.\n\n```yaml\n- operator: record_count\n id: $records_per_usubjid\n group:\n - USUBJID\n```\n\nExample: return the number of records grouped by USUBJID where FLAGVAR = \"Y\".\n\n```yaml\n- operator: record_count\n id: $flagged_records_per_usubjid\n group:\n - USUBJID\n filter:\n FLAGVAR: \"Y\"\n```\n\nExample: return the number of records grouped by USUBJID and IDVARVAL where QNAM = \"TEST1\" and IDVAR = \"GROUPID\", renaming the IDVARVAL column to GROUPID for subsequent merging.\n\n```yaml\n- operator: record_count\n id: $test1_records_per_usubjid_groupid\n group:\n - USUBJID\n - IDVARVAL\n filter:\n QNAM: \"TEST1\"\n IDVAR: \"GROUPID\"\n group_aliases:\n - USUBJID\n - GROUPID\n```\n\nExample: Group the StudyIdentifier dataset by parent_id and merge the result back to the context dataset StudyVersion using StudyVersion.id == StudyIdentifier.parent_id\n\n```yaml\nScope:\n Entities:\n Include:\n - StudyVersion\nOperations:\n - domain: StudyIdentifier\n filter:\n parent_entity: \"StudyVersion\"\n parent_rel: \"studyIdentifiers\"\n rel_type: \"definition\"\n studyIdentifierScope.organizationType.code: \"C70793\"\n studyIdentifierScope.organizationType.codeSystem: \"http://www.cdisc.org\"\n group:\n - parent_id\n group_aliases:\n - id\n id: $num_sponsor_ids\n operator: record_count\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -895,10 +769,7 @@ "markdownDescription": "\nReturns the required variables ( \"Core\" = Req ) for a given domain and standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: required_variables\n id: $required_variables\n```\n\nOutput:\n\n```\n[\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"LBSEQ\", \"LBTESTCD\", \"LBTEST\"]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -908,12 +779,7 @@ "markdownDescription": "\nSplits a dataset column by a given delimiter\n\n```yaml\nOperations:\n - name: PPSPEC\n delimiter: ;\n id: $ppspec_value\n operator: split_by\n```\n" } }, - "required": [ - "id", - "operator", - "delimiter", - "name" - ], + "required": ["id", "operator", "delimiter", "name"], "type": "object" }, { @@ -923,10 +789,7 @@ "markdownDescription": "\nReturns a list of the domains in the study\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -936,10 +799,7 @@ "markdownDescription": "\nReturns a list of the submitted dataset filenames in all uppercase\n\nex. if TS.xpt, AE.xpt, EC.xpt, and SUPPEC.xpt are submitted -> [TS, AE, EC, SUPPEC] will be returned\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -949,10 +809,7 @@ "markdownDescription": "\nReturns a list of valid SDTM domain names from the standard metadata. This can be used to compare extracted suffixes from DOMAIN values or dataset names.\n\nInput\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: standard_domains\n id: $valid_domain_names\n```\n\nOutput\n\n```\n[\"AE\", \"CM\", \"DM\", \"FA\", \"LB\", \"QS\", ...]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -962,10 +819,7 @@ "markdownDescription": "\nReturns the valid terminology package dates for a given standard.\n\nGiven a list of terminology packages:\n\n```\n[\n \"sdtmct-2023-10-26\",\n \"sdtmct-2023-12-13\",\n \"adamct-2023-12-13\",\n \"cdashct-2023-05-19\"\n]\n```\n\nand standard: sdtmig\n\nthe operation will return:\n\n```\n[\"2023-10-26\", \"2023-12-13\"]\n```\n\nBy default, the standard is as specified when running validation - as the validation runtime parameter and/or as specified in the rule header - and the list of terminology packages is obtained from the current cache. If required, the default standard may be overridden using the optional ct_package_types parameter. For example, given the same list of terminology packages, the following operation:\n\n```yaml\nOperations:\n - operator: valid_codelist_dates\n id: $valid_dates\n ct_package_types:\n - SDTM\n - CDASH\n```\n\nwill return:\n\n```\n[\"2023-05-19\", \"2023-10-26\", \"2023-12-13\"]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -975,11 +829,7 @@ "markdownDescription": "\nReturns true if the version of an external dictionary provided in the define.xml file matches the version parsed from the dictionary files.\n\nInput:\n\n```yaml\nOperations:\n - operator: valid_define_external_dictionary_version\n id: $is_valid_loinc_version\n external_dictionary_type: loinc\n```\n\nOutput:\n\n```\n[true, true, true, true]\n```\n" } }, - "required": [ - "id", - "operator", - "external_dictionary_type" - ], + "required": ["id", "operator", "external_dictionary_type"], "type": "object" }, { @@ -1037,10 +887,7 @@ "markdownDescription": "\nDetermines whether the values are valid in the following variables:\n\n- --SOCCD (System Organ Class Code)\n- --HLGTCD (High Level Group Term Code)\n- --HLTCD (High Level Term Code)\n- --PTCD (Preferred Term Code)\n- --LLTCD (Lowest Level Term Code)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_codes\n operator: valid_meddra_code_references\n```\n\nOutput:\n\n```\n[true, false, true, true]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -1050,10 +897,7 @@ "markdownDescription": "\nDetermines whether the values are valid in the following variable pairs:\n\n- --SOCCD, --SOC (System Organ Class Code and Term)\n- --HLGTCD, --HLGT (High Level Group Term Code and Term)\n- --HLTCD, --HLT (High Level Term Code and Term)\n- --PTCD, --DECOD (Preferred Term Code and Dictionary-Derived Term)\n- --LLTCD, --LLT (Lowest Level Term Code and Term)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_pairs\n operator: valid_meddra_code_term_pairs\n```\n\nOutput:\n\n```\n[true, true, false, true, true]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -1063,10 +907,7 @@ "markdownDescription": "\nDetermines whether the values are valid in the following variables:\n\n- --SOC (System Organ Class)\n- --HLGT (High Level Group Term)\n- --HLT (High Level Term)\n- --DECOD (Dictionary-Derived Term)\n- --LLT (Lowest Level Term)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_terms\n operator: valid_meddra_term_references\n```\n\nOutput:\n\n```\n[true, true, false, true, true]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -1076,11 +917,7 @@ "markdownDescription": "\nChecks if a reference to whodrug term in name points to the existing code in Atc Text (INA) file.\n\nInput:\n\n```yaml\nOperations:\n - id: $whodrug_refs_valid\n operator: valid_whodrug_references\n```\n\nOutput:\n\n```\n[true, false, true, true]\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -1090,10 +927,7 @@ "markdownDescription": "\nReturns a mapping of variable names to the number of times that variable appears in a domain within the study.\n\nInput\n\n```\n{\n \"AE\": [\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"AETERM\", \"AEENDTC\"],\n \"LB\": [\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"LBTESTCD\", \"LBENDTC\"]\n}\n```\n\nOutput\n\n```\n{\n \"STUDYID\": 2,\n \"DOMAIN\": 2,\n \"USUBJID\": 2,\n \"--TERM\": 1,\n \"--TESTCD\": 1,\n \"--ENDTC\": 2\n}\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -1103,10 +937,7 @@ "markdownDescription": "\nOperation operates only on original submission datasets regardless of rule type. Flags an error if a column exists is in the submission dataset currently being evaluated.\n\nRule Type: Domain Presence Check\n\n```yaml\nCheck:\n all:\n - name: $MIDS_EXISTS\n operator: equal_to\n value: true\n - name: TM\n operator: not_exists\nOperations:\n - id: $MIDS_EXISTS\n name: MIDS\n operator: variable_exists\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -1116,10 +947,7 @@ "markdownDescription": "\nReturns true if a variable is missing from the dataset or if all values within the variable are null or empty string. This operation first checks if the target variable exists in the dataset, and if it does exist, evaluates whether all its values are null or empty.\nThe operation supports two sources via the `source` parameter:\n\n- **`submission`** : checks against the raw submission dataset\n- **`evaluation`** (default): checks against the evaluation dataset built based on the rule type\n\n```yaml" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -1129,10 +957,7 @@ "markdownDescription": "\nReturns the set of variable names from the library for the given standard. This operation extracts all variable names across all domains in the specified standard's library metadata.\n\nInput:\n\nValidation Standard: sdtmig\nValidation Version: 3-4\n\n```yaml\n- operator: variable_names\n id: $all_variables\n```\n\nOutput:\n\n```\n[\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"SUBJID\", \"RFSTDTC\", \"RFENDTC\", \"SITEID\", \"AGE\", \"AGEU\", \"SEX\", \"RACE\", \"ETHNIC\", \"ARMCD\", \"ARM\", \"ACTARMCD\", \"ACTARM\", \"COUNTRY\", \"DMDTC\", \"DMDY\", \"AETERM\", \"AEDECOD\", \"AECAT\", \"AESCAT\", \"AEPRESP\", \"AEBODSYS\", \"AEBDSYCD\", \"AESOC\", \"AESOCCD\", \"AELLT\", \"AELLTCD\", \"AEHLT\", \"AEHLTCD\", \"AEHLGT\", \"AEHLGTCD\", \"AEPTCD\", \"AESTDTC\", \"AEENDTC\", \"AESTDY\", \"AEENDY\", \"AEDUR\", \"AESER\", \"AESEV\", \"AEACN\", \"AEREL\", \"AEOUT\", \"AESCAN\", \"AESCONG\", \"AESDISAB\", \"AESDTH\", \"AESHOSP\", \"AESLIFE\", \"AESOD\", \"AECONTRT\", \"AETOXGR\", \"LBTESTCD\", \"LBTEST\", \"LBCAT\", \"LBSCAT\", \"LBSPEC\", \"LBMETHOD\", \"LBORRES\", \"LBORRESU\", \"LBORNRLO\", \"LBORNRHI\", \"LBSTRESC\", \"LBSTRESN\", \"LBSTRESU\", \"LBSTNRLO\", \"LBSTNRHI\", \"LBNRIND\", \"LBNAM\", \"LBSPEC\", \"LBANTREG\", \"LBFAST\", \"LBDRVFL\", \"LBTOX\", \"LBTOXGR\", \"LBSTDTC\", \"LBENDTC\", \"LBSTDY\", \"LBENDY\", \"LBTPT\", \"LBTPTNUM\", \"LBELTM\", \"LBTPTREF\", \"LBRFTDTC\", \"...\"]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -1142,11 +967,7 @@ "markdownDescription": "\nGiven a variable name, returns a mapping of variable values to the number of times that value appears in the variable within all datasets in the study.\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -1156,10 +977,7 @@ "markdownDescription": "\nDetermines whether the values are valid and in the correct hierarchical structure in the following variables:\n\n- --DECOD\n- --CLAS\n- --CLASCD\n\nInput:\n\n```yaml\nOperations:\n - id: $valid_whodrug_codes\n operator: whodrug_code_hierarchy\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -1169,12 +987,7 @@ "markdownDescription": "\nValidates XHTML fragments in the target column against the specified namespace.\n\n```yaml\nOperations:\n - id: $xhtml_errors\n name: text\n operator: get_xhtml_errors\n namespace: http://www.cdisc.org/ns/usdm/xhtml/v1.0\n```\n\nNote that a local XSD file is required for validation. The file must be stored in the folder indicated by the value of the `LOCAL_XSD_FILE_DIR` default file path and the mapping between the namespace and the local XSD file's `sub-folder/name` must be included in the value of the `LOCAL_XSD_FILE_MAP` default file path.\n" } }, - "required": [ - "id", - "operator", - "name", - "namespace" - ], + "required": ["id", "operator", "name", "namespace"], "type": "object" }, { @@ -1187,12 +1000,7 @@ "type": "string" }, "on_no_match": { - "enum": [ - "keep_original", - "set_null", - "set_empty", - "error" - ], + "enum": ["keep_original", "set_null", "set_empty", "error"], "type": "string" }, "operator": { @@ -1202,13 +1010,7 @@ "type": "string" } }, - "required": [ - "id", - "operator", - "name", - "find", - "replace" - ], + "required": ["id", "operator", "name", "find", "replace"], "type": "object" } ], @@ -1271,13 +1073,7 @@ "type": "string" }, "dictionary_term_type": { - "enum": [ - "LLT", - "PT", - "HLT", - "HLGT", - "SOC" - ] + "enum": ["LLT", "PT", "HLT", "HLGT", "SOC"] }, "domain": { "anyOf": [ @@ -1290,9 +1086,7 @@ ] }, "external_dictionary_type": { - "enum": [ - "meddra" - ] + "enum": ["meddra"] }, "filter": { "type": "object" @@ -1343,10 +1137,7 @@ "type": "string" }, "level": { - "enum": [ - "codelist", - "term" - ], + "enum": ["codelist", "term"], "type": "string" }, "map": { @@ -1356,9 +1147,7 @@ "type": "string" } }, - "required": [ - "output" - ], + "required": ["output"], "type": "object" }, "type": "array" @@ -1370,12 +1159,7 @@ "type": "string" }, "on_no_match": { - "enum": [ - "keep_original", - "set_null", - "set_empty", - "error" - ], + "enum": ["keep_original", "set_null", "set_empty", "error"], "type": "string" }, "operator": { @@ -1391,11 +1175,7 @@ "type": "string" }, "returntype": { - "enum": [ - "code", - "value", - "pref_term" - ], + "enum": ["code", "value", "pref_term"], "type": "string" }, "source": { @@ -1420,10 +1200,7 @@ "type": "string" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, "Operator.json": { @@ -1435,9 +1212,7 @@ "const": "additional_columns_empty" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1446,9 +1221,7 @@ "const": "additional_columns_not_empty" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1458,10 +1231,7 @@ "markdownDescription": "\nWill return True if the value in `value` is contained within the collection/iterable in the target column, or if there's an exact match for non-iterable data.\n\nThe operator checks if every value in a column is a list or set. If yes, it compares row-by-row. If any value is blank or a different type (like a string or number), it compares each value against the entire column instead.\n\nExample:\n\n```yaml\n- name: \"--TOXGR\" # Column containing lists like ['GRADE', 'SEVERITY', 'ONSET']\n operator: \"contains\"\n value: \"GRADE\" # True if 'GRADE' is an element in the list\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1471,9 +1241,7 @@ "markdownDescription": "\nTrue if all values in `value` are contained within the variable `name`.\n\n> All of ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment') in ACTARM\n\n```yaml\n- name: \"ACTARM\"\n operator: \"contains_all\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n\nThe operator also supports lists:\n\n```yaml\n- name: \"$spec_codelist\"\n operator: \"contains_all\"\n value: \"$ppspec_value\"\n```\n\nWhere:\n\n| $spec_codelist | $ppspec_value |\n| :-------------------------- | :----------------: |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\", \"CODE2\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE2\", \"CODE3\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\"] |\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1483,10 +1251,7 @@ "markdownDescription": "\nTrue if the value in `value` is contained within the collection/iterable in the target column, performing case-insensitive comparison.\n\nExample:\n\n```yaml\n- name: \"--TOXGR\" # Column containing lists like ['Grade', 'Severity', 'Onset']\n operator: \"contains_case_insensitive\"\n value: \"grade\" # True if 'Grade'/'GRADE'/'grade' exists in the list\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1496,10 +1261,7 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified.\n\nThe `date_component` parameter accepts: `\"year\"`, `\"month\"`, `\"day\"`, `\"hour\"`, `\"minute\"`, `\"second\"`, `\"microsecond\"`, or `\"auto\"`.\n\nWhen `date_component: \"auto\"` is used, the operator automatically detects the precision of both dates and compares at the common (less precise) level.\n\n```yaml\n- name: \"AESTDTC\"\n operator: \"date_equal_to\"\n value: \"RFSTDTC\"\n date_component: \"auto\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1509,10 +1271,7 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> Year part of BRTHDTC > 2021\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"date_greater_than\"\n date_component: \"year\"\n value: \"2021\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1522,10 +1281,7 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> Year part of BRTHDTC >= 2021\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"date_greater_than_or_equal_to\"\n date_component: \"year\"\n value: \"2021\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1535,10 +1291,7 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> AEENDTC < AESTDTC\n\n```yaml\n- name: \"AEENDTC\"\n operator: \"date_less_than\"\n value: \"AESTDTC\"\n```\n\n> SSDTC < all DS.DSSTDTC when SSSTRESC = \"DEAD\"\n\n```yaml\nCheck:\n all:\n - name: \"SSSTRESC\"\n operator: \"equal_to\"\n value: \"DEAD\"\n - name: \"SSDTC\"\n operator: \"date_less_than\"\n value: \"$max_ds_dsstdtc\"\nOperations:\n - operator: \"max_date\"\n domain: \"DS\"\n name: \"DSSTDTC\"\n id: \"$max_ds_dsstdtc\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1548,10 +1301,7 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> AEENDTC <= AESTDTC\n\n```yaml\n- name: \"AEENDTC\"\n operator: \"date_less_than_or_equal_to\"\n value: \"AESTDTC\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1561,10 +1311,7 @@ "markdownDescription": "\nComplement of `date_equal_to`\n\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1574,10 +1321,7 @@ "markdownDescription": "\nComplement of `contains`. Returns True when the value is NOT contained within the target collection.\n\n```yaml\n- name: \"--TOXGR\"\n operator: \"does_not_contain\"\n value: \"GRADE\" # True if 'GRADE' is NOT an element in the list\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1587,10 +1331,7 @@ "markdownDescription": "\nComplement of `contains_case_insensitive`. Returns True when the value is NOT contained within the target collection (case-insensitive).\n\nExample:\n\n```yaml\n- name: \"--TOXGR\"\n operator: \"does_not_contain_case_insensitive\"\n value: \"grade\" # True if no case variation of 'grade' exists in the list\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1603,11 +1344,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value", - "regex" - ], + "required": ["operator", "value", "regex"], "type": "object" }, { @@ -1617,12 +1354,7 @@ "markdownDescription": "\nComplement of `has_next_corresponding_record`\n" } }, - "required": [ - "operator", - "ordering", - "value", - "within" - ], + "required": ["operator", "ordering", "value", "within"], "type": "object" }, { @@ -1632,9 +1364,7 @@ "markdownDescription": "\nValue presence\n\n> --OCCUR = null\n\n```yaml\n- name: --OCCUR\n operator: empty\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1644,10 +1374,7 @@ "markdownDescription": "\n> SEENDTC is not empty when it is not the last record, grouped by USUBJID, sorted by SESTDTC\n\n```yaml\n- name: SEENDTC\n operator: empty_within_except_last_row\n ordering: SESTDTC\n value: USUBJID\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1657,10 +1384,7 @@ "markdownDescription": "\nSubstring matching\n\n> DOMAIN ending with 'FOOBAR'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"ends_with\"\n value: \"FOOBAR\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1679,10 +1403,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1701,10 +1422,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1717,11 +1435,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value", - "regex" - ], + "required": ["operator", "value", "regex"], "type": "object" }, { @@ -1731,9 +1445,7 @@ "markdownDescription": "\nTrue if the column exists in the current dataframe. (Works for datasets and variables)\n\n> --OCCUR is present in dataset\n\n```yaml\n- name: \"--OCCUR\"\n operator: \"exists\"\n```\n\n> Domain SJ exists\n\n```yaml\nRule Type: Domain Presence Check\nCheck:\n all:\n - name: \"SJ\"\n operator: \"exists\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1743,10 +1455,7 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL > 0\n\n```yaml\n- name: TSVAL\n operator: greater_than\n value: 0\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1756,10 +1465,7 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL >= 0\n\n```yaml\n- name: TSVAL\n operator: greater_than_or_equal_to\n value: 1\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1769,9 +1475,7 @@ "markdownDescription": "\nComplement of `has_same_values`\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1781,9 +1485,7 @@ "markdownDescription": "\nLength comparison\n\n> Check whether variable values has equal length of another variable.\n\n```yaml\n- name: SEENDTC\n operator: has_equal_length\n value: SESTDTC\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1793,12 +1495,7 @@ "markdownDescription": "\nEnsures that a value of a variable `name` in one record is equal to the value of another variable `value` in the next corresponding record. The rows are grouped by `within` and ordered by `ordering`.\n\n> SEENDTC is equal to the SESTDTC of the next record within a USUBJID. Ordered by SESEQ\n\n```yaml\n- name: SEENDTC\n operator: has_next_corresponding_record\n value: SESTDTC\n within: USUBJID\n ordering: SESEQ\n```\n" } }, - "required": [ - "operator", - "ordering", - "value", - "within" - ], + "required": ["operator", "ordering", "value", "within"], "type": "object" }, { @@ -1808,9 +1505,7 @@ "markdownDescription": "\nComplement of `has_equal_length`\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1820,9 +1515,7 @@ "markdownDescription": "\nTrue if all values in `name` are the same\n\n> Condition: MHCAT ^= null\n> Rule: MHCAT ^= the same value for all records\n\n```yaml\nCheck:\n all:\n - name: MHCAT\n operator: non_empty\n - name: MHCAT\n operator: has_same_values\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1832,10 +1525,7 @@ "markdownDescription": "\nDuration ISO-8601 check, returns True if a duration is not in ISO-8601 format. The negative parameter must be specified to indicate if negative durations are either allowed (True) or disallowed (False)\n\n> DURVAR is invalid (negative durations disallowed)\n\n```yaml\n- name: \"DURVAR\"\n operator: \"invalid_duration\"\n negative: False\n```\n" } }, - "required": [ - "operator", - "negative" - ], + "required": ["operator", "negative"], "type": "object" }, { @@ -1845,9 +1535,7 @@ "markdownDescription": "\nThe operator performs date validation against complete and partial dates with uncertainty in the following order:\n\n1. Attempts to parse using [dateutil.parser.isoparse()](https://dateutil.readthedocs.io/en/stable/parser.html)\n2. If parsing fails and the string contains uncertainty indicators (`/`, `--`, `-:`), validates against an extended ISO 8601 dates regex pattern\n3. If parsing succeeds, dates are still validated against the regex pattern.\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"invalid_date\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1857,9 +1545,7 @@ "markdownDescription": "\nDate check\n\n> DM.RFSTDTC = complete date\n\n```yaml\n- name: \"RFSTDTC\"\n operator: \"is_complete_date\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1869,10 +1555,7 @@ "markdownDescription": "\nValue in `name` compared against a list in `value`. The list can have literal values or be a reference to a `$variable`.\n\nThis operator behaves similarly to `contains`. The key distinction: `contains` checks if comparator \u2208 target, while `is_contained_by` checks if target \u2208 comparator.\n\n> ACTARM in ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment')\n\n```yaml\n- name: \"ACTARM\"\n operator: \"is_contained_by\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1882,10 +1565,7 @@ "markdownDescription": "\nValue in `name` case insensitive compared against a list in `value`. The list can have literal values or be a reference to a `$variable`.\n\n> ACTARM in ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment')\n\n```yaml\n- name: \"ACTARM\"\n operator: \"is_contained_by_case_insensitive\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1895,9 +1575,7 @@ "markdownDescription": "\nComplement of `is_complete_date`\n\nDate check\n\n> DM.RFSTDTC ^= complete date\n\n```yaml\n- name: \"RFSTDTC\"\n operator: \"is_incomplete_date\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1907,10 +1585,7 @@ "markdownDescription": "\nComplement of `is_contained_by`\n\n> ARM not in ('Screen Failure', 'Not Assigned')\n\n```yaml\n- name: \"ARM\"\n operator: \"is_not_contained_by\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1920,10 +1595,7 @@ "markdownDescription": "\nComplement of `is_contained_by_case_insensitive`\n\n> ARM not in ('Screen Failure', 'Not Assigned')\n\n```yaml\n- name: \"ARM\"\n operator: \"is_not_contained_by_case_insensitive\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1933,10 +1605,7 @@ "markdownDescription": "\nComplement of `is_ordered_by`\n" } }, - "required": [ - "operator", - "order" - ], + "required": ["operator", "order"], "type": "object" }, { @@ -1945,10 +1614,7 @@ "const": "is_not_ordered_set" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1958,10 +1624,7 @@ "markdownDescription": "\nComplement of `is_unique_relationship`\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1971,9 +1634,7 @@ "markdownDescription": "\nComplement of `is_unique_set`.\n\n> --SEQ is not unique within DOMAIN, USUBJID, and --TESTCD\n\n```yaml\n- name: \"--SEQ\"\n operator: is_not_unique_set\n value:\n - \"DOMAIN\"\n - \"USUBJID\"\n - \"--TESTCD\"\n```\n\n> `name` can be a variable containing a list of columns and `value` does not need to be present\n\n```yaml\nRule Type: Dataset Contents Check against Define XML\nCheck:\n all:\n - name: define_dataset_key_sequence # contains list of dataset key columns\n operator: is_not_unique_set\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1983,10 +1644,7 @@ "markdownDescription": "\nTrue if the dataset rows are ordered by the values within `name`, given the ordering specified by `order`\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_by\n order: asc\n```\n" } }, - "required": [ - "operator", - "order" - ], + "required": ["operator", "order"], "type": "object" }, { @@ -1996,10 +1654,7 @@ "markdownDescription": "\nTrue if the dataset rows are in ascending order of the values within `name`, grouped by the values within `value`. Value can either be a single column or multiple.\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_set\n value: USUBJID\n```\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_set\n value:\n - USUBJID\n - \"--TESTCD\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2009,10 +1664,7 @@ "markdownDescription": "\nRelationship Integrity Check looking for a 1-1 relationship between name and value. Ensures uniqueness of both name and value.\n\n> AETERM and AEDECOD has a 1-to-1 relationship\n\n```yaml\n- name: AETERM\n operator: is_unique_relationship\n value: AEDECOD\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2022,10 +1674,7 @@ "markdownDescription": "\nChecks if a variable maintains consistent values within groups defined by one or more grouping variables. Groups records by specified value(s) and validates that the target variable maintains the same value within each unique combination of grouping variables. When inconsistency is detected within a group, the operator attempts to identify a majority value. If one value appears more frequently than all others, only the minority records (those not matching the majority value) are flagged. If no single majority exists \u2014 i.e., two or more values are tied for the highest frequency \u2014 all records in that group are flagged.\n\nSingle grouping variable - true if the values of BGSTRESU differ within USUBJID:\n\nIf a regex parameter is provided, it is applied to the values of the target variable before the consistency check. The first capture group of the regex is used as the normalized value for comparison. This can be useful when only part of the value should be considered during comparison (for example, comparing only the date portion of a datetime value).\n\n- regex is optional.\n- The pattern must include at least one capture group(or whole regex will be wrapped to capture group).\n- Only the first capture group is used for comparison.\n- If the pattern does not match a value, the original value is used.\n\n```yaml\n- name: \"BGSTRESU\"\n operator: is_inconsistent_across_dataset\n value: \"USUBJID\"\n```\n\nMultiple grouping variables - true if the values of --STRESU differ within each combination of --TESTCD, --CAT, --SCAT, --SPEC, and --METHOD:\n\n```yaml\n- name: \"--STRESU\"\n operator: is_inconsistent_across_dataset\n value:\n - \"--TESTCD\"\n - \"--CAT\"\n - \"--SCAT\"\n - \"--SPEC\"\n - \"--METHOD\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2035,9 +1684,7 @@ "markdownDescription": "\nRelationship Integrity Check\n\n> --SEQ is unique within DOMAIN, USUBJID, and --TESTCD\n\n```yaml\n- name: \"--SEQ\"\n operator: is_unique_set\n value:\n - \"DOMAIN\"\n - \"USUBJID\"\n - \"--TESTCD\"\n```\n\n> `name` can be a variable containing a list of columns and `value` does not need to be present\n\n> The `regex` parameter allows you to extract portions of values using a regex pattern before checking uniqueness.\n\n> Compare date only (YYYY-MM-DD) for uniqueness\n\n```yaml\n- name: \"--REPNUM\"\n operator: is_not_unique_set\n value:\n - \"USUBJID\"\n - \"--TESTCD\"\n - \"$TIMING_VARIABLES\"\n regex: '^\\d{4}-\\d{2}-\\d{2}'\n```\n\n> Compare by first N characters of a string\n\n```yaml\n- name: \"ITEM_ID\"\n operator: is_not_unique_set\n value:\n - \"USUBJID\"\n - \"CATEGORY\"\n regex: \"^.{2}\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -2047,10 +1694,7 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL < 1\n\n```yaml\n- name: TSVAL\n operator: less_than\n value: 1\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2060,10 +1704,7 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL <= 1\n\n```yaml\n- name: TSVAL\n operator: less_than_or_equal_to\n value: 1\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2073,10 +1714,7 @@ "markdownDescription": "\nLength comparison\n\n> SETCD value length > 8\n\n```yaml\n- name: \"SETCD\"\n operator: \"longer_than\"\n value: 8\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2086,10 +1724,7 @@ "markdownDescription": "\nLength comparison\n\n> TSVAL value length >= 201\n\n```yaml\n- name: \"TSVAL\"\n operator: \"longer_than_or_equal_to\"\n value: 201\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2099,10 +1734,7 @@ "markdownDescription": "\nRegular Expression value matching\n\n- Determine if each string starts with a match of a regular expression. Refer to this pandas documentation: https://pandas.pydata.org/docs/reference/api/pandas.Series.str.match.html\n- To \"search\" for a regex within the entire text, prefix the regex with `.*` and do not use anchors `^` , `$`\n- To do a \"fullmatch\" of a regex with the entire text, suffix the regex with an anchor `$` and do not prefix the regex with `.*`\n- For syntax guide, refer to this Python documentation: [Regular Expression HOWTO](https://docs.python.org/3/howto/regex.html).\n- Suggestion for an on-line regular expression logic. tester: https://regex101.com, choose the Python dialect.\n- For regex token visualization, try https://www.debuggex.com.\n\n> --DOSTXT value is non-numeric\n\n```yaml\n- name: --DOSTXT\n operator: matches_regex\n value: ^\\d*\\.?\\d*$\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2112,9 +1744,7 @@ "markdownDescription": "\nComplement of `empty`\n\n> --OCCUR ^= null\n\n```yaml\n- name: --OCCUR\n operator: non_empty\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -2124,10 +1754,7 @@ "markdownDescription": "\nComplement of `empty_within_except_last_row`\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2137,9 +1764,7 @@ "markdownDescription": "\nComplement of `contains_all`\n\n> All of ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment') not in ACTARM\n\n```yaml\n- name: \"ACTARM\"\n operator: \"not_contains_all\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n\nThe operator also supports lists:\n\n```yaml\n- name: \"$spec_codelist\"\n operator: \"not_contains_all\"\n value: \"$ppspec_value\"\n```\n\nWhere:\n\n| $spec_codelist | $ppspec_value |\n| :-------------------------- | :----------------: |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\", \"CODE2\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE2\", \"CODE3\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\"] |\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -2158,10 +1783,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2180,10 +1802,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2193,9 +1812,7 @@ "markdownDescription": "\nComplement of `exists`\n\n> AEOCCUR not present in dataset\n\n```yaml\n- name: \"AEOCCUR\"\n operator: \"not_exists\"\n```\n\n> Domain SJ does not exist\n\n```yaml\nRule Type: Domain Presence Check\nCheck:\n all:\n - name: \"SJ\"\n operator: \"not_exists\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -2205,10 +1822,7 @@ "markdownDescription": "\nComplement of `matches_regex`\n\n> --TESTCD <= 8 chars and contains only letters, numbers, and underscores and can not start with a number\n\n```yaml\n- name: --TESTCD\n operator: not_matches_regex\n value: ^[A-Z_][A-Z0-9_]{0,7}$\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2218,11 +1832,7 @@ "markdownDescription": "\nComplement of `prefix_matches_regex`\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -2232,10 +1842,7 @@ "markdownDescription": "\nComplement of `present_on_multiple_rows_within`\n\n```yaml\n- operator: \"not_present_on_multiple_rows_within\"\n name: \"RELID\"\n value: 4 (optional)\n within: \"USUBJID\"\n```\n" } }, - "required": [ - "operator", - "within" - ], + "required": ["operator", "within"], "type": "object" }, { @@ -2245,11 +1852,7 @@ "markdownDescription": "\nComplement of `suffix_matches_regex`\n\n> QNAM does not end with numbers\n\n```yaml\n- name: \"QNAM\"\n operator: \"not_suffix_matches_regex\"\n suffix: 2\n value: \"\\d\\d\"\n```\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -2259,11 +1862,7 @@ "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match one of the strings in the list in `value`\n\n> Check if a variable's domain identifier exists in the study\n\n```yaml\n- name: variable_name\n operator: prefix_is_contained_by\n prefix: 2\n value: $study_domains\n```\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -2273,11 +1872,7 @@ "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match the string in `value`\n\n```yaml\n- name: dataset_name\n operator: prefix_equal_to\n prefix: 2\n value: DOMAIN\n```\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -2287,11 +1882,7 @@ "markdownDescription": "\nComplement of `prefix_is_contained_by`\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -2301,11 +1892,7 @@ "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match a regular expression in `value`\n\n```yaml\n- name: DOMAIN\n operator: prefix_matches_regex\n prefix: 2\n value: (AP|ap)\n```\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -2315,11 +1902,7 @@ "markdownDescription": "\nComplement of `prefix_equal_to`\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -2329,10 +1912,7 @@ "markdownDescription": "\nTrue if the same value of `name` is present on multiple rows, grouped by `within`. A maximum allowed number of occurrences can be specified in the value attribute. In this instance the value: 4 means that an error will be flagged if the same value appears more than 4 times within a USUBJID. By default the operator will flag any time a value appears more than once.\n\n```yaml\n- operator: \"present_on_multiple_rows_within\"\n name: \"RELID\"\n value: 4 (optional)\n within: \"USUBJID\"\n```\n" } }, - "required": [ - "operator", - "within" - ], + "required": ["operator", "within"], "type": "object" }, { @@ -2342,10 +1922,7 @@ "markdownDescription": "\nWill raise an issue if at least one of the values in `name` is the same as one of the values in `value`. See [shares_no_elements_with](#shares_no_elements_with).\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2355,10 +1932,7 @@ "markdownDescription": "\nWill raise an issue if exactly one of the values in `name` is the same as one of the values in `value`. See [shares_no_elements_with](#shares_no_elements_with).\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2368,10 +1942,7 @@ "markdownDescription": "\nWill raise an issue if the values in `name` do not share any of the values in `value`\n\n> Check if $dataset_variables shares no elements with $timing_variables\n\n```yaml\nRule Type: Dataset Metadata Check # One record per dataset\nCheck:\n - all:\n name: $dataset_variables\n operator: shares_no_elements_with\n value: $timing_variables\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2381,10 +1952,7 @@ "markdownDescription": "\nLength comparison\n\n> SETCD value length < 9\n\n```yaml\n- name: \"SETCD\"\n operator: \"shorter_than\"\n value: 9\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2394,10 +1962,7 @@ "markdownDescription": "\nLength comparison\n\n> TSVAL value length <= 200\n\n```yaml\n- name: \"TSVAL\"\n operator: \"shorter_than_or_equal_to\"\n value: 201\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2407,9 +1972,7 @@ "markdownDescription": "\nSplits a string by a separator and checks if both parts have equal length. Generic operator for validating paired data formats where both parts must have the same level of detail or precision.\n\nParameters:\n\n- `separator`: The delimiter to split on (default: \"/\")\n\n> Check that string parts separated by a delimiter have equal length\n\n```yaml\n- name: --DTC\n operator: split_parts_have_equal_length\n separator: \"/\"\n```\n\nUse cases:\n\n- **Date/time intervals**: `2003-12-15T10:00/2003-12-15T10:30` \u2192 True (both 16 characters)\n- **Date ranges**: `2003-12-01/2003-12-10` \u2192 True (both 10 characters)\n- **Version ranges**: `1.2.3/2.0.0` \u2192 True (both 5 characters)\n- **Product codes**: `ABC-123/XYZ-789` \u2192 True (both 7 characters)\n\nInvalid example:\n\n- `2003-12-15T10:00/2003-12-15T10:30:15` \u2192 False (16 vs 19 characters - different precision)\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -2419,9 +1982,7 @@ "markdownDescription": "\nComplement of `split_parts_have_equal_length`. Returns True when parts have unequal lengths (indicates a violation).\n\n```yaml\n- name: --DTC\n operator: split_parts_have_unequal_length\n separator: \"/\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -2431,10 +1992,7 @@ "markdownDescription": "\nSubstring matching\n\n> DOMAIN beginning with 'AP'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"starts_with\"\n value: \"AP\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2444,11 +2002,7 @@ "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match the string in `value`\n\n```yaml\n- name: dataset_name\n operator: suffix_equal_to\n prefix: 2\n value: DOMAIN\n```\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -2458,11 +2012,7 @@ "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match one of the strings in the list in `value`\n\n> Check if a supp's parent domain exists in the study\n\n```yaml\n- name: dataset_name\n operator: suffix_is_contained_by\n prefix: 2\n value: $study_domains\n```\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -2472,11 +2022,7 @@ "markdownDescription": "\nComplement of `suffix_is_contained_by`\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -2486,11 +2032,7 @@ "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match a regular expression in `value`\n\n> QNAM ends with numbers\n\n```yaml\n- name: \"QNAM\"\n operator: \"suffix_matches_regex\"\n suffix: 2\n value: \"\\d\\d\"\n```\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -2500,11 +2042,7 @@ "markdownDescription": "\nComplement of `suffix_equal_to`\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -2514,11 +2052,7 @@ "markdownDescription": "\nComplement of `target_is_sorted_by`\n" } }, - "required": [ - "operator", - "value", - "within" - ], + "required": ["operator", "value", "within"], "type": "object" }, { @@ -2528,11 +2062,7 @@ "markdownDescription": "\nTrue if the values in name are ordered according to the values specified by value\nin ascending/descending order, grouped by the values in within. Each value entry\nrequires a variable name, a sort_order of asc or desc, and an optional\nnull_position of first or last (defaults to last) which controls where null/empty\ncomparator values are placed in the expected ordering. Within accepts either a\nsingle column or an ordered list of columns. Columns can be either number or Char\nDates in ISO8601 YYYY-MM-DD format. Date value(s) with different precisions that\noverlap (e.g. 2005-10, 2005-10-3 and 2005-10-08) are all flagged as not sorted as\ntheir order cannot be inferred.\n\nOptionally supports a `regex` parameter that extracts a portion of the target\nvalue for sorting. The regex must contain at least one capturing group. The first\ncaptured group is extracted and converted to numeric if possible, allowing proper\nsorting of sequence numbers (e.g., \"MIDS1\", \"MIDS2\", ..., \"MIDS10\" with regex\n`.*?(\\\\d+)$`). This is particularly useful for variables that end with sequence\nnumbers that may or may not be zero-padded.\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n within:\n - USUBJID\n - MIDSTYPE\n operator: target_is_sorted_by\n value:\n - name: --STDTC\n sort_order: asc\n null_position: last\n```\n\nExample with regex for extracting sequence numbers:\n\n```yaml\nCheck:\n all:\n - name: MIDS\n operator: target_is_sorted_by\n regex: \".*?(\\\\d+)$\" # Extract trailing digits, convert to numeric\n value:\n - name: SMSTDTC\n sort_order: asc\n within:\n - USUBJID\n - MIDSTYPE\n```\n" } }, - "required": [ - "operator", - "value", - "within" - ], + "required": ["operator", "value", "within"], "type": "object" }, { @@ -2542,10 +2072,7 @@ "markdownDescription": "\nComplement of `value_has_multiple_references`\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2555,10 +2082,7 @@ "markdownDescription": "\nTrue if the value in `name` has more than one count in the dictionary defined in `value`\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2568,10 +2092,7 @@ "markdownDescription": "\nChecks for inconsistencies in enumerated columns of a DataFrame. Starting with the smallest/largest enumeration of the given variable, returns True if VARIABLE(N+1) is populated but VARIABLE(N) is not populated. Repeats for all variables belonging to the enumeration. Note that the initial variable will not have an index (VARIABLE) and the next enumerated variable has index 1 (VARIABLE1).\n\nex: Check if there are inconsistencies in the TSVAL columns (TSVAL, TSVAL1, TSVAL2, etc.)\n\n```yaml\nCheck:\n all:\n - name: \"TSVAL\"\n operator: \"inconsistent_enumerated_columns\"\n```\n" } }, - "required": [ - "operator", - "name" - ], + "required": ["operator", "name"], "type": "object" }, { @@ -2581,10 +2102,7 @@ "markdownDescription": "\nChecks if elements in the target list appear in the same relative order in the comparator list.\n\n> Check if dataset column order is a correctly ordered subset of library column order\n\n```yaml\n- name: $column_order_from_dataset\n operator: is_ordered_subset_of\n value: $column_order_from_library\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2594,10 +2112,7 @@ "markdownDescription": "\nComplement of `is_ordered_subset_of`\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -2607,9 +2122,7 @@ "markdownDescription": "\nValidates that variable labels follow proper title case formatting rules using the titlecase PyPi library. Title case capitalizes the first word and all major words, while keeping articles (a, an, the), conjunctions (and, but, or), and prepositions (in, of, for) in lowercase unless they are the first word. \nNOTE: The titlecase library may produce false positives or false negatives in syntactic edge cases (e.g. hyphenated words, slash-separated terms, uncommon prepositions).\n\n> Check that AELABEL values are in proper title case\n\n```yaml\n- name: AELABEL\n operator: is_title_case\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -2619,32 +2132,21 @@ "markdownDescription": "\nComplement of `is_title_case`. Returns True when values are NOT in proper title case.\n\n> Flag AELABEL values that violate title case rules\n\n```yaml\n- name: AELABEL\n operator: is_not_title_case\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" } ], "properties": { "codelistcheck": { - "enum": [ - "code", - "value" - ], + "enum": ["code", "value"], "type": "string" }, "codelistlevel": { - "enum": [ - "term", - "codelist" - ], + "enum": ["term", "codelist"], "type": "string" }, "comparator": { - "type": [ - "number", - "string" - ] + "type": ["number", "string"] }, "context": { "type": "string" @@ -2685,10 +2187,7 @@ "type": "string" }, "order": { - "enum": [ - "asc", - "dsc" - ], + "enum": ["asc", "dsc"], "type": "string" }, "ordering": { @@ -2715,25 +2214,17 @@ "value": { "oneOf": [ { - "type": [ - "boolean", - "number", - "string" - ] + "type": ["boolean", "number", "string"] }, { "items": { - "type": [ - "number" - ] + "type": ["number"] }, "type": "array" }, { "items": { - "type": [ - "string" - ] + "type": ["string"] }, "type": "array" }, @@ -2744,10 +2235,7 @@ "$ref": "#/$defs/Operator.json/properties/name" }, "null_position": { - "enum": [ - "first", - "last" - ], + "enum": ["first", "last"], "type": "string" }, "order": { @@ -2781,9 +2269,7 @@ ] } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, "Organization_CDISC.json": { @@ -2808,9 +2294,7 @@ "const": "Failure" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -2826,9 +2310,7 @@ "type": "object" }, "Version": { - "enum": [ - "5.0" - ] + "enum": ["5.0"] } }, "type": "object" @@ -2837,12 +2319,7 @@ "type": "array" }, "Version": { - "enum": [ - "1.0", - "1.1", - "1.2", - "1.3" - ] + "enum": ["1.0", "1.1", "1.2", "1.3"] } }, "type": "object" @@ -2861,9 +2338,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -2876,19 +2351,13 @@ "type": "string" }, "Version": { - "enum": [ - "1", - "2", - "3" - ] + "enum": ["1", "2", "3"] } }, "type": "object" }, "Version": { - "enum": [ - "2.0" - ] + "enum": ["2.0"] } }, "type": "object" @@ -2897,11 +2366,7 @@ "type": "array" }, "Version": { - "enum": [ - "3.2", - "3.3", - "3.4" - ] + "enum": ["3.2", "3.3", "3.4"] } }, "type": "object" @@ -2920,9 +2385,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -2938,9 +2401,7 @@ "type": "object" }, "Version": { - "enum": [ - "5.0" - ] + "enum": ["5.0"] } }, "type": "object" @@ -2949,11 +2410,7 @@ "type": "array" }, "Version": { - "enum": [ - "3.0", - "3.1", - "3.1.1" - ] + "enum": ["3.0", "3.1", "3.1.1"] } }, "type": "object" @@ -2972,9 +2429,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -2990,9 +2445,7 @@ "type": "object" }, "Version": { - "enum": [ - "5.0" - ] + "enum": ["5.0"] } }, "type": "object" @@ -3001,10 +2454,7 @@ "type": "array" }, "Version": { - "enum": [ - "1.1", - "1.2" - ] + "enum": ["1.1", "1.2"] } }, "type": "object" @@ -3023,9 +2473,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -3041,9 +2489,7 @@ "type": "object" }, "Version": { - "enum": [ - "5.0" - ] + "enum": ["5.0"] } }, "type": "object" @@ -3052,9 +2498,7 @@ "type": "array" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, "type": "object" @@ -3073,9 +2517,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -3091,9 +2533,7 @@ "type": "object" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, "type": "object" @@ -3102,24 +2542,13 @@ "type": "array" }, "Substandard": { - "enum": [ - "SDTM", - "SEND", - "ADaM", - "CDASH" - ] + "enum": ["SDTM", "SEND", "ADaM", "CDASH"] }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, - "required": [ - "Name", - "Version", - "Substandard" - ], + "required": ["Name", "Version", "Substandard"], "type": "object" }, { @@ -3140,17 +2569,13 @@ "type": "string" }, "Version": { - "enum": [ - "1" - ] + "enum": ["1"] } }, "type": "object" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, "type": "object" @@ -3159,10 +2584,7 @@ "type": "array" }, "Version": { - "enum": [ - "3.0", - "4.0" - ] + "enum": ["3.0", "4.0"] } }, "type": "object" @@ -3197,10 +2619,7 @@ }, "OutputType": { "description": "Output type of the rule validation result", - "enum": [ - "Check", - "Listing" - ], + "enum": ["Check", "Listing"], "type": "string" }, "Purpose": { @@ -3241,10 +2660,7 @@ "Organization": { "description": "Name of your custom organization", "not": { - "enum": [ - "CDISC", - "FDA" - ] + "enum": ["CDISC", "FDA"] }, "type": "string" }, @@ -3261,14 +2677,10 @@ "Criteria": { "anyOf": [ { - "required": [ - "Logical Expression" - ] + "required": ["Logical Expression"] }, { - "required": [ - "Plain Language Expression" - ] + "required": ["Plain Language Expression"] } ], "properties": { @@ -3281,25 +2693,18 @@ "type": "string" } }, - "required": [ - "Rule" - ], + "required": ["Rule"], "type": "object" }, "Plain Language Expression": { "type": "string" }, "Type": { - "enum": [ - "Failure", - "Success" - ], + "enum": ["Failure", "Success"], "type": "string" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -3317,9 +2722,7 @@ "type": "string" } }, - "required": [ - "Id" - ], + "required": ["Id"], "type": "object" }, "Version": { @@ -3327,11 +2730,7 @@ "type": "string" } }, - "required": [ - "Origin", - "Rule Identifier", - "Version" - ], + "required": ["Origin", "Rule Identifier", "Version"], "type": "object" }, "minItems": 1, @@ -3346,22 +2745,14 @@ "type": "string" } }, - "required": [ - "Name", - "References", - "Version" - ], + "required": ["Name", "References", "Version"], "type": "object" }, "minItems": 1, "type": "array" } }, - "required": [ - "Organization", - "Standards", - "Category" - ], + "required": ["Organization", "Standards", "Category"], "title": "Custom Organization Schema", "type": "object" }, @@ -3379,11 +2770,7 @@ "const": "SDTMIG" }, "Version": { - "enum": [ - "3.2", - "3.3", - "3.4" - ] + "enum": ["3.2", "3.3", "3.4"] } }, "type": "object" @@ -3394,11 +2781,7 @@ "const": "SENDIG" }, "Version": { - "enum": [ - "3.0", - "3.1", - "3.1.1" - ] + "enum": ["3.0", "3.1", "3.1.1"] } }, "type": "object" @@ -3409,9 +2792,7 @@ "const": "SENDIG-AR" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, "type": "object" @@ -3422,10 +2803,7 @@ "const": "SENDIG-DART" }, "Version": { - "enum": [ - "1.1", - "1.2" - ] + "enum": ["1.1", "1.2"] } }, "type": "object" @@ -3436,9 +2814,7 @@ "const": "SENDIG-GENETOX" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, "type": "object" @@ -3478,10 +2854,7 @@ } } ], - "required": [ - "Document", - "Cited Guidance" - ], + "required": ["Document", "Cited Guidance"], "type": "object" }, "type": "array" @@ -3492,9 +2865,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -3510,9 +2881,7 @@ "type": "object" }, "Version": { - "enum": [ - "1.5" - ] + "enum": ["1.5"] } }, "type": "object" @@ -3766,10 +3135,7 @@ "type": "string" } }, - "required": [ - "Document", - "Cited Guidance" - ], + "required": ["Document", "Cited Guidance"], "type": "object" }, "type": "array" @@ -3778,14 +3144,10 @@ "additionalProperties": false, "anyOf": [ { - "required": [ - "Logical Expression" - ] + "required": ["Logical Expression"] }, { - "required": [ - "Plain Language Expression" - ] + "required": ["Plain Language Expression"] } ], "properties": { @@ -3799,25 +3161,18 @@ "type": "string" } }, - "required": [ - "Rule" - ], + "required": ["Rule"], "type": "object" }, "Plain Language Expression": { "type": "string" }, "Type": { - "enum": [ - "Failure", - "Success" - ], + "enum": ["Failure", "Success"], "type": "string" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -3831,18 +3186,11 @@ "type": "string" }, "Relationship": { - "enum": [ - "Predecessor", - "Related", - "Successor" - ], + "enum": ["Predecessor", "Related", "Successor"], "type": "string" } }, - "required": [ - "Id", - "Relationship" - ], + "required": ["Id", "Relationship"], "type": "object" }, "type": "array" @@ -3860,9 +3208,7 @@ "type": "string" } }, - "required": [ - "Id" - ], + "required": ["Id"], "type": "object" }, "Validator Rule Message": { @@ -3872,11 +3218,7 @@ "type": "string" } }, - "required": [ - "Origin", - "Rule Identifier", - "Version" - ], + "required": ["Origin", "Rule Identifier", "Version"], "type": "object" }, "minItems": 1, @@ -3889,21 +3231,14 @@ "type": "string" } }, - "required": [ - "Name", - "References", - "Version" - ], + "required": ["Name", "References", "Version"], "type": "object" }, "minItems": 1, "type": "array" } }, - "required": [ - "Organization", - "Standards" - ], + "required": ["Organization", "Standards"], "type": "object" }, "minItems": 1, @@ -3938,9 +3273,7 @@ "const": "Published" } }, - "required": [ - "Id" - ] + "required": ["Id"] } ], "properties": { @@ -3948,10 +3281,7 @@ "const": "1" } }, - "required": [ - "Status", - "Version" - ], + "required": ["Status", "Version"], "type": "object" }, "Description": { @@ -4004,9 +3334,7 @@ "type": "string" } }, - "required": [ - "Name" - ], + "required": ["Name"], "type": "object" }, "minItems": 1, @@ -4032,9 +3360,7 @@ "type": "array" } }, - "required": [ - "Message" - ], + "required": ["Message"], "type": "object" }, "Rule Type": { @@ -4044,20 +3370,13 @@ "additionalProperties": false, "oneOf": [ { - "required": [ - "Classes", - "Domains" - ] + "required": ["Classes", "Domains"] }, { - "required": [ - "Data Structures" - ] + "required": ["Data Structures"] }, { - "required": [ - "Entities" - ] + "required": ["Entities"] } ], "properties": { @@ -4070,9 +3389,7 @@ "$ref": "#/$defs/Classes" } }, - "required": [ - "Include" - ], + "required": ["Include"], "type": "object" }, { @@ -4082,9 +3399,7 @@ "$ref": "#/$defs/Classes" } }, - "required": [ - "Exclude" - ], + "required": ["Exclude"], "type": "object" } ] @@ -4098,9 +3413,7 @@ "$ref": "#/$defs/DataStructures" } }, - "required": [ - "Include" - ], + "required": ["Include"], "type": "object" }, { @@ -4110,9 +3423,7 @@ "$ref": "#/$defs/DataStructures" } }, - "required": [ - "Exclude" - ], + "required": ["Exclude"], "type": "object" } ] @@ -4141,14 +3452,10 @@ "additionalProperties": false, "anyOf": [ { - "required": [ - "Exclude" - ] + "required": ["Exclude"] }, { - "required": [ - "Include" - ] + "required": ["Include"] } ], "properties": { @@ -4197,9 +3504,7 @@ "Sensitivity" ], "then": { - "required": [ - "Grouping_Variables" - ] + "required": ["Grouping_Variables"] }, "type": "object" } diff --git a/resources/schema/rule-merged/Operations.json b/resources/schema/rule-merged/Operations.json index ce0152e9b..bc39d94ee 100644 --- a/resources/schema/rule-merged/Operations.json +++ b/resources/schema/rule-merged/Operations.json @@ -10,10 +10,7 @@ "markdownDescription": "\nReturns a Series indicating whether a specified codelist is extensible. Used in conjunction with codelist_terms to determine if values outside the codelist are acceptable. From the above example, $extensible will contain a bool if the codelist PKUDUG is extensible in all rows of the column.\n\nIf ct_package_type, version, and codelist_code parameters are provided, it will instead attach a new column containing the extensible value for each combination provided in the source dataset.\n\nFor example, given the current dataset:\n\n```\nid \tcodeSystemVersion \t$codelist_code\n1 \t2024-09-27 \tC201264\n2 \t2024-09-27 \tC201265\n3 \t2023-03-29 \tC127262\n```\n\nand the following operation:\n\n```yaml\n- id: $codelist_extensible\n operator: codelist_extensible\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n```\n\nThis will result in the following dataset:\n\n```\nid \tcodeSystemVersion \t$codelist_code \t$codelist_extensible\n1 \t2024-09-27 \tC201264 \tfalse\n2 \t2024-09-27 \tC201265 \tfalse\n3 \t2023-03-29 \tC127262 \ttrue\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -23,10 +20,7 @@ "markdownDescription": "\nReturns a list of valid codelist/term values. Used for evaluating whether NCI codes, submission values or NCI preferred terms are valid based on controlled terminology. Expects three parameters: `codelists` which is a list of the codelist submission value(s) to retrieve, `level` which is the level of data (either \"codelist\" or \"term\") at which to return data from, and `returntype` which is the type of values to return: \"code\" for NCI Code(s), \"value\" for submission value(s), or \"pref_term\" for NCI preferred term(s).\n\n```yaml\n- Check:\n - all:\n - name: PPSTRESU\n operator: is_not_contained_by\n value: $terms\n - name: $extensible\n operator: equal_to\n value: true\n- Operations:\n - id: $terms\n operator: codelist_terms\n codelists:\n - PKUDUG\n level: term\n returntype: value\n - id: $extensible\n codelist: PKUDUG\n operator: codelist_extensible\n```\n\nIf `ct_package_type`, `version`, and `codelist_code` parameters are provided, it will instead attach a new column containing the term for each combination provided in the source dataset. If a column name is provided as:\n\n- `term_code`, it will find term information using the term codes in the specified column.\n- `term_value`, it will find term information using the term submission values in the specified column.\n- `term_pref_term`, it will find term information using the term preferred terms in the specified column.\n\nOnly one of `term_code`, `term_value` or `term_pref_term` can be provided. The term information returned will depend on the value of the `returntype` parameter, as described above. If `returntype` is not specified, specifying `term_code` will return the term submission value and specifying either `term_value` or `term_pref_term` will return the term code.\n\nFor example, given the current dataset:\n\n```\nid \tcodeSystemVersion $codelist_code code decode\n1 \t2024-09-27 C201264 C201356 After\n2 \t2024-09-27 C201265 C201352 End to End\n3 \t2023-03-29 C127262 C51282 CLINIC\n```\n\nand the following operations:\n\n```yaml\n- id: $found_term_value\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_code: code\n- id: $found_term_pref_term\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_code: code\n returntype: pref_term\n```\n\nThis will result in the following dataset:\n\n```\n id codeSystemVersion $codelist_code code decode $found_term_value $found_term_pref_term\n 1 2024-09-27 C201264 C201356 After After After Timing Type\n 2 2024-09-27 C201265 C201352 End to End End to End End to End\n 3 2023-03-31 C127262 C51282 CLINIC CLINIC Clinic\n```\n\nConversely, if given the same dataset, and the following operations:\n\n```yaml\n- id: $found_term_code1\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_value: decode\n- id: $found_term_code2\n operator: codelist_terms\n ct_package_type: DDF\n version: codeSystemVersion\n codelist_code: $codelist_code\n term_pref_term: decode\n```\n\nThis will result in the following dataset:\n\n```\n id codeSystemVersion $codelist_code code decode $found_term_code1 $found_term_code2\n 1 2024-09-27 C201264 C201356 After C201356\n 2 2024-09-27 C201265 C201352 End to End C201352 C201352\n 3 2023-03-31 C127262 C51282 CLINIC C51282 C51282\n```\n\nNote that `$found_term_code2` is:\n\n- `null` for the first record because \"After\" does not match any NCI preferred term in the C201264 codelist.\n- populated for the third record because matching is case-insensitive (i.e., \"CLINIC\" matches \"Clinic\").\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -36,11 +30,7 @@ "markdownDescription": "\nReturns a list of valid extensible codelist term's submission values. Used for evaluating whether submission values are valid based on controlled terminology. Expects the parameter codelists which is a list of the codelist submission value(s) to retrieve. If the codelist argument is [\"All\"] will return all extensible terms for the CT in a list.\n\n```yaml\n{\n \"id\": \"$ext_value\",\n \"codelist\": [\"ALL\"],\n \"operator\": \"define_extensible_codelists\",\n}\n```\n" } }, - "required": [ - "id", - "operator", - "codelists" - ], + "required": ["id", "operator", "codelists"], "type": "object" }, { @@ -50,11 +40,7 @@ "markdownDescription": "\nIf a target variable name is specified, returns the specified metadata in the define for the specified target variable.\n\nInput\n\n```yaml\n- operator: define_variable_metadata\n attribute_name: define_variable_label\n name: LBTESTCD\n id: $LBTESTCD_VARIABLE_LABEL\n```\n\nOutput\n\n```\nLaboratory Test Code\n```\n\nIf no target variable name specified, returns a dictionary containing the specified metadata in the define for all variables.\n\nInput\n\n```yaml\n- operator: define_variable_metadata\n attribute_name: define_variable_label\n id: $VARIABLE_LABEL\n```\n\nOutput\n\n```\n{\n \"STUDYID\": \"Study Identifier\",\n \"USUBJID\": \"Unique Subject Identifier\",\n \"LBTESTCD\": \"Laboratory Test Code\",\n \"...\": \"...\"\n}\n```\n" } }, - "required": [ - "id", - "operator", - "attribute_name" - ], + "required": ["id", "operator", "attribute_name"], "type": "object" }, { @@ -64,11 +50,7 @@ "markdownDescription": "\nGet a distinct list of values for the given name.\n\nIf a group list is specified, the distinct value list will be grouped by the variables within group.\nIf a filter object is provided, only values for records that match the filter criteria are included in the distinct values.\nIf `value_is_reference` is set to true, the target column contains the names of other columns, and the operation will check the referenced columns to ensure they exist in the associated dataset before adding them to the distinct list.\n\nIf group is provided, group_aliases may also be provided to assign new grouping variable names so that results grouped by the values in one set of grouping variables can be merged onto a dataset according to the same grouping value(s) stored in different set of grouping variables. When both group and group_aliases are provided, columns are renamed according to corresponding list position (i.e., the 1st column in group is renamed to the 1st column in group_aliases, etc.). If there are more columns listed in group than in group_aliases, only the group columns with corresponding group_aliases columns will be renamed. If there are more columns listed in group_aliases than in group, the extra column names in group_aliases will be ignored. See record_count for an example of the use of group_aliases.\n\n```yaml\nCheck:\n all:\n - name: SSSTRESC\n operator: equal_to\n value: DEAD\n value_is_literal: true\n - name: $ds_dsdecod\n operator: does_not_contain\n value: DEATH\n value_is_literal: true\nOperations:\n - operator: distinct\n domain: DS\n name: DSDECOD\n id: $ds_dsdecod\n group:\n - USUBJID\n filter:\n CAT: \"CATEGORY 1\"\n SCAT: \"SUBCATEGORY A\"\n```\n\n> below, `IDVAR` contains column names, the operation retrieves the value from each column for that row, checks the dataset associated with that column using the CO RDOMAIN. Columns that exist are added to the returns the distinct set.\n\n```yaml\nOperations:\n - domain: CO\n id: $rdomain_variables\n name: IDVAR\n operator: distinct\n value_is_reference: true\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -78,10 +60,7 @@ "markdownDescription": "\nChecks whether the domain is in the set of domains within the provided standard.\n\nInput\n\nTarget Domain: XY\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: domain_is_custom\n id: $domain_is_custom\n```\n\nOutput\n\n```\ntrue\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -91,10 +70,7 @@ "markdownDescription": "\nChecks whether the related domain (for example, the parent domain of a SUPP or RELREC dataset) is not present in the set of standard domains for the provided standard and version. This is useful for determining whether relationships point to non-standard or custom domains.\n\nInput\n\nTarget Domain: SUPPEX\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: related_domain_is_custom\n id: $related_domain_is_custom\n```\n\nOutput\n\n```\ntrue\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -104,10 +80,7 @@ "markdownDescription": "\nReturns the label for the domain the operation is executing on within the provided standard.\n\nInput.\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: domain_label\n id: $domain_label\n```\n\nOutput\n\n```\nLaboratory Test Results\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -117,11 +90,7 @@ "markdownDescription": "\nCalculates the number of days between the DTC and RFSTDTC. The Study Day value is incremented by 1 for each date following RFSTDTC. Dates prior to RFSTDTC are decreased by 1, with the date preceding RFSTDTC designated as Study Day -1 (there is no Study Day 0). All Study Day values are integers. Thus, to calculate Study Day:\n\n- --DY = (date portion of --DTC) - (date portion of RFSTDTC) + 1 if --DTC is on or after RFSTDTC\n- --DY = (date portion of --DTC) - (date portion of RFSTDTC) if --DTC precedes RFSTDTC\n\nThis algorithm should be used across all domains.\n\n```yaml\nCheck:\n all:\n - name: --DY\n operator: non_empty\n - name: --DTC\n operator: is_complete_date\n - name: RFSTDTC\n operator: is_complete_date\n - name: --DY\n operator: not_equal_to\n value: $dy\nOperations:\n - name: --DTC\n operator: dy\n id: $dy\nMatch Datasets:\n - Name: DM\n Keys:\n - USUBJID\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -131,11 +100,7 @@ "markdownDescription": "\nReturns the requested dataset level metadata value for the current dataset. Possible name values are:\n\n- dataset_size\n- dataset_location\n- dataset_name\n- dataset_label\n- domain\n- is_ap\n- ap_suffix\n\nExample\n\nInput:\n\nTarget domain: LB\n\n```yaml\n- name: dataset_label\n operator: extract_metadata\n id: $dataset_label\n```\n\nOutput:\n\n```\nLaboratory Test Results\n```\n\nExample: ap_suffix\n\nExtracts the domain suffix (characters 3-4) from AP-related domains. For example, \"FA\" from \"APFA\" DOMAIN value.\n\nInput:\n\nTarget domain: APFA\n\n```yaml\n- name: ap_suffix\n operator: extract_metadata\n id: $ap_suffix\n```\n\nOutput:\n\n```\nFA\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -145,10 +110,7 @@ "markdownDescription": "\nReturns the expected (\"Core\" = Exp ) variables for the domain in the current standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: expected_variables\n id: $expected_variables\n```\n\nOutput:\n\n```\n[\"LBCAT\", \"LBORRES\", \"LBORRESU\", \"...\"]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -158,13 +120,7 @@ "markdownDescription": "\nFetches controlled terminology attribute values from CT packages based on row-specific CT package and version references. The operation constructs CT package names based on the standard being validated and the values in the `name` and `version` columns (e.g., SDTMIG \u2192 \"sdtmct-{version}\"). When the `name` column contains \"CDISC\" or \"CDISC CT\", it uses the validation run's standard to determine the package prefix and the version found in the cell of the specified column. The operation extracts all codes matching the specified ct_attribute from the package.\n\n**Required Parameters:**\n\n- `ct_attribute`: Attribute to extract - `\"Term CCODE\"`, `\"Codelist CCODE\"`, `\"Term Value\"`, `\"Codelist Value\"`, or `\"Term Preferred Term\"`\n- `name`: Column containing CT reference (e.g., \"TSVCDREF\") - identifies which terminology system is referenced\n- `version`: Column containing CT version (e.g., \"TSVCDVER\")\n\n```yaml\n- id: $VALID_TERM_CODES\n name: TSVCDREF\n operator: get_codelist_attributes\n ct_attribute: Term CCODE\n version: TSVCDVER\n```\n\n**Note:** if using this operator with excel data, you must put the ctpackage versions contained within your data in the library tab for it work properly.\n" } }, - "required": [ - "id", - "operator", - "name", - "ct_attribute", - "version" - ], + "required": ["id", "operator", "name", "ct_attribute", "version"], "type": "object" }, { @@ -174,10 +130,7 @@ "markdownDescription": "\nReturns list of dataset columns in order\n\n```yaml\nCheck:\n all:\n - name: $column_order_from_dataset\n operator: is_not_ordered_by\n value: $column_order_from_library\nOperations:\n - id: $column_order_from_library\n operator: get_column_order_from_library\n - id: $column_order_from_dataset\n operator: get_column_order_from_dataset\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -187,10 +140,7 @@ "markdownDescription": "\nFetches column order for a given domain from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\nOptionally Filters variables based on specified metadata criteria.\n\nRule Type: Variable Metadata Check\n\n```yaml\nCheck:\n all:\n - name: variable_name\n operator: is_not_contained_by\n value: $ig_variables\nOperations:\n - id: $ig_variables\n operator: get_column_order_from_library\n key_name: \"role\" # role, core, etc\n key_value: \"Exp\" # Timing, Req, Exp, Perm, etc\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -200,10 +150,7 @@ "markdownDescription": "\nReturns the list of domains for a given class from the CDISC Library Implementation Guide. This operation retrieves all domains that belong to a specified class (e.g., \"TRIAL DESIGN\", \"FINDINGS\", \"EVENTS\") based on the current standard and version. The operation uses the standard and version from the validation context as well as the optional `domain_class` parameter which is the name of the class to filter by (e.g., \"TRIAL DESIGN\", \"FINDINGS\", \"EVENTS\", \"INTERVENTIONS). NOTE: Class names are case-sensitive and should match the Library metadata format. If no `domain_class` parameter is provided, the operation returns all domains across all classes in the Implementation Guide:\n\n```yaml\n- operator: get_library_class_domains\n id: $trial_design_domains\n domain_class: \"TRIAL DESIGN\"\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -213,10 +160,7 @@ "markdownDescription": "\nFetches column order for a given model class from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\n\nRule Type: Variable Metadata Check\n\n```yaml\nCheck:\n all:\n - name: variable_name\n operator: is_not_contained_by\n value: $model_variables\nOperations:\n - id: $model_variables\n operator: get_model_column_order\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -226,12 +170,7 @@ "markdownDescription": "\nFetches variable level library model properties filtered by the provided key_name and key_value\n\nExample\n\nInput\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: get_model_filtered_variables\n id: $model_filtered_variables\n key_name: \"role\"\n key_value: \"Timing\"\n```\n\nOutput\n\n```\n[\"VISITNUM\", \"VISIT\", \"VISITDY\", \"TAETORD\", \"...\"]\n```\n" } }, - "required": [ - "id", - "operator", - "key_name", - "key_value" - ], + "required": ["id", "operator", "key_name", "key_value"], "type": "object" }, { @@ -241,10 +180,7 @@ "markdownDescription": "\nFetches column order for a given SUPP's parent model class from the CDISC library. The lists with column names are sorted in accordance to \"ordinal\" key of library metadata.\n\n```yaml\nCheck:\n all:\n - operator: is_not_contained_by\n value: $parent_model_variables\nOperations:\n - id: $parent_model_variables\n operator: get_parent_model_column_order\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -254,12 +190,7 @@ "markdownDescription": "\nFilters variables from the dataset based on specified metadata criteria. Returns a list of variable names that exist in the dataset and match the filter criteria.\n\n```yaml\n- operator: get_dataset_filtered_variables\n id: $timing_variables\n key_name: \"role\"\n key_value: \"Timing\"\n```\n" } }, - "required": [ - "id", - "operator", - "key_name", - "key_value" - ], + "required": ["id", "operator", "key_name", "key_value"], "type": "object" }, { @@ -269,11 +200,7 @@ "markdownDescription": "\nGenerates a dataframe where each record in the dataframe is the library ig variable metadata corresponding with the variable label found in the column provided in name. The metadata column names are prefixed with the string provided in `id`.\n\nInput\n\nTarget Dataset: SUPPLB\n\nProduct: sdtmig\n\nVersion: 3-4\n\nDataset:\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QLABEL\": [\"Toxicity\", \"Viscosity\", \"Analysis Method\"]\n}\n```\n\nRule:\n\n```yaml\n- operator: label_referenced_variable_metadata\n id: $qlabel_referenced_variable_metadata\n name: \"QLABEL\"\n```\n\nOutput\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QLABEL\": [\"Toxicity\", \"Viscosity\", \"Analysis Method\"],\n \"$qlabel_referenced_variable_metadata_name\": [\"LBTOX\", null, \"LBANMETH\"],\n \"$qlabel_referenced_variable_metadata_role\": [\n \"Variable Qualifier\",\n null,\n \"Record Qualifier\"\n ],\n \"$qlabel_referenced_variable_metadata_ordinal\": [44, null, 38],\n \"$qlabel_referenced_variable_metadata_core\": [\"Req\", \"Req\", \"Req\"],\n \"$qlabel_referenced_variable_metadata_label\": [\"Toxicity\", null, \"Analysis Method\"]\n}\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -283,11 +210,7 @@ "markdownDescription": "\nAllows the creation of a lookup table to take the values from multiple input columns and map them to values in an output column. The map parameter contains a list of objects. Each dictionary contains column names as properties that match the column names in the source dataset and an output property that will be returned as a result.\n\nIf map has a single object and output is the only property specified on that object, this will function as a direct assignment.\n\nFor example, given the following current dataset:\n\n```\nid \tparent_entity \tparent_rel\n1 \tTiming \trelativeToFrom\n2 \tSomething \trelativeToFrom\n3 \tTiming \ttype\n```\n\nand the following operation:\n\n```yaml\nOperations:\n - id: $codelist_code\n operator: map\n map:\n - parent_entity: Timing\n parent_rel: type\n output: C201264\n - parent_entity: Timing\n parent_rel: relativeToFrom\n output: C201265\n```\n\nThis will result in the following dataset:\n\n```\nid \tparent_entity \tparent_rel \t$codelist_code\n1 \tTiming \trelativeToFrom \tC201265\n2 \tSomething \trelativeToFrom \tNone\n3 \tTiming \ttype \tC201264\n```\n\nThe following operation:\n\n```yaml\nOperations:\n - id: $codelist_code\n operator: map\n map:\n - output: C201264\n```\n\nWill result in the following dataset:\n\n```\nid \tparent_entity \tparent_rel \t$codelist_code\n1 \tTiming \trelativeToFrom \tC201264\n2 \tSomething \trelativeToFrom \tC201264\n3 \tTiming \ttype \tC201264\n```\n" } }, - "required": [ - "id", - "operator", - "map" - ], + "required": ["id", "operator", "map"], "type": "object" }, { @@ -297,11 +220,7 @@ "markdownDescription": "\nIf no group is provided, returns the max value in name. If group is provided, returns the max value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: \"$max_age\"\n operator: \"greater_than\"\n value: \"MAXAGE\"\nOperations:\n - operator: \"max\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$max_age\"\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -311,11 +230,7 @@ "markdownDescription": "\nIf no group is provided, returns the max date value in name. If group is provided, returns the max date value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: USUBJID\n operator: is_contained_by\n value: $ex_usubjid\n - name: RFXENDTC\n operator: not_equal_to\n value: $max_ex_exstdtc\n - name: RFXENDTC\n operator: not_equal_to\n value: $max_ex_exendtc\nOperations:\n - operator: distinct\n domain: EX\n name: USUBJID\n id: $ex_usubjid\n - operator: max_date\n domain: EX\n name: EXSTDTC\n id: $max_ex_exstdtc\n group:\n - USUBJID\n - operator: max_date\n domain: EX\n name: EXENDTC\n id: $max_ex_exendtc\n group:\n - USUBJID\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -325,11 +240,7 @@ "markdownDescription": "\nExample: AAGE > mean(DM.AGE), where AAGE is a fictitious NSV\n\n```yaml\nCheck:\n all:\n - name: \"AAGE\"\n operator: \"greater_than\"\n value: \"$average_age\"\nOperations:\n - operator: \"mean\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$average_age\"\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -339,11 +250,7 @@ "markdownDescription": "\nIf no group is provided, returns the min value in name. If group is provided, returns the min value in name, within each unique set of the grouping variables.\n\n```yaml\nCheck:\n all:\n - name: \"$min_age\"\n operator: \"less_than\"\n value: \"MINAGE\"\nOperations:\n - operator: \"min\"\n domain: \"DM\"\n name: \"AGE\"\n id: \"$min_age\"\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -353,11 +260,7 @@ "markdownDescription": "\nIf no group is provided, returns the min date value in name. If group is provided, returns the min date value in name, within each unique set of the grouping variables.\n\nExample: RFSTDTC is greater than min AE.AESTDTC for the current USUBJID\n\n```yaml\nCheck:\n all:\n - name: \"RFSTDTC\"\n operator: \"date_greater_than\"\n value: \"$ae_aestdtc\"\nOperations:\n - operator: \"min_date\"\n domain: \"AE\"\n name: \"AESTDTC\"\n id: \"$ae_aestdtc\"\n group:\n - USUBJID\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -367,12 +270,7 @@ "markdownDescription": "\nComputes set difference: elements in `name` that are not in `subtract`. By default a standard [set difference]() semantics (A \u2216 B) is applied. Optional `order_insensitive` property allows to have the element order to be taken into consideration and only those `name` elements are removed which follow the same order as in `subtract` . Preserves order from the first list. Both `name` and `subtract` must reference other operation results (e.g., `$expected_variables`, `$dataset_variables`). When `subtract` is empty or missing, returns all elements from `name`. Can be computed and added to output variables to display missing elements in error results.\n\n```yaml\nOperations:\n - id: $expected_variables\n operator: expected_variables\n - id: $dataset_variables\n operator: get_column_order_from_dataset\n - id: $expected_minus_dataset\n name: $expected_variables\n operator: minus\n subtract: $dataset_variables\n order_insensitive: false\n```\n" } }, - "required": [ - "id", - "operator", - "name", - "subtract" - ], + "required": ["id", "operator", "name", "subtract"], "type": "object" }, { @@ -382,11 +280,7 @@ "markdownDescription": "\nGenerates a dataframe where each record in the dataframe is the library ig variable metadata corresponding with the variable name found in the column provided in name. The metadata column names are prefixed with the string provided in `id`.\n\nInput\n\nTarget Dataset: SUPPLB\n\nProduct: sdtmig\n\nVersion: 3-4\n\nDataset:\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QNAM\": [\"Toxicity\", \"LBVISCOS\", \"Analysis Method\"]\n}\n```\n\nRule:\n\n```yaml\n- operator: name_referenced_variable_metadata\n id: $qnam_referenced_variable_metadata\n name: \"QNAM\"\n```\n\nOutput\n\n```\n{\n \"STUDYID\": [\"STUDY1\", \"STUDY1\", \"STUDY1\"],\n \"USUBJID\": [\"SUBJ1\", \"SUBJ1\", \"SUBJ1\"],\n \"QNAM\": [\"LBTOX\", \"LBVISCOS\", \"LBANMETH\"],\n \"$qnam_referenced_variable_metadata_name\": [\"LBTOX\", null, \"LBANMETH\"],\n \"$qnam_referenced_variable_metadata_role\": [\n \"Variable Qualifier\",\n null,\n \"Record Qualifier\"\n ],\n \"$qnam_referenced_variable_metadata_ordinal\": [44, null, 38],\n \"$qnam_referenced_variable_metadata_core\": [\"Req\", \"Req\", \"Req\"],\n \"$qnam_referenced_variable_metadata_label\": [\"Toxicity\", null, \"Analysis Method\"]\n}\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -396,10 +290,7 @@ "markdownDescription": "\nReturns the permissible variables (\"Core\" = Perm ) for a given domain and standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: permissible_variables\n id: $permissible_variables\n```\n\nOutput:\n\n```\n[\"LBGRPID\", \"LBREFID\", \"LBSPID\", \"...\"]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -409,10 +300,7 @@ "markdownDescription": "\nIf no filter or group is provided, returns the number of records in the dataset. If filter is provided, returns the number of records in the dataset that contain the value(s) in the corresponding column(s) provided in the filter. Filter can have a wildcard `&` that when added to the end of the filter value will look for all instances of that prefix (see 4th example below). If group is provided, returns the number of rows matching each unique set of the grouping variables. These can be static column name(s) or can be derived from other operations like get_dataset_filtered_variables.\n\nIf both filter and group are provided, returns the number of records in the dataset that contain the value(s) in the corresponding column(s) provided in the filter that also match each unique set of the grouping variables.\n\n**Wildcard Filtering:** Filter values ending with % will match any records where the column value starts with the specified prefix. For example, RACE% will match RACE1, RACE2, RACE3, etc. This is useful for matching related variables with numeric or alphabetic suffixes.\n\n**Regex Transformation:** If regex is provided along with group, the regex pattern will be applied to transform grouping column values before grouping. The regex is only applied to columns where the pattern matches the data type. For example, using regex `^\\d{4}-\\d{2}-\\d{2}` on a column containing `2022-01-14T08:00` will extract `2022-01-14` for grouping purposes.\n\nIf group is provided, group_aliases may also be provided to assign new grouping variable names so that results grouped by the values in one set of grouping variables can be merged onto a dataset according to the same grouping value(s) stored in different set of grouping variables. When both group and group_aliases are provided, columns are renamed according to corresponding list position (i.e., the 1st column in group is renamed to the 1st column in group_aliases, etc.). If there are more columns listed in group than in group_aliases, only the group columns with corresponding group_aliases columns will be renamed. If there are more columns listed in group_aliases than in group, the extra column names in group_aliases will be ignored.\n\nExample: return the number of records in a dataset.\n\n```yaml\n- operator: record_count\n id: $records_in_dataset\n```\n\nExample: return the number of records where STUDYID = \"CDISC01\" and FLAGVAR = \"Y\".\n\n```yaml\n- operator: record_count\n id: $flagged_cdisc01_records_in_dataset\n filter:\n STUDYID: \"CDISC01\"\n FLAGVAR: \"Y\"\n```\n\nExample: return the number of records grouped by USUBJID and timing variables, extracting only the date portion from datetime values.\n\n```yaml\n- operator: record_count\n id: $records_per_usubjid_date\n group:\n - USUBJID\n - --TESTCD\n - $TIMING_VARIABLES\n regex: \"^\\d{4}-\\d{2}-\\d{2}\"\n```\n\nExample: return the number of records where QNAM starts with \"RACE\" (matches RACE1, RACE2, RACE3, etc.) per USUBJID.\n\n```yaml\n- operator: record_count\n id: $race_records_in_dataset\n filter:\n QNAM: \"RACE&\"\n group:\n - \"USUBJID\"\n```\n\nExample: return the number of records grouped by USUBJID.\n\n```yaml\n- operator: record_count\n id: $records_per_usubjid\n group:\n - USUBJID\n```\n\nExample: return the number of records grouped by USUBJID where FLAGVAR = \"Y\".\n\n```yaml\n- operator: record_count\n id: $flagged_records_per_usubjid\n group:\n - USUBJID\n filter:\n FLAGVAR: \"Y\"\n```\n\nExample: return the number of records grouped by USUBJID and IDVARVAL where QNAM = \"TEST1\" and IDVAR = \"GROUPID\", renaming the IDVARVAL column to GROUPID for subsequent merging.\n\n```yaml\n- operator: record_count\n id: $test1_records_per_usubjid_groupid\n group:\n - USUBJID\n - IDVARVAL\n filter:\n QNAM: \"TEST1\"\n IDVAR: \"GROUPID\"\n group_aliases:\n - USUBJID\n - GROUPID\n```\n\nExample: Group the StudyIdentifier dataset by parent_id and merge the result back to the context dataset StudyVersion using StudyVersion.id == StudyIdentifier.parent_id\n\n```yaml\nScope:\n Entities:\n Include:\n - StudyVersion\nOperations:\n - domain: StudyIdentifier\n filter:\n parent_entity: \"StudyVersion\"\n parent_rel: \"studyIdentifiers\"\n rel_type: \"definition\"\n studyIdentifierScope.organizationType.code: \"C70793\"\n studyIdentifierScope.organizationType.codeSystem: \"http://www.cdisc.org\"\n group:\n - parent_id\n group_aliases:\n - id\n id: $num_sponsor_ids\n operator: record_count\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -422,10 +310,7 @@ "markdownDescription": "\nReturns the required variables ( \"Core\" = Req ) for a given domain and standard Variable Metadata for custom domains will pull from the model while non-custom domains will be from the IG and Model.\n\nInput:\n\nTarget Domain: LB\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\n- operator: required_variables\n id: $required_variables\n```\n\nOutput:\n\n```\n[\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"LBSEQ\", \"LBTESTCD\", \"LBTEST\"]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -435,12 +320,7 @@ "markdownDescription": "\nSplits a dataset column by a given delimiter\n\n```yaml\nOperations:\n - name: PPSPEC\n delimiter: ;\n id: $ppspec_value\n operator: split_by\n```\n" } }, - "required": [ - "id", - "operator", - "delimiter", - "name" - ], + "required": ["id", "operator", "delimiter", "name"], "type": "object" }, { @@ -450,10 +330,7 @@ "markdownDescription": "\nReturns a list of the domains in the study\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -463,10 +340,7 @@ "markdownDescription": "\nReturns a list of the submitted dataset filenames in all uppercase\n\nex. if TS.xpt, AE.xpt, EC.xpt, and SUPPEC.xpt are submitted -> [TS, AE, EC, SUPPEC] will be returned\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -476,10 +350,7 @@ "markdownDescription": "\nReturns a list of valid SDTM domain names from the standard metadata. This can be used to compare extracted suffixes from DOMAIN values or dataset names.\n\nInput\n\nProduct: sdtmig\n\nVersion: 3-4\n\n```yaml\nOperations:\n - operator: standard_domains\n id: $valid_domain_names\n```\n\nOutput\n\n```\n[\"AE\", \"CM\", \"DM\", \"FA\", \"LB\", \"QS\", ...]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -489,10 +360,7 @@ "markdownDescription": "\nReturns the valid terminology package dates for a given standard.\n\nGiven a list of terminology packages:\n\n```\n[\n \"sdtmct-2023-10-26\",\n \"sdtmct-2023-12-13\",\n \"adamct-2023-12-13\",\n \"cdashct-2023-05-19\"\n]\n```\n\nand standard: sdtmig\n\nthe operation will return:\n\n```\n[\"2023-10-26\", \"2023-12-13\"]\n```\n\nBy default, the standard is as specified when running validation - as the validation runtime parameter and/or as specified in the rule header - and the list of terminology packages is obtained from the current cache. If required, the default standard may be overridden using the optional ct_package_types parameter. For example, given the same list of terminology packages, the following operation:\n\n```yaml\nOperations:\n - operator: valid_codelist_dates\n id: $valid_dates\n ct_package_types:\n - SDTM\n - CDASH\n```\n\nwill return:\n\n```\n[\"2023-05-19\", \"2023-10-26\", \"2023-12-13\"]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -502,11 +370,7 @@ "markdownDescription": "\nReturns true if the version of an external dictionary provided in the define.xml file matches the version parsed from the dictionary files.\n\nInput:\n\n```yaml\nOperations:\n - operator: valid_define_external_dictionary_version\n id: $is_valid_loinc_version\n external_dictionary_type: loinc\n```\n\nOutput:\n\n```\n[true, true, true, true]\n```\n" } }, - "required": [ - "id", - "operator", - "external_dictionary_type" - ], + "required": ["id", "operator", "external_dictionary_type"], "type": "object" }, { @@ -564,10 +428,7 @@ "markdownDescription": "\nDetermines whether the values are valid in the following variables:\n\n- --SOCCD (System Organ Class Code)\n- --HLGTCD (High Level Group Term Code)\n- --HLTCD (High Level Term Code)\n- --PTCD (Preferred Term Code)\n- --LLTCD (Lowest Level Term Code)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_codes\n operator: valid_meddra_code_references\n```\n\nOutput:\n\n```\n[true, false, true, true]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -577,10 +438,7 @@ "markdownDescription": "\nDetermines whether the values are valid in the following variable pairs:\n\n- --SOCCD, --SOC (System Organ Class Code and Term)\n- --HLGTCD, --HLGT (High Level Group Term Code and Term)\n- --HLTCD, --HLT (High Level Term Code and Term)\n- --PTCD, --DECOD (Preferred Term Code and Dictionary-Derived Term)\n- --LLTCD, --LLT (Lowest Level Term Code and Term)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_pairs\n operator: valid_meddra_code_term_pairs\n```\n\nOutput:\n\n```\n[true, true, false, true, true]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -590,10 +448,7 @@ "markdownDescription": "\nDetermines whether the values are valid in the following variables:\n\n- --SOC (System Organ Class)\n- --HLGT (High Level Group Term)\n- --HLT (High Level Term)\n- --DECOD (Dictionary-Derived Term)\n- --LLT (Lowest Level Term)\n\nInput:\n\n```yaml\nOperations:\n - id: $is_valid_meddra_terms\n operator: valid_meddra_term_references\n```\n\nOutput:\n\n```\n[true, true, false, true, true]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -603,11 +458,7 @@ "markdownDescription": "\nChecks if a reference to whodrug term in name points to the existing code in Atc Text (INA) file.\n\nInput:\n\n```yaml\nOperations:\n - id: $whodrug_refs_valid\n operator: valid_whodrug_references\n```\n\nOutput:\n\n```\n[true, false, true, true]\n```\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -617,10 +468,7 @@ "markdownDescription": "\nReturns a mapping of variable names to the number of times that variable appears in a domain within the study.\n\nInput\n\n```\n{\n \"AE\": [\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"AETERM\", \"AEENDTC\"],\n \"LB\": [\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"LBTESTCD\", \"LBENDTC\"]\n}\n```\n\nOutput\n\n```\n{\n \"STUDYID\": 2,\n \"DOMAIN\": 2,\n \"USUBJID\": 2,\n \"--TERM\": 1,\n \"--TESTCD\": 1,\n \"--ENDTC\": 2\n}\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -630,10 +478,7 @@ "markdownDescription": "\nOperation operates only on original submission datasets regardless of rule type. Flags an error if a column exists is in the submission dataset currently being evaluated.\n\nRule Type: Domain Presence Check\n\n```yaml\nCheck:\n all:\n - name: $MIDS_EXISTS\n operator: equal_to\n value: true\n - name: TM\n operator: not_exists\nOperations:\n - id: $MIDS_EXISTS\n name: MIDS\n operator: variable_exists\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -643,10 +488,7 @@ "markdownDescription": "\nReturns true if a variable is missing from the dataset or if all values within the variable are null or empty string. This operation first checks if the target variable exists in the dataset, and if it does exist, evaluates whether all its values are null or empty.\nThe operation supports two sources via the `source` parameter:\n\n- **`submission`** : checks against the raw submission dataset\n- **`evaluation`** (default): checks against the evaluation dataset built based on the rule type\n\n```yaml" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -656,10 +498,7 @@ "markdownDescription": "\nReturns the set of variable names from the library for the given standard. This operation extracts all variable names across all domains in the specified standard's library metadata.\n\nInput:\n\nValidation Standard: sdtmig\nValidation Version: 3-4\n\n```yaml\n- operator: variable_names\n id: $all_variables\n```\n\nOutput:\n\n```\n[\"STUDYID\", \"DOMAIN\", \"USUBJID\", \"SUBJID\", \"RFSTDTC\", \"RFENDTC\", \"SITEID\", \"AGE\", \"AGEU\", \"SEX\", \"RACE\", \"ETHNIC\", \"ARMCD\", \"ARM\", \"ACTARMCD\", \"ACTARM\", \"COUNTRY\", \"DMDTC\", \"DMDY\", \"AETERM\", \"AEDECOD\", \"AECAT\", \"AESCAT\", \"AEPRESP\", \"AEBODSYS\", \"AEBDSYCD\", \"AESOC\", \"AESOCCD\", \"AELLT\", \"AELLTCD\", \"AEHLT\", \"AEHLTCD\", \"AEHLGT\", \"AEHLGTCD\", \"AEPTCD\", \"AESTDTC\", \"AEENDTC\", \"AESTDY\", \"AEENDY\", \"AEDUR\", \"AESER\", \"AESEV\", \"AEACN\", \"AEREL\", \"AEOUT\", \"AESCAN\", \"AESCONG\", \"AESDISAB\", \"AESDTH\", \"AESHOSP\", \"AESLIFE\", \"AESOD\", \"AECONTRT\", \"AETOXGR\", \"LBTESTCD\", \"LBTEST\", \"LBCAT\", \"LBSCAT\", \"LBSPEC\", \"LBMETHOD\", \"LBORRES\", \"LBORRESU\", \"LBORNRLO\", \"LBORNRHI\", \"LBSTRESC\", \"LBSTRESN\", \"LBSTRESU\", \"LBSTNRLO\", \"LBSTNRHI\", \"LBNRIND\", \"LBNAM\", \"LBSPEC\", \"LBANTREG\", \"LBFAST\", \"LBDRVFL\", \"LBTOX\", \"LBTOXGR\", \"LBSTDTC\", \"LBENDTC\", \"LBSTDY\", \"LBENDY\", \"LBTPT\", \"LBTPTNUM\", \"LBELTM\", \"LBTPTREF\", \"LBRFTDTC\", \"...\"]\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -669,11 +508,7 @@ "markdownDescription": "\nGiven a variable name, returns a mapping of variable values to the number of times that value appears in the variable within all datasets in the study.\n" } }, - "required": [ - "id", - "operator", - "name" - ], + "required": ["id", "operator", "name"], "type": "object" }, { @@ -683,10 +518,7 @@ "markdownDescription": "\nDetermines whether the values are valid and in the correct hierarchical structure in the following variables:\n\n- --DECOD\n- --CLAS\n- --CLASCD\n\nInput:\n\n```yaml\nOperations:\n - id: $valid_whodrug_codes\n operator: whodrug_code_hierarchy\n```\n" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" }, { @@ -696,12 +528,7 @@ "markdownDescription": "\nValidates XHTML fragments in the target column against the specified namespace.\n\n```yaml\nOperations:\n - id: $xhtml_errors\n name: text\n operator: get_xhtml_errors\n namespace: http://www.cdisc.org/ns/usdm/xhtml/v1.0\n```\n\nNote that a local XSD file is required for validation. The file must be stored in the folder indicated by the value of the `LOCAL_XSD_FILE_DIR` default file path and the mapping between the namespace and the local XSD file's `sub-folder/name` must be included in the value of the `LOCAL_XSD_FILE_MAP` default file path.\n" } }, - "required": [ - "id", - "operator", - "name", - "namespace" - ], + "required": ["id", "operator", "name", "namespace"], "type": "object" }, { @@ -717,25 +544,14 @@ }, "on_no_match": { "type": "string", - "enum": [ - "keep_original", - "set_null", - "set_empty", - "error" - ] + "enum": ["keep_original", "set_null", "set_empty", "error"] }, "flags": { "type": "string", "pattern": "^[ims]*$" } }, - "required": [ - "id", - "operator", - "name", - "find", - "replace" - ], + "required": ["id", "operator", "name", "find", "replace"], "type": "object" } ], @@ -798,13 +614,7 @@ "type": "string" }, "dictionary_term_type": { - "enum": [ - "LLT", - "PT", - "HLT", - "HLGT", - "SOC" - ] + "enum": ["LLT", "PT", "HLT", "HLGT", "SOC"] }, "domain": { "anyOf": [ @@ -817,9 +627,7 @@ ] }, "external_dictionary_type": { - "enum": [ - "meddra" - ] + "enum": ["meddra"] }, "filter": { "type": "object" @@ -864,10 +672,7 @@ }, "level": { "type": "string", - "enum": [ - "codelist", - "term" - ] + "enum": ["codelist", "term"] }, "map": { "type": "array", @@ -878,9 +683,7 @@ "type": "string" } }, - "required": [ - "output" - ] + "required": ["output"] } }, "name": { @@ -903,12 +706,7 @@ }, "on_no_match": { "type": "string", - "enum": [ - "keep_original", - "set_null", - "set_empty", - "error" - ] + "enum": ["keep_original", "set_null", "set_empty", "error"] }, "flags": { "type": "string", @@ -916,11 +714,7 @@ }, "returntype": { "type": "string", - "enum": [ - "code", - "value", - "pref_term" - ] + "enum": ["code", "value", "pref_term"] }, "source": { "type": "string" @@ -947,9 +741,6 @@ "type": "string" } }, - "required": [ - "id", - "operator" - ], + "required": ["id", "operator"], "type": "object" } diff --git a/resources/schema/rule-merged/Operator.json b/resources/schema/rule-merged/Operator.json index f3a269a4d..e1b988e6a 100644 --- a/resources/schema/rule-merged/Operator.json +++ b/resources/schema/rule-merged/Operator.json @@ -9,9 +9,7 @@ "const": "additional_columns_empty" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -20,9 +18,7 @@ "const": "additional_columns_not_empty" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -32,10 +28,7 @@ "markdownDescription": "\nWill return True if the value in `value` is contained within the collection/iterable in the target column, or if there's an exact match for non-iterable data.\n\nThe operator checks if every value in a column is a list or set. If yes, it compares row-by-row. If any value is blank or a different type (like a string or number), it compares each value against the entire column instead.\n\nExample:\n\n```yaml\n- name: \"--TOXGR\" # Column containing lists like ['GRADE', 'SEVERITY', 'ONSET']\n operator: \"contains\"\n value: \"GRADE\" # True if 'GRADE' is an element in the list\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -45,9 +38,7 @@ "markdownDescription": "\nTrue if all values in `value` are contained within the variable `name`.\n\n> All of ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment') in ACTARM\n\n```yaml\n- name: \"ACTARM\"\n operator: \"contains_all\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n\nThe operator also supports lists:\n\n```yaml\n- name: \"$spec_codelist\"\n operator: \"contains_all\"\n value: \"$ppspec_value\"\n```\n\nWhere:\n\n| $spec_codelist | $ppspec_value |\n| :-------------------------- | :----------------: |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\", \"CODE2\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE2\", \"CODE3\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\"] |\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -57,10 +48,7 @@ "markdownDescription": "\nTrue if the value in `value` is contained within the collection/iterable in the target column, performing case-insensitive comparison.\n\nExample:\n\n```yaml\n- name: \"--TOXGR\" # Column containing lists like ['Grade', 'Severity', 'Onset']\n operator: \"contains_case_insensitive\"\n value: \"grade\" # True if 'Grade'/'GRADE'/'grade' exists in the list\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -70,10 +58,7 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified.\n\nThe `date_component` parameter accepts: `\"year\"`, `\"month\"`, `\"day\"`, `\"hour\"`, `\"minute\"`, `\"second\"`, `\"microsecond\"`, or `\"auto\"`.\n\nWhen `date_component: \"auto\"` is used, the operator automatically detects the precision of both dates and compares at the common (less precise) level.\n\n```yaml\n- name: \"AESTDTC\"\n operator: \"date_equal_to\"\n value: \"RFSTDTC\"\n date_component: \"auto\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -83,10 +68,7 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> Year part of BRTHDTC > 2021\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"date_greater_than\"\n date_component: \"year\"\n value: \"2021\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -96,10 +78,7 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> Year part of BRTHDTC >= 2021\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"date_greater_than_or_equal_to\"\n date_component: \"year\"\n value: \"2021\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -109,10 +88,7 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> AEENDTC < AESTDTC\n\n```yaml\n- name: \"AEENDTC\"\n operator: \"date_less_than\"\n value: \"AESTDTC\"\n```\n\n> SSDTC < all DS.DSSTDTC when SSSTRESC = \"DEAD\"\n\n```yaml\nCheck:\n all:\n - name: \"SSSTRESC\"\n operator: \"equal_to\"\n value: \"DEAD\"\n - name: \"SSDTC\"\n operator: \"date_less_than\"\n value: \"$max_ds_dsstdtc\"\nOperations:\n - operator: \"max_date\"\n domain: \"DS\"\n name: \"DSSTDTC\"\n id: \"$max_ds_dsstdtc\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -122,10 +98,7 @@ "markdownDescription": "\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n\n> AEENDTC <= AESTDTC\n\n```yaml\n- name: \"AEENDTC\"\n operator: \"date_less_than_or_equal_to\"\n value: \"AESTDTC\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -135,10 +108,7 @@ "markdownDescription": "\nComplement of `date_equal_to`\n\nDate comparison. Compare `name` to `value`. Compares partial dates if `date_component` is specified. Supports `date_component: \"auto\"`.\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -148,10 +118,7 @@ "markdownDescription": "\nComplement of `contains`. Returns True when the value is NOT contained within the target collection.\n\n```yaml\n- name: \"--TOXGR\"\n operator: \"does_not_contain\"\n value: \"GRADE\" # True if 'GRADE' is NOT an element in the list\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -161,10 +128,7 @@ "markdownDescription": "\nComplement of `contains_case_insensitive`. Returns True when the value is NOT contained within the target collection (case-insensitive).\n\nExample:\n\n```yaml\n- name: \"--TOXGR\"\n operator: \"does_not_contain_case_insensitive\"\n value: \"grade\" # True if no case variation of 'grade' exists in the list\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -177,11 +141,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value", - "regex" - ], + "required": ["operator", "value", "regex"], "type": "object" }, { @@ -191,12 +151,7 @@ "markdownDescription": "\nComplement of `has_next_corresponding_record`\n" } }, - "required": [ - "operator", - "ordering", - "value", - "within" - ], + "required": ["operator", "ordering", "value", "within"], "type": "object" }, { @@ -206,9 +161,7 @@ "markdownDescription": "\nValue presence\n\n> --OCCUR = null\n\n```yaml\n- name: --OCCUR\n operator: empty\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -218,10 +171,7 @@ "markdownDescription": "\n> SEENDTC is not empty when it is not the last record, grouped by USUBJID, sorted by SESTDTC\n\n```yaml\n- name: SEENDTC\n operator: empty_within_except_last_row\n ordering: SESTDTC\n value: USUBJID\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -231,10 +181,7 @@ "markdownDescription": "\nSubstring matching\n\n> DOMAIN ending with 'FOOBAR'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"ends_with\"\n value: \"FOOBAR\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -253,10 +200,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -275,10 +219,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -291,11 +232,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value", - "regex" - ], + "required": ["operator", "value", "regex"], "type": "object" }, { @@ -305,9 +242,7 @@ "markdownDescription": "\nTrue if the column exists in the current dataframe. (Works for datasets and variables)\n\n> --OCCUR is present in dataset\n\n```yaml\n- name: \"--OCCUR\"\n operator: \"exists\"\n```\n\n> Domain SJ exists\n\n```yaml\nRule Type: Domain Presence Check\nCheck:\n all:\n - name: \"SJ\"\n operator: \"exists\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -317,10 +252,7 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL > 0\n\n```yaml\n- name: TSVAL\n operator: greater_than\n value: 0\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -330,10 +262,7 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL >= 0\n\n```yaml\n- name: TSVAL\n operator: greater_than_or_equal_to\n value: 1\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -343,9 +272,7 @@ "markdownDescription": "\nComplement of `has_same_values`\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -355,9 +282,7 @@ "markdownDescription": "\nLength comparison\n\n> Check whether variable values has equal length of another variable.\n\n```yaml\n- name: SEENDTC\n operator: has_equal_length\n value: SESTDTC\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -367,12 +292,7 @@ "markdownDescription": "\nEnsures that a value of a variable `name` in one record is equal to the value of another variable `value` in the next corresponding record. The rows are grouped by `within` and ordered by `ordering`.\n\n> SEENDTC is equal to the SESTDTC of the next record within a USUBJID. Ordered by SESEQ\n\n```yaml\n- name: SEENDTC\n operator: has_next_corresponding_record\n value: SESTDTC\n within: USUBJID\n ordering: SESEQ\n```\n" } }, - "required": [ - "operator", - "ordering", - "value", - "within" - ], + "required": ["operator", "ordering", "value", "within"], "type": "object" }, { @@ -382,9 +302,7 @@ "markdownDescription": "\nComplement of `has_equal_length`\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -394,9 +312,7 @@ "markdownDescription": "\nTrue if all values in `name` are the same\n\n> Condition: MHCAT ^= null\n> Rule: MHCAT ^= the same value for all records\n\n```yaml\nCheck:\n all:\n - name: MHCAT\n operator: non_empty\n - name: MHCAT\n operator: has_same_values\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -406,10 +322,7 @@ "markdownDescription": "\nDuration ISO-8601 check, returns True if a duration is not in ISO-8601 format. The negative parameter must be specified to indicate if negative durations are either allowed (True) or disallowed (False)\n\n> DURVAR is invalid (negative durations disallowed)\n\n```yaml\n- name: \"DURVAR\"\n operator: \"invalid_duration\"\n negative: False\n```\n" } }, - "required": [ - "operator", - "negative" - ], + "required": ["operator", "negative"], "type": "object" }, { @@ -419,9 +332,7 @@ "markdownDescription": "\nThe operator performs date validation against complete and partial dates with uncertainty in the following order:\n\n1. Attempts to parse using [dateutil.parser.isoparse()](https://dateutil.readthedocs.io/en/stable/parser.html)\n2. If parsing fails and the string contains uncertainty indicators (`/`, `--`, `-:`), validates against an extended ISO 8601 dates regex pattern\n3. If parsing succeeds, dates are still validated against the regex pattern.\n\n```yaml\n- name: \"BRTHDTC\"\n operator: \"invalid_date\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -431,9 +342,7 @@ "markdownDescription": "\nDate check\n\n> DM.RFSTDTC = complete date\n\n```yaml\n- name: \"RFSTDTC\"\n operator: \"is_complete_date\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -443,10 +352,7 @@ "markdownDescription": "\nValue in `name` compared against a list in `value`. The list can have literal values or be a reference to a `$variable`.\n\nThis operator behaves similarly to `contains`. The key distinction: `contains` checks if comparator \u2208 target, while `is_contained_by` checks if target \u2208 comparator.\n\n> ACTARM in ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment')\n\n```yaml\n- name: \"ACTARM\"\n operator: \"is_contained_by\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -456,10 +362,7 @@ "markdownDescription": "\nValue in `name` case insensitive compared against a list in `value`. The list can have literal values or be a reference to a `$variable`.\n\n> ACTARM in ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment')\n\n```yaml\n- name: \"ACTARM\"\n operator: \"is_contained_by_case_insensitive\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -469,9 +372,7 @@ "markdownDescription": "\nComplement of `is_complete_date`\n\nDate check\n\n> DM.RFSTDTC ^= complete date\n\n```yaml\n- name: \"RFSTDTC\"\n operator: \"is_incomplete_date\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -481,10 +382,7 @@ "markdownDescription": "\nComplement of `is_contained_by`\n\n> ARM not in ('Screen Failure', 'Not Assigned')\n\n```yaml\n- name: \"ARM\"\n operator: \"is_not_contained_by\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -494,10 +392,7 @@ "markdownDescription": "\nComplement of `is_contained_by_case_insensitive`\n\n> ARM not in ('Screen Failure', 'Not Assigned')\n\n```yaml\n- name: \"ARM\"\n operator: \"is_not_contained_by_case_insensitive\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -507,10 +402,7 @@ "markdownDescription": "\nComplement of `is_ordered_by`\n" } }, - "required": [ - "operator", - "order" - ], + "required": ["operator", "order"], "type": "object" }, { @@ -519,10 +411,7 @@ "const": "is_not_ordered_set" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -532,10 +421,7 @@ "markdownDescription": "\nComplement of `is_unique_relationship`\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -545,9 +431,7 @@ "markdownDescription": "\nComplement of `is_unique_set`.\n\n> --SEQ is not unique within DOMAIN, USUBJID, and --TESTCD\n\n```yaml\n- name: \"--SEQ\"\n operator: is_not_unique_set\n value:\n - \"DOMAIN\"\n - \"USUBJID\"\n - \"--TESTCD\"\n```\n\n> `name` can be a variable containing a list of columns and `value` does not need to be present\n\n```yaml\nRule Type: Dataset Contents Check against Define XML\nCheck:\n all:\n - name: define_dataset_key_sequence # contains list of dataset key columns\n operator: is_not_unique_set\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -557,10 +441,7 @@ "markdownDescription": "\nTrue if the dataset rows are ordered by the values within `name`, given the ordering specified by `order`\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_by\n order: asc\n```\n" } }, - "required": [ - "operator", - "order" - ], + "required": ["operator", "order"], "type": "object" }, { @@ -570,10 +451,7 @@ "markdownDescription": "\nTrue if the dataset rows are in ascending order of the values within `name`, grouped by the values within `value`. Value can either be a single column or multiple.\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_set\n value: USUBJID\n```\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n operator: is_ordered_set\n value:\n - USUBJID\n - \"--TESTCD\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -583,10 +461,7 @@ "markdownDescription": "\nRelationship Integrity Check looking for a 1-1 relationship between name and value. Ensures uniqueness of both name and value.\n\n> AETERM and AEDECOD has a 1-to-1 relationship\n\n```yaml\n- name: AETERM\n operator: is_unique_relationship\n value: AEDECOD\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -596,10 +471,7 @@ "markdownDescription": "\nChecks if a variable maintains consistent values within groups defined by one or more grouping variables. Groups records by specified value(s) and validates that the target variable maintains the same value within each unique combination of grouping variables. When inconsistency is detected within a group, the operator attempts to identify a majority value. If one value appears more frequently than all others, only the minority records (those not matching the majority value) are flagged. If no single majority exists \u2014 i.e., two or more values are tied for the highest frequency \u2014 all records in that group are flagged.\n\nSingle grouping variable - true if the values of BGSTRESU differ within USUBJID:\n\nIf a regex parameter is provided, it is applied to the values of the target variable before the consistency check. The first capture group of the regex is used as the normalized value for comparison. This can be useful when only part of the value should be considered during comparison (for example, comparing only the date portion of a datetime value).\n\n- regex is optional.\n- The pattern must include at least one capture group(or whole regex will be wrapped to capture group).\n- Only the first capture group is used for comparison.\n- If the pattern does not match a value, the original value is used.\n\n```yaml\n- name: \"BGSTRESU\"\n operator: is_inconsistent_across_dataset\n value: \"USUBJID\"\n```\n\nMultiple grouping variables - true if the values of --STRESU differ within each combination of --TESTCD, --CAT, --SCAT, --SPEC, and --METHOD:\n\n```yaml\n- name: \"--STRESU\"\n operator: is_inconsistent_across_dataset\n value:\n - \"--TESTCD\"\n - \"--CAT\"\n - \"--SCAT\"\n - \"--SPEC\"\n - \"--METHOD\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -609,9 +481,7 @@ "markdownDescription": "\nRelationship Integrity Check\n\n> --SEQ is unique within DOMAIN, USUBJID, and --TESTCD\n\n```yaml\n- name: \"--SEQ\"\n operator: is_unique_set\n value:\n - \"DOMAIN\"\n - \"USUBJID\"\n - \"--TESTCD\"\n```\n\n> `name` can be a variable containing a list of columns and `value` does not need to be present\n\n> The `regex` parameter allows you to extract portions of values using a regex pattern before checking uniqueness.\n\n> Compare date only (YYYY-MM-DD) for uniqueness\n\n```yaml\n- name: \"--REPNUM\"\n operator: is_not_unique_set\n value:\n - \"USUBJID\"\n - \"--TESTCD\"\n - \"$TIMING_VARIABLES\"\n regex: '^\\d{4}-\\d{2}-\\d{2}'\n```\n\n> Compare by first N characters of a string\n\n```yaml\n- name: \"ITEM_ID\"\n operator: is_not_unique_set\n value:\n - \"USUBJID\"\n - \"CATEGORY\"\n regex: \"^.{2}\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -621,10 +491,7 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL < 1\n\n```yaml\n- name: TSVAL\n operator: less_than\n value: 1\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -634,10 +501,7 @@ "markdownDescription": "\nValue comparison\n\n> TSVAL <= 1\n\n```yaml\n- name: TSVAL\n operator: less_than_or_equal_to\n value: 1\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -647,10 +511,7 @@ "markdownDescription": "\nLength comparison\n\n> SETCD value length > 8\n\n```yaml\n- name: \"SETCD\"\n operator: \"longer_than\"\n value: 8\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -660,10 +521,7 @@ "markdownDescription": "\nLength comparison\n\n> TSVAL value length >= 201\n\n```yaml\n- name: \"TSVAL\"\n operator: \"longer_than_or_equal_to\"\n value: 201\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -673,10 +531,7 @@ "markdownDescription": "\nRegular Expression value matching\n\n- Determine if each string starts with a match of a regular expression. Refer to this pandas documentation: https://pandas.pydata.org/docs/reference/api/pandas.Series.str.match.html\n- To \"search\" for a regex within the entire text, prefix the regex with `.*` and do not use anchors `^` , `$`\n- To do a \"fullmatch\" of a regex with the entire text, suffix the regex with an anchor `$` and do not prefix the regex with `.*`\n- For syntax guide, refer to this Python documentation: [Regular Expression HOWTO](https://docs.python.org/3/howto/regex.html).\n- Suggestion for an on-line regular expression logic. tester: https://regex101.com, choose the Python dialect.\n- For regex token visualization, try https://www.debuggex.com.\n\n> --DOSTXT value is non-numeric\n\n```yaml\n- name: --DOSTXT\n operator: matches_regex\n value: ^\\d*\\.?\\d*$\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -686,9 +541,7 @@ "markdownDescription": "\nComplement of `empty`\n\n> --OCCUR ^= null\n\n```yaml\n- name: --OCCUR\n operator: non_empty\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -698,10 +551,7 @@ "markdownDescription": "\nComplement of `empty_within_except_last_row`\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -711,9 +561,7 @@ "markdownDescription": "\nComplement of `contains_all`\n\n> All of ('Screen Failure', 'Not Assigned', 'Not Treated', 'Unplanned Treatment') not in ACTARM\n\n```yaml\n- name: \"ACTARM\"\n operator: \"not_contains_all\"\n value:\n - \"Screen Failure\"\n - \"Not Assigned\"\n - \"Not Treated\"\n - \"Unplanned Treatment\"\n```\n\nThe operator also supports lists:\n\n```yaml\n- name: \"$spec_codelist\"\n operator: \"not_contains_all\"\n value: \"$ppspec_value\"\n```\n\nWhere:\n\n| $spec_codelist | $ppspec_value |\n| :-------------------------- | :----------------: |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\", \"CODE2\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE2\", \"CODE3\"] |\n| [\"CODE1\", \"CODE2\", \"CODE3\"] | [\"CODE1\"] |\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -732,10 +580,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -754,10 +599,7 @@ "type": "boolean" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -767,9 +609,7 @@ "markdownDescription": "\nComplement of `exists`\n\n> AEOCCUR not present in dataset\n\n```yaml\n- name: \"AEOCCUR\"\n operator: \"not_exists\"\n```\n\n> Domain SJ does not exist\n\n```yaml\nRule Type: Domain Presence Check\nCheck:\n all:\n - name: \"SJ\"\n operator: \"not_exists\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -779,10 +619,7 @@ "markdownDescription": "\nComplement of `matches_regex`\n\n> --TESTCD <= 8 chars and contains only letters, numbers, and underscores and can not start with a number\n\n```yaml\n- name: --TESTCD\n operator: not_matches_regex\n value: ^[A-Z_][A-Z0-9_]{0,7}$\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -792,11 +629,7 @@ "markdownDescription": "\nComplement of `prefix_matches_regex`\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -806,10 +639,7 @@ "markdownDescription": "\nComplement of `present_on_multiple_rows_within`\n\n```yaml\n- operator: \"not_present_on_multiple_rows_within\"\n name: \"RELID\"\n value: 4 (optional)\n within: \"USUBJID\"\n```\n" } }, - "required": [ - "operator", - "within" - ], + "required": ["operator", "within"], "type": "object" }, { @@ -819,11 +649,7 @@ "markdownDescription": "\nComplement of `suffix_matches_regex`\n\n> QNAM does not end with numbers\n\n```yaml\n- name: \"QNAM\"\n operator: \"not_suffix_matches_regex\"\n suffix: 2\n value: \"\\d\\d\"\n```\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -833,11 +659,7 @@ "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match one of the strings in the list in `value`\n\n> Check if a variable's domain identifier exists in the study\n\n```yaml\n- name: variable_name\n operator: prefix_is_contained_by\n prefix: 2\n value: $study_domains\n```\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -847,11 +669,7 @@ "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match the string in `value`\n\n```yaml\n- name: dataset_name\n operator: prefix_equal_to\n prefix: 2\n value: DOMAIN\n```\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -861,11 +679,7 @@ "markdownDescription": "\nComplement of `prefix_is_contained_by`\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -875,11 +689,7 @@ "markdownDescription": "\nTrue if the `prefix` number of characters beginning a string in `name` match a regular expression in `value`\n\n```yaml\n- name: DOMAIN\n operator: prefix_matches_regex\n prefix: 2\n value: (AP|ap)\n```\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -889,11 +699,7 @@ "markdownDescription": "\nComplement of `prefix_equal_to`\n" } }, - "required": [ - "operator", - "prefix", - "value" - ], + "required": ["operator", "prefix", "value"], "type": "object" }, { @@ -903,10 +709,7 @@ "markdownDescription": "\nTrue if the same value of `name` is present on multiple rows, grouped by `within`. A maximum allowed number of occurrences can be specified in the value attribute. In this instance the value: 4 means that an error will be flagged if the same value appears more than 4 times within a USUBJID. By default the operator will flag any time a value appears more than once.\n\n```yaml\n- operator: \"present_on_multiple_rows_within\"\n name: \"RELID\"\n value: 4 (optional)\n within: \"USUBJID\"\n```\n" } }, - "required": [ - "operator", - "within" - ], + "required": ["operator", "within"], "type": "object" }, { @@ -916,10 +719,7 @@ "markdownDescription": "\nWill raise an issue if at least one of the values in `name` is the same as one of the values in `value`. See [shares_no_elements_with](#shares_no_elements_with).\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -929,10 +729,7 @@ "markdownDescription": "\nWill raise an issue if exactly one of the values in `name` is the same as one of the values in `value`. See [shares_no_elements_with](#shares_no_elements_with).\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -942,10 +739,7 @@ "markdownDescription": "\nWill raise an issue if the values in `name` do not share any of the values in `value`\n\n> Check if $dataset_variables shares no elements with $timing_variables\n\n```yaml\nRule Type: Dataset Metadata Check # One record per dataset\nCheck:\n - all:\n name: $dataset_variables\n operator: shares_no_elements_with\n value: $timing_variables\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -955,10 +749,7 @@ "markdownDescription": "\nLength comparison\n\n> SETCD value length < 9\n\n```yaml\n- name: \"SETCD\"\n operator: \"shorter_than\"\n value: 9\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -968,10 +759,7 @@ "markdownDescription": "\nLength comparison\n\n> TSVAL value length <= 200\n\n```yaml\n- name: \"TSVAL\"\n operator: \"shorter_than_or_equal_to\"\n value: 201\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -981,9 +769,7 @@ "markdownDescription": "\nSplits a string by a separator and checks if both parts have equal length. Generic operator for validating paired data formats where both parts must have the same level of detail or precision.\n\nParameters:\n\n- `separator`: The delimiter to split on (default: \"/\")\n\n> Check that string parts separated by a delimiter have equal length\n\n```yaml\n- name: --DTC\n operator: split_parts_have_equal_length\n separator: \"/\"\n```\n\nUse cases:\n\n- **Date/time intervals**: `2003-12-15T10:00/2003-12-15T10:30` \u2192 True (both 16 characters)\n- **Date ranges**: `2003-12-01/2003-12-10` \u2192 True (both 10 characters)\n- **Version ranges**: `1.2.3/2.0.0` \u2192 True (both 5 characters)\n- **Product codes**: `ABC-123/XYZ-789` \u2192 True (both 7 characters)\n\nInvalid example:\n\n- `2003-12-15T10:00/2003-12-15T10:30:15` \u2192 False (16 vs 19 characters - different precision)\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -993,9 +779,7 @@ "markdownDescription": "\nComplement of `split_parts_have_equal_length`. Returns True when parts have unequal lengths (indicates a violation).\n\n```yaml\n- name: --DTC\n operator: split_parts_have_unequal_length\n separator: \"/\"\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1005,10 +789,7 @@ "markdownDescription": "\nSubstring matching\n\n> DOMAIN beginning with 'AP'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"starts_with\"\n value: \"AP\"\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1018,11 +799,7 @@ "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match the string in `value`\n\n```yaml\n- name: dataset_name\n operator: suffix_equal_to\n prefix: 2\n value: DOMAIN\n```\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -1032,11 +809,7 @@ "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match one of the strings in the list in `value`\n\n> Check if a supp's parent domain exists in the study\n\n```yaml\n- name: dataset_name\n operator: suffix_is_contained_by\n prefix: 2\n value: $study_domains\n```\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -1046,11 +819,7 @@ "markdownDescription": "\nComplement of `suffix_is_contained_by`\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -1060,11 +829,7 @@ "markdownDescription": "\nTrue if the `suffix` number of characters ending a string in `name` match a regular expression in `value`\n\n> QNAM ends with numbers\n\n```yaml\n- name: \"QNAM\"\n operator: \"suffix_matches_regex\"\n suffix: 2\n value: \"\\d\\d\"\n```\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -1074,11 +839,7 @@ "markdownDescription": "\nComplement of `suffix_equal_to`\n" } }, - "required": [ - "operator", - "suffix", - "value" - ], + "required": ["operator", "suffix", "value"], "type": "object" }, { @@ -1088,11 +849,7 @@ "markdownDescription": "\nComplement of `target_is_sorted_by`\n" } }, - "required": [ - "operator", - "value", - "within" - ], + "required": ["operator", "value", "within"], "type": "object" }, { @@ -1102,11 +859,7 @@ "markdownDescription": "\nTrue if the values in name are ordered according to the values specified by value\nin ascending/descending order, grouped by the values in within. Each value entry\nrequires a variable name, a sort_order of asc or desc, and an optional\nnull_position of first or last (defaults to last) which controls where null/empty\ncomparator values are placed in the expected ordering. Within accepts either a\nsingle column or an ordered list of columns. Columns can be either number or Char\nDates in ISO8601 YYYY-MM-DD format. Date value(s) with different precisions that\noverlap (e.g. 2005-10, 2005-10-3 and 2005-10-08) are all flagged as not sorted as\ntheir order cannot be inferred.\n\nOptionally supports a `regex` parameter that extracts a portion of the target\nvalue for sorting. The regex must contain at least one capturing group. The first\ncaptured group is extracted and converted to numeric if possible, allowing proper\nsorting of sequence numbers (e.g., \"MIDS1\", \"MIDS2\", ..., \"MIDS10\" with regex\n`.*?(\\\\d+)$`). This is particularly useful for variables that end with sequence\nnumbers that may or may not be zero-padded.\n\n```yaml\nCheck:\n all:\n - name: --SEQ\n within:\n - USUBJID\n - MIDSTYPE\n operator: target_is_sorted_by\n value:\n - name: --STDTC\n sort_order: asc\n null_position: last\n```\n\nExample with regex for extracting sequence numbers:\n\n```yaml\nCheck:\n all:\n - name: MIDS\n operator: target_is_sorted_by\n regex: \".*?(\\\\d+)$\" # Extract trailing digits, convert to numeric\n value:\n - name: SMSTDTC\n sort_order: asc\n within:\n - USUBJID\n - MIDSTYPE\n```\n" } }, - "required": [ - "operator", - "value", - "within" - ], + "required": ["operator", "value", "within"], "type": "object" }, { @@ -1116,10 +869,7 @@ "markdownDescription": "\nComplement of `value_has_multiple_references`\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1129,10 +879,7 @@ "markdownDescription": "\nTrue if the value in `name` has more than one count in the dictionary defined in `value`\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1142,10 +889,7 @@ "markdownDescription": "\nChecks for inconsistencies in enumerated columns of a DataFrame. Starting with the smallest/largest enumeration of the given variable, returns True if VARIABLE(N+1) is populated but VARIABLE(N) is not populated. Repeats for all variables belonging to the enumeration. Note that the initial variable will not have an index (VARIABLE) and the next enumerated variable has index 1 (VARIABLE1).\n\nex: Check if there are inconsistencies in the TSVAL columns (TSVAL, TSVAL1, TSVAL2, etc.)\n\n```yaml\nCheck:\n all:\n - name: \"TSVAL\"\n operator: \"inconsistent_enumerated_columns\"\n```\n" } }, - "required": [ - "operator", - "name" - ], + "required": ["operator", "name"], "type": "object" }, { @@ -1155,10 +899,7 @@ "markdownDescription": "\nChecks if elements in the target list appear in the same relative order in the comparator list.\n\n> Check if dataset column order is a correctly ordered subset of library column order\n\n```yaml\n- name: $column_order_from_dataset\n operator: is_ordered_subset_of\n value: $column_order_from_library\n```\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1168,10 +909,7 @@ "markdownDescription": "\nComplement of `is_ordered_subset_of`\n" } }, - "required": [ - "operator", - "value" - ], + "required": ["operator", "value"], "type": "object" }, { @@ -1181,9 +919,7 @@ "markdownDescription": "\nValidates that variable labels follow proper title case formatting rules using the titlecase PyPi library. Title case capitalizes the first word and all major words, while keeping articles (a, an, the), conjunctions (and, but, or), and prepositions (in, of, for) in lowercase unless they are the first word. \nNOTE: The titlecase library may produce false positives or false negatives in syntactic edge cases (e.g. hyphenated words, slash-separated terms, uncommon prepositions).\n\n> Check that AELABEL values are in proper title case\n\n```yaml\n- name: AELABEL\n operator: is_title_case\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" }, { @@ -1193,18 +929,13 @@ "markdownDescription": "\nComplement of `is_title_case`. Returns True when values are NOT in proper title case.\n\n> Flag AELABEL values that violate title case rules\n\n```yaml\n- name: AELABEL\n operator: is_not_title_case\n```\n" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" } ], "properties": { "comparator": { - "type": [ - "number", - "string" - ] + "type": ["number", "string"] }, "context": { "type": "string" @@ -1242,27 +973,18 @@ "type": "boolean" }, "codelistcheck": { - "enum": [ - "code", - "value" - ], + "enum": ["code", "value"], "type": "string" }, "codelistlevel": { - "enum": [ - "term", - "codelist" - ], + "enum": ["term", "codelist"], "type": "string" }, "operator": { "type": "string" }, "order": { - "enum": [ - "asc", - "dsc" - ], + "enum": ["asc", "dsc"], "type": "string" }, "ordering": { @@ -1280,25 +1002,17 @@ "value": { "oneOf": [ { - "type": [ - "boolean", - "number", - "string" - ] + "type": ["boolean", "number", "string"] }, { "items": { - "type": [ - "number" - ] + "type": ["number"] }, "type": "array" }, { "items": { - "type": [ - "string" - ] + "type": ["string"] }, "type": "array" }, @@ -1309,10 +1023,7 @@ "$ref": "Operator.json#/properties/name" }, "null_position": { - "enum": [ - "first", - "last" - ], + "enum": ["first", "last"], "type": "string" }, "order": { @@ -1355,8 +1066,6 @@ "type": "string" } }, - "required": [ - "operator" - ], + "required": ["operator"], "type": "object" } diff --git a/resources/schema/rule-merged/Organization_CDISC.json b/resources/schema/rule-merged/Organization_CDISC.json index 9aaef8a76..db1041921 100644 --- a/resources/schema/rule-merged/Organization_CDISC.json +++ b/resources/schema/rule-merged/Organization_CDISC.json @@ -22,9 +22,7 @@ "const": "Failure" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -40,9 +38,7 @@ "type": "object" }, "Version": { - "enum": [ - "5.0" - ] + "enum": ["5.0"] } }, "type": "object" @@ -51,12 +47,7 @@ "type": "array" }, "Version": { - "enum": [ - "1.0", - "1.1", - "1.2", - "1.3" - ] + "enum": ["1.0", "1.1", "1.2", "1.3"] } }, "type": "object" @@ -75,9 +66,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -90,19 +79,13 @@ "type": "string" }, "Version": { - "enum": [ - "1", - "2", - "3" - ] + "enum": ["1", "2", "3"] } }, "type": "object" }, "Version": { - "enum": [ - "2.0" - ] + "enum": ["2.0"] } }, "type": "object" @@ -111,11 +94,7 @@ "type": "array" }, "Version": { - "enum": [ - "3.2", - "3.3", - "3.4" - ] + "enum": ["3.2", "3.3", "3.4"] } }, "type": "object" @@ -134,9 +113,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -152,9 +129,7 @@ "type": "object" }, "Version": { - "enum": [ - "5.0" - ] + "enum": ["5.0"] } }, "type": "object" @@ -163,11 +138,7 @@ "type": "array" }, "Version": { - "enum": [ - "3.0", - "3.1", - "3.1.1" - ] + "enum": ["3.0", "3.1", "3.1.1"] } }, "type": "object" @@ -186,9 +157,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -204,9 +173,7 @@ "type": "object" }, "Version": { - "enum": [ - "5.0" - ] + "enum": ["5.0"] } }, "type": "object" @@ -215,10 +182,7 @@ "type": "array" }, "Version": { - "enum": [ - "1.1", - "1.2" - ] + "enum": ["1.1", "1.2"] } }, "type": "object" @@ -237,9 +201,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -255,9 +217,7 @@ "type": "object" }, "Version": { - "enum": [ - "5.0" - ] + "enum": ["5.0"] } }, "type": "object" @@ -266,9 +226,7 @@ "type": "array" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, "type": "object" @@ -287,9 +245,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -305,9 +261,7 @@ "type": "object" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, "type": "object" @@ -316,24 +270,13 @@ "type": "array" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] }, "Substandard": { - "enum": [ - "SDTM", - "SEND", - "ADaM", - "CDASH" - ] + "enum": ["SDTM", "SEND", "ADaM", "CDASH"] } }, - "required": [ - "Name", - "Version", - "Substandard" - ], + "required": ["Name", "Version", "Substandard"], "type": "object" }, { @@ -354,17 +297,13 @@ "type": "string" }, "Version": { - "enum": [ - "1" - ] + "enum": ["1"] } }, "type": "object" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, "type": "object" @@ -373,10 +312,7 @@ "type": "array" }, "Version": { - "enum": [ - "3.0", - "4.0" - ] + "enum": ["3.0", "4.0"] } }, "type": "object" diff --git a/resources/schema/rule-merged/Organization_Custom.json b/resources/schema/rule-merged/Organization_Custom.json index c5f591394..bf5bd7276 100644 --- a/resources/schema/rule-merged/Organization_Custom.json +++ b/resources/schema/rule-merged/Organization_Custom.json @@ -9,10 +9,7 @@ "type": "string", "description": "Name of your custom organization", "not": { - "enum": [ - "CDISC", - "FDA" - ] + "enum": ["CDISC", "FDA"] } }, "Standards": { @@ -48,9 +45,7 @@ "description": "Version of the rule" } }, - "required": [ - "Id" - ], + "required": ["Id"], "type": "object" }, "Version": { @@ -60,10 +55,7 @@ "Criteria": { "properties": { "Type": { - "enum": [ - "Failure", - "Success" - ], + "enum": ["Failure", "Success"], "type": "string" }, "Plain Language Expression": { @@ -78,46 +70,30 @@ "type": "string" } }, - "required": [ - "Rule" - ], + "required": ["Rule"], "type": "object" } }, - "required": [ - "Type" - ], + "required": ["Type"], "anyOf": [ { - "required": [ - "Logical Expression" - ] + "required": ["Logical Expression"] }, { - "required": [ - "Plain Language Expression" - ] + "required": ["Plain Language Expression"] } ], "type": "object" } }, - "required": [ - "Origin", - "Rule Identifier", - "Version" - ], + "required": ["Origin", "Rule Identifier", "Version"], "type": "object" }, "minItems": 1, "type": "array" } }, - "required": [ - "Name", - "References", - "Version" - ], + "required": ["Name", "References", "Version"], "type": "object" }, "minItems": 1, @@ -165,10 +141,7 @@ }, "OutputType": { "type": "string", - "enum": [ - "Check", - "Listing" - ], + "enum": ["Check", "Listing"], "description": "Output type of the rule validation result" }, "Keywords": { @@ -182,11 +155,7 @@ "additionalProperties": true } }, - "required": [ - "Organization", - "Standards", - "Category" - ], + "required": ["Organization", "Standards", "Category"], "type": "object", "$defs": { "metadata": { diff --git a/resources/schema/rule-merged/Organization_FDA.json b/resources/schema/rule-merged/Organization_FDA.json index 94af54bc4..b0f7de783 100644 --- a/resources/schema/rule-merged/Organization_FDA.json +++ b/resources/schema/rule-merged/Organization_FDA.json @@ -41,10 +41,7 @@ } } ], - "required": [ - "Document", - "Cited Guidance" - ], + "required": ["Document", "Cited Guidance"], "type": "object" }, "type": "array" @@ -55,9 +52,7 @@ "const": "Success" } }, - "required": [ - "Type" - ], + "required": ["Type"], "type": "object" }, "Origin": { @@ -73,9 +68,7 @@ "type": "object" }, "Version": { - "enum": [ - "1.5" - ] + "enum": ["1.5"] } }, "type": "object" @@ -91,11 +84,7 @@ "const": "SDTMIG" }, "Version": { - "enum": [ - "3.2", - "3.3", - "3.4" - ] + "enum": ["3.2", "3.3", "3.4"] } }, "type": "object" @@ -106,11 +95,7 @@ "const": "SENDIG" }, "Version": { - "enum": [ - "3.0", - "3.1", - "3.1.1" - ] + "enum": ["3.0", "3.1", "3.1.1"] } }, "type": "object" @@ -121,9 +106,7 @@ "const": "SENDIG-AR" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, "type": "object" @@ -134,10 +117,7 @@ "const": "SENDIG-DART" }, "Version": { - "enum": [ - "1.1", - "1.2" - ] + "enum": ["1.1", "1.2"] } }, "type": "object" @@ -148,9 +128,7 @@ "const": "SENDIG-GENETOX" }, "Version": { - "enum": [ - "1.0" - ] + "enum": ["1.0"] } }, "type": "object" From c81894d66c8f5d16b200495563b07d20cc13f5ba Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Mon, 27 Jul 2026 12:06:54 -0400 Subject: [PATCH 03/12] Adjusted for PR feedback --- cdisc_rules_engine/operations/regex_find_replace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdisc_rules_engine/operations/regex_find_replace.py b/cdisc_rules_engine/operations/regex_find_replace.py index f37809b91..d0d0ea766 100644 --- a/cdisc_rules_engine/operations/regex_find_replace.py +++ b/cdisc_rules_engine/operations/regex_find_replace.py @@ -85,7 +85,10 @@ def _transform_value(self, value, pattern, replace, on_no_match): text = str(value) match = pattern.search(text) if match: - return pattern.sub(replace, text) + try: + return pattern.sub(replace, text) + except re.error as exc: + raise OperationError(f"Error applying regex pattern '{pattern.pattern}' to value '{text}': {exc}") from exc if on_no_match == "keep_original": return text From b2042a01bc4d8371d798461bc934b7e1abcde70d Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Tue, 28 Jul 2026 10:40:31 -0400 Subject: [PATCH 04/12] Format: apply Black formatting to regex_find_replace.py --- cdisc_rules_engine/operations/regex_find_replace.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cdisc_rules_engine/operations/regex_find_replace.py b/cdisc_rules_engine/operations/regex_find_replace.py index d0d0ea766..b8b7c66bc 100644 --- a/cdisc_rules_engine/operations/regex_find_replace.py +++ b/cdisc_rules_engine/operations/regex_find_replace.py @@ -88,7 +88,9 @@ def _transform_value(self, value, pattern, replace, on_no_match): try: return pattern.sub(replace, text) except re.error as exc: - raise OperationError(f"Error applying regex pattern '{pattern.pattern}' to value '{text}': {exc}") from exc + raise OperationError( + f"Error applying regex pattern '{pattern.pattern}' to value '{text}': {exc}" + ) from exc if on_no_match == "keep_original": return text From 44536bdb7ff8f2b323692eab46a317a2900b6fc6 Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Fri, 31 Jul 2026 14:12:52 -0400 Subject: [PATCH 05/12] Addressed PR feedback --- .../constants/operation_constants.py | 9 +++++++ .../operations/regex_find_replace.py | 22 ++++++++--------- resources/schema/rule-merged/Operations.json | 3 ++- resources/schema/rule/Operations.md | 24 +++++++++++++++++++ 4 files changed, 45 insertions(+), 13 deletions(-) create mode 100644 cdisc_rules_engine/constants/operation_constants.py diff --git a/cdisc_rules_engine/constants/operation_constants.py b/cdisc_rules_engine/constants/operation_constants.py new file mode 100644 index 000000000..7a879b492 --- /dev/null +++ b/cdisc_rules_engine/constants/operation_constants.py @@ -0,0 +1,9 @@ +import re + +NO_MATCH_POLICIES = {"keep_original", "set_null", "set_empty", "error"} + +REGEX_FLAG_MAP = { + "i": re.IGNORECASE, + "m": re.MULTILINE, + "s": re.DOTALL, +} \ No newline at end of file diff --git a/cdisc_rules_engine/operations/regex_find_replace.py b/cdisc_rules_engine/operations/regex_find_replace.py index b8b7c66bc..57371267e 100644 --- a/cdisc_rules_engine/operations/regex_find_replace.py +++ b/cdisc_rules_engine/operations/regex_find_replace.py @@ -3,17 +3,15 @@ from cdisc_rules_engine.operations.base_operation import BaseOperation from cdisc_rules_engine.exceptions.custom_exceptions import OperationError +from cdisc_rules_engine.constants.operation_constants import NO_MATCH_POLICIES, REGEX_FLAG_MAP class RegexFindReplace(BaseOperation): - _NO_MATCH_POLICIES = {"keep_original", "set_null", "set_empty", "error"} - _FLAG_MAP = { - "i": re.IGNORECASE, - "m": re.MULTILINE, - "s": re.DOTALL, - } - def _execute_operation(self): + """ + Finds and replaces text in a target column using regex pattern matching. + Returns a Series with transformed values based on the replace pattern and no_match policy. + """ operation_id = self.params.operation_id target = self.params.target find = getattr(self.params, "find", None) or getattr(self.params, "regex", None) @@ -54,22 +52,22 @@ def _validate_required( raise OperationError("regex_find_replace requires find (or regex)") if replace is None: raise OperationError("regex_find_replace requires replace") - if on_no_match not in self._NO_MATCH_POLICIES: + if on_no_match not in NO_MATCH_POLICIES: raise OperationError( f"Invalid on_no_match: {on_no_match}. " - f"Must be one of {sorted(self._NO_MATCH_POLICIES)}" + f"Must be one of {sorted(NO_MATCH_POLICIES)}" ) - invalid_flags = [f for f in flags_str if f not in self._FLAG_MAP] + invalid_flags = [f for f in flags_str if f not in REGEX_FLAG_MAP] if invalid_flags: raise OperationError( f"Invalid flags: {''.join(invalid_flags)}. " - f"Allowed flags: {''.join(sorted(self._FLAG_MAP.keys()))}" + f"Allowed flags: {''.join(sorted(REGEX_FLAG_MAP.keys()))}" ) def _parse_flags(self, flags_str): flags = 0 for ch in flags_str: - flags |= self._FLAG_MAP[ch] + flags |= REGEX_FLAG_MAP[ch] return flags def _compile_pattern(self, find, flags): diff --git a/resources/schema/rule-merged/Operations.json b/resources/schema/rule-merged/Operations.json index 0a2666286..661a36697 100644 --- a/resources/schema/rule-merged/Operations.json +++ b/resources/schema/rule-merged/Operations.json @@ -534,7 +534,8 @@ { "properties": { "operator": { - "const": "regex_find_replace" + "const": "regex_find_replace", + "markdownDescription": "\nFinds and replaces text in a dataset column using regex pattern matching.\n\n**Parameters:**\n- `name` (required): Column name to perform the replacement on\n- `find` or `regex` (required): Regex pattern to search for\n- `replace` (required): Replacement string (can include backreferences like `\\1`, `\\2`)\n- `on_no_match` (optional): Policy when pattern doesn't match. Options: `keep_original` (default), `set_null`, `set_empty`, `error`\n- `flags` (optional): Regex flags as a string: `i` (case-insensitive), `m` (multiline), `s` (dotall)\n\n**Example:**\n\n```yaml\nOperations:\n - id: $normalized_value\n name: PPSTRESC\n operator: regex_find_replace\n find: '(\\d+)\\s*mg'\n replace: '\\1 milligrams'\n on_no_match: keep_original\n flags: i\n" }, "find": { "type": "string" diff --git a/resources/schema/rule/Operations.md b/resources/schema/rule/Operations.md index f5eb48cbd..d8c8fc93f 100644 --- a/resources/schema/rule/Operations.md +++ b/resources/schema/rule/Operations.md @@ -1386,6 +1386,30 @@ Operations: Note that a local XSD file is required for validation. The file must be stored in the folder indicated by the value of the `LOCAL_XSD_FILE_DIR` default file path and the mapping between the namespace and the local XSD file's `sub-folder/name` must be included in the value of the `LOCAL_XSD_FILE_MAP` default file path. +### regex_find_replace + +Finds and replaces text in a dataset column using regex pattern matching. + +**Parameters:** +- `name` (required): Column name to perform the replacement on +- `find` or `regex` (required): Regex pattern to search for +- `replace` (required): Replacement string (can include backreferences like `\1`, `\2`) +- `on_no_match` (optional): Policy when pattern doesn't match. Options: `keep_original` (default), `set_null`, `set_empty`, `error` +- `flags` (optional): Regex flags as a string: `i` (case-insensitive), `m` (multiline), `s` (dotall) + +**Example:** + +```yaml +Operations: + - id: $normalized_value + name: PPSTRESC + operator: regex_find_replace + find: '(\d+)\s*mg' + replace: '\1 milligrams' + on_no_match: keep_original + flags: i +``` + ### split_by Splits a dataset column by a given delimiter From 32bf47ec685b38613da263d6e410067cced63088 Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Fri, 31 Jul 2026 14:40:04 -0400 Subject: [PATCH 06/12] Adjusted to fix script errors --- cdisc_rules_engine/constants/operation_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdisc_rules_engine/constants/operation_constants.py b/cdisc_rules_engine/constants/operation_constants.py index 7a879b492..39de598c0 100644 --- a/cdisc_rules_engine/constants/operation_constants.py +++ b/cdisc_rules_engine/constants/operation_constants.py @@ -6,4 +6,4 @@ "i": re.IGNORECASE, "m": re.MULTILINE, "s": re.DOTALL, -} \ No newline at end of file +} From b4752dbf7fd01d53514e961907418e0e5798bc5c Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Fri, 31 Jul 2026 15:08:14 -0400 Subject: [PATCH 07/12] Fix W292: add missing newline at end of file --- cdisc_rules_engine/constants/operation_constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cdisc_rules_engine/constants/operation_constants.py b/cdisc_rules_engine/constants/operation_constants.py index 39de598c0..5d9abf957 100644 --- a/cdisc_rules_engine/constants/operation_constants.py +++ b/cdisc_rules_engine/constants/operation_constants.py @@ -7,3 +7,4 @@ "m": re.MULTILINE, "s": re.DOTALL, } + From 126caf43eff3bdf8528cb0f3e387329c32ce794f Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Fri, 31 Jul 2026 15:08:57 -0400 Subject: [PATCH 08/12] Added newline --- cdisc_rules_engine/constants/operation_constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cdisc_rules_engine/constants/operation_constants.py b/cdisc_rules_engine/constants/operation_constants.py index 5d9abf957..39de598c0 100644 --- a/cdisc_rules_engine/constants/operation_constants.py +++ b/cdisc_rules_engine/constants/operation_constants.py @@ -7,4 +7,3 @@ "m": re.MULTILINE, "s": re.DOTALL, } - From 28fbaf10e0b6eb6acb19f31b9d0629629a84d797 Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Fri, 31 Jul 2026 15:14:34 -0400 Subject: [PATCH 09/12] Fix W292: re-add missing newline at end of file --- cdisc_rules_engine/constants/operation_constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cdisc_rules_engine/constants/operation_constants.py b/cdisc_rules_engine/constants/operation_constants.py index 39de598c0..5d9abf957 100644 --- a/cdisc_rules_engine/constants/operation_constants.py +++ b/cdisc_rules_engine/constants/operation_constants.py @@ -7,3 +7,4 @@ "m": re.MULTILINE, "s": re.DOTALL, } + From 94e8c2ba605b33d9f552a4183dd08ed9b7201f77 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 3 Aug 2026 12:19:52 +0000 Subject: [PATCH 10/12] Update merged schema files with markdown descriptions --- resources/schema/rule-merged/Operations.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/schema/rule-merged/Operations.json b/resources/schema/rule-merged/Operations.json index 661a36697..4ebe3ae4b 100644 --- a/resources/schema/rule-merged/Operations.json +++ b/resources/schema/rule-merged/Operations.json @@ -535,7 +535,7 @@ "properties": { "operator": { "const": "regex_find_replace", - "markdownDescription": "\nFinds and replaces text in a dataset column using regex pattern matching.\n\n**Parameters:**\n- `name` (required): Column name to perform the replacement on\n- `find` or `regex` (required): Regex pattern to search for\n- `replace` (required): Replacement string (can include backreferences like `\\1`, `\\2`)\n- `on_no_match` (optional): Policy when pattern doesn't match. Options: `keep_original` (default), `set_null`, `set_empty`, `error`\n- `flags` (optional): Regex flags as a string: `i` (case-insensitive), `m` (multiline), `s` (dotall)\n\n**Example:**\n\n```yaml\nOperations:\n - id: $normalized_value\n name: PPSTRESC\n operator: regex_find_replace\n find: '(\\d+)\\s*mg'\n replace: '\\1 milligrams'\n on_no_match: keep_original\n flags: i\n" + "markdownDescription": "\nFinds and replaces text in a dataset column using regex pattern matching.\n\n**Parameters:**\n- `name` (required): Column name to perform the replacement on\n- `find` or `regex` (required): Regex pattern to search for\n- `replace` (required): Replacement string (can include backreferences like `\\1`, `\\2`)\n- `on_no_match` (optional): Policy when pattern doesn't match. Options: `keep_original` (default), `set_null`, `set_empty`, `error`\n- `flags` (optional): Regex flags as a string: `i` (case-insensitive), `m` (multiline), `s` (dotall)\n\n**Example:**\n\n```yaml\nOperations:\n - id: $normalized_value\n name: PPSTRESC\n operator: regex_find_replace\n find: '(\\d+)\\s*mg'\n replace: '\\1 milligrams'\n on_no_match: keep_original\n flags: i\n```\n" }, "find": { "type": "string" From f3ec9285345a70eb63121f5a2619ab5c50db52fb Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Mon, 3 Aug 2026 08:37:43 -0400 Subject: [PATCH 11/12] Fix W391: remove extra blank line at end of file --- cdisc_rules_engine/constants/operation_constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cdisc_rules_engine/constants/operation_constants.py b/cdisc_rules_engine/constants/operation_constants.py index 5d9abf957..39de598c0 100644 --- a/cdisc_rules_engine/constants/operation_constants.py +++ b/cdisc_rules_engine/constants/operation_constants.py @@ -7,4 +7,3 @@ "m": re.MULTILINE, "s": re.DOTALL, } - From e63ac1f9fc8b10c832bf9a8f67d87379e4e52364 Mon Sep 17 00:00:00 2001 From: "Niemes, Adam" Date: Mon, 3 Aug 2026 08:43:55 -0400 Subject: [PATCH 12/12] Apply black formatting to regex_find_replace --- cdisc_rules_engine/operations/regex_find_replace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdisc_rules_engine/operations/regex_find_replace.py b/cdisc_rules_engine/operations/regex_find_replace.py index 57371267e..4fa99b180 100644 --- a/cdisc_rules_engine/operations/regex_find_replace.py +++ b/cdisc_rules_engine/operations/regex_find_replace.py @@ -3,7 +3,10 @@ from cdisc_rules_engine.operations.base_operation import BaseOperation from cdisc_rules_engine.exceptions.custom_exceptions import OperationError -from cdisc_rules_engine.constants.operation_constants import NO_MATCH_POLICIES, REGEX_FLAG_MAP +from cdisc_rules_engine.constants.operation_constants import ( + NO_MATCH_POLICIES, + REGEX_FLAG_MAP, +) class RegexFindReplace(BaseOperation):