diff --git a/cdisc_rules_engine/check_operators/dataframe_operators.py b/cdisc_rules_engine/check_operators/dataframe_operators.py index 151ab41a7..1cf37ede4 100644 --- a/cdisc_rules_engine/check_operators/dataframe_operators.py +++ b/cdisc_rules_engine/check_operators/dataframe_operators.py @@ -337,6 +337,7 @@ def equal_to_case_insensitive(self, other_value): round_values=round_values, ), axis=1, + meta=(None, "bool"), ) @log_operator_execution diff --git a/cdisc_rules_engine/enums/execution_status.py b/cdisc_rules_engine/enums/execution_status.py index d1b6b58bb..ab8c1740e 100644 --- a/cdisc_rules_engine/enums/execution_status.py +++ b/cdisc_rules_engine/enums/execution_status.py @@ -5,3 +5,16 @@ class ExecutionStatus(BaseEnum): SUCCESS = "success" SKIPPED = "skipped" EXECUTION_ERROR = "execution_error" + ISSUE_REPORTED = "issue_reported" + UNKNOWN_STATUS = "unknown_status" + + +class SkippedReason(BaseEnum): + COLUMN_NOT_FOUND_IN_DATA = "Column not found in data" + DOMAIN_NOT_FOUND = "Domain not found" + SCHEMA_VALIDATION_IS_OFF = "Schema validation is off" + OUTSIDE_SCOPE = "Outside scope" + + +class ExecutionError(BaseEnum): + AN_UNKNOWN_EXCEPTION_HAS_OCCURRED = "An unknown exception has occurred" diff --git a/cdisc_rules_engine/exceptions/custom_exceptions.py b/cdisc_rules_engine/exceptions/custom_exceptions.py index 94da31a67..e3e5b79d5 100644 --- a/cdisc_rules_engine/exceptions/custom_exceptions.py +++ b/cdisc_rules_engine/exceptions/custom_exceptions.py @@ -100,3 +100,8 @@ class OperationError(EngineError): class DatasetBuilderError(EngineError): description = "Error occurred during dataset building" + + +class DateTimeParserError(EngineError): + code = 400 + description = "Failure to parse a datetime string" diff --git a/cdisc_rules_engine/rules_engine.py b/cdisc_rules_engine/rules_engine.py index 1c50f3493..9df700be1 100644 --- a/cdisc_rules_engine/rules_engine.py +++ b/cdisc_rules_engine/rules_engine.py @@ -3,12 +3,16 @@ from dateutil.parser._parser import ParserError from business_rules import export_rule_data from business_rules.engine import run -import os from cdisc_rules_engine.config import config as default_config -from cdisc_rules_engine.enums.execution_status import ExecutionStatus +from cdisc_rules_engine.enums.execution_status import ( + ExecutionError, + ExecutionStatus, + SkippedReason, +) from cdisc_rules_engine.enums.rule_types import RuleTypes from cdisc_rules_engine.exceptions.custom_exceptions import ( DatasetNotFoundError, + DateTimeParserError, DomainNotFoundInDefineXMLError, InvalidJSONFormat, RuleFormatError, @@ -245,13 +249,22 @@ def validate_single_dataset( logger.info( f"Skipped dataset {dataset_metadata.name}. Reason: {reason}" ) - error_obj = ValidationErrorContainer( - status=ExecutionStatus.SKIPPED.value, - message=reason, + error_obj = FailedValidationEntity( dataset=dataset_metadata.filename, - domain=dataset_metadata.domain or dataset_metadata.rdomain or "", + error=SkippedReason.OUTSIDE_SCOPE.value, + message=reason, ) - return [error_obj.to_representation()] + return [ + ValidationErrorContainer( + status=ExecutionStatus.SKIPPED.value, + message=reason, + dataset=dataset_metadata.filename, + domain=dataset_metadata.domain + or dataset_metadata.rdomain + or "", + errors=[error_obj], + ).to_representation() + ] except Exception as e: logger.trace(e) logger.error( @@ -265,7 +278,7 @@ def validate_single_dataset( """ ) error_obj: ValidationErrorContainer = self.handle_validation_exceptions( - e, dataset_metadata.full_path, dataset_metadata.full_path + e, dataset_metadata.filename ) error_obj.domain = dataset_metadata.domain or dataset_metadata.rdomain or "" # this wrapping into a list is necessary to keep return type consistent @@ -462,120 +475,134 @@ def get_define_xml_value_level_metadata( return define_xml_reader.extract_value_level_metadata(domain_name=domain_name) def handle_validation_exceptions( # noqa - self, exception, dataset_path, file_name + self, exception, filename: str ) -> ValidationErrorContainer: if isinstance(exception, DatasetNotFoundError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error="Dataset Not Found", message=exception.message, ) message = "rule execution error" elif isinstance(exception, RuleFormatError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error="Rule format error", message=exception.message, ) message = "rule execution error" elif isinstance(exception, AssertionError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error="Rule format error", message="Rule contains invalid operator", ) message = "rule execution error" - elif isinstance(exception, (KeyError, ParserError)): + elif isinstance(exception, KeyError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), - error="Column not found in data", + dataset=filename, + error=SkippedReason.COLUMN_NOT_FOUND_IN_DATA.value, + message=exception.args[0], + ) + message = "rule execution error" + errors = [error_obj] + return ValidationErrorContainer( + dataset=filename, + errors=errors, + message=message, + status=ExecutionStatus.SKIPPED.value, + ) + elif isinstance(exception, ParserError): + error_obj = FailedValidationEntity( + dataset=filename, + error=DateTimeParserError.description, message=exception.args[0], ) message = "rule execution error" elif isinstance(exception, DomainNotFoundInDefineXMLError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error=DomainNotFoundInDefineXMLError.description, message=exception.args[0], ) message = "rule execution error" elif isinstance(exception, SchemaNotFoundError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error=SchemaNotFoundError.description, message=exception.args[0], ) message = "rule execution error" elif isinstance(exception, InvalidSchemaProvidedError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error=InvalidSchemaProvidedError.description, message=exception.args[0], ) message = "rule execution error" elif isinstance(exception, VariableMetadataNotFoundError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error=VariableMetadataNotFoundError.description, message=exception.args[0], ) message = "rule execution error" elif isinstance(exception, InvalidJSONFormat): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error=InvalidJSONFormat.description, message=exception.args[0], ) message = "rule execution error" elif isinstance(exception, PreprocessingError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error=PreprocessingError.description, message=str(exception), ) message = "rule evaluation error - preprocessing failed" errors = [error_obj] return ValidationErrorContainer( - dataset=os.path.basename(dataset_path), + dataset=filename, errors=errors, message=message, - status=ExecutionStatus.SKIPPED.value, + status=ExecutionStatus.EXECUTION_ERROR.value, ) elif isinstance(exception, OperationError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error=OperationError.description, message=str(exception), ) message = "rule evaluation error - operation failed" errors = [error_obj] return ValidationErrorContainer( - dataset=os.path.basename(dataset_path), + dataset=filename, errors=errors, message=message, - status=ExecutionStatus.SKIPPED.value, + status=ExecutionStatus.EXECUTION_ERROR.value, ) elif isinstance(exception, DatasetBuilderError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error=DatasetBuilderError.description, message=str(exception), ) message = "rule evaluation error - evaluation dataset failed to build" errors = [error_obj] return ValidationErrorContainer( - dataset=os.path.basename(dataset_path), + dataset=filename, errors=errors, message=message, - status=ExecutionStatus.SKIPPED.value, + status=ExecutionStatus.EXECUTION_ERROR.value, ) elif isinstance(exception, FailedSchemaValidation): if self.validate_xml: error_obj = FailedValidationEntity( error=FailedSchemaValidation.description, message=exception.args[0], - dataset=os.path.basename(dataset_path), + dataset=filename, ) message = "Schema Validation Error" errors = [error_obj] @@ -583,32 +610,32 @@ def handle_validation_exceptions( # noqa errors=errors, message=message, status=ExecutionStatus.SUCCESS.value, - dataset=os.path.basename(dataset_path), + dataset=filename, ) else: message = "Skipped because schema validation is off" error_obj = FailedValidationEntity( - error="Schema validation is off", + error=SkippedReason.SCHEMA_VALIDATION_IS_OFF.value, message=message, - dataset=os.path.basename(dataset_path), + dataset=filename, ) errors = [error_obj] return ValidationErrorContainer( - dataset=os.path.basename(dataset_path), + dataset=filename, errors=errors, message=message, status=ExecutionStatus.SKIPPED.value, ) elif isinstance(exception, DomainNotFoundError): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), - error="Domain not found", + dataset=filename, + error=SkippedReason.DOMAIN_NOT_FOUND.value, message=str(exception), ) message = "rule evaluation skipped - operation domain not found" errors = [error_obj] return ValidationErrorContainer( - dataset=os.path.basename(dataset_path), + dataset=filename, errors=errors, message=message, status=ExecutionStatus.SKIPPED.value, @@ -617,28 +644,28 @@ def handle_validation_exceptions( # noqa exception, AttributeError ) and "'NoneType' object has no attribute" in str(exception): error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), + dataset=filename, error="Missing field during execution", message="Missing field during execution, rule may not be applicable- unable to process dataset", ) message = "rule evaluation skipped - missing metadata" errors = [error_obj] return ValidationErrorContainer( - dataset=os.path.basename(dataset_path), + dataset=filename, errors=errors, message=message, - status=ExecutionStatus.SKIPPED.value, + status=ExecutionStatus.EXECUTION_ERROR.value, ) else: error_obj = FailedValidationEntity( - dataset=os.path.basename(dataset_path), - error="An unknown exception has occurred", + dataset=filename, + error=ExecutionError.AN_UNKNOWN_EXCEPTION_HAS_OCCURRED.value, message=str(exception), ) message = "rule execution error" errors = [error_obj] return ValidationErrorContainer( - dataset=os.path.basename(dataset_path), + dataset=filename, errors=errors, message=message, status=ExecutionStatus.EXECUTION_ERROR.value, diff --git a/cdisc_rules_engine/services/reporting/sdtm_report_data.py b/cdisc_rules_engine/services/reporting/sdtm_report_data.py index a6af212f8..b589f98cf 100644 --- a/cdisc_rules_engine/services/reporting/sdtm_report_data.py +++ b/cdisc_rules_engine/services/reporting/sdtm_report_data.py @@ -217,39 +217,106 @@ def get_summary_data(self) -> list[dict]: """ summary_data = [] for validation_result in self._results: - if validation_result.execution_status == "success": - for result in validation_result.results or []: - dataset = result.get("dataset") - if ( - result.get("errors") - and result.get("executionStatus") == "success" - ): - summary_item = { - "dataset": dataset, - "core_id": validation_result.id, - "message": result.get("message"), - "issues": len(result.get("errors")), - } - summary_data.append(summary_item) + for result in validation_result.results or []: + dataset = result.get("dataset") + if ( + result.get("errors") + and result.get("executionStatus") != ExecutionStatus.SKIPPED.value + ): + summary_item = { + "dataset": dataset, + "core_id": validation_result.id, + "message": result.get("message"), + "issues": len(result.get("errors")), + } + summary_data.append(summary_item) return sorted( summary_data, key=lambda x: (x["dataset"], x["core_id"]), ) - def get_detailed_data(self, excel=False) -> list[dict]: + def get_detailed_data(self) -> list[dict]: detailed_data = [] for validation_result in self._results: detailed_data = detailed_data + self._generate_error_details( - validation_result, excel + validation_result ) return sorted( detailed_data, key=lambda x: (x["core_id"], x["dataset"]), ) + def _issue_details( + self, validation_result: RuleValidationResult, result: dict + ) -> list[dict]: + """ + Generates the Issue details data that goes into the excel export. + Each row is represented by a list or a dict containing the following + information: + return [ + "CORE-ID", + "Message", + "Executability", + "Dataset Name" + "USUBJID", + "Record", + "Sequence", + "Variable(s)", + "Value(s)" + ] + """ + errors = [] + variables = result.get("variables", []) + for error in [ + error + for error in result.get("errors") + if result.get("executionStatus") == ExecutionStatus.ISSUE_REPORTED.value + ]: + values = [] + for variable in variables: + raw_value = error.get("value", {}).get(variable) + if raw_value is None: + values.append(None) + else: + values.append(str(raw_value)) + error_item = { + "core_id": validation_result.id, + "message": result.get("message"), + "executability": validation_result.executability, + "dataset": error.get("dataset"), + "USUBJID": error.get("USUBJID", ""), + "row": error.get("row", ""), + "SEQ": error.get("SEQ", ""), + "variables": variables, + "values": self.process_values(values), + } + errors.append(error_item) + return errors + + def _error_details(self, validation_result: RuleValidationResult, result: dict): + errors = [] + for error in [ + error + for error in result.get("errors") + if result.get("executionStatus") == ExecutionStatus.EXECUTION_ERROR.value + ]: + error_item = { + "core_id": validation_result.id, + "message": (f"{result.get('message')} - {error.get('error')}"), + "executability": validation_result.executability, + "dataset": error.get("dataset"), + "USUBJID": "", + "row": "", + "SEQ": "", + "variables": "", + "values": error.get("message"), + } + errors.append(error_item) + return errors + def _generate_error_details( - self, validation_result: RuleValidationResult, excel + self, validation_result: RuleValidationResult ) -> list[dict]: """ Generates the Issue details data that goes into the excel export. @@ -269,28 +336,11 @@ def _generate_error_details( """ errors = [] for result in validation_result.results or []: - if result.get("errors", []) and result.get("executionStatus") == "success": - variables = result.get("variables", []) - for error in result.get("errors"): - values = [] - for variable in variables: - raw_value = error.get("value", {}).get(variable) - if raw_value is None: - values.append(None) - else: - values.append(str(raw_value)) - error_item = { - "core_id": validation_result.id, - "message": result.get("message"), - "executability": validation_result.executability, - "dataset": error.get("dataset"), - "USUBJID": error.get("USUBJID", ""), - "row": error.get("row", ""), - "SEQ": error.get("SEQ", ""), - "variables": variables, - "values": self.process_values(values), - } - errors.append(error_item) + errors = ( + errors + + self._issue_details(validation_result, result) + + self._error_details(validation_result, result) + ) return errors def get_rules_report_data(self) -> list[dict]: @@ -315,12 +365,9 @@ def get_rules_report_data(self) -> list[dict]: "cdisc_rule_id": validation_result.cdisc_rule_id, "fda_rule_id": validation_result.fda_rule_id, "message": validation_result.message, - "status": ( - ExecutionStatus.SUCCESS.value.upper() - if validation_result.execution_status - == ExecutionStatus.SUCCESS.value - else ExecutionStatus.SKIPPED.value.upper() - ), + "status": ExecutionStatus( + validation_result.execution_status + ).value.upper(), } rules_report.append(rules_item) return sorted( diff --git a/cdisc_rules_engine/services/reporting/usdm_report_data.py b/cdisc_rules_engine/services/reporting/usdm_report_data.py index 41cb5c21a..01f0fbb28 100644 --- a/cdisc_rules_engine/services/reporting/usdm_report_data.py +++ b/cdisc_rules_engine/services/reporting/usdm_report_data.py @@ -119,40 +119,109 @@ def get_summary_data(self) -> list[dict]: """ summary_data = [] for validation_result in self._results: - if validation_result.execution_status == "success": - for result in validation_result.results or []: - if ( - result.get("errors") - and result.get("executionStatus") == "success" - ): - summary_item = { - "entity": result.get("entity") - or (result.get("dataset", "") or "").replace(".json", ""), - "core_id": validation_result.id, - "cdisc_rule_id": validation_result.cdisc_rule_id, - "message": result.get("message"), - "issues": len(result.get("errors")), - } - summary_data.append(summary_item) + for result in validation_result.results or []: + if ( + result.get("errors") + and result.get("executionStatus") != ExecutionStatus.SKIPPED.value + ): + summary_item = { + "entity": result.get("entity") + or (result.get("dataset", "") or "").replace(".json", ""), + "core_id": validation_result.id, + "cdisc_rule_id": validation_result.cdisc_rule_id, + "message": result.get("message"), + "issues": len(result.get("errors")), + } + summary_data.append(summary_item) return sorted( summary_data, key=lambda x: (x.get("entity"), x["core_id"]), ) - def get_detailed_data(self, excel=False) -> list[dict]: + def get_detailed_data(self) -> list[dict]: detailed_data = [] for validation_result in self._results: detailed_data = detailed_data + self._generate_error_details( - validation_result, excel + validation_result ) return sorted( detailed_data, key=lambda x: (x["core_id"], x.get("entity")), ) + def _issue_details( + self, validation_result: RuleValidationResult, result: dict + ) -> list[dict]: + """ + Generates the Issue details data that goes into the excel export. + Each row is represented by a list or a dict containing the following + information: + return [ + "CORE-ID", + "Message", + "Executability", + "Dataset Name" + "USUBJID", + "Record", + "Sequence", + "Variable(s)", + "Value(s)" + ] + """ + errors = [] + variables = result.get("variables", []) + for error in [ + error + for error in result.get("errors") + if result.get("executionStatus") == ExecutionStatus.ISSUE_REPORTED.value + ]: + values = [] + for variable in variables: + raw_value = error.get("value", {}).get(variable) + if raw_value is None: + values.append(None) + else: + values.append(str(raw_value)) + error_item = { + "core_id": validation_result.id, + "cdisc_rule_id": validation_result.cdisc_rule_id, + "message": result.get("message"), + "executability": validation_result.executability, + "entity": error.get("entity") + or error.get("dataset", "").replace(".json", ""), + "instance_id": error.get("instance_id"), + "path": error.get("path"), + "attributes": variables, + "values": self.process_values(values), + } + errors.append(error_item) + return errors + + def _error_details(self, validation_result: RuleValidationResult, result: dict): + errors = [] + for error in [ + error + for error in result.get("errors") + if result.get("executionStatus") == ExecutionStatus.EXECUTION_ERROR.value + ]: + error_item = { + "core_id": validation_result.id, + "cdisc_rule_id": validation_result.cdisc_rule_id, + "message": (f"{result.get('message')} - {error.get('error')}"), + "executability": validation_result.executability, + "entity": error.get("entity") + or error.get("dataset", "").replace(".json", ""), + "instance_id": "", + "path": "", + "attributes": "", + "values": error.get("message"), + } + errors.append(error_item) + return errors + def _generate_error_details( - self, validation_result: RuleValidationResult, excel + self, validation_result: RuleValidationResult ) -> list[dict]: """ Generates the Issue details data that goes into the excel export. @@ -172,29 +241,11 @@ def _generate_error_details( """ errors = [] for result in validation_result.results or []: - if result.get("errors", []) and result.get("executionStatus") == "success": - variables = result.get("variables", []) - for error in result.get("errors"): - values = [] - for variable in variables: - raw_value = error.get("value", {}).get(variable) - if raw_value is None: - values.append(None) - else: - values.append(str(raw_value)) - error_item = { - "core_id": validation_result.id, - "cdisc_rule_id": validation_result.cdisc_rule_id, - "message": result.get("message"), - "executability": validation_result.executability, - "entity": error.get("entity") - or error.get("dataset", "").replace(".json", ""), - "instance_id": error.get("instance_id"), - "path": error.get("path"), - "attributes": variables, - "values": self.process_values(values), - } - errors.append(error_item) + errors = ( + errors + + self._issue_details(validation_result, result) + + self._error_details(validation_result, result) + ) return errors def get_rules_report_data(self) -> list[dict]: @@ -217,12 +268,9 @@ def get_rules_report_data(self) -> list[dict]: "version": "1", "cdisc_rule_id": validation_result.cdisc_rule_id, "message": validation_result.message, - "status": ( - ExecutionStatus.SUCCESS.value.upper() - if validation_result.execution_status - == ExecutionStatus.SUCCESS.value - else ExecutionStatus.SKIPPED.value.upper() - ), + "status": ExecutionStatus( + validation_result.execution_status + ).value.upper(), } rules_report.append(rules_item) diff --git a/cdisc_rules_engine/utilities/dataset_preprocessor.py b/cdisc_rules_engine/utilities/dataset_preprocessor.py index 0b59eb229..7c55a191b 100644 --- a/cdisc_rules_engine/utilities/dataset_preprocessor.py +++ b/cdisc_rules_engine/utilities/dataset_preprocessor.py @@ -585,6 +585,9 @@ def _merge_datasets( # noqa right_dataset_domain_details.get("join_type", "inner") ), ) + except KeyError as e: + # Handle with COLUMN_NOT_FOUND_IN_DATA in rules_engine + raise e except Exception as e: raise PreprocessingError( f"Failed to merge datasets. " diff --git a/cdisc_rules_engine/utilities/utils.py b/cdisc_rules_engine/utilities/utils.py index 2b4a47163..41d79a317 100644 --- a/cdisc_rules_engine/utilities/utils.py +++ b/cdisc_rules_engine/utilities/utils.py @@ -50,25 +50,62 @@ def convert_file_size(size_in_bytes: int, desired_unit: str) -> float: def get_execution_status(results): """ - If all results have skipped status, return skipped. - Else return success + If any result has an execution error, return execution error + Else if any result has an issue reported, return issue reported + Else if any result is successful, return issue successful + Else, result should have all skips, return issue skipped """ if len(results) == 0: return ExecutionStatus.SUCCESS.value - if isinstance(results[0], BaseValidationEntity): - successful_results = [ - entity for entity in results if entity.status == ExecutionStatus.SUCCESS - ] - else: - successful_results = [ - result - for result in results - if result.get("executionStatus") == ExecutionStatus.SUCCESS.value - ] - if successful_results: + status = ( + { + ExecutionStatus.SUCCESS: [], + ExecutionStatus.EXECUTION_ERROR: [], + ExecutionStatus.ISSUE_REPORTED: results, + ExecutionStatus.SKIPPED: [], + } + if isinstance(results[0], BaseValidationEntity) + else { + ExecutionStatus.SUCCESS: [ + result + for result in results + if result.get("executionStatus") == ExecutionStatus.SUCCESS.value + ], + ExecutionStatus.EXECUTION_ERROR: [ + result + for result in results + if result.get("executionStatus") + == ExecutionStatus.EXECUTION_ERROR.value + ], + ExecutionStatus.ISSUE_REPORTED: [ + result + for result in results + if result.get("executionStatus") == ExecutionStatus.ISSUE_REPORTED.value + ], + ExecutionStatus.SKIPPED: [ + result + for result in results + if result.get("executionStatus") == ExecutionStatus.SKIPPED.value + ], + } + ) + if len(results) != ( + len(status[ExecutionStatus.SUCCESS]) + + len(status[ExecutionStatus.EXECUTION_ERROR]) + + len(status[ExecutionStatus.ISSUE_REPORTED]) + + len(status[ExecutionStatus.SKIPPED]) + ): + return ExecutionStatus.UNKNOWN_STATUS.value + elif status[ExecutionStatus.EXECUTION_ERROR]: + return ExecutionStatus.EXECUTION_ERROR.value + elif status[ExecutionStatus.ISSUE_REPORTED]: + return ExecutionStatus.ISSUE_REPORTED.value + elif status[ExecutionStatus.SUCCESS]: return ExecutionStatus.SUCCESS.value - else: + elif status[ExecutionStatus.SKIPPED]: return ExecutionStatus.SKIPPED.value + else: + return ExecutionStatus.UNKNOWN_STATUS.value def get_standard_codelist_cache_key(standard: str, version: str) -> str: diff --git a/resources/templates/report-template.xlsx b/resources/templates/report-template.xlsx index e2e47a628..41b3eb81a 100644 Binary files a/resources/templates/report-template.xlsx and b/resources/templates/report-template.xlsx differ diff --git a/resources/templates/usdm-report-template.xlsx b/resources/templates/usdm-report-template.xlsx index d2b273321..998b8f8bd 100644 Binary files a/resources/templates/usdm-report-template.xlsx and b/resources/templates/usdm-report-template.xlsx differ diff --git a/tests/conftest.py b/tests/conftest.py index 5c6e66f06..5563eb1a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -984,7 +984,7 @@ def mock_validation_results() -> list[RuleValidationResult]: { "domain": "AE", "variables": ["AESTDY", "DOMAIN"], - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "errors": [ { "row": 1, @@ -1043,7 +1043,7 @@ def mock_validation_results() -> list[RuleValidationResult]: { "domain": "TT", "variables": ["TTVAR1", "TTVAR2"], - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "errors": [ { "row": 1, diff --git a/tests/unit/test_rule_tester/test_rule_tester.py b/tests/unit/test_rule_tester/test_rule_tester.py index c2e46b6d5..de381880f 100644 --- a/tests/unit/test_rule_tester/test_rule_tester.py +++ b/tests/unit/test_rule_tester/test_rule_tester.py @@ -201,7 +201,7 @@ def test_rule_skipped(): data = run_single_rule_validation(datasets, rule) assert "LB" in data assert len(data["LB"]) == 1 - assert len(data["LB"][0]["errors"]) == 0 + assert len(data["LB"][0]["errors"]) == 1 assert data["LB"][0]["executionStatus"] == "skipped" diff --git a/tests/unit/test_rules_engine.py b/tests/unit/test_rules_engine.py index c19c209d8..ffae7f45c 100644 --- a/tests/unit/test_rules_engine.py +++ b/tests/unit/test_rules_engine.py @@ -101,7 +101,7 @@ def test_validate_rule_invalid_suffix( ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "bundle", "domain": "AE", "variables": ["AESTDY"], @@ -149,7 +149,7 @@ def test_validate_rule_invalid_prefix( ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "bundle", "domain": "AE", "variables": ["AESTDY"], @@ -261,7 +261,7 @@ def test_validate_rule_cross_dataset_check( ).validate_single_dataset(dataset_rule_equal_to, datasets, datasets[0]) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "ec.xpt", "domain": "EC", "variables": ["ECSTDY"], @@ -366,7 +366,7 @@ def test_validate_one_to_one_rel_across_datasets(dataset_rule_one_to_one_related ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "ec.xpt", "domain": "EC", "variables": ["VISITNUM"], @@ -412,7 +412,7 @@ def test_validate_rule_single_dataset_check(dataset_rule_greater_than: dict): ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "domain": "EC", "dataset": "bundle", "variables": ["ECCOOLVAR"], @@ -458,7 +458,7 @@ def test_validate_rule_equal_length(dataset_rule_has_equal_length: dict): ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "domain": "EC", "dataset": "bundle", "variables": ["ECCOOLVAR"], @@ -508,7 +508,7 @@ def test_validate_is_contained_by_distinct(mock_rule_distinct_operation: dict): ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "ae.xpt", "domain": "AE", "variables": ["AESTDY"], @@ -551,7 +551,7 @@ def test_validate_rule_not_equal_length(dataset_rule_has_not_equal_length: dict) ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "domain": "EC", "dataset": "bundle", "variables": ["ECCOOLVAR"], @@ -594,7 +594,7 @@ def test_validate_rule_multiple_conditions(dataset_rule_multiple_conditions: dic ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "domain": "EC", "dataset": "bundle", "variables": ["ECCOOLVAR"], @@ -636,7 +636,7 @@ def test_validate_record_rule_numbers_separated_by_dash_pattern(): ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "bundle", "domain": "AE", "variables": ["AESTDY"], @@ -676,7 +676,7 @@ def test_validate_record_rule_semi_colon_delimited_pattern(): ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "domain": "AE", "dataset": "bundle", "variables": ["AESTDY"], @@ -718,7 +718,7 @@ def test_validate_record_rule_no_letters_numbers_underscores(): ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "bundle", "domain": "AE", "variables": ["AESTDY"], @@ -774,7 +774,7 @@ def test_validate_dataset_metadata( "domain": "EC", "dataset": "bundle", "errors": [], - "executionStatus": "success", + "executionStatus": ExecutionStatus.SUCCESS.value, "message": None, "variables": [], } @@ -825,7 +825,7 @@ def test_validate_dataset_metadata_wrong_metadata( { "domain": "EC", "dataset": "bundle", - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "variables": ["dataset_label", "dataset_name", "dataset_size"], "errors": [ { @@ -881,7 +881,7 @@ def test_validate_variable_metadata( "domain": "EC", "dataset": "bundle", "errors": [], - "executionStatus": "success", + "executionStatus": ExecutionStatus.SUCCESS.value, "message": None, "variables": [], } @@ -904,7 +904,7 @@ def test_validate_variable_metadata( "domain": "EC", "dataset": "bundle", "errors": [], - "executionStatus": "success", + "executionStatus": ExecutionStatus.SUCCESS.value, "message": None, "variables": [], } @@ -953,7 +953,7 @@ def test_validate_variable_metadata_wrong_metadata( "domain": "EC", "dataset": "bundle", "variables": ["variable_name", "variable_label", "variable_data_type"], - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "errors": [ { "dataset": "bundle", @@ -1022,7 +1022,7 @@ def test_rule_with_domain_prefix_replacement(mock_get_dataset: MagicMock): ).validate_single_dataset(rule, [dataset_metadata], dataset_metadata) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "bundle", "domain": "AE", "variables": ["AESTDY"], @@ -1048,7 +1048,7 @@ def test_rule_with_domain_prefix_replacement(mock_get_dataset: MagicMock): ], [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "bundle", "domain": "AE", "variables": ["AE"], @@ -1066,7 +1066,7 @@ def test_rule_with_domain_prefix_replacement(mock_get_dataset: MagicMock): "domain": "AE", "dataset": "bundle", "errors": [], - "executionStatus": "success", + "executionStatus": ExecutionStatus.SUCCESS.value, "message": None, "variables": [], } @@ -1150,7 +1150,7 @@ def test_validate_single_dataset(dataset_rule_equal_to_error_objects: dict): { "domain": "AE", "dataset": "bundle", - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "variables": ["AESTDY"], "errors": [ { @@ -1235,7 +1235,7 @@ def test_validate_single_dataset_not_equal_to( { "domain": "AE", "dataset": "data_bundle", - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "variables": ["AESTDY"], "errors": [ { @@ -1286,7 +1286,7 @@ def test_validate_single_dataset_not_equal_to( ), [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "ae.xpt", "domain": "AE", "variables": ["dataset_label", "dataset_name", "dataset_location"], @@ -1327,7 +1327,7 @@ def test_validate_single_dataset_not_equal_to( "domain": "AE", "dataset": "ae.xpt", "errors": [], - "executionStatus": "success", + "executionStatus": ExecutionStatus.SUCCESS.value, "message": None, "variables": [], } @@ -1408,7 +1408,7 @@ def test_validate_dataset_metadata_against_define_xml( { "domain": "AE", "dataset": "test", - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "variables": ["variable_size"], "errors": [ {"dataset": "test", "row": 1, "value": {"variable_size": 30}} @@ -1451,7 +1451,7 @@ def test_validate_dataset_metadata_against_define_xml( { "domain": "AE", "dataset": "test", - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "variables": ["variable_size"], "errors": [ {"dataset": "test", "row": 1, "value": {"variable_size": 30}} @@ -1560,7 +1560,7 @@ def filter_func(row): { "domain": "AE", "dataset": "test", - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "variables": [ "AETERM", ], @@ -1601,7 +1601,13 @@ def filter_func(row): "executionStatus": ExecutionStatus.SKIPPED.value, "variables": [], "message": "Rule skipped - doesn't apply to domain for rule id=MockRule, dataset=", - "errors": [], + "errors": [ + { + "dataset": "ae_2.xpt", + "error": "Outside scope", + "message": "Rule skipped - doesn't apply to domain for rule id=MockRule, dataset=", + } + ], } ], ), @@ -1612,7 +1618,7 @@ def filter_func(row): { "domain": "AE", "dataset": "ae_1.xpt, ae_2.xpt", - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "variables": ["AESTDY"], "errors": [ { @@ -1810,7 +1816,7 @@ def test_validate_split_dataset_metadata( { "domain": "EC", "dataset": "ec_1.xpt", - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "errors": [ { "dataset": "ec_1.xpt", @@ -1886,7 +1892,7 @@ def test_validate_split_dataset_variables_metadata( { "domain": "EC", "dataset": "ec_2.xpt", - "executionStatus": ExecutionStatus.SUCCESS.value, + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "variables": ["variable_name", "variable_label", "variable_data_type"], "errors": [ { @@ -2008,7 +2014,7 @@ def test_validate_record_in_parent_domain( ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "domain": "EC", "dataset": "ec.xpt", "variables": ["ECREASOC", "ECPRESP"], @@ -2073,7 +2079,7 @@ def test_validate_additional_columns( ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "ts.xpt", "domain": "TS", "variables": ["TSVAL"], @@ -2183,7 +2189,7 @@ def test_validate_dataset_contents_against_define_and_library_variable_metadata( ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "filename", "domain": "AE", "variables": [ @@ -2297,7 +2303,7 @@ def test_validate_single_dataset_operation_dataset_larger_than_target_dataset( ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.SUCCESS.value, "dataset": "ie.xpt", "domain": "IE", "variables": [], @@ -2372,7 +2378,7 @@ def test_validate_extract_metadata_operation( ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "suppec.xpt", "domain": "SUPPEC", "variables": [ @@ -2464,7 +2470,7 @@ def test_dataset_references_invalid_whodrug_terms( ) assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "domain": "AE", "dataset": "dataset_path", "variables": [ @@ -2680,7 +2686,7 @@ def mock_cached_method(*args, **kwargs): ) assert result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "dataset_path", "domain": "AE", "variables": [ diff --git a/tests/unit/test_services/test_reporting/test_sdtm_report.py b/tests/unit/test_services/test_reporting/test_sdtm_report.py index 95a9beddb..2191af35d 100644 --- a/tests/unit/test_services/test_reporting/test_sdtm_report.py +++ b/tests/unit/test_services/test_reporting/test_sdtm_report.py @@ -21,7 +21,7 @@ def test_get_rules_report_data(mock_validation_results): "cdisc_rule_id": result.cdisc_rule_id, "fda_rule_id": result.fda_rule_id, "message": result.message, - "status": ExecutionStatus.SUCCESS.value.upper(), + "status": ExecutionStatus.ISSUE_REPORTED.value.upper(), } ) expected_reports = sorted(expected_reports, key=lambda x: x["core_id"]) diff --git a/tests/unit/test_services/test_reporting/test_usdm_report.py b/tests/unit/test_services/test_reporting/test_usdm_report.py index 22a4642d2..ab804df7d 100644 --- a/tests/unit/test_services/test_reporting/test_usdm_report.py +++ b/tests/unit/test_services/test_reporting/test_usdm_report.py @@ -20,7 +20,7 @@ def test_get_rules_report_data(mock_validation_results): "version": "1", "cdisc_rule_id": result.cdisc_rule_id, "message": result.message, - "status": ExecutionStatus.SUCCESS.value.upper(), + "status": ExecutionStatus.ISSUE_REPORTED.value.upper(), } ) expected_reports = sorted(expected_reports, key=lambda x: x["core_id"]) diff --git a/tests/unit/test_usdm_data.py b/tests/unit/test_usdm_data.py index c2bdfa426..85d9eff80 100644 --- a/tests/unit/test_usdm_data.py +++ b/tests/unit/test_usdm_data.py @@ -1,5 +1,6 @@ from typing import List from unittest.mock import patch, MagicMock +from cdisc_rules_engine.enums.execution_status import ExecutionStatus from cdisc_rules_engine.models.sdtm_dataset_metadata import SDTMDatasetMetadata from cdisc_rules_engine.models.dataset.pandas_dataset import PandasDataset from cdisc_rules_engine.rules_engine import RulesEngine @@ -107,7 +108,7 @@ def test_validate_rule_single_dataset_check(dataset_rule_greater_than: dict): assert validation_result == [ { - "executionStatus": "success", + "executionStatus": ExecutionStatus.ISSUE_REPORTED.value, "dataset": "USDM_EliLilly_NCT03421379_Diabetes.json", "domain": "EC", "variables": ["ECCOOLVAR"],