diff --git a/README.md b/README.md index 8bc42b805..93f275e89 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Run `python core.py validate --help` to see the list of validation options. -v, --version TEXT Standard version to validate against [required] -ss, --substandard TEXT Substandard to validate against - [required for TIG] + [required for TIG] -ct, --controlled-terminology-package TEXT Controlled terminology package to validate against, can provide more than one @@ -143,6 +143,13 @@ Run `python core.py validate --help` to see the list of validation options. local rule yml and/or json rule files. -cs, --custom-standard Adding this flag tells engine to use a custom standard specified with -s and -v that has been uploaded to the cache using update-cache + -cse, --custom-standard-encoding TEXT + Explicitly specify the file encoding to use + when reading custom standard files (JSON). + If not provided, the engine will attempt to + automatically detect the encoding by trying + common options (utf-8-sig, utf-8, system + default). -vo, --verbose-output Specify this option to print rules as they are completed -p, --progress [verbose_output|disabled|percents|bar] diff --git a/cdisc_rules_engine/services/cache/cache_populator_service.py b/cdisc_rules_engine/services/cache/cache_populator_service.py index a51a060bd..efb15a28a 100644 --- a/cdisc_rules_engine/services/cache/cache_populator_service.py +++ b/cdisc_rules_engine/services/cache/cache_populator_service.py @@ -1,6 +1,5 @@ import asyncio import pickle -import json from functools import partial from typing import Iterable, List, Optional import os @@ -17,6 +16,7 @@ get_library_variables_metadata_cache_key, get_standard_details_cache_key, get_model_details_cache_key, + load_json_with_optional_encoding, ) from scripts.script_utils import load_and_parse_rule from cdisc_rules_engine.constants.cache_constants import PUBLISHED_CT_PACKAGES @@ -32,6 +32,7 @@ def __init__( remove_custom_rules=None, update_custom_rule=None, custom_standards=None, + custom_standards_encoding: str = None, remove_custom_standards=None, cache_path="", ): @@ -42,6 +43,7 @@ def __init__( self.remove_custom_rules = remove_custom_rules self.update_custom_rule = update_custom_rule self.custom_standards = custom_standards + self.custom_standards_encoding = custom_standards_encoding self.remove_custom_standards = remove_custom_standards self.cache_path = cache_path @@ -598,8 +600,9 @@ def add_custom_standard_to_cache(self): """Add or update a custom standard to the cache.""" if not os.path.isfile(self.custom_standards): raise ValueError("Invalid standard filepath") - with open(self.custom_standards, "r") as f: - new_standard = json.load(f) + new_standard = load_json_with_optional_encoding( + self.custom_standards, self.custom_standards_encoding + ) # Validate the input format if not isinstance(new_standard, dict): diff --git a/cdisc_rules_engine/utilities/utils.py b/cdisc_rules_engine/utilities/utils.py index f3e001cee..f3b8906a5 100644 --- a/cdisc_rules_engine/utilities/utils.py +++ b/cdisc_rules_engine/utilities/utils.py @@ -4,6 +4,8 @@ """ import copy +import json +import locale import os import re import ast @@ -481,3 +483,34 @@ def set_max_errors_per_rule(args): per_dataset = bool(env_per_dataset or cli_per_dataset) return max_errors_per_rule, per_dataset + + +def load_json_with_optional_encoding(path: str, encoding: str | None = None) -> dict: + tried = [] + if encoding: + # If the file contains only ASCII characters, incorrect encodings may still succeed + try: + with open(path, "r", encoding=encoding) as f: + return json.load(f) + except (UnicodeDecodeError, json.JSONDecodeError) as e: + tried.append((encoding, e)) + + fallback_encodings = [ + "utf-8-sig", # UTF-8 + BOM + "utf-8", + locale.getpreferredencoding(False), + ] + + for enc in fallback_encodings: + if enc == encoding: + continue + + try: + with open(path, "r", encoding=enc) as f: + return json.load(f) + except (UnicodeDecodeError, json.JSONDecodeError) as e: + tried.append((enc, e)) + + tried_msg = ", ".join(enc for enc, _ in tried) + + raise ValueError(f"Unable to load JSON file '{path}'. Tried encodings: {tried_msg}") diff --git a/core.py b/core.py index 3504abc8b..3d4d663fd 100644 --- a/core.py +++ b/core.py @@ -551,10 +551,15 @@ def validate( "Will update the standard if it already exists." ), ) +@click.option( + "-cse", + "--custom-standard-encoding", + help="Encoding for custom standard details. ", +) @click.option( "-rcs", "--remove-custom-standard", - help=("Removes a custom standard and version from the cache. "), + help="Removes a custom standard and version from the cache. ", multiple=True, ) @click.pass_context @@ -567,6 +572,7 @@ def update_cache( remove_custom_rules: str, update_custom_rule: str, custom_standard: str, + custom_standard_encoding: str, remove_custom_standard: str, ): cache = CacheServiceFactory(config).get_cache_service() @@ -579,6 +585,7 @@ def update_cache( remove_custom_rules, update_custom_rule, custom_standard, + custom_standard_encoding, remove_custom_standard, cache_path, ) diff --git a/tests/unit/test_utilities/test_json_load_with_fallback.py b/tests/unit/test_utilities/test_json_load_with_fallback.py new file mode 100644 index 000000000..184cb1222 --- /dev/null +++ b/tests/unit/test_utilities/test_json_load_with_fallback.py @@ -0,0 +1,83 @@ +import json +import locale +import os +import tempfile +from unittest import mock + +import pytest + +from cdisc_rules_engine.utilities.utils import load_json_with_optional_encoding + + +def test_user_encoding_used_successfully(): + data = {"日本語": [1, 2, 3]} + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "data.json") + + with open(path, "w", encoding="cp932") as f: + json.dump(data, f, ensure_ascii=False) + + result = load_json_with_optional_encoding(path, encoding="cp932") + + assert result == data + + +def test_user_encoding_fails_then_fallback_to_utf8(): + data = {"key": "value"} + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "data.json") + + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f) + + result = load_json_with_optional_encoding(path, encoding="cp932") + + assert result == data + + +def test_no_encoding_utf8_with_bom(): + data = {"STD_BOM": [1]} + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "data.json") + + with open(path, "w", encoding="utf-8-sig") as f: + json.dump(data, f) + + result = load_json_with_optional_encoding(path) + + assert result == data + + +def test_system_encoding_fallback_mocked(): + data = {"日本語": [1, 2]} + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "data.json") + + with open(path, "w", encoding="cp932") as f: + json.dump(data, f, ensure_ascii=False) + + with mock.patch.object( + locale, + "getpreferredencoding", + return_value="cp932", + ): + result = load_json_with_optional_encoding(path) + + assert result == data + + +def test_all_encodings_fail(): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "broken.json") + + with open(path, "wb") as f: + f.write(b"\xff\xfe\xfa\xfb") + + with pytest.raises(ValueError) as exc: + load_json_with_optional_encoding(path, encoding="utf-8") + + assert "Unable to load JSON file" in str(exc.value)