1174: json load fallback logic. update-cache encoding parameter.

README.md

@@ -72,7 +72,7 @@ Run `python core.py validate --help` to see the list of validation options.
   -v, --version TEXT              Standard version to validate against
                                   [required]
   -ss, --substandard TEXT         Substandard to validate against
-                                  [required for TIG]  
+                                  [required for TIG]
   -ct, --controlled-terminology-package TEXT
                                   Controlled terminology package to validate
                                   against, can provide more than one
@@ -143,6 +143,13 @@ Run `python core.py validate --help` to see the list of validation options.
                                   local rule yml and/or json rule files.
   -cs, --custom-standard       Adding this flag tells engine to use a custom standard specified with -s and -v
                                   that has been uploaded to the cache using update-cache
+  -cse, --custom-standard-encoding TEXT
+                                  Explicitly specify the file encoding to use
+                                  when reading custom standard files (JSON).
+                                  If not provided, the engine will attempt to
+                                  automatically detect the encoding by trying
+                                  common options (utf-8-sig, utf-8, system
+                                  default).
   -vo, --verbose-output           Specify this option to print rules as they
                                   are completed
   -p, --progress [verbose_output|disabled|percents|bar]

cdisc_rules_engine/services/cache/cache_populator_service.py

@@ -1,6 +1,5 @@
 import asyncio
 import pickle
-import json
 from functools import partial
 from typing import Iterable, List, Optional
 import os
@@ -17,6 +16,7 @@
     get_library_variables_metadata_cache_key,
     get_standard_details_cache_key,
     get_model_details_cache_key,
+    load_json_with_optional_encoding,
 )
 from scripts.script_utils import load_and_parse_rule
 from cdisc_rules_engine.constants.cache_constants import PUBLISHED_CT_PACKAGES
@@ -32,6 +32,7 @@ def __init__(
         remove_custom_rules=None,
         update_custom_rule=None,
         custom_standards=None,
+        custom_standards_encoding: str = None,
         remove_custom_standards=None,
         cache_path="",
     ):
@@ -42,6 +43,7 @@ def __init__(
         self.remove_custom_rules = remove_custom_rules
         self.update_custom_rule = update_custom_rule
         self.custom_standards = custom_standards
+        self.custom_standards_encoding = custom_standards_encoding
         self.remove_custom_standards = remove_custom_standards
         self.cache_path = cache_path
 
@@ -598,8 +600,9 @@ def add_custom_standard_to_cache(self):
         """Add or update a custom standard to the cache."""
         if not os.path.isfile(self.custom_standards):
             raise ValueError("Invalid standard filepath")
-        with open(self.custom_standards, "r") as f:
-            new_standard = json.load(f)
+        new_standard = load_json_with_optional_encoding(
+            self.custom_standards, self.custom_standards_encoding
+        )
 
         # Validate the input format
         if not isinstance(new_standard, dict):

cdisc_rules_engine/utilities/utils.py

@@ -4,6 +4,8 @@
 """
 
 import copy
+import json
+import locale
 import os
 import re
 import ast
@@ -481,3 +483,34 @@ def set_max_errors_per_rule(args):
 
     per_dataset = bool(env_per_dataset or cli_per_dataset)
     return max_errors_per_rule, per_dataset
+
+
+def load_json_with_optional_encoding(path: str, encoding: str | None = None) -> dict:
+    tried = []
+    if encoding:
+        # If the file contains only ASCII characters, incorrect encodings may still succeed
+        try:
+            with open(path, "r", encoding=encoding) as f:
+                return json.load(f)
+        except (UnicodeDecodeError, json.JSONDecodeError) as e:
+            tried.append((encoding, e))
+
+    fallback_encodings = [
+        "utf-8-sig",  # UTF-8 + BOM
+        "utf-8",
+        locale.getpreferredencoding(False),
+    ]
+
+    for enc in fallback_encodings:
+        if enc == encoding:
+            continue
+
+        try:
+            with open(path, "r", encoding=enc) as f:
+                return json.load(f)
+        except (UnicodeDecodeError, json.JSONDecodeError) as e:
+            tried.append((enc, e))
+
+    tried_msg = ", ".join(enc for enc, _ in tried)
+
+    raise ValueError(f"Unable to load JSON file '{path}'. Tried encodings: {tried_msg}")

core.py

@@ -551,10 +551,15 @@ def validate(
         "Will update the standard if it already exists."
     ),
 )
+@click.option(
+    "-cse",
+    "--custom-standard-encoding",
+    help="Encoding for custom standard details. ",
+)
 @click.option(
     "-rcs",
     "--remove-custom-standard",
-    help=("Removes a custom standard and version from the cache. "),
+    help="Removes a custom standard and version from the cache. ",
     multiple=True,
 )
 @click.pass_context
@@ -567,6 +572,7 @@ def update_cache(
     remove_custom_rules: str,
     update_custom_rule: str,
     custom_standard: str,
+    custom_standard_encoding: str,
     remove_custom_standard: str,
 ):
     cache = CacheServiceFactory(config).get_cache_service()
@@ -579,6 +585,7 @@ def update_cache(
         remove_custom_rules,
         update_custom_rule,
         custom_standard,
+        custom_standard_encoding,
         remove_custom_standard,
         cache_path,
     )

tests/unit/test_utilities/test_json_load_with_fallback.py

@@ -0,0 +1,83 @@
+import json
+import locale
+import os
+import tempfile
+from unittest import mock
+
+import pytest
+
+from cdisc_rules_engine.utilities.utils import load_json_with_optional_encoding
+
+
+def test_user_encoding_used_successfully():
+    data = {"日本語": [1, 2, 3]}
+
+    with tempfile.TemporaryDirectory() as tmp:
+        path = os.path.join(tmp, "data.json")
+
+        with open(path, "w", encoding="cp932") as f:
+            json.dump(data, f, ensure_ascii=False)
+
+        result = load_json_with_optional_encoding(path, encoding="cp932")
+
+        assert result == data
+
+
+def test_user_encoding_fails_then_fallback_to_utf8():
+    data = {"key": "value"}
+
+    with tempfile.TemporaryDirectory() as tmp:
+        path = os.path.join(tmp, "data.json")
+
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(data, f)
+
+        result = load_json_with_optional_encoding(path, encoding="cp932")
+
+        assert result == data
+
+
+def test_no_encoding_utf8_with_bom():
+    data = {"STD_BOM": [1]}
+
+    with tempfile.TemporaryDirectory() as tmp:
+        path = os.path.join(tmp, "data.json")
+
+        with open(path, "w", encoding="utf-8-sig") as f:
+            json.dump(data, f)
+
+        result = load_json_with_optional_encoding(path)
+
+        assert result == data
+
+
+def test_system_encoding_fallback_mocked():
+    data = {"日本語": [1, 2]}
+
+    with tempfile.TemporaryDirectory() as tmp:
+        path = os.path.join(tmp, "data.json")
+
+        with open(path, "w", encoding="cp932") as f:
+            json.dump(data, f, ensure_ascii=False)
+
+        with mock.patch.object(
+            locale,
+            "getpreferredencoding",
+            return_value="cp932",
+        ):
+            result = load_json_with_optional_encoding(path)
+
+        assert result == data
+
+
+def test_all_encodings_fail():
+    with tempfile.TemporaryDirectory() as tmp:
+        path = os.path.join(tmp, "broken.json")
+
+        with open(path, "wb") as f:
+            f.write(b"\xff\xfe\xfa\xfb")
+
+        with pytest.raises(ValueError) as exc:
+            load_json_with_optional_encoding(path, encoding="utf-8")
+
+        assert "Unable to load JSON file" in str(exc.value)