fix(prism): stop treating C++ digit separators as char-literal openers

gitgalaxy/core/detector.py

@@ -1889,7 +1889,15 @@ def fast_shield(m):
 
         # Rust uses single quotes for lifetimes (e.g. 'a), so a greedy string match corrupts ASTs.
         single_quote = r"'(?:\\.|[^'\\])*'"
-        if lang_id in ("rust", "zig"):
+        if lang_id == "cpp":
+            # #1718: C++14+ digit separators (512'000, 1'000'000, 0xDE'AD) use ' inside
+            # numeric literals. The unbounded branch read a separator as a char-literal opener
+            # and paired it with the next unrelated ' anywhere later in the file, blanking every
+            # real function body in between from the brace scan. Consume separators as their own
+            # alternative (same shape as prism.py's CPP_LITERAL_MASK_PATTERN) and bound the branch
+            # to 64 chars, matching #1302/#1426.
+            single_quote = r"[0-9a-fA-F]'[0-9a-fA-F]|(?<!\\)'(?:\\.|[^'\\]){0,64}'"
+        elif lang_id in ("rust", "zig"):
             # #1426: zig's char literals ('a', '\n', '\u{1F600}') are just as short-lived
             # as rust's, but zig ALSO has multi-line `\\`-prefixed string literals that are
             # never shielded at all here (a separate, pre-existing gap) -- so a real

gitgalaxy/core/prism.py

@@ -99,6 +99,26 @@ def __init__(
         # Defends against catastrophic backtracking and logic erosion inside strings
         self.LITERAL_MASK_PATTERN = PRISM_CONFIG.get("SHIELD_PATTERN", "")
 
+        # #1718: C++ (C++14+) uses a single quote as a digit separator inside
+        # numeric literals (512'000, 1'000'000'000, 0xDE'AD). The shared
+        # SHIELD_PATTERN's single-quote branch is unbounded, so a separator
+        # `'` is mistaken for the opening quote of a char literal and pairs
+        # with the NEXT unrelated `'` anywhere later in the file (re.S lets
+        # [^'\\] span newlines), swallowing every real // and /* */ comment
+        # in between as one giant "literal" -- the code stream then carries
+        # comment text into the detector and coding_loc is inflated.
+        # C++ char literals are short, but C++23 named escapes (\\N{...}) can
+        # run much longer than 10 chars, so the branch is bounded to 64 -- wide
+        # enough for any real literal, still far too short for a cross-file
+        # cascade. Kept per-language because the shared pattern must stay
+        # unbounded for JS/PHP single-quoted strings.
+        self.CPP_LITERAL_MASK_PATTERN = (
+            r'((?<!\\)"(?:\\.|[^"\\])*"'
+            r"|[0-9a-fA-F]'[0-9a-fA-F]"
+            r"|(?<!\\)'(?:\\.|[^'\\]){0,64}'"
+            r"|(?<!\\)`(?:\\.|[^`\\])*`)"
+        )
+
         # #1271: detects a quote that opens but never closes before end-of-
         # line -- a backslash-newline-continued literal (legal in both Ruby
         # and Python) -- so _strip_single_line_comments can carry that
@@ -136,6 +156,13 @@ def __init__(
         # --- TIER 2: REGEX PRE-COMPILATION ---
         self.REGEX_MATRIX: dict[str, re.Pattern] = self._compile_regex_matrix()
 
+        # #1718: C++ digit separators (512'000) must not pair with a later
+        # `'` as a char literal, so C++ uses a bounded single-quote shield
+        # in the generic standard_block stripper (see _strip_segment_comments).
+        self.CPP_REGEX_MATRIX: dict[str, re.Pattern] = self._compile_regex_matrix(
+            literal_pattern=self.CPP_LITERAL_MASK_PATTERN
+        )
+
         # #697: _strip_single_line_comments() used to hardcode `#|--|;|//`
         # regardless of what a given family's real delimiters are. #1193:
         # a single shared pattern for the whole "line_exclusive" family was
@@ -323,6 +350,15 @@ def _strip_segment_comments(self, text: str, lang_id: str, family: str) -> tuple
 
         # 3. GENERIC STRIPPER
         pattern = self.REGEX_MATRIX.get(family)
+        if lang_id == "cpp" and family == "standard_block":
+            # #1718: C++ digit separators (512'000) use `'` as a digit
+            # separator, which the unbounded shared single-quote branch
+            # misreads as a char literal opener that pairs with the next
+            # unrelated `'` anywhere later in the file -- swallowing every
+            # real comment in between. Route C++ through the bounded
+            # CPP_REGEX_MATRIX so separators can't cascade into a false
+            # literal (JS/PHP keep the unbounded shared pattern).
+            pattern = self.CPP_REGEX_MATRIX.get(family) or pattern
         if not pattern:
             return text, "\n".join(lits)
 
@@ -339,7 +375,7 @@ def strip_callback(m: re.Match) -> str:
         code = pattern.sub(strip_callback, text)
         return code, "\n".join(lits)
 
-    def _compile_regex_matrix(self) -> dict[str, re.Pattern]:
+    def _compile_regex_matrix(self, literal_pattern: Optional[str] = None) -> dict[str, re.Pattern]:
         """Safely pre-compiles the standard regex matrix based on dynamic config lengths."""
         matrix = {}
 
@@ -438,7 +474,8 @@ def _compile_regex_matrix(self) -> dict[str, re.Pattern]:
                 try:
                     # ---> THE FIX: Strip any rogue inline flags injected by the config <---
                     p = p.replace("(?i)", "").replace("(?m)", "").replace("(?s)", "")
-                    full_pattern = f"{self.LITERAL_MASK_PATTERN}|{p}"
+                    literal_mask = literal_pattern or self.LITERAL_MASK_PATTERN
+                    full_pattern = f"{literal_mask}|{p}"
 
                     flags = re.S | re.M
                     if fam_key == "line_exclusive":

tests/core_engine/test_detector_issue_1718.py

@@ -0,0 +1,72 @@
+"""
+Regression tests for issue #1718: C++14+ digit separators (512'000,
+1'000'000'000, 0xDE'AD'BE'EF) use a single quote inside numeric literals.
+
+_build_brace_safe_stream's single-quote branch used to be unbounded for cpp,
+so a separator quote was read as the opener of a char literal and paired with
+the NEXT unrelated apostrophe anywhere later in the file -- blanking every real
+{/} in between, including real function bodies, and desyncing the brace scan so
+functions after the separator were dropped entirely. The digit separator must be
+consumed as its own alternative and the char-literal branch bounded to 64 chars,
+matching the bound #1302/#1426 already apply to rust/zig and prism.py.
+"""
+from gitgalaxy.core.detector import StructuralExtractor
+from gitgalaxy.standards.language_standards import LANGUAGE_DEFINITIONS
+
+
+def test_detector_cpp_digit_separator_does_not_blank_function_bodies():
+    code = (
+        "constexpr long long KB = 512'000;\n"
+        "\n"
+        "bool firstFunction(const String &p_state) {\n"
+        "    return !force_background;\n"
+        "}\n"
+        "\n"
+        "// a comment with a stray apostrophe: it's here\n"
+        "int secondFunction(int y) {\n"
+        "    return y * 2;\n"
+        "}\n"
+        "\n"
+        "char trigger = 'q';\n"
+        "\n"
+        "int thirdFunction(int z) {\n"
+        "    return z * 3;\n"
+        "}\n"
+    )
+    detector = StructuralExtractor("cpp", LANGUAGE_DEFINITIONS)
+    rules = LANGUAGE_DEFINITIONS["cpp"]["rules"]
+    satellites, _ = detector._slice_by_braces(code, "cpp", rules, 0, {})
+    names = [s["name"] for s in satellites]
+    assert "firstFunction" in names, f"firstFunction should be found: {names}"
+    assert "secondFunction" in names, f"secondFunction must not be swallowed: {names}"
+    assert "thirdFunction" in names, f"thirdFunction must survive after the digit separator: {names}"
+    second = next(s for s in satellites if s["name"] == "secondFunction")
+    assert second["loc"] <= 3, f"secondFunction body must stay bounded: loc={second['loc']}"
+    third = next(s for s in satellites if s["name"] == "thirdFunction")
+    assert third["loc"] <= 3, f"thirdFunction body must stay bounded: loc={third['loc']}"
+
+
+def test_issue_1718_cpp23_named_escape_literal_stays_intact():
+    r"""
+    C++23 named character escapes (\N{...}) are much longer than 10 chars
+    and contain braces. The C++ char-literal branch must stay wide enough
+    to shield them whole, so a real (if rare) literal isn't clipped and
+    doesn't desync the brace tracker.
+    """
+    code = (
+        "int main() {\n"
+        "    char32_t c = '\\N{LATIN CAPITAL LETTER A}';\n"
+        "    // a comment after the named escape\n"
+        "    return 0;\n"
+        "}\n"
+        "int nextFunction() {\n"
+        "    return 1;\n"
+        "}\n"
+    )
+    detector = StructuralExtractor("cpp", LANGUAGE_DEFINITIONS)
+    rules = LANGUAGE_DEFINITIONS["cpp"]["rules"]
+    satellites, _ = detector._slice_by_braces(code, "cpp", rules, 0, {})
+    names = [s["name"] for s in satellites]
+
+    assert "main" in names, f"main should be found: {names}"
+    assert "nextFunction" in names, f"nextFunction should be found: {names}"

tests/core_engine/test_prism_issue_1718.py

@@ -0,0 +1,181 @@
+from gitgalaxy.core.prism import Prism
+
+LANG_DEFS = {
+    "cpp": {"lexical_family": "standard_block"},
+    "c": {"lexical_family": "standard_block"},
+    "javascript": {"lexical_family": "standard_block"},
+    "php": {"lexical_family": "standard_block"},
+}
+
+CONFIG = {
+    "lexical_families": {
+        "standard_block": {"delimiters": ["//", "/*", "*/"]},
+    }
+}
+
+
+def test_issue_1718_digit_separator_does_not_pair_with_far_away_quote():
+    """
+    Regression test for #1718: C++ digit separators (512'000, 1'000'000'000)
+    use a single quote that the unbounded shared literal shield misread as a
+    char-literal opener, pairing it with the next unrelated `'` anywhere later
+    in the file -- so every real // and /* */ comment in between was swallowed
+    as one giant "literal" and never stripped.
+    """
+    prism = Prism(CONFIG, LANG_DEFS)
+
+    code = """
+constexpr long long KB = 512'000;
+
+// this comment should be stripped
+int firstFunction(int x) {
+    return x + 1;
+}
+
+/* this block comment should be stripped too */
+int secondFunction(int y) {
+    return y * 2;
+}
+
+char trigger = 'q';
+"""
+
+    result = prism.split_streams(code, "cpp")
+
+    assert "// this comment should be stripped" not in result["code_stream"]  # noqa: S101
+    assert "/* this block comment should be stripped too */" not in result["code_stream"]  # noqa: S101
+    assert "firstFunction" in result["code_stream"]  # noqa: S101
+    assert "secondFunction" in result["code_stream"]  # noqa: S101
+    assert "512'000" in result["code_stream"]  # noqa: S101
+
+
+def test_issue_1718_hex_and_multiple_digit_separators_kept():
+    """Hex (0xDE'AD'BE'EF) and multi-group (1'000'000'000) separators stay intact."""
+    prism = Prism(CONFIG, LANG_DEFS)
+
+    code = """
+constexpr unsigned long long V = 0xDE'AD'BE'EF;
+constexpr long long LARGE = 1'000'000'000;
+
+// separator line comment
+int first(int a) { return a; }
+"""
+
+    result = prism.split_streams(code, "cpp")
+
+    assert "0xDE'AD'BE'EF" in result["code_stream"]  # noqa: S101
+    assert "1'000'000'000" in result["code_stream"]  # noqa: S101
+    assert "// separator line comment" not in result["code_stream"]  # noqa: S101
+
+
+def test_issue_1718_comment_apostrophe_within_bound_still_stripped():
+    """
+    A comment apostrophe close to a digit separator ("it's") must not be
+    re-paired: the separator is consumed as its own alternative first, so
+    the comment line is still stripped.
+    """
+    prism = Prism(CONFIG, LANG_DEFS)
+
+    code = """
+int f() {
+    long long x = 512'000; // it's a lot
+    return (int)x;
+}
+"""
+
+    result = prism.split_streams(code, "cpp")
+
+    assert "// it's a lot" not in result["code_stream"]  # noqa: S101
+    assert "512'000" in result["code_stream"]  # noqa: S101
+
+
+def test_issue_1718_real_char_literals_still_shielded():
+    """The bound must not break genuine short char literals -- they stay intact."""
+    prism = Prism(CONFIG, LANG_DEFS)
+
+    code = """
+int f() {
+    char a = 'x';
+    char nl = '\n';
+    char q = '\'';
+    return a;
+}
+"""
+
+    result = prism.split_streams(code, "cpp")
+
+    assert "'x'" in result["code_stream"]  # noqa: S101
+    assert "'\n'" in result["code_stream"]  # noqa: S101
+    assert "'''" in result["code_stream"]  # noqa: S101
+
+
+def test_issue_1718_prefixed_u8_char_literal_still_shielded():
+    """u8-prefixed char literals must still shield like ordinary ones."""
+    prism = Prism(CONFIG, LANG_DEFS)
+
+    code = """
+int f() {
+    char8_t c = u8'x'; // still a comment
+    return c == u8'x' ? 1 : 0;
+}
+"""
+
+    result = prism.split_streams(code, "cpp")
+
+    assert "u8'x'" in result["code_stream"]  # noqa: S101
+    assert "// still a comment" not in result["code_stream"]  # noqa: S101
+
+
+def test_issue_1718_other_languages_keep_unbounded_single_quotes():
+    """
+    The fix is scoped to C++ only: JS/PHP (and C) single-quoted strings may be
+    arbitrarily long and must keep the unbounded shared shield. A long string
+    containing comment markers must survive whole.
+    """
+    prism = Prism(CONFIG, LANG_DEFS)
+
+    js_code = """
+const s = 'this is a long single-quoted string with // not a comment and /* also not */ inside';
+function foo() { return s; }
+"""
+
+    js_result = prism.split_streams(js_code, "javascript")
+    assert (
+        "'this is a long single-quoted string with // not a comment and /* also not */ inside'"
+        in js_result["code_stream"]
+    )  # noqa: S101
+
+    php_code = """
+<?php
+$s = 'a php string with // inside and /* not comment */';
+function foo() { return $s; }
+"""
+
+    php_result = prism.split_streams(php_code, "php")
+    assert "'a php string with // inside and /* not comment */'" in php_result["code_stream"]  # noqa: S101
+
+    c_code = """
+void f(void) {
+    char *s = 'single quoted text // still one literal';
+}
+"""
+
+    c_result = prism.split_streams(c_code, "c")
+    assert "'single quoted text // still one literal'" in c_result["code_stream"]  # noqa: S101
+
+
+def test_issue_1718_cpp23_named_escape_literal_stays_intact():
+    """
+    C++23 named character escapes (\\N{...}) are much longer than 10 chars
+    and contain braces. The C++ char-literal branch must stay wide enough
+    to shield them whole, so a real (if rare) literal isn't clipped.
+    """
+    prism = Prism(CONFIG, LANG_DEFS)
+
+    code = "int main() {\n    char32_t c = '\\N{LATIN CAPITAL LETTER A}';\n    // a comment after the named escape\n    return 0;\n}\n"
+
+    result = prism.split_streams(code, "cpp")
+
+    assert r"\N{LATIN CAPITAL LETTER A}" in result["code_stream"]  # noqa: S101
+    assert "// a comment after the named escape" not in result["code_stream"]  # noqa: S101
+    assert "int main" in result["code_stream"]  # noqa: S101