diff --git a/gitgalaxy/core/detector.py b/gitgalaxy/core/detector.py index c1656388..1fcf0113 100644 --- a/gitgalaxy/core/detector.py +++ b/gitgalaxy/core/detector.py @@ -1889,7 +1889,15 @@ def fast_shield(m): # Rust uses single quotes for lifetimes (e.g. 'a), so a greedy string match corrupts ASTs. single_quote = r"'(?:\\.|[^'\\])*'" - if lang_id in ("rust", "zig"): + if lang_id == "cpp": + # #1718: C++14+ digit separators (512'000, 1'000'000, 0xDE'AD) use ' inside + # numeric literals. The unbounded branch read a separator as a char-literal opener + # and paired it with the next unrelated ' anywhere later in the file, blanking every + # real function body in between from the brace scan. Consume separators as their own + # alternative (same shape as prism.py's CPP_LITERAL_MASK_PATTERN) and bound the branch + # to 64 chars, matching #1302/#1426. + single_quote = r"[0-9a-fA-F]'[0-9a-fA-F]|(? tuple # 3. GENERIC STRIPPER pattern = self.REGEX_MATRIX.get(family) + if lang_id == "cpp" and family == "standard_block": + # #1718: C++ digit separators (512'000) use `'` as a digit + # separator, which the unbounded shared single-quote branch + # misreads as a char literal opener that pairs with the next + # unrelated `'` anywhere later in the file -- swallowing every + # real comment in between. Route C++ through the bounded + # CPP_REGEX_MATRIX so separators can't cascade into a false + # literal (JS/PHP keep the unbounded shared pattern). + pattern = self.CPP_REGEX_MATRIX.get(family) or pattern if not pattern: return text, "\n".join(lits) @@ -339,7 +375,7 @@ def strip_callback(m: re.Match) -> str: code = pattern.sub(strip_callback, text) return code, "\n".join(lits) - def _compile_regex_matrix(self) -> dict[str, re.Pattern]: + def _compile_regex_matrix(self, literal_pattern: Optional[str] = None) -> dict[str, re.Pattern]: """Safely pre-compiles the standard regex matrix based on dynamic config lengths.""" matrix = {} @@ -438,7 +474,8 @@ def _compile_regex_matrix(self) -> dict[str, re.Pattern]: try: # ---> THE FIX: Strip any rogue inline flags injected by the config <--- p = p.replace("(?i)", "").replace("(?m)", "").replace("(?s)", "") - full_pattern = f"{self.LITERAL_MASK_PATTERN}|{p}" + literal_mask = literal_pattern or self.LITERAL_MASK_PATTERN + full_pattern = f"{literal_mask}|{p}" flags = re.S | re.M if fam_key == "line_exclusive": diff --git a/tests/core_engine/test_detector_issue_1718.py b/tests/core_engine/test_detector_issue_1718.py new file mode 100644 index 00000000..b678937e --- /dev/null +++ b/tests/core_engine/test_detector_issue_1718.py @@ -0,0 +1,72 @@ +""" +Regression tests for issue #1718: C++14+ digit separators (512'000, +1'000'000'000, 0xDE'AD'BE'EF) use a single quote inside numeric literals. + +_build_brace_safe_stream's single-quote branch used to be unbounded for cpp, +so a separator quote was read as the opener of a char literal and paired with +the NEXT unrelated apostrophe anywhere later in the file -- blanking every real +{/} in between, including real function bodies, and desyncing the brace scan so +functions after the separator were dropped entirely. The digit separator must be +consumed as its own alternative and the char-literal branch bounded to 64 chars, +matching the bound #1302/#1426 already apply to rust/zig and prism.py. +""" +from gitgalaxy.core.detector import StructuralExtractor +from gitgalaxy.standards.language_standards import LANGUAGE_DEFINITIONS + + +def test_detector_cpp_digit_separator_does_not_blank_function_bodies(): + code = ( + "constexpr long long KB = 512'000;\n" + "\n" + "bool firstFunction(const String &p_state) {\n" + " return !force_background;\n" + "}\n" + "\n" + "// a comment with a stray apostrophe: it's here\n" + "int secondFunction(int y) {\n" + " return y * 2;\n" + "}\n" + "\n" + "char trigger = 'q';\n" + "\n" + "int thirdFunction(int z) {\n" + " return z * 3;\n" + "}\n" + ) + detector = StructuralExtractor("cpp", LANGUAGE_DEFINITIONS) + rules = LANGUAGE_DEFINITIONS["cpp"]["rules"] + satellites, _ = detector._slice_by_braces(code, "cpp", rules, 0, {}) + names = [s["name"] for s in satellites] + assert "firstFunction" in names, f"firstFunction should be found: {names}" + assert "secondFunction" in names, f"secondFunction must not be swallowed: {names}" + assert "thirdFunction" in names, f"thirdFunction must survive after the digit separator: {names}" + second = next(s for s in satellites if s["name"] == "secondFunction") + assert second["loc"] <= 3, f"secondFunction body must stay bounded: loc={second['loc']}" + third = next(s for s in satellites if s["name"] == "thirdFunction") + assert third["loc"] <= 3, f"thirdFunction body must stay bounded: loc={third['loc']}" + + +def test_issue_1718_cpp23_named_escape_literal_stays_intact(): + r""" + C++23 named character escapes (\N{...}) are much longer than 10 chars + and contain braces. The C++ char-literal branch must stay wide enough + to shield them whole, so a real (if rare) literal isn't clipped and + doesn't desync the brace tracker. + """ + code = ( + "int main() {\n" + " char32_t c = '\\N{LATIN CAPITAL LETTER A}';\n" + " // a comment after the named escape\n" + " return 0;\n" + "}\n" + "int nextFunction() {\n" + " return 1;\n" + "}\n" + ) + detector = StructuralExtractor("cpp", LANGUAGE_DEFINITIONS) + rules = LANGUAGE_DEFINITIONS["cpp"]["rules"] + satellites, _ = detector._slice_by_braces(code, "cpp", rules, 0, {}) + names = [s["name"] for s in satellites] + + assert "main" in names, f"main should be found: {names}" + assert "nextFunction" in names, f"nextFunction should be found: {names}" \ No newline at end of file diff --git a/tests/core_engine/test_prism_issue_1718.py b/tests/core_engine/test_prism_issue_1718.py new file mode 100644 index 00000000..408c994c --- /dev/null +++ b/tests/core_engine/test_prism_issue_1718.py @@ -0,0 +1,181 @@ +from gitgalaxy.core.prism import Prism + +LANG_DEFS = { + "cpp": {"lexical_family": "standard_block"}, + "c": {"lexical_family": "standard_block"}, + "javascript": {"lexical_family": "standard_block"}, + "php": {"lexical_family": "standard_block"}, +} + +CONFIG = { + "lexical_families": { + "standard_block": {"delimiters": ["//", "/*", "*/"]}, + } +} + + +def test_issue_1718_digit_separator_does_not_pair_with_far_away_quote(): + """ + Regression test for #1718: C++ digit separators (512'000, 1'000'000'000) + use a single quote that the unbounded shared literal shield misread as a + char-literal opener, pairing it with the next unrelated `'` anywhere later + in the file -- so every real // and /* */ comment in between was swallowed + as one giant "literal" and never stripped. + """ + prism = Prism(CONFIG, LANG_DEFS) + + code = """ +constexpr long long KB = 512'000; + +// this comment should be stripped +int firstFunction(int x) { + return x + 1; +} + +/* this block comment should be stripped too */ +int secondFunction(int y) { + return y * 2; +} + +char trigger = 'q'; +""" + + result = prism.split_streams(code, "cpp") + + assert "// this comment should be stripped" not in result["code_stream"] # noqa: S101 + assert "/* this block comment should be stripped too */" not in result["code_stream"] # noqa: S101 + assert "firstFunction" in result["code_stream"] # noqa: S101 + assert "secondFunction" in result["code_stream"] # noqa: S101 + assert "512'000" in result["code_stream"] # noqa: S101 + + +def test_issue_1718_hex_and_multiple_digit_separators_kept(): + """Hex (0xDE'AD'BE'EF) and multi-group (1'000'000'000) separators stay intact.""" + prism = Prism(CONFIG, LANG_DEFS) + + code = """ +constexpr unsigned long long V = 0xDE'AD'BE'EF; +constexpr long long LARGE = 1'000'000'000; + +// separator line comment +int first(int a) { return a; } +""" + + result = prism.split_streams(code, "cpp") + + assert "0xDE'AD'BE'EF" in result["code_stream"] # noqa: S101 + assert "1'000'000'000" in result["code_stream"] # noqa: S101 + assert "// separator line comment" not in result["code_stream"] # noqa: S101 + + +def test_issue_1718_comment_apostrophe_within_bound_still_stripped(): + """ + A comment apostrophe close to a digit separator ("it's") must not be + re-paired: the separator is consumed as its own alternative first, so + the comment line is still stripped. + """ + prism = Prism(CONFIG, LANG_DEFS) + + code = """ +int f() { + long long x = 512'000; // it's a lot + return (int)x; +} +""" + + result = prism.split_streams(code, "cpp") + + assert "// it's a lot" not in result["code_stream"] # noqa: S101 + assert "512'000" in result["code_stream"] # noqa: S101 + + +def test_issue_1718_real_char_literals_still_shielded(): + """The bound must not break genuine short char literals -- they stay intact.""" + prism = Prism(CONFIG, LANG_DEFS) + + code = """ +int f() { + char a = 'x'; + char nl = '\n'; + char q = '\''; + return a; +} +""" + + result = prism.split_streams(code, "cpp") + + assert "'x'" in result["code_stream"] # noqa: S101 + assert "'\n'" in result["code_stream"] # noqa: S101 + assert "'''" in result["code_stream"] # noqa: S101 + + +def test_issue_1718_prefixed_u8_char_literal_still_shielded(): + """u8-prefixed char literals must still shield like ordinary ones.""" + prism = Prism(CONFIG, LANG_DEFS) + + code = """ +int f() { + char8_t c = u8'x'; // still a comment + return c == u8'x' ? 1 : 0; +} +""" + + result = prism.split_streams(code, "cpp") + + assert "u8'x'" in result["code_stream"] # noqa: S101 + assert "// still a comment" not in result["code_stream"] # noqa: S101 + + +def test_issue_1718_other_languages_keep_unbounded_single_quotes(): + """ + The fix is scoped to C++ only: JS/PHP (and C) single-quoted strings may be + arbitrarily long and must keep the unbounded shared shield. A long string + containing comment markers must survive whole. + """ + prism = Prism(CONFIG, LANG_DEFS) + + js_code = """ +const s = 'this is a long single-quoted string with // not a comment and /* also not */ inside'; +function foo() { return s; } +""" + + js_result = prism.split_streams(js_code, "javascript") + assert ( + "'this is a long single-quoted string with // not a comment and /* also not */ inside'" + in js_result["code_stream"] + ) # noqa: S101 + + php_code = """ +