fix(tree-sitter-qmd): accept Unicode \p{Po}/\p{Pc} punctuation as Str (bd-6kewx)

crates/tree-sitter-qmd/tree-sitter-markdown/grammar.js

@@ -34,6 +34,34 @@ const PANDOC_VALID_SYMBOLS =
 
 const PANDOC_ALPHA_NUM = "0-9A-Za-z\\p{L}\\p{N}";
 const PANDOC_PUNCTUATION = "\\p{Pd}#%&()/:+\\u{2026}";
+
+// Po (Other Punctuation) and Pc (Connector Punctuation), NON-ASCII only.
+// Pandoc folds these into the surrounding `Str` (e.g. `§`, `¶`, `·`, `†`,
+// `‡`, `•`, `‣`, `‰`). The grammar previously enumerated the symbol
+// categories (`Sm`/`Sk`/`So`/`Sc`) and dash punctuation (`\p{Pd}`) but not
+// these, so a bare `Po` char in prose produced a parse ERROR (bd-6kewx).
+// ASCII chars in these categories are excluded (codepoints >= U+0080) because
+// they are either already handled above or reserved as qmd inline delimiters
+// (`* _ ~ ^ ' " @ \` etc.). Generated by scripts/unicode-ranges.py.
+const PANDOC_VALID_OTHER_PUNCTUATION =
+    "\\u{00A1}\\u{00A7}\\u{00B6}-\\u{00B7}\\u{00BF}\\u{037E}\\u{0387}\\u{055A}-\\u{055F}\\u{0589}\\u{05C0}\\u{05C3}"
+  + "\\u{05C6}\\u{05F3}-\\u{05F4}\\u{0609}-\\u{060A}\\u{060C}-\\u{060D}\\u{061B}\\u{061E}-\\u{061F}\\u{066A}-\\u{066D}\\u{06D4}\\u{0700}-\\u{070D}\\u{07F7}-\\u{07F9}"
+  + "\\u{0830}-\\u{083E}\\u{085E}\\u{0964}-\\u{0965}\\u{0970}\\u{09FD}\\u{0A76}\\u{0AF0}\\u{0C77}\\u{0C84}\\u{0DF4}"
+  + "\\u{0E4F}\\u{0E5A}-\\u{0E5B}\\u{0F04}-\\u{0F12}\\u{0F14}\\u{0F85}\\u{0FD0}-\\u{0FD4}\\u{0FD9}-\\u{0FDA}\\u{104A}-\\u{104F}\\u{10FB}\\u{1360}-\\u{1368}"
+  + "\\u{166E}\\u{16EB}-\\u{16ED}\\u{1735}-\\u{1736}\\u{17D4}-\\u{17D6}\\u{17D8}-\\u{17DA}\\u{1800}-\\u{1805}\\u{1807}-\\u{180A}\\u{1944}-\\u{1945}\\u{1A1E}-\\u{1A1F}\\u{1AA0}-\\u{1AA6}"
+  + "\\u{1AA8}-\\u{1AAD}\\u{1B5A}-\\u{1B60}\\u{1BFC}-\\u{1BFF}\\u{1C3B}-\\u{1C3F}\\u{1C7E}-\\u{1C7F}\\u{1CC0}-\\u{1CC7}\\u{1CD3}\\u{2016}-\\u{2017}\\u{2020}-\\u{2027}\\u{2030}-\\u{2038}"
+  + "\\u{203B}-\\u{203E}\\u{2041}-\\u{2043}\\u{2047}-\\u{2051}\\u{2053}\\u{2055}-\\u{205E}\\u{2CF9}-\\u{2CFC}\\u{2CFE}-\\u{2CFF}\\u{2D70}\\u{2E00}-\\u{2E01}\\u{2E06}-\\u{2E08}"
+  + "\\u{2E0B}\\u{2E0E}-\\u{2E16}\\u{2E18}-\\u{2E19}\\u{2E1B}\\u{2E1E}-\\u{2E1F}\\u{2E2A}-\\u{2E2E}\\u{2E30}-\\u{2E39}\\u{2E3C}-\\u{2E3F}\\u{2E41}\\u{2E43}-\\u{2E4F}"
+  + "\\u{2E52}\\u{3001}-\\u{3003}\\u{303D}\\u{30FB}\\u{A4FE}-\\u{A4FF}\\u{A60D}-\\u{A60F}\\u{A673}\\u{A67E}\\u{A6F2}-\\u{A6F7}\\u{A874}-\\u{A877}"
+  + "\\u{A8CE}-\\u{A8CF}\\u{A8F8}-\\u{A8FA}\\u{A8FC}\\u{A92E}-\\u{A92F}\\u{A95F}\\u{A9C1}-\\u{A9CD}\\u{A9DE}-\\u{A9DF}\\u{AA5C}-\\u{AA5F}\\u{AADE}-\\u{AADF}\\u{AAF0}-\\u{AAF1}"
+  + "\\u{ABEB}\\u{FE10}-\\u{FE16}\\u{FE19}\\u{FE30}\\u{FE45}-\\u{FE46}\\u{FE49}-\\u{FE4C}\\u{FE50}-\\u{FE52}\\u{FE54}-\\u{FE57}\\u{FE5F}-\\u{FE61}\\u{FE68}"
+  + "\\u{FE6A}-\\u{FE6B}\\u{FF01}-\\u{FF03}\\u{FF05}-\\u{FF07}\\u{FF0A}\\u{FF0C}\\u{FF0E}-\\u{FF0F}\\u{FF1A}-\\u{FF1B}\\u{FF1F}-\\u{FF20}\\u{FF3C}\\u{FF61}"
+  + "\\u{FF64}-\\u{FF65}\\u{10100}-\\u{10102}\\u{1039F}\\u{103D0}\\u{1056F}\\u{10857}\\u{1091F}\\u{1093F}\\u{10A50}-\\u{10A58}\\u{10A7F}"
+  + "\\u{10AF0}-\\u{10AF6}\\u{10B39}-\\u{10B3F}\\u{10B99}-\\u{10B9C}\\u{10F55}-\\u{10F59}\\u{11047}-\\u{1104D}\\u{110BB}-\\u{110BC}\\u{110BE}-\\u{110C1}\\u{11140}-\\u{11143}\\u{11174}-\\u{11175}\\u{111C5}-\\u{111C8}"
+  + "\\u{111CD}\\u{111DB}\\u{111DD}-\\u{111DF}\\u{11238}-\\u{1123D}\\u{112A9}\\u{1144B}-\\u{1144F}\\u{1145A}-\\u{1145B}\\u{1145D}\\u{114C6}\\u{115C1}-\\u{115D7}"
+  + "\\u{11641}-\\u{11643}\\u{11660}-\\u{1166C}\\u{1173C}-\\u{1173E}\\u{1183B}\\u{11944}-\\u{11946}\\u{119E2}\\u{11A3F}-\\u{11A46}\\u{11A9A}-\\u{11A9C}\\u{11A9E}-\\u{11AA2}\\u{11C41}-\\u{11C45}"
+  + "\\u{11C70}-\\u{11C71}\\u{11EF7}-\\u{11EF8}\\u{11FFF}\\u{12470}-\\u{12474}\\u{16A6E}-\\u{16A6F}\\u{16AF5}\\u{16B37}-\\u{16B3B}\\u{16B44}\\u{16E97}-\\u{16E9A}\\u{16FE2}"
+  + "\\u{1BC9F}\\u{1DA87}-\\u{1DA8B}\\u{1E95E}-\\u{1E95F}\\u{203F}-\\u{2040}\\u{2054}\\u{FE33}-\\u{FE34}\\u{FE4D}-\\u{FE4F}\\u{FF3F}";
 // Smart quotes that are allowed in pandoc_str
 // U+2018 = ' (left single quotation mark)
 // U+2019 = ' (right single quotation mark / apostrophe)
@@ -83,6 +111,7 @@ const PANDOC_REGEX_STR =
             KEYCAP_EMOJI_REGEX,
             EMOJI_REGEX,
             "[" + PANDOC_PUNCTUATION + "]",
+            "[" + PANDOC_VALID_OTHER_PUNCTUATION + "]",
             "[" + PANDOC_VALID_SYMBOLS + "]",
             "[>.,;!?]",
             startStrRegex +

crates/tree-sitter-qmd/tree-sitter-markdown/scripts/unicode-ranges.py

@@ -94,4 +94,17 @@ def generate_ranges_excluding(category, exclude_codes):
     print("\n" + "=" * 60)
     print("So (Other Symbol) - no exclusions")
     print("=" * 60)
-    so_result = generate_ranges_excluding('So', [])
+    so_result = generate_ranges_excluding('So', [])
+
+    # bd-6kewx: Po (Other Punctuation) and Pc (Connector Punctuation),
+    # NON-ASCII only. Pandoc folds these into Str; the grammar previously
+    # omitted them, so a bare § / ¶ / • etc. in prose produced a parse ERROR.
+    # ASCII chars in these categories are excluded because they are either
+    # already handled or reserved as qmd inline delimiters (* _ ~ ^ ' " @ \).
+    for cat in ('Po', 'Pc'):
+        print("\n" + "=" * 60)
+        print(f"{cat} (non-ASCII, codepoints >= U+0080) - for PANDOC_VALID_OTHER_PUNCTUATION")
+        print("=" * 60)
+        ascii_in_cat = [c for c in range(0x80)
+                        if unicodedata.category(chr(c)) == cat]
+        generate_ranges_excluding(cat, ascii_in_cat)

crates/tree-sitter-qmd/tree-sitter-markdown/src/grammar.json

@@ -3598,7 +3598,7 @@
       "members": [
         {
           "type": "PATTERN",
-          "value": "(?:\\\\.|([0-9#]\\uFE0F?\\u20E3)|(\\p{Extended_Pictographic}(\\p{Emoji_Modifier}|️)?(‍\\p{Extended_Pictographic}(\\p{Emoji_Modifier}|️)?)*)|[\\p{Pd}#%&()/:+\\u{2026}]|[+=¬±×÷϶؆-؈⁄⁒⁺-⁼₊-₌℘⅀-⅄⅋←-↔↚-↛↠↣↦↮⇎-⇏⇒⇔⇴-⋿⌠-⌡⍼⎛-⎳⏜-⏡▷◁◸-◿♯⟀-⟄⟇-⟥⟰-⟿⤀-⦂⦙-⧗⧜-⧻⧾-⫿⬰-⭄⭇-⭌﬩﹢﹤-﹦＋＜-＞｜～￢￩-￬𝛁𝛛𝛻𝜕𝜵𝝏𝝯𝞉𝞩𝟃𞻰-𞻱¨¯´¸˂-˅˒-˟˥-˫˭˯-˿͵΄-΅࢈᾽᾿-῁῍-῏῝-῟῭-`´-῾゛-゜꜀-꜖꜠-꜡꞉-꞊꭛꭪-꭫﮲-﯂＾｀￣🏻-🏿\\p{So}¢£¤¥֏؋߾߿৲৳৻૱௹฿៛₠₡₢₣₤₥₦₧₨₩₪₫€₭₮₯₱₲₳₴₵₶₷₸₹₺₻₼₽₾₿⃀꠸﷼﹩＄￠￡￥￦𑿝𑿞𑿟𑿠𞋿𞲰]|[>.,;!?]|(?:[\\u{00A0}\\u{1680}\\u{2000}-\\u{200A}\\u{2028}\\u{2029}\\u{202F}\\u{205F}\\u{3000}0-9A-Za-z\\p{L}\\p{N}\\u{2018}\\u{2019}\\u{201A}\\u{201B}\\u{201C}\\u{201D}\\u{201E}\\u{201F}\\u{2039}\\u{203A}\\u{00AB}\\u{00BB}-])(?:[!,.;?\\u{00A0}\\u{1680}\\u{2000}-\\u{200A}\\u{2028}\\u{2029}\\u{202F}\\u{205F}\\u{3000}0-9A-Za-z\\p{L}\\p{N}\\u{2018}\\u{2019}\\u{201A}\\u{201B}\\u{201C}\\u{201D}\\u{201E}\\u{201F}\\u{2039}\\u{203A}\\u{00AB}\\u{00BB}-]|['\\u{2018}\\u{2019}][\\p{L}\\p{N}]|(?:[_][0-9A-Za-z\\p{L}\\p{N}]))*)",
+          "value": "(?:\\\\.|([0-9#]\\uFE0F?\\u20E3)|(\\p{Extended_Pictographic}(\\p{Emoji_Modifier}|️)?(‍\\p{Extended_Pictographic}(\\p{Emoji_Modifier}|️)?)*)|[\\p{Pd}#%&()/:+\\u{2026}]|[\\u{00A1}\\u{00A7}\\u{00B6}-\\u{00B7}\\u{00BF}\\u{037E}\\u{0387}\\u{055A}-\\u{055F}\\u{0589}\\u{05C0}\\u{05C3}\\u{05C6}\\u{05F3}-\\u{05F4}\\u{0609}-\\u{060A}\\u{060C}-\\u{060D}\\u{061B}\\u{061E}-\\u{061F}\\u{066A}-\\u{066D}\\u{06D4}\\u{0700}-\\u{070D}\\u{07F7}-\\u{07F9}\\u{0830}-\\u{083E}\\u{085E}\\u{0964}-\\u{0965}\\u{0970}\\u{09FD}\\u{0A76}\\u{0AF0}\\u{0C77}\\u{0C84}\\u{0DF4}\\u{0E4F}\\u{0E5A}-\\u{0E5B}\\u{0F04}-\\u{0F12}\\u{0F14}\\u{0F85}\\u{0FD0}-\\u{0FD4}\\u{0FD9}-\\u{0FDA}\\u{104A}-\\u{104F}\\u{10FB}\\u{1360}-\\u{1368}\\u{166E}\\u{16EB}-\\u{16ED}\\u{1735}-\\u{1736}\\u{17D4}-\\u{17D6}\\u{17D8}-\\u{17DA}\\u{1800}-\\u{1805}\\u{1807}-\\u{180A}\\u{1944}-\\u{1945}\\u{1A1E}-\\u{1A1F}\\u{1AA0}-\\u{1AA6}\\u{1AA8}-\\u{1AAD}\\u{1B5A}-\\u{1B60}\\u{1BFC}-\\u{1BFF}\\u{1C3B}-\\u{1C3F}\\u{1C7E}-\\u{1C7F}\\u{1CC0}-\\u{1CC7}\\u{1CD3}\\u{2016}-\\u{2017}\\u{2020}-\\u{2027}\\u{2030}-\\u{2038}\\u{203B}-\\u{203E}\\u{2041}-\\u{2043}\\u{2047}-\\u{2051}\\u{2053}\\u{2055}-\\u{205E}\\u{2CF9}-\\u{2CFC}\\u{2CFE}-\\u{2CFF}\\u{2D70}\\u{2E00}-\\u{2E01}\\u{2E06}-\\u{2E08}\\u{2E0B}\\u{2E0E}-\\u{2E16}\\u{2E18}-\\u{2E19}\\u{2E1B}\\u{2E1E}-\\u{2E1F}\\u{2E2A}-\\u{2E2E}\\u{2E30}-\\u{2E39}\\u{2E3C}-\\u{2E3F}\\u{2E41}\\u{2E43}-\\u{2E4F}\\u{2E52}\\u{3001}-\\u{3003}\\u{303D}\\u{30FB}\\u{A4FE}-\\u{A4FF}\\u{A60D}-\\u{A60F}\\u{A673}\\u{A67E}\\u{A6F2}-\\u{A6F7}\\u{A874}-\\u{A877}\\u{A8CE}-\\u{A8CF}\\u{A8F8}-\\u{A8FA}\\u{A8FC}\\u{A92E}-\\u{A92F}\\u{A95F}\\u{A9C1}-\\u{A9CD}\\u{A9DE}-\\u{A9DF}\\u{AA5C}-\\u{AA5F}\\u{AADE}-\\u{AADF}\\u{AAF0}-\\u{AAF1}\\u{ABEB}\\u{FE10}-\\u{FE16}\\u{FE19}\\u{FE30}\\u{FE45}-\\u{FE46}\\u{FE49}-\\u{FE4C}\\u{FE50}-\\u{FE52}\\u{FE54}-\\u{FE57}\\u{FE5F}-\\u{FE61}\\u{FE68}\\u{FE6A}-\\u{FE6B}\\u{FF01}-\\u{FF03}\\u{FF05}-\\u{FF07}\\u{FF0A}\\u{FF0C}\\u{FF0E}-\\u{FF0F}\\u{FF1A}-\\u{FF1B}\\u{FF1F}-\\u{FF20}\\u{FF3C}\\u{FF61}\\u{FF64}-\\u{FF65}\\u{10100}-\\u{10102}\\u{1039F}\\u{103D0}\\u{1056F}\\u{10857}\\u{1091F}\\u{1093F}\\u{10A50}-\\u{10A58}\\u{10A7F}\\u{10AF0}-\\u{10AF6}\\u{10B39}-\\u{10B3F}\\u{10B99}-\\u{10B9C}\\u{10F55}-\\u{10F59}\\u{11047}-\\u{1104D}\\u{110BB}-\\u{110BC}\\u{110BE}-\\u{110C1}\\u{11140}-\\u{11143}\\u{11174}-\\u{11175}\\u{111C5}-\\u{111C8}\\u{111CD}\\u{111DB}\\u{111DD}-\\u{111DF}\\u{11238}-\\u{1123D}\\u{112A9}\\u{1144B}-\\u{1144F}\\u{1145A}-\\u{1145B}\\u{1145D}\\u{114C6}\\u{115C1}-\\u{115D7}\\u{11641}-\\u{11643}\\u{11660}-\\u{1166C}\\u{1173C}-\\u{1173E}\\u{1183B}\\u{11944}-\\u{11946}\\u{119E2}\\u{11A3F}-\\u{11A46}\\u{11A9A}-\\u{11A9C}\\u{11A9E}-\\u{11AA2}\\u{11C41}-\\u{11C45}\\u{11C70}-\\u{11C71}\\u{11EF7}-\\u{11EF8}\\u{11FFF}\\u{12470}-\\u{12474}\\u{16A6E}-\\u{16A6F}\\u{16AF5}\\u{16B37}-\\u{16B3B}\\u{16B44}\\u{16E97}-\\u{16E9A}\\u{16FE2}\\u{1BC9F}\\u{1DA87}-\\u{1DA8B}\\u{1E95E}-\\u{1E95F}\\u{203F}-\\u{2040}\\u{2054}\\u{FE33}-\\u{FE34}\\u{FE4D}-\\u{FE4F}\\u{FF3F}]|[+=¬±×÷϶؆-؈⁄⁒⁺-⁼₊-₌℘⅀-⅄⅋←-↔↚-↛↠↣↦↮⇎-⇏⇒⇔⇴-⋿⌠-⌡⍼⎛-⎳⏜-⏡▷◁◸-◿♯⟀-⟄⟇-⟥⟰-⟿⤀-⦂⦙-⧗⧜-⧻⧾-⫿⬰-⭄⭇-⭌﬩﹢﹤-﹦＋＜-＞｜～￢￩-￬𝛁𝛛𝛻𝜕𝜵𝝏𝝯𝞉𝞩𝟃𞻰-𞻱¨¯´¸˂-˅˒-˟˥-˫˭˯-˿͵΄-΅࢈᾽᾿-῁῍-῏῝-῟῭-`´-῾゛-゜꜀-꜖꜠-꜡꞉-꞊꭛꭪-꭫﮲-﯂＾｀￣🏻-🏿\\p{So}¢£¤¥֏؋߾߿৲৳৻૱௹฿៛₠₡₢₣₤₥₦₧₨₩₪₫€₭₮₯₱₲₳₴₵₶₷₸₹₺₻₼₽₾₿⃀꠸﷼﹩＄￠￡￥￦𑿝𑿞𑿟𑿠𞋿𞲰]|[>.,;!?]|(?:[\\u{00A0}\\u{1680}\\u{2000}-\\u{200A}\\u{2028}\\u{2029}\\u{202F}\\u{205F}\\u{3000}0-9A-Za-z\\p{L}\\p{N}\\u{2018}\\u{2019}\\u{201A}\\u{201B}\\u{201C}\\u{201D}\\u{201E}\\u{201F}\\u{2039}\\u{203A}\\u{00AB}\\u{00BB}-])(?:[!,.;?\\u{00A0}\\u{1680}\\u{2000}-\\u{200A}\\u{2028}\\u{2029}\\u{202F}\\u{205F}\\u{3000}0-9A-Za-z\\p{L}\\p{N}\\u{2018}\\u{2019}\\u{201A}\\u{201B}\\u{201C}\\u{201D}\\u{201E}\\u{201F}\\u{2039}\\u{203A}\\u{00AB}\\u{00BB}-]|['\\u{2018}\\u{2019}][\\p{L}\\p{N}]|(?:[_][0-9A-Za-z\\p{L}\\p{N}]))*)",
           "flags": "u"
         },
         {