From 0844f973e51be2d2bbc65a33531335cb097505c4 Mon Sep 17 00:00:00 2001 From: Johnson Chu Date: Sat, 6 Jun 2026 16:19:07 +0800 Subject: [PATCH 1/3] Add RedCMD TextMate diagnostics guard --- .gitmodules | 3 + javascript.tmLanguage.json | 7 - javascriptreact.tmLanguage.json | 7 - package.json | 1 + src/gen-tm.ts | 6 +- test/redcmd-tm-diagnostics.ts | 185 ++++++++++++++++++++ vendor/RedCMD-TmLanguage-Syntax-Highlighter | 1 + yaml.tmLanguage.json | 7 - 8 files changed, 193 insertions(+), 24 deletions(-) create mode 100644 .gitmodules create mode 100644 test/redcmd-tm-diagnostics.ts create mode 160000 vendor/RedCMD-TmLanguage-Syntax-Highlighter diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..f73452b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vendor/RedCMD-TmLanguage-Syntax-Highlighter"] + path = vendor/RedCMD-TmLanguage-Syntax-Highlighter + url = https://github.com/RedCMD/TmLanguage-Syntax-Highlighter.git diff --git a/javascript.tmLanguage.json b/javascript.tmLanguage.json index 5db51c9..3dbed5a 100644 --- a/javascript.tmLanguage.json +++ b/javascript.tmLanguage.json @@ -2086,13 +2086,6 @@ } ] }, - "type": { - "patterns": [ - { - "include": "#simple-type" - } - ] - }, "qstring-double": { "name": "string.quoted.double.js", "begin": "\"", diff --git a/javascriptreact.tmLanguage.json b/javascriptreact.tmLanguage.json index ca701f0..7e036d4 100644 --- a/javascriptreact.tmLanguage.json +++ b/javascriptreact.tmLanguage.json @@ -2574,13 +2574,6 @@ } ] }, - "type": { - "patterns": [ - { - "include": "#simple-type" - } - ] - }, "qstring-double": { "name": "string.quoted.double.js.jsx", "begin": "\"", diff --git a/package.json b/package.json index 3dffa30..3bfbe2f 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "conformance:tsx": "node test/tsx-conformance.ts", "conformance:jsx": "node test/jsx-conformance.ts", "conformance:html": "node test/html-conformance.ts", + "test:tm-diagnostics": "node test/redcmd-tm-diagnostics.ts", "spike:html-lexer": "node test/html-lexer-spike.ts", "bench:html-official": "node test/html-bench.ts", "bench:html-embed": "node test/html-embed-js.ts", diff --git a/src/gen-tm.ts b/src/gen-tm.ts index 9ba6e55..7349f93 100644 --- a/src/gen-tm.ts +++ b/src/gen-tm.ts @@ -4937,7 +4937,7 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra // reference it via `{ include: '#type-inner' }`. No shared mutable array; // later injections rebuild the patterns array non-destructively. // Type operators are derived from @type rule literals. - const typeInnerPats: (TmPattern | { include: string })[] = [ + const typeInnerPats: (TmPattern | { include: string })[] = hasTypeAnnotations ? [ ...(repository['generic-type'] ? [{ include: '#generic-type' }] : []), ...(repository['type-object-type'] ? [{ include: '#type-object-type' }] : []), ...(repository['type-paren'] ? [{ include: '#type-paren' }] : []), @@ -4947,7 +4947,7 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra // swallowed by the surrounding type region's name. ...literalTypeIncludes, { include: '#simple-type' }, - ]; + ] : []; // Union/intersection operators — only if present in @type rules const typeUnionOps = ['|', '&'].filter(op => typeLiterals.has(op)); if (typeUnionOps.length > 0) { @@ -5028,7 +5028,7 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra typeInnerPats.splice(idx === -1 ? typeInnerPats.length : idx, 0, { include: '#type-conditional' }); } - repository['type-inner'] = { patterns: typeInnerPats }; + if (hasTypeAnnotations) repository['type-inner'] = { patterns: typeInnerPats }; // Wire up deferred type-paren pattern (basic wiring; patched after type injections) if (repository['type-paren']) { diff --git a/test/redcmd-tm-diagnostics.ts b/test/redcmd-tm-diagnostics.ts new file mode 100644 index 0000000..38cb868 --- /dev/null +++ b/test/redcmd-tm-diagnostics.ts @@ -0,0 +1,185 @@ +// redcmd-tm-diagnostics.ts -- focused CLI guard for the RedCMD TextMate diagnostics that +// reported issue #12 (`TextMate(include)` / `TextMate(dead)`). The upstream extension is +// VS Code-bound, so this test mirrors the broken-include/dead-rule subset over JSON data and +// fingerprints the vendored source that defines the user-facing diagnostics. +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; + +type JsonRecord = Record; + +type Diagnostic = { + file: string; + path: string; + source: 'TextMate'; + code: 'include' | 'dead'; + severity: 'warning' | 'error' | 'hint'; + message: string; +}; + +const upstreamSubmodule = 'vendor/RedCMD-TmLanguage-Syntax-Highlighter'; +const upstreamDiagnostics = `${upstreamSubmodule}/src/DiagnosticCollection.ts`; + +function assertUpstreamDiagnosticFingerprint(): void { + if (!existsSync(upstreamDiagnostics)) { + throw new Error(`Missing RedCMD diagnostics submodule. Run: git submodule update --init ${upstreamSubmodule}`); + } + const source = readFileSync(upstreamDiagnostics, 'utf8'); + const required = [ + 'function diagnosticsBrokenIncludes', + "Cannot find repo name '${text}'", + 'The entire parent rule is nullified because all "#includes" failed.', + "source: 'TextMate'", + "code: 'include'", + "code: 'dead'", + ]; + const missing = required.filter((needle) => !source.includes(needle)); + if (missing.length) throw new Error(`RedCMD diagnostics fingerprint changed; missing: ${missing.join(', ')}`); +} + +function isRecord(value: unknown): value is JsonRecord { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function repositoryKeys(rule: JsonRecord): Set { + const repository = rule.repository; + return isRecord(repository) ? new Set(Object.keys(repository)) : new Set(); +} + +function visibleRepositories(rootRepository: Set, repositoryStack: Set[]): Set { + return new Set([...rootRepository, ...repositoryStack.flatMap((items) => [...items])]); +} + +function missingInclude(rule: JsonRecord, visible: Set): string | undefined { + const include = rule.include; + if (typeof include !== 'string' || !include.startsWith('#') || include.length <= 1) return; + const name = include.slice(1); + return visible.has(name) ? undefined : name; +} + +function includeDiagnostic(file: string, path: string, name: string, severity: 'warning' | 'error'): Diagnostic { + return { + file, + path, + source: 'TextMate', + code: 'include', + severity, + message: `Cannot find repo name '${name}'`, + }; +} + +function shouldReportDeadRule(rule: JsonRecord, path: string): boolean { + return path !== '$' && !('match' in rule) && !('begin' in rule) && !('include' in rule); +} + +function collectDiagnostics(file: string, grammar: JsonRecord): Diagnostic[] { + const diagnostics: Diagnostic[] = []; + const rootRepository = repositoryKeys(grammar); + + function walk(value: unknown, path: string, repositoryStack: Set[]): void { + if (Array.isArray(value)) { + value.forEach((item, index) => walk(item, `${path}[${index}]`, repositoryStack)); + return; + } + if (!isRecord(value)) return; + + const localRepositories = repositoryKeys(value); + const nextStack = localRepositories.size ? [...repositoryStack, localRepositories] : repositoryStack; + const visible = visibleRepositories(rootRepository, nextStack); + const missing = missingInclude(value, visible); + if (missing) diagnostics.push(includeDiagnostic(file, path, missing, 'warning')); + + for (const [key, child] of Object.entries(value)) { + if (key === 'patterns' && Array.isArray(child)) { + walkPatterns(value, child, `${path}.patterns`, nextStack, path); + } + else { + walk(child, `${path}.${key}`, nextStack); + } + } + } + + function walkPatterns(parentRule: JsonRecord, patterns: unknown[], path: string, repositoryStack: Set[], parentPath: string): void { + const deferredIncludes: Diagnostic[] = []; + let invalidIncludeOnlyCount = 0; + + patterns.forEach((pattern, index) => { + if (!isRecord(pattern)) { + walk(pattern, `${path}[${index}]`, repositoryStack); + return; + } + + const patternRepositories = repositoryKeys(pattern); + const patternStack = patternRepositories.size ? [...repositoryStack, patternRepositories] : repositoryStack; + const missing = missingInclude(pattern, visibleRepositories(rootRepository, patternStack)); + const includeOnly = missing && !('match' in pattern) && !('begin' in pattern); + if (includeOnly) { + invalidIncludeOnlyCount++; + deferredIncludes.push(includeDiagnostic(file, `${path}[${index}]`, missing, 'warning')); + return; + } + + walk(pattern, `${path}[${index}]`, repositoryStack); + }); + + if (!deferredIncludes.length) return; + const allPatternsFailed = invalidIncludeOnlyCount === patterns.length; + diagnostics.push(...deferredIncludes.map((diagnostic) => ({ + ...diagnostic, + severity: allPatternsFailed ? 'error' as const : diagnostic.severity, + }))); + if (allPatternsFailed && shouldReportDeadRule(parentRule, parentPath)) { + diagnostics.push({ + file, + path: parentPath, + source: 'TextMate', + code: 'dead', + severity: 'hint', + message: 'The entire parent rule is nullified because all "#includes" failed.', + }); + } + } + + walk(grammar, '$', []); + return diagnostics; +} + +function assertDetectorSelfTest(): void { + const grammar = { + scopeName: 'source.self-test', + patterns: [{ include: '#bad' }], + repository: { + bad: { + patterns: [{ include: '#missing' }], + }, + }, + }; + const diagnostics = collectDiagnostics('', grammar); + if (!diagnostics.some((diagnostic) => diagnostic.code === 'include' && diagnostic.severity === 'error')) { + throw new Error('Self-test failed to report TextMate(include) for a missing repository include.'); + } + if (!diagnostics.some((diagnostic) => diagnostic.code === 'dead')) { + throw new Error('Self-test failed to report TextMate(dead) for a nullified parent rule.'); + } +} + +assertUpstreamDiagnosticFingerprint(); +assertDetectorSelfTest(); + +const grammarFiles = readdirSync(process.cwd()) + .filter((name) => name.endsWith('.tmLanguage.json')) + .sort(); + +const failures = grammarFiles.flatMap((file) => { + const grammar = JSON.parse(readFileSync(join(process.cwd(), file), 'utf8')) as JsonRecord; + return collectDiagnostics(file, grammar); +}); + +if (failures.length) { + console.error(`RedCMD TextMate diagnostics found ${failures.length} issue(s):`); + for (const diagnostic of failures) { + console.error(` ${diagnostic.file} ${diagnostic.path} ${diagnostic.source}(${diagnostic.code}) ${diagnostic.severity}: ${diagnostic.message}`); + } + process.exit(1); +} + +console.log(`RedCMD TextMate diagnostics: ${grammarFiles.length} top-level grammars clean.`); \ No newline at end of file diff --git a/vendor/RedCMD-TmLanguage-Syntax-Highlighter b/vendor/RedCMD-TmLanguage-Syntax-Highlighter new file mode 160000 index 0000000..95c5cf0 --- /dev/null +++ b/vendor/RedCMD-TmLanguage-Syntax-Highlighter @@ -0,0 +1 @@ +Subproject commit 95c5cf0025211b1d0e33588cbc293ea2086cc1ab diff --git a/yaml.tmLanguage.json b/yaml.tmLanguage.json index 13c0ec8..e818b28 100644 --- a/yaml.tmLanguage.json +++ b/yaml.tmLanguage.json @@ -525,13 +525,6 @@ } ] }, - "type-inner": { - "patterns": [ - { - "include": "#simple-type" - } - ] - }, "punctuation": { "match": ":|\\?|-|\\{|,|\\}|\\[|\\]", "name": "punctuation.yaml" From 2311ea906cb8d2cab1c81d163df1af0559c5a65c Mon Sep 17 00:00:00 2001 From: Johnson Chu Date: Sat, 6 Jun 2026 16:26:47 +0800 Subject: [PATCH 2/3] Run TextMate diagnostics in CI --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cf5d16..4baafe7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true # Node 24+ runs the .ts sources directly (native type stripping) — no build, no tsx. - uses: actions/setup-node@v4 @@ -44,6 +46,7 @@ jobs: node test/js-conformance.ts node test/tsx-conformance.ts node test/jsx-conformance.ts + node test/redcmd-tm-diagnostics.ts node test/html-lexer-spike.ts node test/html-conformance.ts node test/html-monarch.ts From ce0b1759de3abe550835d071c653185c1fcc9691 Mon Sep 17 00:00:00 2001 From: Johnson Chu Date: Sat, 6 Jun 2026 16:43:46 +0800 Subject: [PATCH 3/3] Add Onigmo TextMate diagnostics guard --- javascript.tmLanguage.json | 37 ++++++++- javascriptreact.tmLanguage.json | 37 ++++++++- package-lock.json | 8 ++ package.json | 1 + src/gen-tm.ts | 68 ++++++++++------ test/redcmd-tm-diagnostics.ts | 140 ++++++++++++++++++++++++++++++-- typescript.tmLanguage.json | 43 +++++++++- typescriptreact.tmLanguage.json | 41 +++++++++- yaml.monarch.json | 4 +- yaml.tmLanguage.json | 4 +- yaml.ts | 7 +- 11 files changed, 344 insertions(+), 46 deletions(-) diff --git a/javascript.tmLanguage.json b/javascript.tmLanguage.json index 3dbed5a..64e591b 100644 --- a/javascript.tmLanguage.json +++ b/javascript.tmLanguage.json @@ -18,6 +18,9 @@ { "include": "#blockcomment" }, + { + "include": "#regex-literal-prefix-ops" + }, { "include": "#regex" }, @@ -224,6 +227,35 @@ } ], "repository": { + "regex-literal-prefix-ops": { + "name": "string.regexp.js", + "begin": "(?:(?<=[=|\\^&<>+\\-*%~(,.\\[?:{;])|(?<=\\binstanceof)|(?<=\\bin)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\byield)|(?<=\\bget)|(?<=\\bset)|(?<=\\basync)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\bstatic)|(?<=\\baccessor)|(?<=\\btypeof)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=^))s*([!](?:s*[!])*)s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", + "beginCaptures": { + "1": { + "name": "keyword.operator.logical.prefix.js" + }, + "2": { + "name": "comment.block.js" + }, + "3": { + "name": "punctuation.definition.string.begin.regexp.js" + } + }, + "end": "(/)([gimsuydv]*)", + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.regexp.js" + }, + "2": { + "name": "keyword.other.regexp.js" + } + }, + "patterns": [ + { + "include": "#regexp" + } + ] + }, "regexp": { "patterns": [ { @@ -1922,6 +1954,9 @@ { "include": "#blockcomment" }, + { + "include": "#regex-literal-prefix-ops" + }, { "include": "#regex" }, @@ -2142,7 +2177,7 @@ }, "regex": { "name": "string.regexp.js", - "begin": "(?:(?<=[=|\\^&<>+\\-*%~(,.\\[?:{;])|(?<=\\binstanceof)|(?<=\\bin)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\byield)|(?<=\\bget)|(?<=\\bset)|(?<=\\basync)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\bstatic)|(?<=\\baccessor)|(?<=\\btypeof)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=(?:[=|\\^&<>+\\-*%~(,.\\[?:{;]|\\binstanceof|\\bin|\\bnew|\\bextends|\\byield|\\bget|\\bset|\\basync|\\belse|\\bdo|\\breturn|\\bthrow|\\btry|\\bfinally|\\bcatch|\\bof|\\bcase|\\bexport|\\bdefault|\\bimport|\\bstatic|\\baccessor|\\btypeof|\\bvoid|\\bdelete|\\bawait|^)\\s*[!](?:\\s*[!])*)|(?<=^))\\s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", + "begin": "(?:(?<=[=|\\^&<>+\\-*%~(,.\\[?:{;])|(?<=\\binstanceof)|(?<=\\bin)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\byield)|(?<=\\bget)|(?<=\\bset)|(?<=\\basync)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\bstatic)|(?<=\\baccessor)|(?<=\\btypeof)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=^))\\s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", "beginCaptures": { "1": { "name": "comment.block.js" diff --git a/javascriptreact.tmLanguage.json b/javascriptreact.tmLanguage.json index 7e036d4..ca9efbb 100644 --- a/javascriptreact.tmLanguage.json +++ b/javascriptreact.tmLanguage.json @@ -27,6 +27,9 @@ { "include": "#blockcomment" }, + { + "include": "#regex-literal-prefix-ops" + }, { "include": "#regex" }, @@ -703,6 +706,35 @@ } ] }, + "regex-literal-prefix-ops": { + "name": "string.regexp.js.jsx", + "begin": "(?:(?<=[=|\\^&<>+\\-*%~(,.\\[?:{;])|(?<=\\binstanceof)|(?<=\\bin)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\byield)|(?<=\\bget)|(?<=\\bset)|(?<=\\basync)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\bstatic)|(?<=\\baccessor)|(?<=\\btypeof)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=^))s*([!](?:s*[!])*)s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", + "beginCaptures": { + "1": { + "name": "keyword.operator.logical.prefix.js.jsx" + }, + "2": { + "name": "comment.block.js.jsx" + }, + "3": { + "name": "punctuation.definition.string.begin.regexp.js.jsx" + } + }, + "end": "(/)([gimsuydv]*)", + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.regexp.js.jsx" + }, + "2": { + "name": "keyword.other.regexp.js.jsx" + } + }, + "patterns": [ + { + "include": "#regexp" + } + ] + }, "regexp": { "patterns": [ { @@ -2410,6 +2442,9 @@ { "include": "#blockcomment" }, + { + "include": "#regex-literal-prefix-ops" + }, { "include": "#regex" }, @@ -2630,7 +2665,7 @@ }, "regex": { "name": "string.regexp.js.jsx", - "begin": "(?:(?<=[=|\\^&<>+\\-*%~(,.\\[?:{;])|(?<=\\binstanceof)|(?<=\\bin)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\byield)|(?<=\\bget)|(?<=\\bset)|(?<=\\basync)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\bstatic)|(?<=\\baccessor)|(?<=\\btypeof)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=(?:[=|\\^&<>+\\-*%~(,.\\[?:{;]|\\binstanceof|\\bin|\\bnew|\\bextends|\\byield|\\bget|\\bset|\\basync|\\belse|\\bdo|\\breturn|\\bthrow|\\btry|\\bfinally|\\bcatch|\\bof|\\bcase|\\bexport|\\bdefault|\\bimport|\\bstatic|\\baccessor|\\btypeof|\\bvoid|\\bdelete|\\bawait|^)\\s*[!](?:\\s*[!])*)|(?<=^))\\s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", + "begin": "(?:(?<=[=|\\^&<>+\\-*%~(,.\\[?:{;])|(?<=\\binstanceof)|(?<=\\bin)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\byield)|(?<=\\bget)|(?<=\\bset)|(?<=\\basync)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\bstatic)|(?<=\\baccessor)|(?<=\\btypeof)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=^))\\s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", "beginCaptures": { "1": { "name": "comment.block.js.jsx" diff --git a/package-lock.json b/package-lock.json index 4ae993c..47f54ac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "parse5": "^8.0.1", "tree-sitter-cli": "^0.26.9", "typescript": "^5.6.0", + "vscode-onigmo": "^2.0.1", "vscode-oniguruma": "^2.0.1", "vscode-textmate": "^9.3.2", "vscode-tmlanguage-snapshot": "^1.0.1", @@ -313,6 +314,13 @@ "dev": true, "license": "MIT" }, + "node_modules/vscode-onigmo": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/vscode-onigmo/-/vscode-onigmo-2.0.1.tgz", + "integrity": "sha512-qxCk1RMffB3H6wGO2qeGuqag9yj6X5mHV//MdRmiYKmVSAZwFMBdAkfG0azUHZ5djnv0eV+RKIN0UT/4D8k7QQ==", + "dev": true, + "license": "MIT" + }, "node_modules/vscode-oniguruma": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-2.0.1.tgz", diff --git a/package.json b/package.json index 3bfbe2f..ea234d6 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "parse5": "^8.0.1", "tree-sitter-cli": "^0.26.9", "typescript": "^5.6.0", + "vscode-onigmo": "^2.0.1", "vscode-oniguruma": "^2.0.1", "vscode-textmate": "^9.3.2", "vscode-tmlanguage-snapshot": "^1.0.1", diff --git a/src/gen-tm.ts b/src/gen-tm.ts index 7349f93..4e4b8b0 100644 --- a/src/gen-tm.ts +++ b/src/gen-tm.ts @@ -833,6 +833,14 @@ function buildOperandStartClass(grammar: CstGrammar, identToken: TokenDecl | und return `[[:alpha:][:digit:]${cls}]`; } +function notAfterValueWithOptionalWhitespace(valueCharClass: string, maxWhitespace = 16): string { + const assertions: string[] = []; + for (let spaces = 0; spaces <= maxWhitespace; spaces++) { + assertions.push(`(?` named group `(?…)` arrowParamShape: string; // the arrow-shaped `(` confirm after `>` + close: string; // the generic close delimiter (`>` for TS/TSX) // Lookbehind body asserting the `>` just left closes a type-param LIST that carried a // top-level comma or constraint keyword (``, ``) — i.e. the SAME // generic-arrow disambiguation signal as `topTypeParam`, but for matching a `(` that @@ -1126,7 +1135,7 @@ function jsxDisambigDelims(grammar: CstGrammar, identRegex: string, separator: s ? `|${skip}\\b(?:${constraintKeywords.map(escapeRegex).join('|')})\\b` : ''; const typeParamCloseBehind = `${escapeRegex(open)}(?:${topComma}${behindKw})${skip}${escapeRegex(close)}`; - return { topComma, topTypeParam, balancedAngles, arrowParamShape, typeParamCloseBehind }; + return { topComma, topTypeParam, balancedAngles, arrowParamShape, close, typeParamCloseBehind }; } /** @@ -1916,12 +1925,12 @@ function generateTypeCastPattern( const tpEnd = `punctuation.definition.typeparameters.end.${langName}`; // `<` only at expression-start. A prefix cast's `<` is never preceded by a value // OPERAND; a comparison's `<` always is (`a < b`). Reject the cast when `<` is - // preceded — across any whitespace — by an operand-ending char: an identifier + // preceded — across bounded whitespace — by an operand-ending char: an identifier // char, `)`, `]`, a numeric/quote tail. This keeps `a < b > c`, `f() < g`, - // `x] < y` as comparisons (variable-length lookbehind; Oniguruma supports it). + // `x] < y` as comparisons while staying compatible with TextMate 2.0 Onigmo. // Casts after a keyword that ends in a letter (`return x`) stay a comparison // here — rare, and never a regression (they were unhighlighted before too). - const notAfter = `(? c` comparison — whose operands are arbitrary expressions — // is not swallowed). `\g` recurses for nested generics like `>`. @@ -2445,26 +2454,7 @@ function generateRegexLiteralPatterns( // Also match at start of line const startOfLine = '(?<=^)'; - // Ambiguous postfix/prefix op chars (TS `!`): a `/` may follow one ONLY when the op-run is - // the PREFIX form — i.e. the run is itself in a regex-start position (`= !/re/`, `!!/re/`, - // `return !/x/`), NOT the postfix non-null form (`x! / y` → division). We can't decide that - // from the single char before `/` (it's the op either way), so look back PAST the op-run and - // re-apply the same regex-start test there. The inner context is the SAME char-class + - // keywords + line-start used above, but un-wrapped (it sits inside this lookbehind), and the - // op-run is `[ops](?:\s*[ops])*` (chained `!!` allowed). Because these chars were excluded - // from `charLookbehind`, a postfix op (preceded by a value) matches NONE of the alternatives - // → the `/` falls through to the division operator. - const innerCtx = [ - charEsc ? `[${charEsc}]` : null, - ...info.preceedingKeywords.map(kw => `\\b${escapeRegex(kw)}`), - '^', - ].filter(Boolean).join('|'); - const opRun = info.postfixAmbiguousChars.map(escapeRegex).join(''); - const postfixBangLookbehind = opRun - ? `(?<=(?:${innerCtx})\\s*[${opRun}](?:\\s*[${opRun}])*)` - : ''; - - const lbAlts = [charLookbehind, keywordLookbehinds, postfixBangLookbehind, startOfLine] + const lbAlts = [charLookbehind, keywordLookbehinds, startOfLine] .filter(Boolean).join('|'); const fullLookbehind = `(?:${lbAlts})`; @@ -2491,6 +2481,31 @@ function generateRegexLiteralPatterns( }; if (commentBody) beginCaptures['1'] = { name: `comment.block.${langName}` }; + // Ambiguous postfix/prefix op chars (TS `!`): a `/` may follow one ONLY when the op-run is + // the PREFIX form (`= !/re/`, `return !!/x/`), not postfix non-null (`x! / y`). TextMate 2.0's + // Onigmo rejects the old variable-length lookbehind that looked past the whole op-run, so this + // separate pattern anchors on the fixed-width expression-start context and consumes the op-run. + const prefixOpClass = info.postfixAmbiguousChars.map(escapeForCharClass).join(''); + if (prefixOpClass) { + const prefixSlashGroup = commentBody ? '3' : '2'; + const prefixCaptures: Record = { + '1': { name: `keyword.operator.logical.prefix.${langName}` }, + [prefixSlashGroup]: { name: `punctuation.definition.string.begin.regexp.${langName}` }, + }; + if (commentBody) prefixCaptures['2'] = { name: `comment.block.${langName}` }; + result['regex-literal-prefix-ops'] = { + name: `string.regexp.${langName}`, + begin: `${fullLookbehind}\s*([${prefixOpClass}](?:\s*[${prefixOpClass}])*)\s*${commentPrefix}(/)${commentExclude}`, + beginCaptures: prefixCaptures, + end: `(/)(${info.flagsPattern})`, + endCaptures: { + '1': { name: `punctuation.definition.string.end.regexp.${langName}` }, + '2': { name: `keyword.other.regexp.${langName}` }, + }, + patterns: [{ include: '#regexp' }], + }; + } + result['regex-literal'] = { name: `string.regexp.${langName}`, begin: `${fullLookbehind}\\s*${commentPrefix}(/)${commentExclude}`, @@ -4382,6 +4397,7 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra for (const [key, pattern] of Object.entries(rlPatterns)) { repository[key] = pattern; } + if (rlPatterns['regex-literal-prefix-ops']) topPatterns.push({ include: '#regex-literal-prefix-ops' }); topPatterns.push({ include: '#regex-literal' }); } @@ -5405,7 +5421,7 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra if (angleBracket && angleDisambig) { const balancedAngles = angleDisambig.balancedAngles; const arrowParamShape = angleDisambig.arrowParamShape; - const arrowPos = `(?:(?<=\\basync\\s)|(?(…` // is a JSX element, so a generic-arrow type-param list is only recognised // when it carries a TOP-LEVEL comma inside the `<…>` (``, ``, @@ -6366,7 +6382,7 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra if (angleBracket && angleDisambig) { repository['arrow-function-params-generic'] = { name: `meta.parameters.arrow.${langName}`, - begin: `(?<=${angleDisambig.typeParamCloseBehind})\\s*(\\()\\s*$`, + begin: `(?<=${escapeRegex(angleDisambig.close)})\\s*(\\()\\s*$`, beginCaptures: { '1': { name: `punctuation.definition.parameters.begin.${langName}` }, }, diff --git a/test/redcmd-tm-diagnostics.ts b/test/redcmd-tm-diagnostics.ts index 38cb868..8c9e8ad 100644 --- a/test/redcmd-tm-diagnostics.ts +++ b/test/redcmd-tm-diagnostics.ts @@ -1,23 +1,39 @@ // redcmd-tm-diagnostics.ts -- focused CLI guard for the RedCMD TextMate diagnostics that -// reported issue #12 (`TextMate(include)` / `TextMate(dead)`). The upstream extension is -// VS Code-bound, so this test mirrors the broken-include/dead-rule subset over JSON data and -// fingerprints the vendored source that defines the user-facing diagnostics. +// reported issue #12 (`TextMate(include)` / `TextMate(dead)`) plus the TextMate 2.0 +// Onigmo regex compatibility diagnostic. The upstream extension is VS Code-bound, so this +// test mirrors the relevant JSON-data subset and fingerprints the vendored source that +// defines the user-facing diagnostics. import { existsSync, readFileSync, readdirSync } from 'node:fs'; -import { join } from 'node:path'; +import { createRequire } from 'node:module'; +import { dirname, join } from 'node:path'; +import type * as Onigmo from 'vscode-onigmo'; type JsonRecord = Record; +type TextMateRegexKey = 'match' | 'begin' | 'end' | 'while'; + +type OnigmoBinding = { + UTF8ToString(ptr: number): string; + _getLastOnigError(): number; +}; + +type OnigmoScannerWithBinding = Onigmo.OnigScanner & { + readonly _onigBinding?: OnigmoBinding; +}; type Diagnostic = { file: string; path: string; source: 'TextMate'; - code: 'include' | 'dead'; + code: 'include' | 'dead' | 'Onigmo'; severity: 'warning' | 'error' | 'hint'; message: string; }; const upstreamSubmodule = 'vendor/RedCMD-TmLanguage-Syntax-Highlighter'; const upstreamDiagnostics = `${upstreamSubmodule}/src/DiagnosticCollection.ts`; +const textmateRegexKeys = new Set(['match', 'begin', 'end', 'while']); +const require = createRequire(import.meta.url); +const textmateOnigmo = require('vscode-onigmo') as typeof Onigmo; function assertUpstreamDiagnosticFingerprint(): void { if (!existsSync(upstreamDiagnostics)) { @@ -26,20 +42,111 @@ function assertUpstreamDiagnosticFingerprint(): void { const source = readFileSync(upstreamDiagnostics, 'utf8'); const required = [ 'function diagnosticsBrokenIncludes', + 'function diagnosticsRegularExpressionErrors', "Cannot find repo name '${text}'", 'The entire parent rule is nullified because all "#includes" failed.', + 'Regex incompatible with TextMate 2.0', "source: 'TextMate'", "code: 'include'", "code: 'dead'", + "code: 'Onigmo'", ]; const missing = required.filter((needle) => !source.includes(needle)); if (missing.length) throw new Error(`RedCMD diagnostics fingerprint changed; missing: ${missing.join(', ')}`); } +async function loadOnigmo(): Promise { + const wasmPath = join(dirname(require.resolve('vscode-onigmo')), 'onigmo.wasm'); + await textmateOnigmo.loadWASM(readFileSync(wasmPath)); +} + function isRecord(value: unknown): value is JsonRecord { return typeof value === 'object' && value !== null && !Array.isArray(value); } +function onigmoError(pattern: string): string | undefined { + let scanner: OnigmoScannerWithBinding | undefined; + try { + scanner = new textmateOnigmo.OnigScanner([pattern]) as OnigmoScannerWithBinding; + const binding = scanner._onigBinding; + const lastError = binding?.UTF8ToString(binding._getLastOnigError()) ?? ''; + return normalizeOnigmoError(lastError); + } + catch (error: unknown) { + return normalizeOnigmoError(error instanceof Error ? error.message : String(error)); + } + finally { + scanner?.dispose(); + } +} + +function normalizeOnigmoError(error: string): string | undefined { + const message = error.replace(/^Error: /, '').replace(/^undefined error code$/, '').trim(); + return message || undefined; +} + +function countCapturingGroups(pattern: string): number { + let count = 0; + let inCharacterClass = false; + + for (let index = 0; index < pattern.length; index++) { + const char = pattern[index]; + if (char === '\\') { + index++; + continue; + } + if (char === '[') { + inCharacterClass = true; + continue; + } + if (char === ']' && inCharacterClass) { + inCharacterClass = false; + continue; + } + if (char !== '(' || inCharacterClass) continue; + + const next = pattern[index + 1]; + if (next !== '?') { + count++; + continue; + } + + const marker = pattern[index + 2]; + if (marker === '<') { + const lookbehindMarker = pattern[index + 3]; + if (lookbehindMarker !== '=' && lookbehindMarker !== '!') count++; + } + else if (marker === "'") { + count++; + } + } + + return count; +} + +function replaceBeginBackreferencesForTextMate(pattern: string, begin: string | undefined): string { + if (!begin || !/\\[0-9]/.test(pattern)) return pattern; + const captureCount = countCapturingGroups(begin); + return pattern.replace(/\\\\|\\([0-9])/g, (match, digit: string | undefined) => { + if (!digit) return match; + const index = Number(digit); + return index > 0 && index <= captureCount ? '' : match; + }); +} + +function onigmoDiagnostic(file: string, path: string, pattern: string): Diagnostic | undefined { + const error = onigmoError(pattern); + if (!error) return; + return { + file, + path, + source: 'TextMate', + code: 'Onigmo', + severity: 'warning', + message: `Regex incompatible with TextMate 2.0 (Onigmo v5.13.5)\n${error}`, + }; +} + function repositoryKeys(rule: JsonRecord): Set { const repository = rule.repository; return isRecord(repository) ? new Set(Object.keys(repository)) : new Set(); @@ -88,6 +195,15 @@ function collectDiagnostics(file: string, grammar: JsonRecord): Diagnostic[] { const missing = missingInclude(value, visible); if (missing) diagnostics.push(includeDiagnostic(file, path, missing, 'warning')); + for (const key of textmateRegexKeys) { + const pattern = value[key]; + if (typeof pattern !== 'string') continue; + const begin = key === 'end' || key === 'while' ? value.begin : undefined; + const replacedPattern = replaceBeginBackreferencesForTextMate(pattern, typeof begin === 'string' ? begin : undefined); + const diagnostic = onigmoDiagnostic(file, `${path}.${key}`, replacedPattern); + if (diagnostic) diagnostics.push(diagnostic); + } + for (const [key, child] of Object.entries(value)) { if (key === 'patterns' && Array.isArray(child)) { walkPatterns(value, child, `${path}.patterns`, nextStack, path); @@ -160,9 +276,23 @@ function assertDetectorSelfTest(): void { if (!diagnostics.some((diagnostic) => diagnostic.code === 'dead')) { throw new Error('Self-test failed to report TextMate(dead) for a nullified parent rule.'); } + const onigmoDiagnostics = collectDiagnostics('', { + scopeName: 'source.regex-self-test', + patterns: [{ include: '#bad-regex' }], + repository: { + 'bad-regex': { + begin: '(?<=(?:^|=)\\s*!*)/', + end: '/', + }, + }, + }); + if (!onigmoDiagnostics.some((diagnostic) => diagnostic.code === 'Onigmo')) { + throw new Error('Self-test failed to report TextMate(Onigmo) for an incompatible regex.'); + } } assertUpstreamDiagnosticFingerprint(); +await loadOnigmo(); assertDetectorSelfTest(); const grammarFiles = readdirSync(process.cwd()) diff --git a/typescript.tmLanguage.json b/typescript.tmLanguage.json index 3793b39..4bad517 100644 --- a/typescript.tmLanguage.json +++ b/typescript.tmLanguage.json @@ -33,6 +33,9 @@ { "include": "#blockcomment" }, + { + "include": "#regex-literal-prefix-ops" + }, { "include": "#regex" }, @@ -451,6 +454,35 @@ "match": "[<>]", "name": "keyword.operator.relational.ts" }, + "regex-literal-prefix-ops": { + "name": "string.regexp.ts", + "begin": "(?:(?<=[=|\\^&<>+\\-*%~,\\[(?:{;.])|(?<=\\bis)|(?<=\\bkeyof)|(?<=\\btypeof)|(?<=\\breadonly)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\bin)|(?<=\\bas)|(?<=\\binstanceof)|(?<=\\bclass)|(?<=\\basync)|(?<=\\byield)|(?<=\\bsatisfies)|(?<=\\bfunction)|(?<=\\bget)|(?<=\\bset)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bpublic)|(?<=\\bprivate)|(?<=\\bprotected)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bdeclare)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\btype)|(?<=\\bstatic)|(?<=\\babstract)|(?<=\\boverride)|(?<=\\baccessor)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=^))s*([!](?:s*[!])*)s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", + "beginCaptures": { + "1": { + "name": "keyword.operator.logical.prefix.ts" + }, + "2": { + "name": "comment.block.ts" + }, + "3": { + "name": "punctuation.definition.string.begin.regexp.ts" + } + }, + "end": "(/)([gimsuydv]*)", + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.regexp.ts" + }, + "2": { + "name": "keyword.other.regexp.ts" + } + }, + "patterns": [ + { + "include": "#regexp" + } + ] + }, "regexp": { "patterns": [ { @@ -1475,7 +1507,7 @@ }, "arrow-type-parameters": { "name": "meta.type.parameters.ts", - "begin": "(?:(?<=\\basync\\s)|(?[^<>]*(?:<\\g>[^<>]*)*)>\\s*\\(\\s*(?:\\)|\\.\\.\\.|(?:[a-zA-Z_$\\p{L}\\p{Nl}]|\\\\u[0-9A-Fa-f]{4}|\\\\u\\{[0-9A-Fa-f]+\\})(?:[a-zA-Z0-9_$\\p{L}\\p{Nl}\\p{Nd}\\p{Mn}\\p{Mc}\\p{Pc}]|\\\\u[0-9A-Fa-f]{4}|\\\\u\\{[0-9A-Fa-f]+\\})*\\s*[:,?)]|[{\\[]|$))", + "begin": "(?:(?<=\\basync\\s)|(?[^<>]*(?:<\\g>[^<>]*)*)>\\s*\\(\\s*(?:\\)|\\.\\.\\.|(?:[a-zA-Z_$\\p{L}\\p{Nl}]|\\\\u[0-9A-Fa-f]{4}|\\\\u\\{[0-9A-Fa-f]+\\})(?:[a-zA-Z0-9_$\\p{L}\\p{Nl}\\p{Nd}\\p{Mn}\\p{Mc}\\p{Pc}]|\\\\u[0-9A-Fa-f]{4}|\\\\u\\{[0-9A-Fa-f]+\\})*\\s*[:,?)]|[{\\[]|$))", "beginCaptures": { "1": { "name": "punctuation.definition.typeparameters.begin.ts" @@ -2290,7 +2322,7 @@ }, "arrow-function-params-generic": { "name": "meta.parameters.arrow.ts", - "begin": "(?<=<(?:(?:[^<>{}\"']|\\{[^{}]*\\}|\"[^\"]*\"|'[^']*')*,|(?:[^<>{}\"']|\\{[^{}]*\\}|\"[^\"]*\"|'[^']*')*\\b(?:extends)\\b)(?:[^<>{}\"']|\\{[^{}]*\\}|\"[^\"]*\"|'[^']*')*>)\\s*(\\()\\s*$", + "begin": "(?<=>)\\s*(\\()\\s*$", "beginCaptures": { "1": { "name": "punctuation.definition.parameters.begin.ts" @@ -2821,6 +2853,9 @@ { "include": "#blockcomment" }, + { + "include": "#regex-literal-prefix-ops" + }, { "include": "#regex" }, @@ -3209,7 +3244,7 @@ }, "regex": { "name": "string.regexp.ts", - "begin": "(?:(?<=[=|\\^&<>+\\-*%~,\\[(?:{;.])|(?<=\\bis)|(?<=\\bkeyof)|(?<=\\btypeof)|(?<=\\breadonly)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\bin)|(?<=\\bas)|(?<=\\binstanceof)|(?<=\\bclass)|(?<=\\basync)|(?<=\\byield)|(?<=\\bsatisfies)|(?<=\\bfunction)|(?<=\\bget)|(?<=\\bset)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bpublic)|(?<=\\bprivate)|(?<=\\bprotected)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bdeclare)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\btype)|(?<=\\bstatic)|(?<=\\babstract)|(?<=\\boverride)|(?<=\\baccessor)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=(?:[=|\\^&<>+\\-*%~,\\[(?:{;.]|\\bis|\\bkeyof|\\btypeof|\\breadonly|\\bnew|\\bextends|\\bin|\\bas|\\binstanceof|\\bclass|\\basync|\\byield|\\bsatisfies|\\bfunction|\\bget|\\bset|\\belse|\\bdo|\\breturn|\\bthrow|\\btry|\\bfinally|\\bcatch|\\bpublic|\\bprivate|\\bprotected|\\bof|\\bcase|\\bdeclare|\\bexport|\\bdefault|\\bimport|\\btype|\\bstatic|\\babstract|\\boverride|\\baccessor|\\bvoid|\\bdelete|\\bawait|^)\\s*[!](?:\\s*[!])*)|(?<=^))\\s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", + "begin": "(?:(?<=[=|\\^&<>+\\-*%~,\\[(?:{;.])|(?<=\\bis)|(?<=\\bkeyof)|(?<=\\btypeof)|(?<=\\breadonly)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\bin)|(?<=\\bas)|(?<=\\binstanceof)|(?<=\\bclass)|(?<=\\basync)|(?<=\\byield)|(?<=\\bsatisfies)|(?<=\\bfunction)|(?<=\\bget)|(?<=\\bset)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bpublic)|(?<=\\bprivate)|(?<=\\bprotected)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bdeclare)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\btype)|(?<=\\bstatic)|(?<=\\babstract)|(?<=\\boverride)|(?<=\\baccessor)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=^))\\s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", "beginCaptures": { "1": { "name": "comment.block.ts" @@ -3387,7 +3422,7 @@ }, "cast": { "name": "meta.cast.expr.ts", - "begin": "(?[\\w$.,\\[\\]\\s|&]*(?:<\\g>[\\w$.,\\[\\]\\s|&]*)*)>\\s*[[:alpha:][:digit:]_$\"`({\\[\\-])", + "begin": "(?[\\w$.,\\[\\]\\s|&]*(?:<\\g>[\\w$.,\\[\\]\\s|&]*)*)>\\s*[[:alpha:][:digit:]_$\"`({\\[\\-])", "beginCaptures": { "1": { "name": "punctuation.definition.typeparameters.begin.ts" diff --git a/typescriptreact.tmLanguage.json b/typescriptreact.tmLanguage.json index ee94ba9..224ffbb 100644 --- a/typescriptreact.tmLanguage.json +++ b/typescriptreact.tmLanguage.json @@ -39,6 +39,9 @@ { "include": "#blockcomment" }, + { + "include": "#regex-literal-prefix-ops" + }, { "include": "#regex" }, @@ -956,6 +959,35 @@ } ] }, + "regex-literal-prefix-ops": { + "name": "string.regexp.tsx", + "begin": "(?:(?<=[=|\\^&<>+\\-*%~,\\[(?:{;.])|(?<=\\bis)|(?<=\\bkeyof)|(?<=\\btypeof)|(?<=\\breadonly)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\bin)|(?<=\\bas)|(?<=\\binstanceof)|(?<=\\bclass)|(?<=\\basync)|(?<=\\byield)|(?<=\\bsatisfies)|(?<=\\bfunction)|(?<=\\bget)|(?<=\\bset)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bpublic)|(?<=\\bprivate)|(?<=\\bprotected)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bdeclare)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\btype)|(?<=\\bstatic)|(?<=\\babstract)|(?<=\\boverride)|(?<=\\baccessor)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=^))s*([!](?:s*[!])*)s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", + "beginCaptures": { + "1": { + "name": "keyword.operator.logical.prefix.tsx" + }, + "2": { + "name": "comment.block.tsx" + }, + "3": { + "name": "punctuation.definition.string.begin.regexp.tsx" + } + }, + "end": "(/)([gimsuydv]*)", + "endCaptures": { + "1": { + "name": "punctuation.definition.string.end.regexp.tsx" + }, + "2": { + "name": "keyword.other.regexp.tsx" + } + }, + "patterns": [ + { + "include": "#regexp" + } + ] + }, "regexp": { "patterns": [ { @@ -1980,7 +2012,7 @@ }, "arrow-type-parameters": { "name": "meta.type.parameters.tsx", - "begin": "(?:(?<=\\basync\\s)|(?{}\"']|\\{[^{}]*\\}|\"[^\"]*\"|'[^']*')*,|(?:[^<>{}\"']|\\{[^{}]*\\}|\"[^\"]*\"|'[^']*')*\\bextends\\b\\s*(?!=)))(?=(?[^<>]*(?:<\\g>[^<>]*)*)>\\s*\\(\\s*(?:\\)|\\.\\.\\.|(?:[a-zA-Z_$\\p{L}\\p{Nl}]|\\\\u[0-9A-Fa-f]{4}|\\\\u\\{[0-9A-Fa-f]+\\})(?:[a-zA-Z0-9_$\\p{L}\\p{Nl}\\p{Nd}\\p{Mn}\\p{Mc}\\p{Pc}]|\\\\u[0-9A-Fa-f]{4}|\\\\u\\{[0-9A-Fa-f]+\\})*\\s*[:,?)]|[{\\[]|$))", + "begin": "(?:(?<=\\basync\\s)|(?{}\"']|\\{[^{}]*\\}|\"[^\"]*\"|'[^']*')*,|(?:[^<>{}\"']|\\{[^{}]*\\}|\"[^\"]*\"|'[^']*')*\\bextends\\b\\s*(?!=)))(?=(?[^<>]*(?:<\\g>[^<>]*)*)>\\s*\\(\\s*(?:\\)|\\.\\.\\.|(?:[a-zA-Z_$\\p{L}\\p{Nl}]|\\\\u[0-9A-Fa-f]{4}|\\\\u\\{[0-9A-Fa-f]+\\})(?:[a-zA-Z0-9_$\\p{L}\\p{Nl}\\p{Nd}\\p{Mn}\\p{Mc}\\p{Pc}]|\\\\u[0-9A-Fa-f]{4}|\\\\u\\{[0-9A-Fa-f]+\\})*\\s*[:,?)]|[{\\[]|$))", "beginCaptures": { "1": { "name": "punctuation.definition.typeparameters.begin.tsx" @@ -2795,7 +2827,7 @@ }, "arrow-function-params-generic": { "name": "meta.parameters.arrow.tsx", - "begin": "(?<=<(?:(?:[^<>{}\"']|\\{[^{}]*\\}|\"[^\"]*\"|'[^']*')*,|(?:[^<>{}\"']|\\{[^{}]*\\}|\"[^\"]*\"|'[^']*')*\\b(?:extends)\\b)(?:[^<>{}\"']|\\{[^{}]*\\}|\"[^\"]*\"|'[^']*')*>)\\s*(\\()\\s*$", + "begin": "(?<=>)\\s*(\\()\\s*$", "beginCaptures": { "1": { "name": "punctuation.definition.parameters.begin.tsx" @@ -3332,6 +3364,9 @@ { "include": "#blockcomment" }, + { + "include": "#regex-literal-prefix-ops" + }, { "include": "#regex" }, @@ -3720,7 +3755,7 @@ }, "regex": { "name": "string.regexp.tsx", - "begin": "(?:(?<=[=|\\^&<>+\\-*%~,\\[(?:{;.])|(?<=\\bis)|(?<=\\bkeyof)|(?<=\\btypeof)|(?<=\\breadonly)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\bin)|(?<=\\bas)|(?<=\\binstanceof)|(?<=\\bclass)|(?<=\\basync)|(?<=\\byield)|(?<=\\bsatisfies)|(?<=\\bfunction)|(?<=\\bget)|(?<=\\bset)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bpublic)|(?<=\\bprivate)|(?<=\\bprotected)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bdeclare)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\btype)|(?<=\\bstatic)|(?<=\\babstract)|(?<=\\boverride)|(?<=\\baccessor)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=(?:[=|\\^&<>+\\-*%~,\\[(?:{;.]|\\bis|\\bkeyof|\\btypeof|\\breadonly|\\bnew|\\bextends|\\bin|\\bas|\\binstanceof|\\bclass|\\basync|\\byield|\\bsatisfies|\\bfunction|\\bget|\\bset|\\belse|\\bdo|\\breturn|\\bthrow|\\btry|\\bfinally|\\bcatch|\\bpublic|\\bprivate|\\bprotected|\\bof|\\bcase|\\bdeclare|\\bexport|\\bdefault|\\bimport|\\btype|\\bstatic|\\babstract|\\boverride|\\baccessor|\\bvoid|\\bdelete|\\bawait|^)\\s*[!](?:\\s*[!])*)|(?<=^))\\s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", + "begin": "(?:(?<=[=|\\^&<>+\\-*%~,\\[(?:{;.])|(?<=\\bis)|(?<=\\bkeyof)|(?<=\\btypeof)|(?<=\\breadonly)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\bin)|(?<=\\bas)|(?<=\\binstanceof)|(?<=\\bclass)|(?<=\\basync)|(?<=\\byield)|(?<=\\bsatisfies)|(?<=\\bfunction)|(?<=\\bget)|(?<=\\bset)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bpublic)|(?<=\\bprivate)|(?<=\\bprotected)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bdeclare)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\btype)|(?<=\\bstatic)|(?<=\\babstract)|(?<=\\boverride)|(?<=\\baccessor)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=^))\\s*(?:((?:/\\*\\*(?!/)[\\s\\S]*?\\*/|/\\*[\\s\\S]*?\\*/)\\s*))?(/)(?![*/])", "beginCaptures": { "1": { "name": "comment.block.tsx" diff --git a/yaml.monarch.json b/yaml.monarch.json index 119fce7..e80fa26 100644 --- a/yaml.monarch.json +++ b/yaml.monarch.json @@ -96,7 +96,7 @@ } ], [ - "(? + notPrecededBy(seq('!', repeat(hspace, index + 1, index + 1))), +)); // Numeric plain scalars (YAML 1.2 core schema): decimal / octal / hex integers, floats, ±.inf, // .nan. Anything outside the core schema (binary `0b…`, dates, `12:34:56`) stays a plain string, // matching what the `yaml` oracle resolves to a number.