diff --git a/README.md b/README.md index e5a67b3..05f3df7 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,19 @@ import * as js from '@speed-highlight/core/languages/js.js'; loadLanguage('js', js); ``` +Tokenize without the language registry, so a bundler only keeps the languages you import + +```js +import { tokenize } from '@speed-highlight/core/tokenize'; +import html from '@speed-highlight/core/languages/html.js'; +import css from '@speed-highlight/core/languages/css.js'; +import js from '@speed-highlight/core/languages/js.js'; + +tokenize(code, { sub: html }, (str, type) => { /* ... */ }, { languages: { css, js } }); +``` + +This entry is synchronous and a `sub` referring to a language not given in `languages` is emitted as plain text. + --- #### CDN diff --git a/package.json b/package.json index 230b05f..8015f98 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,16 @@ "require": "./dist/node/terminal.js", "types": "./dist/terminal.d.ts" }, + "./tokenize": { + "import": "./dist/tokenize.js", + "require": "./dist/node/tokenize.js", + "types": "./dist/tokenize.d.ts" + }, + "./common.js": { + "import": "./dist/common.js", + "require": "./dist/node/common.js", + "types": "./dist/common.d.ts" + }, "./languages/*.js": { "import": "./dist/languages/*.js", "require": "./dist/node/languages/*.js", @@ -30,7 +40,8 @@ "example": "examples" }, "scripts": { - "build": "bash .github/workflows/build.sh" + "build": "bash .github/workflows/build.sh", + "test": "node --test tests/*.test.js" }, "repository": { "type": "git", diff --git a/src/index.js b/src/index.js index 93e69e5..37ab620 100644 --- a/src/index.js +++ b/src/index.js @@ -30,7 +30,7 @@ * @typedef {('deleted'|'err'|'var'|'section'|'kwd'|'class'|'cmnt'|'insert'|'type'|'func'|'bool'|'num'|'oper'|'str'|'esc')} ShjToken */ -import expandData from './common.js'; +import { tokenizer } from './tokenize.js'; const langs = {}, sanitize = (str = '') => @@ -58,56 +58,20 @@ const langs = {}, * * the type of the token */ export async function tokenize(src, lang, token) { - try { - let m, - part, - first = {}, - match, - cache = [], - i = 0, - data = typeof lang === 'string' ? (await (langs[lang] ??= import(`./languages/${lang}.js`))) : lang, - // make a fast shallow copy to bee able to splice lang without change the original one - arr = [...typeof lang === 'string' ? data.default : lang.sub]; - - while (i < src.length) { - first.index = null; - for (m = arr.length; m-- > 0;) { - part = arr[m].expand ? expandData[arr[m].expand] : arr[m]; - // do not call again exec if the previous result is sufficient - if (cache[m] === undefined || cache[m].match.index < i) { - part.match.lastIndex = i; - match = part.match.exec(src); - if (match === null) { - // no more match with this regex can be disposed - arr.splice(m, 1); - cache.splice(m, 1); - continue; - } - // save match for later use to decrease performance cost - cache[m] = { match, lastIndex: part.match.lastIndex }; - } - // check if it the first match in the string - if (cache[m].match[0] && (cache[m].match.index <= first.index || first.index === null)) - first = { - part: part, - index: cache[m].match.index, - match: cache[m].match[0], - end: cache[m].lastIndex - } - } - if (first.index === null) - break; - token(src.slice(i, first.index), data.type); - i = first.end; - if (first.part.sub) - await tokenize(first.match, typeof first.part.sub === 'string' ? first.part.sub : (typeof first.part.sub === 'function' ? first.part.sub(first.match) : first.part), token); - else - token(first.match, first.part.type); + let data, + it = tokenizer(src, lang, token), + res = it.next(); + + while (!res.done) { + try { + // bundlers can make this throw synchronously, so it cannot be a catch on the promise + data = await (langs[res.value] ??= import(`./languages/${res.value}.js`)); } - token(src.slice(i, src.length), data.type); - } - catch { - token(src); + catch { + // an unknown language is left undefined, the tokenizer emits its source untouched + data = undefined; + } + res = it.next(data); } } diff --git a/src/tokenize.js b/src/tokenize.js new file mode 100644 index 0000000..8e4c4d2 --- /dev/null +++ b/src/tokenize.js @@ -0,0 +1,121 @@ +/** + * @module tokenize + * (Registry free tokenizer) +*/ + +/** + * @typedef {import('./index.js').ShjToken} ShjToken + */ + +/** + * @typedef {import('./index.js').ShjLanguageDefinition} ShjLanguageDefinition + */ + +/** + * A language, either the module exporting it or the definition itself + * @typedef {{ default: ShjLanguageDefinition, type?: ShjToken }|ShjLanguageDefinition} ShjLanguageModule + */ + +/** + * @typedef {Object} ShjTokenizeOptions + * @property {Object} [languages={}] The languages a `sub` can refer to by name + */ + +import expandData from './common.js'; + +/** + * Find the tokens in the given code, yielding the name of every language + * it needs and expecting its definition to be sent back + * + * @generator + * @function tokenizer + * @param {string} src The code + * @param {string|ShjLanguageDefinition|{ sub: ShjLanguageDefinition }} lang The language of the code + * @param {function(string, ShjToken=):void} token The callback function + * @yields {string} The name of a language to resolve + * @returns {Generator} + */ +export function* tokenizer(src, lang, token) { + try { + let m, + part, + first = {}, + match, + cache = [], + i = 0, + data = typeof lang === 'string' ? yield lang : lang, + // make a fast shallow copy to bee able to splice lang without change the original one + arr = [...typeof lang === 'string' ? data.default : lang.sub]; + + while (i < src.length) { + first.index = null; + for (m = arr.length; m-- > 0;) { + part = arr[m].expand ? expandData[arr[m].expand] : arr[m]; + // do not call again exec if the previous result is sufficient + if (cache[m] === undefined || cache[m].match.index < i) { + part.match.lastIndex = i; + match = part.match.exec(src); + if (match === null) { + // no more match with this regex can be disposed + arr.splice(m, 1); + cache.splice(m, 1); + continue; + } + // save match for later use to decrease performance cost + cache[m] = { match, lastIndex: part.match.lastIndex }; + } + // check if it the first match in the string + if (cache[m].match[0] && (cache[m].match.index <= first.index || first.index === null)) + first = { + part: part, + index: cache[m].match.index, + match: cache[m].match[0], + end: cache[m].lastIndex + } + } + if (first.index === null) + break; + token(src.slice(i, first.index), data.type); + i = first.end; + if (first.part.sub) + yield* tokenizer(first.match, typeof first.part.sub === 'string' ? first.part.sub : (typeof first.part.sub === 'function' ? first.part.sub(first.match) : first.part), token); + else + token(first.match, first.part.type); + } + token(src.slice(i, src.length), data.type); + } + catch { + token(src); + } +} + +/** + * Find the tokens in the given code and call the given callback, + * without loading anything: every language used has to be given by the caller + * + * @example + * import json from '@speed-highlight/core/languages/json.js'; + * import { tokenize } from '@speed-highlight/core/tokenize'; + * + * tokenize(src, { sub: json }, (str, type) => process.stdout.write(str)); + * + * @function tokenize + * @param {string} src The code + * @param {string|ShjLanguageDefinition|{ sub: ShjLanguageDefinition }} lang The language of the code + * @param {function(string, ShjToken=):void} token The callback function + * this function will be given + * * the text of the token + * * the type of the token + * @param {ShjTokenizeOptions} [opt={}] Customization options + */ +export function tokenize(src, lang, token, opt = {}) { + let lng, + it = tokenizer(src, lang, token), + res = it.next(); + + while (!res.done) { + lng = opt.languages?.[res.value]; + res = it.next(Array.isArray(lng) ? { default: lng } : lng); + } +} + diff --git a/tests/tokenize.test.js b/tests/tokenize.test.js new file mode 100644 index 0000000..f0e8ca0 --- /dev/null +++ b/tests/tokenize.test.js @@ -0,0 +1,75 @@ +import { test } from 'node:test'; +import { deepStrictEqual } from 'node:assert'; +import { readFileSync } from 'node:fs'; +import { tokenize as tokenizeAsync } from '../src/index.js'; +import { tokenize } from '../src/tokenize.js'; +import * as css from '../src/languages/css.js'; +import * as html from '../src/languages/html.js'; +import * as js from '../src/languages/js.js'; +import * as js_template_literals from '../src/languages/js_template_literals.js'; +import * as jsdoc from '../src/languages/jsdoc.js'; +import * as json from '../src/languages/json.js'; +import * as regex from '../src/languages/regex.js'; +import * as todo from '../src/languages/todo.js'; + +let fixtures = new URL('../examples/languages/', import.meta.url), + languages = { css, html, js, js_template_literals, jsdoc, json, regex, todo }, + read = file => readFileSync(new URL(file, fixtures), 'utf8'), + collect = (src, lang, opt) => { + let tokens = []; + tokenize(src, lang, (str, token) => tokens.push([token, str]), opt); + return tokens; + }, + collectAsync = async (src, lang) => { + let tokens = []; + await tokenizeAsync(src, lang, (str, token) => tokens.push([token, str])); + return tokens; + }; + +test('a definition given as a sub needs no registry', async () => { + let src = read('test.json'); + + deepStrictEqual(collect(src, { sub: json.default }), await collectAsync(src, 'json')); +}); + +test('a nested sub is resolved from the given languages', async () => { + let src = read('test.html'); + + deepStrictEqual(collect(src, { sub: html.default }, { languages }), await collectAsync(src, 'html')); +}); + +test('a language can be given by name', async () => { + let src = read('test.js'); + + deepStrictEqual(collect(src, 'js', { languages }), await collectAsync(src, 'js')); +}); + +test('a language can be given as a definition or as its module', async () => { + let src = read('test.json'); + + deepStrictEqual( + collect(src, 'json', { languages: { json: json.default } }), + collect(src, 'json', { languages })); +}); + +test('the type of a language applies to the text it does not match', () => { + deepStrictEqual(collect('// TODO stuff', { sub: js.default }, { languages }), [ + [undefined, ''], + ['cmnt', '// '], + ['err', 'TODO'], + ['cmnt', ' stuff'], + [undefined, ''] + ]); +}); + +test('a sub that is not given is emitted as plain text', () => { + deepStrictEqual(collect('// TODO stuff', { sub: js.default }), [ + [undefined, ''], + [undefined, '// TODO stuff'], + [undefined, ''] + ]); +}); + +test('a language that is not given is emitted as plain text', () => { + deepStrictEqual(collect('{}', 'json'), [[undefined, '{}']]); +});