feat: expose registry-free tokenize function

README.md

@@ -68,6 +68,19 @@ import * as js from '@speed-highlight/core/languages/js.js';
 loadLanguage('js', js);
 ```
 
+Tokenize without the language registry, so a bundler only keeps the languages you import
+
+```js
+import { tokenize } from '@speed-highlight/core/tokenize';
+import html from '@speed-highlight/core/languages/html.js';
+import css from '@speed-highlight/core/languages/css.js';
+import js from '@speed-highlight/core/languages/js.js';
+
+tokenize(code, { sub: html }, (str, type) => { /* ... */ }, { languages: { css, js } });
+```
+
+This entry is synchronous and a `sub` referring to a language not given in `languages` is emitted as plain text.
+
 ---
 
 #### CDN

package.json

@@ -19,6 +19,16 @@
       "require": "./dist/node/terminal.js",
       "types": "./dist/terminal.d.ts"
     },
+    "./tokenize": {
+      "import": "./dist/tokenize.js",
+      "require": "./dist/node/tokenize.js",
+      "types": "./dist/tokenize.d.ts"
+    },
+    "./common.js": {
+      "import": "./dist/common.js",
+      "require": "./dist/node/common.js",
+      "types": "./dist/common.d.ts"
+    },
     "./languages/*.js": {
       "import": "./dist/languages/*.js",
       "require": "./dist/node/languages/*.js",
@@ -30,7 +40,8 @@
     "example": "examples"
   },
   "scripts": {
-    "build": "bash .github/workflows/build.sh"
+    "build": "bash .github/workflows/build.sh",
+    "test": "node --test tests/*.test.js"
   },
   "repository": {
     "type": "git",

src/index.js

@@ -30,7 +30,7 @@
  * @typedef {('deleted'|'err'|'var'|'section'|'kwd'|'class'|'cmnt'|'insert'|'type'|'func'|'bool'|'num'|'oper'|'str'|'esc')} ShjToken
  */
 
-import expandData from './common.js';
+import { tokenizer } from './tokenize.js';
 
 const langs = {},
 	sanitize = (str = '') =>
@@ -58,56 +58,20 @@ const langs = {},
  * * the type of the token
  */
 export async function tokenize(src, lang, token) {
-	try {
-		let m,
-			part,
-			first = {},
-			match,
-			cache = [],
-			i = 0,
-			data = typeof lang === 'string' ? (await (langs[lang] ??= import(`./languages/${lang}.js`))) : lang,
-			// make a fast shallow copy to bee able to splice lang without change the original one
-			arr = [...typeof lang === 'string' ? data.default : lang.sub];
-
-		while (i < src.length) {
-			first.index = null;
-			for (m = arr.length; m-- > 0;) {
-				part = arr[m].expand ? expandData[arr[m].expand] : arr[m];
-				// do not call again exec if the previous result is sufficient
-				if (cache[m] === undefined || cache[m].match.index < i) {
-					part.match.lastIndex = i;
-					match = part.match.exec(src);
-					if (match === null) {
-						// no more match with this regex can be disposed
-						arr.splice(m, 1);
-						cache.splice(m, 1);
-						continue;
-					}
-					// save match for later use to decrease performance cost
-					cache[m] = { match, lastIndex: part.match.lastIndex };
-				}
-				// check if it the first match in the string
-				if (cache[m].match[0] && (cache[m].match.index <= first.index || first.index === null))
-					first = {
-						part: part,
-						index: cache[m].match.index,
-						match: cache[m].match[0],
-						end: cache[m].lastIndex
-					}
-			}
-			if (first.index === null)
-				break;
-			token(src.slice(i, first.index), data.type);
-			i = first.end;
-			if (first.part.sub)
-				await tokenize(first.match, typeof first.part.sub === 'string' ? first.part.sub : (typeof first.part.sub === 'function' ? first.part.sub(first.match) : first.part), token);
-			else
-				token(first.match, first.part.type);
+	let data,
+		it = tokenizer(src, lang, token),
+		res = it.next();
+
+	while (!res.done) {
+		try {
+			// bundlers can make this throw synchronously, so it cannot be a catch on the promise
+			data = await (langs[res.value] ??= import(`./languages/${res.value}.js`));
 		}
-		token(src.slice(i, src.length), data.type);
-	}
-	catch {
-		token(src);
+		catch {
+			// an unknown language is left undefined, the tokenizer emits its source untouched
+			data = undefined;
+		}
+		res = it.next(data);
 	}
 }
 

src/tokenize.js

@@ -0,0 +1,121 @@
+/**
+ * @module tokenize
+ * (Registry free tokenizer)
+*/
+
+/**
+ * @typedef {import('./index.js').ShjToken} ShjToken
+ */
+
+/**
+ * @typedef {import('./index.js').ShjLanguageDefinition} ShjLanguageDefinition
+ */
+
+/**
+ * A language, either the module exporting it or the definition itself
+ * @typedef {{ default: ShjLanguageDefinition, type?: ShjToken }|ShjLanguageDefinition} ShjLanguageModule
+ */
+
+/**
+ * @typedef {Object} ShjTokenizeOptions
+ * @property {Object<string, ShjLanguageModule>} [languages={}] The languages a `sub` can refer to by name
+ */
+
+import expandData from './common.js';
+
+/**
+ * Find the tokens in the given code, yielding the name of every language
+ * it needs and expecting its definition to be sent back
+ *
+ * @generator
+ * @function tokenizer
+ * @param {string} src The code
+ * @param {string|ShjLanguageDefinition|{ sub: ShjLanguageDefinition }} lang The language of the code
+ * @param {function(string, ShjToken=):void} token The callback function
+ * @yields {string} The name of a language to resolve
+ * @returns {Generator<string, void, { default: ShjLanguageDefinition, type?: ShjToken }|undefined>}
+ */
+export function* tokenizer(src, lang, token) {
+	try {
+		let m,
+			part,
+			first = {},
+			match,
+			cache = [],
+			i = 0,
+			data = typeof lang === 'string' ? yield lang : lang,
+			// make a fast shallow copy to bee able to splice lang without change the original one
+			arr = [...typeof lang === 'string' ? data.default : lang.sub];
+
+		while (i < src.length) {
+			first.index = null;
+			for (m = arr.length; m-- > 0;) {
+				part = arr[m].expand ? expandData[arr[m].expand] : arr[m];
+				// do not call again exec if the previous result is sufficient
+				if (cache[m] === undefined || cache[m].match.index < i) {
+					part.match.lastIndex = i;
+					match = part.match.exec(src);
+					if (match === null) {
+						// no more match with this regex can be disposed
+						arr.splice(m, 1);
+						cache.splice(m, 1);
+						continue;
+					}
+					// save match for later use to decrease performance cost
+					cache[m] = { match, lastIndex: part.match.lastIndex };
+				}
+				// check if it the first match in the string
+				if (cache[m].match[0] && (cache[m].match.index <= first.index || first.index === null))
+					first = {
+						part: part,
+						index: cache[m].match.index,
+						match: cache[m].match[0],
+						end: cache[m].lastIndex
+					}
+			}
+			if (first.index === null)
+				break;
+			token(src.slice(i, first.index), data.type);
+			i = first.end;
+			if (first.part.sub)
+				yield* tokenizer(first.match, typeof first.part.sub === 'string' ? first.part.sub : (typeof first.part.sub === 'function' ? first.part.sub(first.match) : first.part), token);
+			else
+				token(first.match, first.part.type);
+		}
+		token(src.slice(i, src.length), data.type);
+	}
+	catch {
+		token(src);
+	}
+}
+
+/**
+ * Find the tokens in the given code and call the given callback,
+ * without loading anything: every language used has to be given by the caller
+ *
+ * @example
+ * import json from '@speed-highlight/core/languages/json.js';
+ * import { tokenize } from '@speed-highlight/core/tokenize';
+ *
+ * tokenize(src, { sub: json }, (str, type) => process.stdout.write(str));
+ *
+ * @function tokenize
+ * @param {string} src The code
+ * @param {string|ShjLanguageDefinition|{ sub: ShjLanguageDefinition }} lang The language of the code
+ * @param {function(string, ShjToken=):void} token The callback function
+ * this function will be given
+ * * the text of the token
+ * * the type of the token
+ * @param {ShjTokenizeOptions} [opt={}] Customization options
+ */
+export function tokenize(src, lang, token, opt = {}) {
+	let lng,
+		it = tokenizer(src, lang, token),
+		res = it.next();
+
+	while (!res.done) {
+		lng = opt.languages?.[res.value];
+		res = it.next(Array.isArray(lng) ? { default: lng } : lng);
+	}
+}
+

tests/tokenize.test.js

@@ -0,0 +1,75 @@
+import { test } from 'node:test';
+import { deepStrictEqual } from 'node:assert';
+import { readFileSync } from 'node:fs';
+import { tokenize as tokenizeAsync } from '../src/index.js';
+import { tokenize } from '../src/tokenize.js';
+import * as css from '../src/languages/css.js';
+import * as html from '../src/languages/html.js';
+import * as js from '../src/languages/js.js';
+import * as js_template_literals from '../src/languages/js_template_literals.js';
+import * as jsdoc from '../src/languages/jsdoc.js';
+import * as json from '../src/languages/json.js';
+import * as regex from '../src/languages/regex.js';
+import * as todo from '../src/languages/todo.js';
+
+let fixtures = new URL('../examples/languages/', import.meta.url),
+	languages = { css, html, js, js_template_literals, jsdoc, json, regex, todo },
+	read = file => readFileSync(new URL(file, fixtures), 'utf8'),
+	collect = (src, lang, opt) => {
+		let tokens = [];
+		tokenize(src, lang, (str, token) => tokens.push([token, str]), opt);
+		return tokens;
+	},
+	collectAsync = async (src, lang) => {
+		let tokens = [];
+		await tokenizeAsync(src, lang, (str, token) => tokens.push([token, str]));
+		return tokens;
+	};
+
+test('a definition given as a sub needs no registry', async () => {
+	let src = read('test.json');
+
+	deepStrictEqual(collect(src, { sub: json.default }), await collectAsync(src, 'json'));
+});
+
+test('a nested sub is resolved from the given languages', async () => {
+	let src = read('test.html');
+
+	deepStrictEqual(collect(src, { sub: html.default }, { languages }), await collectAsync(src, 'html'));
+});
+
+test('a language can be given by name', async () => {
+	let src = read('test.js');
+
+	deepStrictEqual(collect(src, 'js', { languages }), await collectAsync(src, 'js'));
+});
+
+test('a language can be given as a definition or as its module', async () => {
+	let src = read('test.json');
+
+	deepStrictEqual(
+		collect(src, 'json', { languages: { json: json.default } }),
+		collect(src, 'json', { languages }));
+});
+
+test('the type of a language applies to the text it does not match', () => {
+	deepStrictEqual(collect('// TODO stuff', { sub: js.default }, { languages }), [
+		[undefined, ''],
+		['cmnt', '// '],
+		['err', 'TODO'],
+		['cmnt', ' stuff'],
+		[undefined, '']
+	]);
+});
+
+test('a sub that is not given is emitted as plain text', () => {
+	deepStrictEqual(collect('// TODO stuff', { sub: js.default }), [
+		[undefined, ''],
+		[undefined, '// TODO stuff'],
+		[undefined, '']
+	]);
+});
+
+test('a language that is not given is emitted as plain text', () => {
+	deepStrictEqual(collect('{}', 'json'), [[undefined, '{}']]);
+});