Preserve markup during syntax highlighting

Syntax/CodeElement.js

@@ -1,9 +1,67 @@
 import Syntax from '../Syntax.js';
+import {Match} from './Match.js';
 
 const supportsAdopted =
 	typeof CSSStyleSheet !== 'undefined' &&
 	'adoptedStyleSheets' in Document.prototype;
 
+// These values are defined by the DOM standard. Keep them local so this code
+// does not depend on a global `Node`, which may be unavailable in non-browser
+// DOM implementations.
+const ELEMENT_NODE = 1;
+const TEXT_NODE = 3;
+const CDATA_SECTION_NODE = 4;
+
+/**
+ * Extract the source text and existing markup as source-aligned matches.
+ *
+ * The highlighting pipeline can then insert these matches into the syntax
+ * tree, preserving elements such as links while allowing their contents to
+ * receive syntax highlighting.
+ */
+function extractCode(root) {
+	let text = '';
+	const matches = [];
+
+	function extract(node) {
+		if (node.nodeType === TEXT_NODE || node.nodeType === CDATA_SECTION_NODE) {
+			text += node.nodeValue.replace(/\r/g, '');
+			return;
+		}
+
+		if (node.nodeType !== ELEMENT_NODE) {
+			return;
+		}
+
+		if (node.tagName === 'BR') {
+			text += '\n';
+			return;
+		}
+
+		const offset = text.length;
+		let match = null;
+
+		if (node !== root) {
+			match = new Match(offset, 0, {element: node, force: true, allow: '*'}, '');
+			matches.push(match);
+		}
+
+		for (const child of node.childNodes) {
+			extract(child);
+		}
+
+		if (match) {
+			match.length = text.length - offset;
+			match.endOffset = text.length;
+			match.value = text.slice(offset);
+		}
+	}
+
+	extract(root);
+
+	return {text, matches: matches.filter(match => match.length > 0)};
+}
+
 /**
  * CodeElement - Web Component for syntax highlighting with isolated styles
  *
@@ -178,16 +236,16 @@ export class CodeElement extends HTMLElement {
 	}
 
 	/**
-	 * Get the code content to highlight
+	 * Get the source text and existing markup to highlight.
 	 */
 	#getCodeContent() {
 		// Check if there's a <code> child element
 		const codeElement = this.querySelector('code');
 		if (codeElement) {
-			return codeElement.textContent;
+			return extractCode(codeElement);
 		}
 
-		return this.textContent;
+		return extractCode(this);
 	}
 
 	/**
@@ -249,7 +307,7 @@ export class CodeElement extends HTMLElement {
 	async #render() {
 		try {
 			const languageName = this.language;
-			const code = this.#getCodeContent();
+			const {text: code, matches} = this.#getCodeContent();
 
 			if (!languageName) {
 				console.warn('<syntax-code>: No language specified');
@@ -271,7 +329,12 @@ export class CodeElement extends HTMLElement {
 
 			// Highlight off-DOM so the original source remains visible while all
 			// asynchronous work is in progress:
-			const highlighted = await language.process(this.syntax, code);
+			const highlighted = await language.process(
+				this.syntax,
+				code,
+				undefined,
+				matches
+			);
 
 			// Swap the completed rendering in synchronously. On the first render,
 			// the slot keeps the light-DOM source visible. On subsequent renders,
@@ -337,8 +400,11 @@ export function upgradeAll(selector, syntax = null) {
 			wrapper.setAttribute('language', language);
 		}
 
-		// Copy the code content into the wrapper
-		wrapper.textContent = element.textContent;
+		// Move the source content into the wrapper so existing markup remains
+		// available for extraction and re-rendering.
+		while (element.firstChild) {
+			wrapper.appendChild(element.firstChild);
+		}
 
 		// Replace <code> with <syntax-code>, leaving <pre> parent in place
 		const parent = element.parentElement;

Syntax/Language.js

@@ -239,9 +239,12 @@ export class Language {
 
 	/**
 	 * Build a syntax tree and process it into HTML.
+	 *
+	 * Additional matches can preserve source annotations, such as links, by
+	 * inserting them into the syntax tree before it is reduced to HTML.
 	 */
-	async process(syntax, text, options) {
-		const top = await this.buildTree(syntax, text, 0);
+	async process(syntax, text, options, additionalMatches) {
+		const top = await this.buildTree(syntax, text, 0, additionalMatches);
 
 		const lines = top.splitLines();
 

Syntax/Match.js

@@ -108,6 +108,7 @@ export class Match {
 
 		for (const child of this.children) {
 			const end = child.offset;
+			child.parent = this;
 
 			if (child.offset < this.offset) {
 				console.warn(
@@ -284,8 +285,13 @@ export class Match {
 		if (parts[1]) {
 			match.children = [];
 
-			// Update the match's expression based on the current position in the tree:
-			if (this.expression && this.expression.owner) {
+			// Element-backed matches describe authored markup and must retain their
+			// original expression so reduction can recreate that element.
+			if (
+				this.expression &&
+				this.expression.owner &&
+				!match.expression.element
+			) {
 				match.expression =
 					this.expression.owner.getRuleForType(match.expression.type) ||
 					match.expression;

Syntax/Rule.js

@@ -288,6 +288,22 @@ export class Rule {
 	 */
 	static webLinkProcess(baseUrl) {
 		return function (container, match, options) {
+			// Authored links take precedence over generated documentation links.
+			// Depending on the source ranges, the authored link may be either an
+			// ancestor or a descendant of this syntax match.
+			let current = match;
+			while (current) {
+				if (current.expression?.element?.tagName === 'A') {
+					return container;
+				}
+
+				current = current.parent;
+			}
+
+			if (container.matches('a') || container.querySelector('a')) {
+				return container;
+			}
+
 			// Replace the span with an anchor element
 			const anchor = document.createElement('a');
 

test/Syntax/CodeElement.js

@@ -242,6 +242,99 @@ test('upgradeAll can handle standalone <code> blocks with custom selector', asyn
 	);
 });
 
+test('upgradeAll preserves and highlights code containing markup', async () => {
+	const {upgradeAll} = await import('../../Syntax/CodeElement.js');
+
+	document.body.innerHTML = `
+		<code class="language-ruby">class <a href="/source/Foo">Foo</a></code>
+	`;
+
+	upgradeAll('code[class*="language-"]');
+
+	const element = document.querySelector('syntax-code');
+	await element.ready;
+
+	const sourceLink = element.querySelector('a');
+	const renderedLink = element.shadowRoot.querySelector('a');
+
+	assert.equal(element.textContent, 'class Foo');
+	assert.equal(sourceLink.getAttribute('href'), '/source/Foo');
+	assert.equal(renderedLink.getAttribute('href'), '/source/Foo');
+	assert.equal(renderedLink.textContent, 'Foo');
+	assert.ok(
+		renderedLink.closest('.type'),
+		'linked source should still receive syntax highlighting'
+	);
+});
+
+test('upgradeAll preserves nested markup structure', async () => {
+	const {upgradeAll} = await import('../../Syntax/CodeElement.js');
+
+	document.body.innerHTML = `
+		<code class="language-ruby"><a href="/source/Foo"><strong>Foo</strong>::Bar</a></code>
+	`;
+
+	upgradeAll('code[class*="language-"]');
+
+	const element = document.querySelector('syntax-code');
+	await element.ready;
+
+	const renderedLink = element.shadowRoot.querySelector('a');
+	const renderedStrong = renderedLink.querySelector('strong');
+
+	assert.equal(renderedLink.getAttribute('href'), '/source/Foo');
+	assert.equal(renderedLink.textContent, 'Foo::Bar');
+	assert.equal(renderedStrong.textContent, 'Foo');
+	assert.ok(renderedStrong.closest('.type'));
+	assert.equal(renderedLink.querySelectorAll('.type').length, 2);
+});
+
+test('upgradeAll preserves markup spanning multiple lines', async () => {
+	const {upgradeAll} = await import('../../Syntax/CodeElement.js');
+
+	document.body.innerHTML =
+		'<code class="language-ruby"><a href="/source/Foo">Foo\nBar</a></code>';
+
+	upgradeAll('code[class*="language-"]');
+
+	const element = document.querySelector('syntax-code');
+	await element.ready;
+
+	const renderedLinks = [...element.shadowRoot.querySelectorAll('a')];
+
+	assert.equal(element.lineCount, 2);
+	assert.deepEqual(
+		renderedLinks.map(link => link.textContent),
+		['Foo\n', 'Bar']
+	);
+	assert.ok(
+		renderedLinks.every(link => link.getAttribute('href') === '/source/Foo')
+	);
+});
+
+test('syntax-code preserves markup when re-rendering', async () => {
+	await import('../../Syntax/CodeElement.js');
+
+	document.body.innerHTML =
+		'<syntax-code language="ruby"><a href="/source/Foo" data-kind="class">Foo</a></syntax-code>';
+
+	const element = document.querySelector('syntax-code');
+	await element.ready;
+
+	const firstLink = element.shadowRoot.querySelector('a');
+	assert.equal(firstLink.getAttribute('href'), '/source/Foo');
+	assert.equal(firstLink.dataset.kind, 'class');
+
+	element.language = 'python';
+	await element.ready;
+
+	const secondLink = element.shadowRoot.querySelector('a');
+	assert.notEqual(secondLink, firstLink);
+	assert.equal(secondLink.getAttribute('href'), '/source/Foo');
+	assert.equal(secondLink.dataset.kind, 'class');
+	assert.equal(secondLink.textContent, 'Foo');
+});
+
 test('syntax-code behaves semantically like <code> when inline', async () => {
 	const {CodeElement} = await import('../../Syntax/CodeElement.js');
 

test/Syntax/Rule.js

@@ -249,3 +249,38 @@ test('webLinkProcess preserves nested HTML', () => {
 	);
 	assert.strictEqual(result.className, 'function');
 });
+
+test('webLinkProcess preserves a nested authored link', () => {
+	const process = Rule.webLinkProcess('http://docs.example.com/');
+	const container = document.createElement('span');
+	container.innerHTML = '<a href="/source/Foo">Foo</a>';
+
+	const match = {
+		value: 'Foo',
+		expression: {type: 'type'}
+	};
+
+	assert.strictEqual(process(container, match, {}), container);
+	assert.strictEqual(
+		container.querySelector('a').getAttribute('href'),
+		'/source/Foo'
+	);
+});
+
+test('webLinkProcess preserves an authored link from a parent match', () => {
+	const process = Rule.webLinkProcess('http://docs.example.com/');
+	const container = document.createElement('span');
+	container.textContent = 'Foo';
+
+	const sourceLink = document.createElement('a');
+	const match = {
+		value: 'Foo',
+		expression: {type: 'type'},
+		parent: {
+			expression: {element: sourceLink},
+			parent: null
+		}
+	};
+
+	assert.strictEqual(process(container, match, {}), container);
+});