From 92b8d9c0caa00ae747ddc18ff314ac0dff8c6795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Andr=C3=A9?= Date: Thu, 16 Apr 2026 00:16:28 +0200 Subject: [PATCH] test: improve Pygments test reliability and diagnostics - Use proc_open to capture stdout/stderr and debug "unknown" CI failures. - Improve Python/Pygments detection to ensure tests skip correctly when missing. - Include process output in error messages for better troubleshooting. --- bin/pygments-tokenize | 178 ++++++++++++++++++++++++++ tests/Unit/PygmentsComparisonTest.php | 120 ++++++++--------- 2 files changed, 228 insertions(+), 70 deletions(-) create mode 100755 bin/pygments-tokenize diff --git a/bin/pygments-tokenize b/bin/pygments-tokenize new file mode 100755 index 0000000..29a9e26 --- /dev/null +++ b/bin/pygments-tokenize @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +""" +Tokenize source code using Pygments and output JSON. + +Usage: + pygments-tokenize < input.code + pygments-tokenize + +Output: JSON array of [token_type, token_text] pairs. + +This script is used by the test suite to compare the library's output +against Pygments, the de-facto standard for syntax highlighting. + +Requirements: + pip install pygments +""" + +import json +import sys + +try: + from pygments.lexers import get_lexer_by_name + from pygments.token import Token +except ImportError: + print(json.dumps({"error": "pygments is not installed. Install it with: pip install pygments"})) + sys.exit(1) + +# Map alto language identifiers to Pygments lexer names +LANGUAGE_MAP = { + "php": ("php", {"startinline": False}), + "html": ("html", {}), + "svg": ("html", {}), + "xml": ("xml", {}), + "yaml": ("yaml", {}), + "sql": ("sql", {}), + "json": ("json", {}), + "css": ("css", {}), + "scss": ("scss", {}), + "markdown": ("markdown", {}), + "javascript": ("javascript", {}), + "typescript": ("typescript", {}), + "twig": ("twig", {}), + "makefile": ("makefile", {}), + "bash": ("bash", {}), + "ini": ("ini", {}), + "http": ("http", {}), + "go": ("go", {}), + "rust": ("rust", {}), + "ruby": ("ruby", {}), + "swift": ("swift", {}), + "python": ("python3", {}), + "java": ("java", {}), + "csharp": ("csharp", {}), + "dockerfile": ("docker", {}), + "diff": ("diff", {}), + "dotenv": ("bash", {}), +} + +# Map Pygments token types to alto scope categories +SCOPE_CATEGORY_MAP = { + Token.Comment: "comment", + Token.Comment.Single: "comment", + Token.Comment.Multiline: "comment", + Token.Comment.Preproc: "comment", + Token.Comment.PreprocFile: "comment", + Token.Comment.Special: "comment", + Token.Comment.Hashbang: "comment", + Token.Keyword: "keyword", + Token.Keyword.Declaration: "keyword", + Token.Keyword.Namespace: "keyword", + Token.Keyword.Pseudo: "keyword", + Token.Keyword.Reserved: "keyword", + Token.Keyword.Type: "keyword", + Token.Keyword.Constant: "keyword", + Token.Name.Function: "function", + Token.Name.Function.Magic: "function", + Token.Name.Class: "type", + Token.Name.Decorator: "attribute", + Token.Name.Variable: "variable", + Token.Name.Variable.Class: "variable", + Token.Name.Variable.Global: "variable", + Token.Name.Variable.Instance: "variable", + Token.Name.Variable.Magic: "variable", + Token.Name.Attribute: "property", + Token.Name.Builtin: "builtin", + Token.Name.Builtin.Pseudo: "builtin", + Token.Name.Tag: "tag", + Token.Name.Namespace: "namespace", + Token.Name.Constant: "constant", + Token.Literal.String: "string", + Token.Literal.String.Single: "string", + Token.Literal.String.Double: "string", + Token.Literal.String.Backtick: "string", + Token.Literal.String.Doc: "string", + Token.Literal.String.Escape: "string", + Token.Literal.String.Heredoc: "string", + Token.Literal.String.Interpol: "string", + Token.Literal.String.Other: "string", + Token.Literal.String.Regex: "regexp", + Token.Literal.String.Symbol: "string", + Token.Literal.String.Affix: "string", + Token.Literal.Number: "number", + Token.Literal.Number.Bin: "number", + Token.Literal.Number.Float: "number", + Token.Literal.Number.Hex: "number", + Token.Literal.Number.Integer: "number", + Token.Literal.Number.Integer.Long: "number", + Token.Literal.Number.Oct: "number", + Token.Operator: "operator", + Token.Operator.Word: "operator", + Token.Punctuation: "punctuation", + Token.Punctuation.Marker: "punctuation", + Token.Generic.Inserted: "diff.added", + Token.Generic.Deleted: "diff.removed", + Token.Generic.Heading: "section", + Token.Generic.Subheading: "section", + Token.Name.Label: "label", + Token.Name.Entity: "entity", +} + + +def get_scope_category(token_type): + """Map a Pygments token type to a simplified scope category.""" + # Try exact match first, then walk up the token hierarchy + current = token_type + while current: + if current in SCOPE_CATEGORY_MAP: + return SCOPE_CATEGORY_MAP[current] + if current.parent: + current = current.parent + else: + break + return "other" + + +def tokenize(language, code): + """Tokenize code with Pygments and return categorized tokens.""" + if language not in LANGUAGE_MAP: + return {"error": f"Unsupported language: {language}"} + + lexer_name, options = LANGUAGE_MAP[language] + lexer = get_lexer_by_name(lexer_name, **options) + + tokens = [] + for tok_type, tok_value in lexer.get_tokens(code): + # Skip pure whitespace + if tok_type in Token.Text or not tok_value.strip(): + continue + + category = get_scope_category(tok_type) + tokens.append({ + "text": tok_value, + "pygments_type": str(tok_type), + "category": category, + }) + + return {"tokens": tokens, "language": language} + + +def main(): + if len(sys.argv) < 2: + print(json.dumps({"error": "Usage: pygments-tokenize [file]"})) + sys.exit(1) + + language = sys.argv[1].lower() + + if len(sys.argv) >= 3: + with open(sys.argv[2], "r") as f: + code = f.read() + else: + code = sys.stdin.read() + + result = tokenize(language, code) + print(json.dumps(result, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/tests/Unit/PygmentsComparisonTest.php b/tests/Unit/PygmentsComparisonTest.php index 28d15d9..6f36ae3 100644 --- a/tests/Unit/PygmentsComparisonTest.php +++ b/tests/Unit/PygmentsComparisonTest.php @@ -37,12 +37,10 @@ final class PygmentsComparisonTest extends TestCase { private static string $pygmentsScript; + private static ?string $pythonCommand = null; + /** * Map of Alto Scope → simplified category for comparison. - * - * The categories are intentionally coarse to account for legitimate - * differences between highlighters (e.g., one may call `echo` a keyword, - * another a builtin function). */ private const array SCOPE_TO_CATEGORY = [ // Comments @@ -123,41 +121,26 @@ final class PygmentsComparisonTest extends TestCase /** * Tokens that may legitimately differ between highlighters. - * - * For example, `echo` in PHP could be classified as a keyword or a builtin function, - * and `true` could be a keyword or a boolean literal. These mappings define which - * category combinations are considered equivalent. */ private const array EQUIVALENT_CATEGORIES = [ - // Boolean literals may be classified as keywords or constants ['constant', 'keyword'], - // Some identifiers may be classified as functions or variables ['function', 'variable'], - // Builtin types/functions might be constants or keywords ['type', 'constant'], ['type', 'keyword'], ['function', 'keyword'], - // Some things may be classified as attributes or properties ['attribute', 'property'], ['attribute', 'constant'], ['attribute', 'variable'], ['attribute', 'keyword'], - // Tags ['tag', 'keyword'], - // Punctuation vs operators (common disagreement between tokenizers) ['punctuation', 'operator'], - // Punctuation vs comments (e.g. */ private static function getLanguageSamples(): array { @@ -268,22 +251,43 @@ public static function setUpBeforeClass(): void private static function isPygmentsAvailable(): bool { - $output = []; - $returnCode = 0; - exec('python3 -c "import pygments" 2>&1', $output, $returnCode); + if (null !== self::$pythonCommand) { + return true; + } + + if (!function_exists('proc_open')) { + return false; + } + + foreach (['python3', 'python'] as $cmd) { + $process = @proc_open( + [$cmd, '-c', 'import pygments'], + [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], + $pipes + ); + + if (is_resource($process)) { + fclose($pipes[1]); + fclose($pipes[2]); + if (0 === proc_close($process)) { + self::$pythonCommand = $cmd; + + return true; + } + } + } - return 0 === $returnCode; + return false; } - /** - * Get Pygments tokens for a code sample. - * - * @return list - */ private static function getPygmentsTokens(string $language, string $code): array { + if (!self::isPygmentsAvailable()) { + self::fail('Pygments is not available'); + } + $process = proc_open( - ['python3', self::$pygmentsScript, $language], + [self::$pythonCommand, self::$pygmentsScript, $language], [ 0 => ['pipe', 'r'], 1 => ['pipe', 'w'], @@ -300,24 +304,32 @@ private static function getPygmentsTokens(string $language, string $code): array fclose($pipes[0]); $output = stream_get_contents($pipes[1]); + $errorOutput = stream_get_contents($pipes[2]); fclose($pipes[1]); fclose($pipes[2]); - proc_close($process); + $exitCode = proc_close($process); - $result = json_decode($output, true); + $result = json_decode((string) $output, true); if (!is_array($result) || isset($result['error'])) { - self::fail('Pygments error: '.($result['error'] ?? 'unknown')); + $message = $result['error'] ?? 'unknown error (no JSON output)'; + if (!is_array($result)) { + $message .= sprintf( + "\nOutput: %s\nStderr: %s\nExit Code: %d\nCommand: %s %s %s", + $output ?: '(empty)', + $errorOutput ?: '(empty)', + $exitCode, + self::$pythonCommand, + self::$pygmentsScript, + $language + ); + } + self::fail('Pygments error: '.$message); } return $result['tokens']; } - /** - * Get Alto tokens for a code sample. - * - * @return list - */ private static function getAltoTokens(string $language, string $code): array { $languages = Languages::getDefaultLanguages(); @@ -355,9 +367,6 @@ private static function getAltoTokens(string $language, string $code): array return $tokens; } - /** - * Check if two categories are considered equivalent. - */ private static function areCategoriesEquivalent(string $a, string $b): bool { if ($a === $b) { @@ -373,9 +382,6 @@ private static function areCategoriesEquivalent(string $a, string $b): bool return false; } - /** - * @return array - */ public static function languageProvider(): array { $samples = self::getLanguageSamples(); @@ -386,13 +392,6 @@ public static function languageProvider(): array ); } - /** - * Compare token-level output with Pygments for each language. - * - * This test doesn't require exact token-by-token match (tokenizers may split - * text differently), but verifies that both highlighters agree on the semantic - * category of significant tokens (keywords, strings, comments, numbers). - */ #[DataProvider('languageProvider')] public function testKeyTokensMatchPygments(string $language): void { @@ -406,7 +405,6 @@ public function testKeyTokensMatchPygments(string $language): void $altoTokens = self::getAltoTokens($language, $code); $pygmentsTokens = self::getPygmentsTokens($language, $code); - // Build a map of text → category for Pygments (significant tokens only) $pygmentsMap = []; $significantCategories = ['keyword', 'string', 'comment', 'number', 'operator']; foreach ($pygmentsTokens as $token) { @@ -418,7 +416,6 @@ public function testKeyTokensMatchPygments(string $language): void } } - // Check that Alto agrees with Pygments on significant tokens $mismatches = []; foreach ($altoTokens as $token) { $text = trim($token['text']); @@ -440,11 +437,6 @@ public function testKeyTokensMatchPygments(string $language): void } } - // Also check that Alto doesn't miss significant tokens that Pygments found. - // Note: tokenizers may split text differently (e.g., Pygments may emit `"` - // and `Hello` as separate string tokens, while Alto emits `"Hello"` as one). - // We check whether the Pygments token text is *contained* in any Alto token - // of a compatible category. $altoTexts = array_map(fn ($t) => trim($t['text']), $altoTokens); $missingFromAlto = []; foreach ($pygmentsMap as $text => $category) { @@ -453,7 +445,6 @@ public function testKeyTokensMatchPygments(string $language): void continue; } - // Check if any Alto token contains this text in a compatible category $foundInSubstring = false; foreach ($altoTokens as $altoToken) { if (str_contains($altoToken['text'], $text) @@ -478,8 +469,6 @@ public function testKeyTokensMatchPygments(string $language): void } if (!empty($errorMessages)) { - // Append missing-token info as additional context (tokenizers may - // legitimately split text differently, so these are informational). if (!empty($missingFromAlto)) { $errorMessages[] = "Note: tokens from Pygments not matched in Alto output:\n".implode("\n", $missingFromAlto); } @@ -490,9 +479,6 @@ public function testKeyTokensMatchPygments(string $language): void self::assertEmpty($mismatches, "No mismatches for '{$language}'"); } - /** - * Verify that all languages produce non-empty token streams. - */ #[DataProvider('languageProvider')] public function testLanguageProducesTokens(string $language): void { @@ -504,11 +490,6 @@ public function testLanguageProducesTokens(string $language): void self::assertNotEmpty($tokens, "Language '{$language}' produced no tokens for sample code"); } - /** - * Verify that all tokens emitted by each language concatenate back to the original code. - * - * This catches bugs where tokens are dropped or duplicated during parsing. - */ #[DataProvider('languageProvider')] public function testTokensCoverEntireInput(string $language): void { @@ -535,8 +516,7 @@ public function testTokensCoverEntireInput(string $language): void self::assertSame( $code, $reconstructed, - "Token stream for '{$language}' does not reconstruct the original input. ". - 'Some tokens may be dropped or duplicated during parsing.', + "Token stream for '{$language}' does not reconstruct the original input.", ); } }