diff --git a/.github/workflows/parity-tests.yml b/.github/workflows/parity-tests.yml new file mode 100644 index 0000000..fe10b3d --- /dev/null +++ b/.github/workflows/parity-tests.yml @@ -0,0 +1,160 @@ +name: Generator Parity Tests + +on: + # Run on PRs to ensure parity before merge + pull_request: + branches: [main, master] + paths: + - 'build_docs.py' + - 'build_docs.hml' + - 'test_generator_parity.py' + - '.github/workflows/parity-tests.yml' + + # Run on pushes to main + push: + branches: [main, master] + paths: + - 'build_docs.py' + - 'build_docs.hml' + - 'test_generator_parity.py' + - '.github/workflows/parity-tests.yml' + + # Allow manual trigger + workflow_dispatch: + inputs: + language: + description: 'Language to test (en, zh, de, es, fr, ja, pt, or all)' + required: false + default: 'en' + verbose: + description: 'Enable verbose output' + required: false + default: 'false' + type: boolean + +permissions: + contents: read + +jobs: + parity-test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Test English by default, can be overridden by workflow_dispatch + language: ${{ github.event_name == 'workflow_dispatch' && fromJson(format('["{0}"]', inputs.language || 'en')) || fromJson('["en"]') }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libev4 + + - name: Install Hemlock + run: | + curl -fsSL https://raw.githubusercontent.com/hemlang/hemlock/main/install.sh | bash + echo "$HOME/.hemlock/bin" >> $GITHUB_PATH + + - name: Install hpm + run: | + curl -fsSL https://raw.githubusercontent.com/hemlang/hpm/main/install.sh | sh + + - name: Verify installation + run: | + export PATH="$HOME/.hemlock/bin:$PATH" + hemlock --version + hpm --version + + - name: Install dependencies + run: | + export PATH="$HOME/.hemlock/bin:$PATH" + hpm install + + - name: Run parity tests + run: | + export PATH="$HOME/.hemlock/bin:$PATH" + VERBOSE_FLAG="" + if [ "${{ inputs.verbose }}" = "true" ]; then + VERBOSE_FLAG="--verbose" + fi + python test_generator_parity.py --lang ${{ matrix.language }} $VERBOSE_FLAG + + - name: Upload test artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: parity-test-outputs-${{ matrix.language }} + path: | + docs*.html + llms*.txt + retention-days: 7 + + # Run full language tests on schedule or manual trigger + full-parity-test: + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' && inputs.language == 'all' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libev4 + + - name: Install Hemlock + run: | + curl -fsSL https://raw.githubusercontent.com/hemlang/hemlock/main/install.sh | bash + echo "$HOME/.hemlock/bin" >> $GITHUB_PATH + + - name: Install hpm + run: | + curl -fsSL https://raw.githubusercontent.com/hemlang/hpm/main/install.sh | sh + + - name: Verify installation + run: | + export PATH="$HOME/.hemlock/bin:$PATH" + hemlock --version + hpm --version + + - name: Install dependencies + run: | + export PATH="$HOME/.hemlock/bin:$PATH" + hpm install + + - name: Run full parity tests + run: | + export PATH="$HOME/.hemlock/bin:$PATH" + VERBOSE_FLAG="" + if [ "${{ inputs.verbose }}" = "true" ]; then + VERBOSE_FLAG="--verbose" + fi + python test_generator_parity.py --lang all $VERBOSE_FLAG + + - name: Upload test artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: parity-test-outputs-all + path: | + docs*.html + llms*.txt + retention-days: 7 diff --git a/Makefile b/Makefile index 1abf8e9..d72100d 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ HPM ?= hpm PYTHON ?= python3 VERSION := 1.0.5 -.PHONY: all deps docs docs-py server package dist clean help +.PHONY: all deps docs docs-py server package dist clean help test test-parity all: docs @@ -30,6 +30,23 @@ docs-py: @echo "Done: docs.html ($(shell wc -c < docs.html | tr -d ' ') bytes)" @echo "Done: llms.txt ($(shell wc -c < llms.txt | tr -d ' ') bytes)" +# Run parity tests between Python and Hemlock generators +test: test-parity + +test-parity: + @echo "Running generator parity tests..." + @$(PYTHON) test_generator_parity.py + +# Run parity tests for all languages +test-parity-all: + @echo "Running generator parity tests for all languages..." + @$(PYTHON) test_generator_parity.py --lang all + +# Run parity tests with verbose output +test-parity-verbose: + @echo "Running generator parity tests (verbose)..." + @$(PYTHON) test_generator_parity.py --verbose + # Package the documentation server server: docs @echo "Packaging documentation server..." @@ -59,14 +76,18 @@ help: @echo "Hemlock Documentation $(VERSION)" @echo "" @echo "Usage:" - @echo " make deps - Install dependencies via hpm" - @echo " make docs - Generate docs.html and llms.txt using Hemlock" - @echo " make docs-py - Generate docs.html and llms.txt using Python (fallback)" - @echo " make server - Package the documentation server executable" - @echo " make dist - Create distribution zip (server + docs + llms.txt)" - @echo " make run - Run the documentation server locally" - @echo " make clean - Remove build artifacts" - @echo " make help - Show this help message" + @echo " make deps - Install dependencies via hpm" + @echo " make docs - Generate docs.html and llms.txt using Hemlock" + @echo " make docs-py - Generate docs.html and llms.txt using Python (fallback)" + @echo " make test - Run generator parity tests (English only)" + @echo " make test-parity - Run generator parity tests (English only)" + @echo " make test-parity-all - Run generator parity tests for all languages" + @echo " make test-parity-verbose - Run parity tests with verbose output" + @echo " make server - Package the documentation server executable" + @echo " make dist - Create distribution zip (server + docs + llms.txt)" + @echo " make run - Run the documentation server locally" + @echo " make clean - Remove build artifacts" + @echo " make help - Show this help message" @echo "" @echo "Output files:" @echo " docs.html - Interactive HTML documentation" diff --git a/test_generator_parity.py b/test_generator_parity.py new file mode 100644 index 0000000..adb905d --- /dev/null +++ b/test_generator_parity.py @@ -0,0 +1,530 @@ +#!/usr/bin/env python3 +""" +Test suite for verifying parity between Python and Hemlock documentation generators. + +This script runs both generators and compares their outputs to ensure they produce +functionally equivalent documentation. It checks: +- HTML structure and content +- LLM text output +- Navigation structure +- Page content and ordering +- Language support + +Usage: + python test_generator_parity.py # Test English only + python test_generator_parity.py --lang all # Test all languages + python test_generator_parity.py --lang zh # Test specific language + python test_generator_parity.py --verbose # Show detailed diff output +""" + +import argparse +import difflib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Optional + + +class Colors: + """ANSI color codes for terminal output.""" + GREEN = '\033[92m' + RED = '\033[91m' + YELLOW = '\033[93m' + BLUE = '\033[94m' + RESET = '\033[0m' + BOLD = '\033[1m' + + +def colorize(text: str, color: str) -> str: + """Apply color to text if stdout is a terminal.""" + if sys.stdout.isatty(): + return f"{color}{text}{Colors.RESET}" + return text + + +class ParityTestResult: + """Represents the result of a single parity test.""" + + def __init__(self, name: str, passed: bool, message: str = "", details: str = ""): + self.name = name + self.passed = passed + self.message = message + self.details = details + + def __str__(self) -> str: + status = colorize("PASS", Colors.GREEN) if self.passed else colorize("FAIL", Colors.RED) + result = f" [{status}] {self.name}" + if self.message: + result += f": {self.message}" + return result + + +class GeneratorParityTester: + """Tests parity between Python and Hemlock documentation generators.""" + + def __init__(self, project_dir: Path, verbose: bool = False): + self.project_dir = project_dir + self.verbose = verbose + self.temp_dir = None + self.results: list[ParityTestResult] = [] + + def setup(self) -> bool: + """Set up temporary directories for test outputs.""" + self.temp_dir = Path(tempfile.mkdtemp(prefix="generator_parity_")) + self.py_output_dir = self.temp_dir / "python" + self.hml_output_dir = self.temp_dir / "hemlock" + self.py_output_dir.mkdir() + self.hml_output_dir.mkdir() + return True + + def cleanup(self): + """Clean up temporary directories.""" + if self.temp_dir and self.temp_dir.exists(): + shutil.rmtree(self.temp_dir) + + def run_python_generator(self, lang: str = "en") -> bool: + """Run the Python documentation generator.""" + try: + env = os.environ.copy() + result = subprocess.run( + [sys.executable, "build_docs.py", "--lang", lang], + cwd=self.project_dir, + capture_output=True, + text=True, + timeout=120 + ) + + if result.returncode != 0: + print(colorize(f"Python generator failed: {result.stderr}", Colors.RED)) + return False + + # Copy output files to temp directory + html_file = f"docs.html" if lang == "en" else f"docs-{lang}.html" + llm_file = f"llms.txt" if lang == "en" else f"llms-{lang}.txt" + + src_html = self.project_dir / html_file + src_llm = self.project_dir / llm_file + + if src_html.exists(): + shutil.copy(src_html, self.py_output_dir / html_file) + if src_llm.exists(): + shutil.copy(src_llm, self.py_output_dir / llm_file) + + return True + except subprocess.TimeoutExpired: + print(colorize("Python generator timed out", Colors.RED)) + return False + except Exception as e: + print(colorize(f"Error running Python generator: {e}", Colors.RED)) + return False + + def run_hemlock_generator(self, lang: str = "en") -> bool: + """Run the Hemlock documentation generator.""" + try: + result = subprocess.run( + ["hemlock", "build_docs.hml", "--lang", lang], + cwd=self.project_dir, + capture_output=True, + text=True, + timeout=120 + ) + + if result.returncode != 0: + print(colorize(f"Hemlock generator failed: {result.stderr}", Colors.RED)) + return False + + # Copy output files to temp directory + html_file = f"docs.html" if lang == "en" else f"docs-{lang}.html" + llm_file = f"llms.txt" if lang == "en" else f"llms-{lang}.txt" + + src_html = self.project_dir / html_file + src_llm = self.project_dir / llm_file + + if src_html.exists(): + shutil.copy(src_html, self.hml_output_dir / html_file) + if src_llm.exists(): + shutil.copy(src_llm, self.hml_output_dir / llm_file) + + return True + except subprocess.TimeoutExpired: + print(colorize("Hemlock generator timed out", Colors.RED)) + return False + except FileNotFoundError: + print(colorize("Hemlock interpreter not found. Make sure 'hemlock' is in PATH.", Colors.RED)) + return False + except Exception as e: + print(colorize(f"Error running Hemlock generator: {e}", Colors.RED)) + return False + + def extract_pages_json(self, html_content: str) -> Optional[dict]: + """Extract the pages JSON data from HTML content.""" + # Look for the pages data in the script section + match = re.search(r'const\s+pages\s*=\s*(\{[\s\S]*?\});?\s*\n\s*(?:const|let|var|function|//)', html_content) + if match: + try: + # Clean up the JSON - handle trailing commas and other issues + json_str = match.group(1) + # Remove trailing commas before closing braces/brackets + json_str = re.sub(r',(\s*[}\]])', r'\1', json_str) + return json.loads(json_str) + except json.JSONDecodeError: + pass + return None + + def extract_nav_structure(self, html_content: str) -> list[tuple[str, str]]: + """Extract navigation structure (section, title) pairs from HTML.""" + nav_items = [] + # Find all nav links + pattern = r']*href="#([^"]+)"[^>]*>([^<]+)' + for match in re.finditer(pattern, html_content): + page_id = match.group(1) + title = match.group(2).strip() + nav_items.append((page_id, title)) + return nav_items + + def extract_section_headers(self, html_content: str) -> list[str]: + """Extract section headers from navigation.""" + pattern = r'