diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml new file mode 100644 index 0000000..83d4009 --- /dev/null +++ b/.github/workflows/js.yml @@ -0,0 +1,210 @@ +name: JS + +# Single, unified pipeline that subsumes the previous test.yml, pages.yml, +# and links.yml workflows. Structured to fail fast: cheap checks +# (syntax-check, unit tests, link audit) gate the slower jobs (e2e against +# a locally-served build, Pages deploy, e2e against the deployed site). +# +# Layout adopted from link-foundation/js-ai-driven-development-pipeline-template +# (release.yml), trimmed for this repo: +# * no npm publish path — this is a static site, not a library +# * no changeset PR / manual release jobs +# * no Docker publish job +# The template's `detect-changes` matrix is intentionally omitted: every job +# below already either runs in <1 minute or is gated by a `needs:` chain, +# so partial skipping wouldn't meaningfully improve wall time and would +# add a lot of yaml. + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + +# Cancel in-flight runs for the same ref so the latest commit wins on +# `main` and on PR force-pushes. Matches the template's policy. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref == 'refs/heads/main' }} + +# Default permissions are read-only; Pages deploy job widens scope below. +permissions: + contents: read + +jobs: + # === Fast checks ============================================================ + + syntax-check: + name: Syntax check (node --check) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '20.x' + - name: Walk repo and run node --check + run: node js/scripts/check-mjs-syntax.mjs + + unit-tests: + name: Unit tests (Node) + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: [syntax-check] + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '20.x' + - name: Run unit and integration suites + # `run-unit-tests.mjs` distinguishes gating (zero-dep node:test + # suites: routing.test.mjs, ipa.test.mjs) from informational + # (live-Wikidata cache/transformer scripts). Only gating failures + # set a non-zero exit code. + run: node js/scripts/run-unit-tests.mjs + + link-check: + name: Broken link check (lychee) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - name: Check links with lychee + id: lychee + uses: lycheeverse/lychee-action@v2 + with: + args: >- + --verbose + --no-progress + --cache + --max-cache-age 1d + --max-retries 3 + --timeout 30 + --exclude-path docs/case-studies + './**/*.md' + './**/*.html' + fail: false + output: lychee/out.md + jobSummary: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Check broken links against Web Archive + if: steps.lychee.outputs.exit_code != 0 + id: webarchive + run: node js/scripts/check-web-archive.mjs + env: + LYCHEE_OUTPUT: lychee/out.md + - name: Fail if broken links found and no web archive fallback + if: steps.lychee.outputs.exit_code != 0 && steps.webarchive.outputs.all_archived != 'true' + run: | + echo "::error::Broken links were detected with no Web Archive fallback available." + echo "See the 'Check links with lychee' step for the full list and lychee/out.md." + exit 1 + + # === E2E against locally-served build (PR + push gate) ===================== + + e2e-local: + name: E2E (Playwright @ localhost) + runs-on: ubuntu-latest + timeout-minutes: 15 + needs: [unit-tests] + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '20.x' + - name: Install dependencies + # We don't have a deep dependency tree (only @playwright/test as a + # devDep), so a vanilla `npm install` is fast and produces a clean + # node_modules. `npm ci` would require a committed lockfile that + # matches package.json exactly — overkill for this repo. + run: npm install + - name: Install Playwright browsers + run: npx playwright install --with-deps chromium + - name: Run e2e suite (boots js/scripts/serve-static.mjs) + run: node js/scripts/run-e2e-local.mjs + - name: Upload Playwright report on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: playwright-report/ + retention-days: 7 + + # === GitHub Pages build & deploy (main only) =============================== + + pages-build: + name: Pages build (Jekyll) + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: [syntax-check, unit-tests, e2e-local, link-check] + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + steps: + - uses: actions/checkout@v6 + - name: Setup Pages + uses: actions/configure-pages@v5 + - name: Build with Jekyll + uses: actions/jekyll-build-pages@v1 + with: + source: ./ + destination: ./_site + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + + pages-deploy: + name: Pages deploy + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: [pages-build] + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + permissions: + contents: read + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + outputs: + # Exposed so the e2e-deployed job can reuse the freshly deployed URL. + page_url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + + # === Post-deploy e2e against the live GitHub Pages site ==================== + # Runs the same Playwright suite against the just-deployed site. Most + # assertions are wiring/DOM checks that don't depend on live Wikidata, + # so they're stable on the public Pages URL. + + e2e-deployed: + name: E2E (Playwright @ deployed Pages) + runs-on: ubuntu-latest + timeout-minutes: 15 + needs: [pages-deploy] + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '20.x' + - name: Install dependencies + run: npm install + - name: Install Playwright browsers + run: npx playwright install --with-deps chromium + - name: Wait for new content to propagate + # Pages CDN can lag the deploy step by a few seconds. + run: sleep 10 + - name: Run e2e suite against the deployed site + env: + BASE_URL: ${{ needs.pages-deploy.outputs.page_url || 'https://link-assistant.github.io/human-language' }} + run: npx playwright test --config js/tests/e2e/playwright.config.mjs + - name: Upload Playwright report on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-report-deployed + path: playwright-report/ + retention-days: 7 diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml deleted file mode 100644 index 3e910dc..0000000 --- a/.github/workflows/links.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Broken Link Checker - -on: - push: - branches: - - main - paths: - - '**.md' - - '**.html' - - '.github/workflows/links.yml' - pull_request: - types: [opened, synchronize, reopened] - paths: - - '**.md' - - '**.html' - - '.github/workflows/links.yml' - workflow_dispatch: - -jobs: - link-checker: - name: Check Links - runs-on: ubuntu-latest - # Typical run: <1min with lychee cache. 10min prevents slow - # external hosts or Wayback Machine probes from hanging the workflow. - timeout-minutes: 10 - permissions: - contents: read - steps: - - uses: actions/checkout@v6 - - - name: Check links with lychee - id: lychee - uses: lycheeverse/lychee-action@v2 - with: - # Check all Markdown and HTML files - # Exclude case-studies directory - these are research documents from - # external repos with references to files and issues that don't exist - # in this repository (similar exclusion pattern as eslint.config.js) - args: >- - --verbose - --no-progress - --cache - --max-cache-age 1d - --max-retries 3 - --timeout 30 - --exclude-path docs/case-studies - './**/*.md' - './**/*.html' - # Don't fail the workflow immediately - we want to check web archive first - fail: false - # Output file for broken links report (used by check-web-archive.mjs) - output: lychee/out.md - # Write a job summary - jobSummary: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Check broken links against Web Archive - if: steps.lychee.outputs.exit_code != 0 - id: webarchive - run: node scripts/check-web-archive.mjs - env: - LYCHEE_OUTPUT: lychee/out.md - - - name: Fail if broken links found and no web archive fallback - if: steps.lychee.outputs.exit_code != 0 && steps.webarchive.outputs.all_archived != 'true' - run: | - echo "::error::Broken links were detected with no Web Archive fallback available." - echo "" - echo "What happened:" - echo " lychee found one or more broken links in the *.md and *.html files of this repository." - echo " The Web Archive (Wayback Machine) check found no archived versions for some of them." - echo "" - echo "How to fix:" - echo " 1. Review the 'Check links with lychee' step above for a full list of broken links." - echo " 2. For links marked with a '::notice::' annotation above, a Web Archive version exists." - echo " Replace those broken links with the suggested archive.org URL." - echo " 3. For links with no archive version, either:" - echo " a. Find an updated URL that points to the same or equivalent content." - echo " b. Remove the link if the content is no longer relevant." - echo " c. Add the URL to .lycheeignore if it is a known false positive." - echo "" - echo "Report location: lychee/out.md (available as a workflow artifact if configured)." - exit 1 diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml deleted file mode 100644 index 83b979c..0000000 --- a/.github/workflows/pages.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: GitHub Pages - -# Publishes the static SPA (app.html, app/, app.css, the existing demos, and -# the redirect shells) to GitHub Pages. The site is a Jekyll site because -# _config.yml is present; we let `actions/configure-pages@v5` set up the -# right Jekyll inputs and `actions/upload-pages-artifact@v3` package it. - -on: - push: - branches: - - main - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -# Cancel in-flight publishes so the latest commit always wins. -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - build: - name: Build - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v6 - - name: Setup Pages - uses: actions/configure-pages@v5 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@v1 - with: - source: ./ - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - - deploy: - name: Deploy - runs-on: ubuntu-latest - needs: build - timeout-minutes: 10 - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 70558ff..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Tests - -# Runs the Node-runnable test scripts that live at the repo root. This is the -# minimal CI gate adapted from the JS template's release.yml — we only need -# the test matrix because we don't publish to npm. - -on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened] - workflow_dispatch: - -jobs: - test: - name: Test (Bun) - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - steps: - - uses: actions/checkout@v6 - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - name: Run text-transformer tests - # The transformer test suite asserts on live Wikidata results, so a - # subset of assertions flaps based on what Wikidata returns at run - # time. Run it for the rich logs but don't gate the build on these - # assertions — the same applies to the cache/API sanity scripts - # below. - continue-on-error: true - run: bun run-tests.mjs - - - name: Run cache/API sanity scripts - # These scripts hit the live Wikidata API; treat them as informational - # because external availability could flap. They print rich logs but - # do not gate the build. - continue-on-error: true - run: | - bun cache-test.mjs || true - bun unified-cache-test.mjs || true - - - name: Syntax-check unified SPA source files - # Plain JS modules (no JSX) — these *do* gate the build, since they - # are loaded directly by `app.html` in production. - run: | - node --check app/routing.js - node --check app/ipa.js diff --git a/.gitignore b/.gitignore index 7b944ed..2b509bc 100644 --- a/.gitignore +++ b/.gitignore @@ -140,3 +140,7 @@ vite.config.ts.timestamp-* # Playwright MCP runtime artifacts .playwright-mcp/ + +# Playwright e2e artifacts +test-results/ +playwright-report/ diff --git a/README.md b/README.md index 3ef391e..3220c59 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,12 @@ The whole project is published to GitHub Pages as a single unified single-page a | Mode | Description | Source | | ---- | ----------- | ------ | -| [Alphabet](https://link-assistant.github.io/human-language/app.html#mode=alphabet) | Each letter at ≥50 % of the viewport, with IPA pronunciation and keyboard navigation. | [`app/modes/alphabet.jsx`](app/modes/alphabet.jsx) | -| [Dictionary](https://link-assistant.github.io/human-language/app.html#mode=dictionary) | Definitions merged from the Free Dictionary API and Wiktionary, with an IPA-only display toggle. | [`app/modes/dictionary.jsx`](app/modes/dictionary.jsx) | -| [Ontology](https://link-assistant.github.io/human-language/app.html#mode=ontology) | Walk Wikidata's *subclass of* tree, rooted at entity (Q35120). Cycles are allowed and flagged. | [`app/modes/ontology.jsx`](app/modes/ontology.jsx) | -| [Entities](https://link-assistant.github.io/human-language/app.html#mode=entity) | Browse any Wikidata Q-id, with IPA toggle and an inline test runner. | [`app/modes/entity.jsx`](app/modes/entity.jsx) | -| [Properties](https://link-assistant.github.io/human-language/app.html#mode=property) | Same lens as Entities but for P-ids. | [`app/modes/property.jsx`](app/modes/property.jsx) | -| [Text → Q/P Transformer](https://link-assistant.github.io/human-language/app.html#mode=transformer) | Turn English text into a sequence of Wikidata entities (Q) and properties (P), with n-gram support. | [`app/modes/transformer.jsx`](app/modes/transformer.jsx) | +| [Alphabet](https://link-assistant.github.io/human-language/app.html#mode=alphabet) | Each letter at ≥50 % of the viewport, with IPA pronunciation and keyboard navigation. | [`js/src/app/modes/alphabet.jsx`](js/src/app/modes/alphabet.jsx) | +| [Dictionary](https://link-assistant.github.io/human-language/app.html#mode=dictionary) | Definitions merged from the Free Dictionary API and Wiktionary, with an IPA-only display toggle. | [`js/src/app/modes/dictionary.jsx`](js/src/app/modes/dictionary.jsx) | +| [Ontology](https://link-assistant.github.io/human-language/app.html#mode=ontology) | Walk Wikidata's *subclass of* tree, rooted at entity (Q35120). Cycles are allowed and flagged. | [`js/src/app/modes/ontology.jsx`](js/src/app/modes/ontology.jsx) | +| [Entities](https://link-assistant.github.io/human-language/app.html#mode=entity) | Browse any Wikidata Q-id, with IPA toggle and an inline test runner. | [`js/src/app/modes/entity.jsx`](js/src/app/modes/entity.jsx) | +| [Properties](https://link-assistant.github.io/human-language/app.html#mode=property) | Same lens as Entities but for P-ids. | [`js/src/app/modes/property.jsx`](js/src/app/modes/property.jsx) | +| [Text → Q/P Transformer](https://link-assistant.github.io/human-language/app.html#mode=transformer) | Turn English text into a sequence of Wikidata entities (Q) and properties (P), with n-gram support. | [`js/src/app/modes/transformer.jsx`](js/src/app/modes/transformer.jsx) | The legacy URLs (`entities.html`, `properties.html`, `transformation/index.html`, and bare hashes like `entities.html#Q35120`) still work — they now redirect into the unified SPA preserving any parameters. @@ -170,27 +170,27 @@ Based on our [GitHub issues](https://github.com/link-assistant/human-language/is ### Core Components -1. **Wikidata API Client** (`wikidata-api.js`) +1. **Wikidata API Client** (`js/src/wikidata-api.js`) - Handles all Wikidata API interactions - Configurable caching strategies - Batch request optimization -2. **Text Transformer** (`transformation/text-to-qp-transformer.js`) +2. **Text Transformer** (`js/src/transformation/text-to-qp-transformer.js`) - N-gram generation and matching - Parallel search execution - Priority-based result merging -3. **Search Utilities** (`wikidata-api.js`) +3. **Search Utilities** (`js/src/wikidata-api.js`) - Exact and fuzzy search algorithms - Context-aware ranking system - Multi-language support -4. **Caching System** (`unified-cache.js`) +4. **Caching System** (`js/src/unified-cache.js`) - Factory pattern for cache creation - File system cache for Node.js - IndexedDB cache for browsers -5. **UI Components** (`statements.jsx`, `loading.jsx`) +5. **UI Components** (`js/src/statements.jsx`, `js/src/loading.jsx`) - React 19 components with JSX - No build step required (Babel in-browser) - Responsive and theme-aware design @@ -235,20 +235,21 @@ User Input → Text Transformer → N-gram Generator → Parallel Search ### For Developers ```bash -# Run tests -bun run-tests.mjs +# Run gating unit tests (zero deps) +npm run test:unit -# Test n-gram features -bun transformation/test-ngram-demo.mjs +# Run gating E2E tests (boots a local static server + Playwright) +npm run test:e2e:local -# Run comprehensive tests -bun comprehensive-test.mjs +# Syntax-check every .mjs/.js module +npm run test:syntax -# Run E2E tests -bun e2e-test.mjs - -# Check limitations -bun limitation-test.mjs +# Live integration test runners (hit the real Wikidata API) +node js/scripts/run-tests.mjs +node js/src/transformation/test-ngram-demo.mjs +node js/scripts/comprehensive-test.mjs +node js/scripts/e2e-test.mjs +node js/scripts/limitation-test.mjs ``` ### Interactive Demos diff --git a/_config.yml b/_config.yml index 0bfce04..35d9a8c 100644 --- a/_config.yml +++ b/_config.yml @@ -20,9 +20,11 @@ exclude: - data - api-patterns.json - limitations-found.json - - "*.mjs" + # CI/CD and integration-test scripts (Node only, never loaded by a page). + - js/scripts + - js/tests # Node-only modules (use the *-browser.js counterparts in the browser). - - persistent-cache.js - - unified-cache.js - - wikidata-api.js + - js/src/persistent-cache.js + - js/src/unified-cache.js + - js/src/wikidata-api.js - .gitkeep diff --git a/app.html b/app.html index e7d2cd0..05f6ede 100644 --- a/app.html +++ b/app.html @@ -23,15 +23,23 @@ + text/babel mode files need on window.HumanLanguageApp. + + NOTE: The mode `.jsx` files run through Babel-standalone, which compiles + dynamic `import('./x.js')` calls into `require('./x.js')` — and `require` + is not defined in the browser. So every ES module the mode files need + has to be imported here (in a real ` - - + + - - - - - - - - + + + + + + + + + +``` + +```jsx +// js/src/app/modes/transformer.jsx — babel-compiled, no dynamic import +const { TextToQPTransformer, TextTransformerTest, demonstrateTransformer } = + (window.HumanLanguageApp || {}); + +React.useEffect(() => { + try { + if (!TextToQPTransformer) throw new Error('TextToQPTransformer not loaded by app.html'); + setTransformer(new TextToQPTransformer()); + } catch (e) { + setError(`Failed to load transformer: ${e?.message || e}`); + } +}, []); +``` + +`window.HumanLanguageApp` was already a pattern in the codebase (the +ontology & dictionary modes use it for the same reason), so this fix +follows the surrounding style. + +## Why this won't silently break again + +Three layers of defense: + +1. **No dynamic `import()` inside any `.jsx` file** — the file + `transformer.jsx` only consumes pre-loaded globals. If someone + reintroduces `import()`, the same banner will appear, and: +2. **A Playwright regression test** asserts the body never contains + the strings `"Failed to load transformer"` or + `"require is not defined"`. The test runs in the PR-gate workflow + on every push, so the failure surfaces before merge. +3. **A second Playwright test** clicks Transform and waits for a + `Q/P Sequence` panel — that fails if the transformer module is + unloaded for any reason (not just the `require` bug). + +## Why we didn't tell babel-standalone to keep `import()` + +You can ask `@babel/preset-env` to leave dynamic imports alone: + +```html + +``` + +…or pass `{ targets: { esmodules: true } }` so `env` recognises that +the target supports native `import()`. Both work, but both spread the +fix into a configuration string that's easy to lose during a future +refactor. The window-handoff pattern keeps the babel-compiled file +free of any runtime feature that depends on a specific preset +configuration, which is more robust. diff --git a/docs/case-studies/issue-35/solution-plans.md b/docs/case-studies/issue-35/solution-plans.md new file mode 100644 index 0000000..80b3cec --- /dev/null +++ b/docs/case-studies/issue-35/solution-plans.md @@ -0,0 +1,110 @@ +# Solution plans + +Each top-level requirement from [`requirements.md`](./requirements.md) +was a small design choice in its own right. The chosen approach is +listed first, followed by the alternatives that were considered and +rejected. + +## 1. Fix the transformer + +**Chosen:** Hoist the transformer modules out of `app/modes/transformer.jsx` +into a real `