From 2058273841ecbb8e6ce218cb198cec43e68b496a Mon Sep 17 00:00:00 2001 From: Johannes Stein Date: Fri, 24 Jul 2026 18:37:09 +0100 Subject: [PATCH 1/5] feat(compliance): add native TOML and JSON5 runners --- .github/workflows/ci.yml | 213 +++---- .github/workflows/json5-test-bump.yml | 79 +++ .github/workflows/toml-test-bump.yml | 6 +- build.pas | 36 ++ docs/build-system.md | 10 +- docs/built-ins-data-formats.md | 13 +- docs/contributing/tooling.md | 10 +- docs/interpreter.md | 4 +- docs/testing.md | 38 +- scripts/GocciaJSON5Check.dpr | 210 ------- scripts/GocciaTOMLCheck.dpr | 138 ----- ..._cases.js => regenerate-json5-manifest.js} | 20 +- scripts/run_json5_test_suite.py | 466 --------------- scripts/run_toml_test_suite.py | 375 ------------ scripts/suite-bump-pin.ts | 28 +- source/app/compliance/Goccia.Compliance.pas | 477 +++++++++++++++ .../GocciaJSON5ComplianceRunner.dpr | 487 ++++++++++++++++ .../compliance/GocciaTOMLComplianceRunner.dpr | 544 ++++++++++++++++++ tests/compliance/json5-manifest.json | 1 + tests/compliance/json5.pin | 1 + tests/compliance/toml-test.pin | 1 + 21 files changed, 1802 insertions(+), 1355 deletions(-) create mode 100644 .github/workflows/json5-test-bump.yml delete mode 100644 scripts/GocciaJSON5Check.dpr delete mode 100644 scripts/GocciaTOMLCheck.dpr rename scripts/{extract_json5_cases.js => regenerate-json5-manifest.js} (88%) delete mode 100644 scripts/run_json5_test_suite.py delete mode 100644 scripts/run_toml_test_suite.py create mode 100644 source/app/compliance/Goccia.Compliance.pas create mode 100644 source/app/compliance/GocciaJSON5ComplianceRunner.dpr create mode 100644 source/app/compliance/GocciaTOMLComplianceRunner.dpr create mode 100644 tests/compliance/json5-manifest.json create mode 100644 tests/compliance/json5.pin create mode 100644 tests/compliance/toml-test.pin diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aeccf36c6..cdb091e33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,29 +201,22 @@ jobs: echo "${name} built successfully for ${TARGET}" done - echo "" - echo "=== Building GocciaTOMLCheck for ${TARGET} ===" - "$CROSS_FPC" -T"${OS}" -O4 -dPRODUCTION -Xs -CX -XX -B \ - -Fu./source/units -Fu./source/generated -Fu./source/shared -Fu./source/app -Fu./souffle \ - -Fi./source/units -Fi./source/shared -Fi./souffle \ - -Fu"$RTL_DIR" -Fu"$OBJPAS_DIR" -Fu"$GEN_DIR" -Fu"$FCL_DIR" -Fu"$FCL_BASE_SRC" -Fu"$FCL_NET_SRC" -Fu"$OPENSSL_SRC" \ - -FU"build/compiled" -FE"build" \ - $EXTRA_FLAGS \ - -dFPC_SOFT_FPUX80 \ - "./scripts/GocciaTOMLCheck.dpr" - echo "GocciaTOMLCheck built successfully for ${TARGET}" - - echo "" - echo "=== Building GocciaJSON5Check for ${TARGET} ===" - "$CROSS_FPC" -T"${OS}" -O4 -dPRODUCTION -Xs -CX -XX -B \ - -Fu./source/units -Fu./source/generated -Fu./source/shared -Fu./source/app -Fu./souffle \ - -Fi./source/units -Fi./source/shared -Fi./souffle \ - -Fu"$RTL_DIR" -Fu"$OBJPAS_DIR" -Fu"$GEN_DIR" -Fu"$FCL_DIR" -Fu"$FCL_BASE_SRC" -Fu"$FCL_NET_SRC" -Fu"$OPENSSL_SRC" \ - -FU"build/compiled" -FE"build" \ - $EXTRA_FLAGS \ - -dFPC_SOFT_FPUX80 \ - "./scripts/GocciaJSON5Check.dpr" - echo "GocciaJSON5Check built successfully for ${TARGET}" + # Compliance runners are explicit CI targets. They live outside the + # release-entrypoint glob so they cannot enter release archives. + for src in ./source/app/compliance/*.dpr; do + name="$(basename "${src%.dpr}")" + echo "" + echo "=== Building ${name} for ${TARGET} ===" + "$CROSS_FPC" -T"${OS}" -O4 -dPRODUCTION -Xs -CX -XX -B \ + -Fu./source/units -Fu./source/generated -Fu./source/shared -Fu./source/app -Fu./source/app/compliance -Fu./souffle \ + -Fi./source/units -Fi./source/shared -Fi./souffle \ + -Fu"$RTL_DIR" -Fu"$OBJPAS_DIR" -Fu"$GEN_DIR" -Fu"$FCL_DIR" -Fu"$FCL_BASE_SRC" -Fu"$FCL_NET_SRC" -Fu"$OPENSSL_SRC" \ + -FU"build/compiled" -FE"build" \ + $EXTRA_FLAGS \ + -dFPC_SOFT_FPUX80 \ + "$src" + echo "${name} built successfully for ${TARGET}" + done echo "" echo "=== Build artifacts for ${TARGET} ===" @@ -232,33 +225,21 @@ jobs: - name: Stage build artifacts run: bash ./.github/scripts/stage-build-artifacts.sh build ci-artifacts --include-tests --strip - - name: Stage TOML compliance harness + - name: Stage compliance runners run: | - copied=0 - for candidate in build/GocciaTOMLCheck build/GocciaTOMLCheck.exe; do - if [ -f "$candidate" ]; then - cp "$candidate" ci-artifacts/ - copied=1 + for name in GocciaTOMLComplianceRunner GocciaJSON5ComplianceRunner; do + copied=0 + for candidate in "build/$name" "build/$name.exe"; do + if [ -f "$candidate" ]; then + cp "$candidate" ci-artifacts/ + copied=1 + fi + done + if [ "$copied" -eq 0 ]; then + echo "::error::Missing compiled $name binary in build/" + exit 1 fi done - if [ "$copied" -eq 0 ]; then - echo "::error::Missing compiled GocciaTOMLCheck binary in build/" - exit 1 - fi - - - name: Stage JSON5 compliance harness - run: | - copied=0 - for candidate in build/GocciaJSON5Check build/GocciaJSON5Check.exe; do - if [ -f "$candidate" ]; then - cp "$candidate" ci-artifacts/ - copied=1 - fi - done - if [ "$copied" -eq 0 ]; then - echo "::error::Missing compiled GocciaJSON5Check binary in build/" - exit 1 - fi - name: Upload build artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -432,39 +413,39 @@ jobs: - name: Run TOML 1.1.0 compliance suite run: | - PYTHON_BIN="$(command -v python3 || command -v python)" - HARNESS="./build/GocciaTOMLCheck" - if [ -f "./build/GocciaTOMLCheck.exe" ]; then - HARNESS="./build/GocciaTOMLCheck.exe" + pin="$(tr -d '\r\n' < tests/compliance/toml-test.pin)" + suite_dir="$RUNNER_TEMP/toml-test" + git init "$suite_dir" + git -C "$suite_dir" remote add origin https://github.com/toml-lang/toml-test.git + git -C "$suite_dir" fetch --depth 1 origin "$pin" + git -C "$suite_dir" checkout --detach "$pin" + runner="./build/GocciaTOMLComplianceRunner" + if [ -f "$runner.exe" ]; then + runner="$runner.exe" fi - "$PYTHON_BIN" scripts/run_toml_test_suite.py --harness="$HARNESS" --output=toml-test-results-${{ matrix.target }}.json + "$runner" --suite-dir="$suite_dir" --jobs=4 \ + --output=toml-test-results-${{ matrix.target }}.json - name: Check TOML compliance summary if: always() run: | - PYTHON_BIN="$(command -v python3 || command -v python)" - "$PYTHON_BIN" - <<'PY' - import json - import sys - from pathlib import Path - - path = Path("toml-test-results-${{ matrix.target }}.json") - if not path.exists(): - print(f"::error::Missing TOML report: {path}") - sys.exit(1) - - summary = json.loads(path.read_text())["summary"] - expected_zero = ["failed", "false_accepts", "false_rejects", "valid_mismatches", "timeouts"] - for key in expected_zero: - if summary.get(key) != 0: - print(f"::error::{key}={summary.get(key)} in {path}") - print(json.dumps(summary, indent=2)) - sys.exit(1) - if summary.get("total") != summary.get("passed"): - print(f"::error::passed={summary.get('passed')} total={summary.get('total')} in {path}") - print(json.dumps(summary, indent=2)) - sys.exit(1) - PY + node - <<'JS' + const fs = require("fs"); + const path = "toml-test-results-${{ matrix.target }}.json"; + if (!fs.existsSync(path)) { + throw new Error(`Missing TOML report: ${path}`); + } + const report = JSON.parse(fs.readFileSync(path, "utf8")); + if ( + report.reportVersion !== 1 || + report.runner?.name !== "GocciaTOMLComplianceRunner" || + report.suite?.name !== "toml-test" || + typeof report.summary?.failed !== "number" || + !Array.isArray(report.cases) + ) { + throw new Error(`Unexpected TOML report shape: ${path}`); + } + JS - name: Upload TOML compliance report if: always() @@ -514,59 +495,47 @@ jobs: - name: Run JSON5 compliance suite run: | - PYTHON_BIN="$(command -v python3 || command -v python)" - HARNESS="./build/GocciaJSON5Check" - TEST_RUNNER="./build/GocciaTestRunner" - if [ -f "./build/GocciaJSON5Check.exe" ]; then - HARNESS="./build/GocciaJSON5Check.exe" + pin="$(tr -d '\r\n' < tests/compliance/json5.pin)" + suite_dir="$RUNNER_TEMP/json5" + git init "$suite_dir" + git -C "$suite_dir" remote add origin https://github.com/json5/json5.git + git -C "$suite_dir" fetch --depth 1 origin "$pin" + git -C "$suite_dir" checkout --detach "$pin" + cp tests/compliance/json5-manifest.json \ + "$suite_dir/goccia-json5-cases.json" + runner="./build/GocciaJSON5ComplianceRunner" + test_runner="./build/GocciaTestRunner" + if [ -f "$runner.exe" ]; then + runner="$runner.exe" fi - if [ -f "./build/GocciaTestRunner.exe" ]; then - TEST_RUNNER="./build/GocciaTestRunner.exe" + if [ -f "$test_runner.exe" ]; then + test_runner="$test_runner.exe" fi - "$PYTHON_BIN" scripts/run_json5_test_suite.py --harness="$HARNESS" --test-runner="$TEST_RUNNER" --output=json5-test-results-${{ matrix.target }}.json + "$runner" --suite-dir="$suite_dir" --test-runner="$test_runner" \ + --jobs=4 --output=json5-test-results-${{ matrix.target }}.json - name: Check JSON5 compliance summary if: always() run: | - PYTHON_BIN="$(command -v python3 || command -v python)" - "$PYTHON_BIN" - <<'PY' - import json - import sys - from pathlib import Path - - path = Path("json5-test-results-${{ matrix.target }}.json") - if not path.exists(): - print(f"::error::Missing JSON5 report: {path}") - sys.exit(1) - - summary = json.loads(path.read_text()) - parse_summary = summary["summary"]["parse"] - stringify_summary = summary["summary"]["stringify"] - - expected_parse_zero = ["failed", "false_accepts", "false_rejects", "valid_mismatches", "timeouts"] - for key in expected_parse_zero: - if parse_summary.get(key) != 0: - print(f"::error::parse.{key}={parse_summary.get(key)} in {path}") - print(json.dumps(summary, indent=2)) - sys.exit(1) - - if parse_summary.get("total") != parse_summary.get("passed"): - print(f"::error::parse.passed={parse_summary.get('passed')} parse.total={parse_summary.get('total')} in {path}") - print(json.dumps(summary, indent=2)) - sys.exit(1) - - expected_stringify_zero = ["failed", "skipped"] - for key in expected_stringify_zero: - if stringify_summary.get(key) != 0: - print(f"::error::stringify.{key}={stringify_summary.get(key)} in {path}") - print(json.dumps(summary, indent=2)) - sys.exit(1) - - if stringify_summary.get("totalTests") != stringify_summary.get("passed"): - print(f"::error::stringify.passed={stringify_summary.get('passed')} stringify.totalTests={stringify_summary.get('totalTests')} in {path}") - print(json.dumps(summary, indent=2)) - sys.exit(1) - PY + node - <<'JS' + const fs = require("fs"); + const path = "json5-test-results-${{ matrix.target }}.json"; + if (!fs.existsSync(path)) { + throw new Error(`Missing JSON5 report: ${path}`); + } + const report = JSON.parse(fs.readFileSync(path, "utf8")); + if ( + report.reportVersion !== 1 || + report.runner?.name !== "GocciaJSON5ComplianceRunner" || + report.suite?.name !== "json5" || + typeof report.summary?.failed !== "number" || + !Array.isArray(report.cases) || + typeof report.sections?.parse?.summary?.failed !== "number" || + typeof report.sections?.stringify?.testRunnerReport !== "object" + ) { + throw new Error(`Unexpected JSON5 report shape: ${path}`); + } + JS - name: Upload JSON5 compliance report if: always() diff --git a/.github/workflows/json5-test-bump.yml b/.github/workflows/json5-test-bump.yml new file mode 100644 index 000000000..49cd0b445 --- /dev/null +++ b/.github/workflows/json5-test-bump.yml @@ -0,0 +1,79 @@ +name: JSON5 suite weekly bump + +# Updates the pinned json5/json5 revision and deterministically regenerates +# the executable parser-case manifest from a prepared checkout. + +on: + schedule: + - cron: '45 6 * * 1' + workflow_dispatch: + +jobs: + bump: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + + - name: Install FPC + run: sudo apt-get update && sudo apt-get install fpc -qq > /dev/null + + - id: latest + run: | + sha=$(gh api repos/json5/json5/commits/main --jq .sha) + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "short=${sha:0:8}" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Update pin + run: bun scripts/suite-bump-pin.ts tests/compliance/json5.pin ${{ steps.latest.outputs.sha }} + + - name: Prepare pinned checkout + run: | + git init "$RUNNER_TEMP/json5" + git -C "$RUNNER_TEMP/json5" remote add origin https://github.com/json5/json5.git + git -C "$RUNNER_TEMP/json5" fetch --depth 1 origin "${{ steps.latest.outputs.sha }}" + git -C "$RUNNER_TEMP/json5" checkout --detach "${{ steps.latest.outputs.sha }}" + + - name: Regenerate manifest + run: | + ./build.pas json5compliancerunner + ./build/GocciaJSON5ComplianceRunner \ + --regenerate-manifest \ + --suite-dir="$RUNNER_TEMP/json5" \ + --manifest=tests/compliance/json5-manifest.json + + - id: changed + run: | + if git diff --quiet -- tests/compliance/json5.pin tests/compliance/json5-manifest.json; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "JSON5 suite already pinned to ${{ steps.latest.outputs.short }} — no PR to open." + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Open or update PR + if: steps.changed.outputs.changed == 'true' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + branch: chore/json5-test-bump + commit-message: "chore(json5): bump suite pin to ${{ steps.latest.outputs.short }}" + title: "chore(json5): bump suite pin to ${{ steps.latest.outputs.short }}" + body: | + Automated bump of the json5/json5 SHA pin to `${{ steps.latest.outputs.sha }}`. + + The executable parser-case manifest was regenerated from that + exact checkout. CI will run parser and stringify compliance. + + Cron: weekly, Mondays 06:45 UTC. + labels: | + json5 + automated diff --git a/.github/workflows/toml-test-bump.yml b/.github/workflows/toml-test-bump.yml index 5f46be4d6..93881703b 100644 --- a/.github/workflows/toml-test-bump.yml +++ b/.github/workflows/toml-test-bump.yml @@ -1,7 +1,7 @@ name: toml-test weekly bump # Pulls the latest toml-lang/toml-test main SHA, updates the pin in -# scripts/run_toml_test_suite.py, and opens a PR. Merge once the +# tests/compliance/toml-test.pin, and opens a PR. Merge once the # compliance delta is acceptable. # # Cadence: weekly (Mondays 06:30 UTC). Manual trigger available via @@ -34,11 +34,11 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Update pin in suite runner - run: bun scripts/suite-bump-pin.ts scripts/run_toml_test_suite.py ${{ steps.latest.outputs.sha }} + run: bun scripts/suite-bump-pin.ts tests/compliance/toml-test.pin ${{ steps.latest.outputs.sha }} - id: changed run: | - if git diff --quiet -- scripts/run_toml_test_suite.py; then + if git diff --quiet -- tests/compliance/toml-test.pin; then echo "changed=false" >> "$GITHUB_OUTPUT" echo "toml-test already pinned to ${{ steps.latest.outputs.short }} — no PR to open." else diff --git a/build.pas b/build.pas index 89ea3f8e3..df700e7c7 100755 --- a/build.pas +++ b/build.pas @@ -448,6 +448,36 @@ procedure BuildTestRunner; WriteLn('GocciaTestRunner built successfully'); end; +procedure BuildTOMLComplianceRunner; +var + Output: string; +begin + WriteLn('Building GocciaTOMLComplianceRunner...'); + if not RunCommand('fpc', FPCArgs( + 'source/app/compliance/GocciaTOMLComplianceRunner.dpr', + EnsureUnitOutputDirectory( + TargetUnitOutputDirectory('tomlcompliancerunner'))), Output) then + PrintBuildFailureAndExit(Output, + 'GocciaTOMLComplianceRunner build failed', 'tomlcompliancerunner'); + WriteLn(Output); + WriteLn('GocciaTOMLComplianceRunner built successfully'); +end; + +procedure BuildJSON5ComplianceRunner; +var + Output: string; +begin + WriteLn('Building GocciaJSON5ComplianceRunner...'); + if not RunCommand('fpc', FPCArgs( + 'source/app/compliance/GocciaJSON5ComplianceRunner.dpr', + EnsureUnitOutputDirectory( + TargetUnitOutputDirectory('json5compliancerunner'))), Output) then + PrintBuildFailureAndExit(Output, + 'GocciaJSON5ComplianceRunner build failed', 'json5compliancerunner'); + WriteLn(Output); + WriteLn('GocciaJSON5ComplianceRunner built successfully'); +end; + procedure BuildWasmTestRunner; var Output: string; @@ -549,6 +579,10 @@ procedure Build(const ATrigger: string); BuildTestRunner; BuildFFIFixture; end + else if ATrigger = 'tomlcompliancerunner' then + BuildTOMLComplianceRunner + else if ATrigger = 'json5compliancerunner' then + BuildJSON5ComplianceRunner else if ATrigger = 'wasmtestrunner' then BuildWasmTestRunner else if ATrigger = 'benchmarkrunner' then @@ -607,6 +641,8 @@ procedure Build(const ATrigger: string); BuildTriggers.Add('loaderbare'); BuildTriggers.Add('sandboxrunner'); BuildTriggers.Add('testrunner'); + BuildTriggers.Add('tomlcompliancerunner'); + BuildTriggers.Add('json5compliancerunner'); BuildTriggers.Add('wasmtestrunner'); BuildTriggers.Add('benchmarkrunner'); BuildTriggers.Add('bundler'); diff --git a/docs/build-system.md b/docs/build-system.md index c4aaf862e..8471d24db 100644 --- a/docs/build-system.md +++ b/docs/build-system.md @@ -47,7 +47,7 @@ The build script supports two modes via `--dev` (default) and `--prod` flags: ./build.pas --prod # Production build of all components ``` -Builds all components in order: tests, loader, loaderbare, sandboxrunner, testrunner, benchmarkrunner, bundler, repl. The default full build does not clean first; pass `--clean` explicitly when you need to remove stale build artifacts. +Builds all components in order: tests, loader, loaderbare, sandboxrunner, testrunner, TOML and JSON5 compliance runners, benchmarkrunner, bundler, and repl. The default full build does not clean first; pass `--clean` explicitly when you need to remove stale build artifacts. ### Build Specific Components @@ -57,6 +57,7 @@ Builds all components in order: tests, loader, loaderbare, sandboxrunner, testru ./build.pas loaderbare # Bare Script Loader (core engine only) ./build.pas sandboxrunner # Sandbox Runner with virtual filesystem ./build.pas testrunner # JavaScript test runner + native FFI fixture +./build.pas tomlcompliancerunner json5compliancerunner # Compliance runners ./build.pas benchmarkrunner # Performance benchmark runner ./build.pas bundler # Bundler (compile to .gbc) ./build.pas tests # Pascal unit tests @@ -482,6 +483,8 @@ All compiled binaries go to the `build/` directory: | `build/GocciaScriptLoaderBare` | `source/app/GocciaScriptLoaderBare.dpr` | Execute file or stdin source with the core engine and CLI-local `print`; no loader runtime profile. `--test262-host` exposes private conformance hooks for the test262 runner | | `build/GocciaSandboxRunner` | `source/app/GocciaSandboxRunner.dpr` | Execute sandbox entry paths inside a seeded virtual filesystem | | `build/GocciaTestRunner` | `source/app/GocciaTestRunner.dpr` | JavaScript test runner | +| `build/GocciaTOMLComplianceRunner` | `source/app/compliance/GocciaTOMLComplianceRunner.dpr` | Native pinned `toml-test` runner | +| `build/GocciaJSON5ComplianceRunner` | `source/app/compliance/GocciaJSON5ComplianceRunner.dpr` | Pinned JSON5 parser/stringify runner | | `build/GocciaBenchmarkRunner` | `source/app/GocciaBenchmarkRunner.dpr` | Performance benchmark runner for files or stdin input | | `build/GocciaBundler` | `source/app/GocciaBundler.dpr` | Bundler (source to `.gbc`) | | `build/Goccia.Values.Primitives.Test` | `*.Test.pas` | Pascal unit test binaries | @@ -654,9 +657,8 @@ Runs on the full platform matrix: **`test`** (needs build) — Runs all JavaScript tests and Pascal unit tests on all platforms. -**`toml-compliance`** — Downloads the prebuilt `GocciaTOMLCheck` harness from each matrix build artifact, runs the official `toml-test` TOML 1.1.0 suite (pinned to a specific SHA) on every CI platform via `python3 scripts/run_toml_test_suite.py --harness=...`, validates that the JSON summary reports zero failures, and uploads the per-platform JSON report. The pin is bumped weekly by `.github/workflows/toml-test-bump.yml`. - -**`json5-compliance`** — Downloads the prebuilt `GocciaJSON5Check` harness and `GocciaTestRunner` binary from each matrix build artifact, runs `python3 scripts/run_json5_test_suite.py --harness=... --test-runner=...` on every CI platform, validates both the parser and stringify summaries, and uploads the per-platform JSON report. +**`toml-compliance`** — Downloads the prebuilt `GocciaTOMLComplianceRunner`, prepares the pinned `toml-test` checkout, trusts the runner exit status, validates the report envelope, and uploads the per-platform JSON report. The pin is bumped weekly by `.github/workflows/toml-test-bump.yml`. +**`json5-compliance`** — Downloads `GocciaJSON5ComplianceRunner` and `GocciaTestRunner`, prepares the pinned JSON5 checkout with its matching manifest, trusts the runner exit status, validates the shared report envelope, and uploads the per-platform report. **`test262`** (needs build, ubuntu-latest x64 only, **non-blocking**) — Downloads the `gocciascript-x86_64-linux` build, checks out [`tc39/test262`](https://github.com/tc39/test262) at the SHA pinned in `scripts/test262-suite-sha.txt`, runs `bun scripts/run_test262_suite.ts --suite-dir test262-suite --mode=bytecode --jobs=4 --timeout-ms=20000 --output=test262-results.json`, and uploads the report as a 30-day workflow artifact. The run step uses `continue-on-error: true` because the conformance lane is allowed to carry known steady-state failures while the engine closes compatibility gaps. On main, the JSON is also stashed via `actions/cache/save` under `test262-baseline-` so the PR workflow can compute Δ vs main, and `cd website && bun run publish-test262 ../test262-results.json` publishes the compressed run report plus its UTC daily dashboard pointer to Vercel Blob when `BLOB_READ_WRITE_TOKEN` is configured. Main runs also upload the `test262-profile` artifact and publish matching aggregate/detail profile payloads under the separate `test262-profiles/` Blob namespace for weekly performance review. The one-off `cd website && bun run backfill-test262` command seeds retained artifact reports and reruns expired historical days directly into Blob. The pin is bumped weekly by `.github/workflows/test262-bump.yml`. See [docs/test262.md](test262.md) for the harness and profile report contracts. diff --git a/docs/built-ins-data-formats.md b/docs/built-ins-data-formats.md index 53c7ccb3a..466ac6965 100644 --- a/docs/built-ins-data-formats.md +++ b/docs/built-ins-data-formats.md @@ -71,7 +71,12 @@ After `import * as JSON5 from "goccia:json5"`, `JSON5.parse` delegates to the st The `stringify` export delegates to `TGocciaJSON5Stringifier`, which reuses the same shared serialization core as strict JSON but switches to JSON5 formatting rules. That means unquoted identifier keys, single- or double-quoted strings (with optional `{ quote: "'" | '"' }` override), preserved `Infinity` / `-Infinity` / `NaN`, trailing commas when pretty-printing, `toJSON5()` preference over `toJSON()`, and the same replacer / space semantics as JSON plus the upstream JSON5 options-object form `{ replacer, space, quote }`. -Compatibility goal: GocciaScript is targeting full JSON5 parser compatibility plus upstream-aligned stringify behavior. `python3 scripts/run_json5_test_suite.py` now runs both the official `json5/json5` parser corpus and the local upstream-aligned stringify suite in one command, and `python3 scripts/run_json5_test_suite.py --harness=./build/GocciaJSON5Check` reuses a prebuilt parser decoder when you already have it. +Compatibility goal: GocciaScript is targeting full JSON5 parser compatibility +plus upstream-aligned stringify behavior. The native +`GocciaJSON5ComplianceRunner` runs the pinned upstream parser manifest through +`GocciaTestRunner` and then runs the local upstream-aligned stringify suite. +See [Testing](testing.md#running-tests) for checkout preparation and +commands. ## YAML (`goccia:yaml`, `Goccia.Builtins.YAML.pas`) @@ -141,4 +146,8 @@ After `import * as TOML from "goccia:toml"`, `TOML.parse` delegates to the stand TOML date/time values currently map to validated string scalars rather than Temporal values. This keeps the runtime and module-import behavior stable for v1 while leaving room for future Temporal-aware interop. -Compatibility goal: GocciaScript is targeting full TOML 1.1.0 support over time. Historical decision context lives in [Architecture Decision Records](adr/), and the official `toml-test` rerun command is `python3 scripts/run_toml_test_suite.py` or `python3 scripts/run_toml_test_suite.py --harness=./build/GocciaTOMLCheck` when you already have the decoder harness built. +Compatibility goal: GocciaScript is targeting full TOML 1.1.0 support over +time. Historical decision context lives in +[Architecture Decision Records](adr/). The native +`GocciaTOMLComplianceRunner` verifies and runs a prepared pinned `toml-test` +checkout; see [Testing](testing.md#running-tests). diff --git a/docs/contributing/tooling.md b/docs/contributing/tooling.md index 5e8de14bd..83cd3caaa 100644 --- a/docs/contributing/tooling.md +++ b/docs/contributing/tooling.md @@ -110,11 +110,11 @@ from a clean build. FPC 3.2.2 aborts with `Fatal: Internal error 200611011` when a second program is compiled against the `.ppu` files another program left in a shared `-FU` unit-output directory (the inliner trips while recompiling a unit it loaded -from the first program's build). This is why `build.pas` compiles every target -into its own `build/compiled/targets/` directory, and why -`scripts/run_json5_test_suite.py` gives each program its own subdirectory of -its build dir. Any script that compiles more than one program with -`fpc @config.cfg` must use a separate `-FU` directory per program. +from the first program's build). This is why `build.pas` compiles every target, +including the TOML and JSON5 compliance runners, into its own +`build/compiled/targets/` directory. Any script that compiles more than +one program with `fpc @config.cfg` must use a separate `-FU` directory per +program. ### `Int64` to `Double` Conversion on FPC 3.2.2 diff --git a/docs/interpreter.md b/docs/interpreter.md index 60639e35d..d5c88d653 100644 --- a/docs/interpreter.md +++ b/docs/interpreter.md @@ -103,12 +103,12 @@ Scopes form a tree with parent pointers, implementing lexical scoping: - **Standalone JSON utilities** — `Goccia.JSON` provides `TGocciaJSONParser` and `TGocciaJSONStringifier` as dependency-free utility classes that convert between JSON text and `TGocciaValue` types. `Goccia.Builtins.JSON` (the `JSON.parse`/`JSON.stringify` built-in) delegates to these, keeping the built-in a thin adapter. This separation allows the interpreter and any other component to parse JSON without instantiating a built-in. - **Capability-driven JSON parsing** — `JSONParser.pas` now owns one event-driven parser core plus a `TJSONParserCapabilities` set. Strict JSON uses the empty capability set, while JSON5 opts into comments, trailing commas, single-quoted strings, identifier keys, hexadecimal numbers, signed numbers, `Infinity` / `NaN`, line continuations, and ECMAScript whitespace extensions. This keeps JSON and JSON5 behavior aligned on the shared grammar machinery instead of maintaining two diverging parser implementations. - **JSON5 parser/stringifier split** — `Goccia.JSON5` provides standalone `TGocciaJSON5Parser` and `TGocciaJSON5Stringifier` utilities. The parser reuses the same core capability-driven parser engine as strict JSON but enables the JSON5 capability set, while the stringifier reuses the shared JSON serialization engine in JSON5 mode instead of maintaining a second formatter. `Goccia.Builtins.JSON5` backs the named exports of `goccia:json5` (`parse`, `stringify`), the module loader reuses the parser for `.json5` imports, and globals injection reuses it for `--globals=file.json5` and embedding helpers. -- **JSON5 compatibility target** — The project goal for JSON5 is full parser compatibility with the reference `json5/json5` implementation plus upstream-aligned stringify behavior. The local rerun command is `python3 scripts/run_json5_test_suite.py`, or `python3 scripts/run_json5_test_suite.py --harness=./build/GocciaJSON5Check` if you already built the parser decoder harness. That command runs both the upstream parser corpus and the local upstream-aligned stringify suite. A rerun on 2026-07-21 against upstream commit `b935d4a280eafa8835e6182551b63809e61243b0` matched 84 of 84 extracted parser cases; the local stringify suite passed all 42 upstream-aligned tests covering special numeric values, quote handling, replacers, boxed primitives, options objects, and pretty-print trailing commas. +- **JSON5 compatibility target** — The project goal for JSON5 is full parser compatibility with the reference `json5/json5` implementation plus upstream-aligned stringify behavior. `GocciaJSON5ComplianceRunner` verifies the pinned checkout and generated manifest, runs each parser case through `GocciaTestRunner`, and then runs the local stringify suite. A rerun on 2026-07-21 against upstream commit `b935d4a280eafa8835e6182551b63809e61243b0` matched 84 of 84 extracted parser cases; the local stringify suite passed all 42 upstream-aligned tests covering special numeric values, quote handling, replacers, boxed primitives, options objects, and pretty-print trailing commas. - **JSONL parser split** — `Goccia.JSONL` provides a standalone `TGocciaJSONLParser` utility that builds on `TGocciaJSONParser` one line at a time, preserving JSONL source line numbers in parse errors and supporting Bun-style chunked parsing through `ParseChunk(...)`. `Goccia.Builtins.JSONL` backs the named exports of `goccia:jsonl` (`parse`, `parseChunk`), while the module loader reuses the same parser for `.jsonl` imports. - **JSONL module imports** — `.jsonl` modules intentionally expose each non-empty line as a zero-based string-indexed named export (`"0"`, `"1"`, ...). This keeps the structured-data import surface consistent with the existing string-literal named import/export work and means namespace imports can reuse the same export table without introducing a JSONL-specific synthetic wrapper object just for modules. - **Text asset module imports** — `.txt` and `.md` modules bypass the script parser and expose a small named-export surface: `content` is the UTF-8 file text with source newlines canonicalized to LF (`\n`), and `metadata` is a frozen object containing `kind`, `path`, `fileName`, `extension`, and `byteLength`. Text assets do not invent a default export wrapper just for plain text files, which keeps imported text stable across Windows and non-Windows hosts. - **TOML parser split** — `Goccia.TOML` provides a standalone `TGocciaTOMLParser` utility that converts TOML 1.1.0 text into `TGocciaValue` trees. `Goccia.Builtins.TOML` backs the `parse` named export of `goccia:toml`, and the module loader reuses the same utility for `.toml` imports and TOML-backed globals injection. TOML module imports expose each root-table key as a named export, and namespace imports project that same export table into a frozen namespace object. TOML date/time values currently map to validated string scalars rather than Temporal values. For compliance work, the parser also exposes `ParseDocument(...)`, which preserves TOML scalar kinds and canonical values in a recursive TOML node tree without changing the public TOML runtime API. -- **TOML compatibility target** — The project goal for TOML is full TOML 1.1.0 compatibility. The official `toml-test` TOML 1.1.0 suite is part of CI and is rerun across the supported platform matrix. The local rerun command is `python3 scripts/run_toml_test_suite.py`, or `python3 scripts/run_toml_test_suite.py --harness=./build/GocciaTOMLCheck` if you already built the decoder harness. +- **TOML compatibility target** — The project goal for TOML is full TOML 1.1.0 compatibility. The official `toml-test` TOML 1.1.0 suite is part of CI and is rerun across the supported platform matrix through the native `GocciaTOMLComplianceRunner`. - **YAML parser split** — `Goccia.YAML` provides a standalone `TGocciaYAMLParser` utility that converts YAML text into `TGocciaValue` trees. `Goccia.Builtins.YAML` backs the named exports of `goccia:yaml`: `parse`, which follows Bun-style stream semantics by returning an array whenever explicit `---` document markers are present, and `parseDocuments` for callers that always want an array. The module loader reuses the same utility for `.yaml` and `.yml` imports: a single top-level mapping still exports its keys directly, while multi-document streams expose each document as a string-indexed named export (`"0"`, `"1"`, ...). Namespace imports for YAML file modules simply project that same export table into a frozen namespace object. - **YAML anchor handling** — Anchors are tracked per document during parsing, aliases resolve to the anchored node, and `<<:` merge keys fill only missing keys so explicit mapping entries always win and earlier entries in merge sequences keep precedence over later ones. - **YAML block scalars** — Literal (`|`) and folded (`>`) block scalars are parsed directly in `Goccia.YAML`, including chomping modifiers and indentation indicators, so common multi-line configuration text works the same through the `goccia:yaml` `parse` export and `.yaml`/`.yml` module imports. diff --git a/docs/testing.md b/docs/testing.md index 37821027e..d94f0a3a5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -420,18 +420,16 @@ This check is intentionally parse-only: it compares whether each suite case shou **Official TOML Suite** -For TOML 1.1.0 checks against the official `toml-test` corpus (pinned to a specific SHA in the script), run: +For TOML 1.1.0 checks against a prepared checkout of the official `toml-test` corpus, run: ```bash -python3 scripts/run_toml_test_suite.py -python3 scripts/run_toml_test_suite.py --output=tmp/toml-suite-results.json -python3 scripts/run_toml_test_suite.py --suite-dir=/path/to/toml-test -python3 scripts/run_toml_test_suite.py --harness=./build/GocciaTOMLCheck --output=tmp/toml-suite-results.json +./build.pas tomlcompliancerunner +./build/GocciaTOMLComplianceRunner --suite-dir=/path/to/toml-test --output=tmp/toml-suite-results.json ``` -Unlike the YAML script, this harness compares both parse/fail behavior and the official tagged JSON fixtures for valid cases. It uses a Pascal decoder built around `TGocciaTOMLParser.ParseDocument(...)` so TOML scalar kinds like `integer`, `float`, `datetime`, `datetime-local`, `date-local`, and `time-local` remain visible during compliance checks even though the normal TOML runtime API still maps date/time values to strings. +The runner verifies the checkout against `tests/compliance/toml-test.pin` without fetching or cloning. It launches its private worker mode once per case, applies a bounded `--jobs` limit, and classifies mismatches, false accepts, false rejects, timeouts, crashes, and infrastructure failures. Valid cases are compared with the official tagged JSON fixtures through `TGocciaTOMLParser.ParseDocument(...)`, preserving scalar distinctions such as `integer`, `float`, `datetime`, `datetime-local`, `date-local`, and `time-local`. -The TOML runner exits non-zero when any case fails or times out, so it is safe to use directly in CI. When `--harness` is omitted it compiles `scripts/GocciaTOMLCheck.dpr` automatically; CI uses a prebuilt harness from the matrix build artifacts instead. +The runner prints a human summary, optionally writes the shared compliance JSON envelope with `--output`, and exits non-zero for compliance or infrastructure failures. The harness and any file-backed parser regression must read source text as UTF-8 bytes first and decode it strictly before handing UTF-16 text to the @@ -442,16 +440,23 @@ identical on Windows and non-Windows hosts and prevent mojibake such as **Official JSON5 Suite** -For JSON5 parser compatibility checks against the official `json5/json5` parser test corpus, run: +For JSON5 parser compatibility checks against a prepared checkout of the official `json5/json5` parser test corpus, copy the committed manifest into the checkout and run: ```bash -python3 scripts/run_json5_test_suite.py -python3 scripts/run_json5_test_suite.py --output=tmp/json5-suite-results.json -python3 scripts/run_json5_test_suite.py --suite-dir=/path/to/json5 -python3 scripts/run_json5_test_suite.py --harness=./build/GocciaJSON5Check --output=tmp/json5-suite-results.json +./build.pas json5compliancerunner testrunner +cp tests/compliance/json5-manifest.json /path/to/json5/goccia-json5-cases.json +./build/GocciaJSON5ComplianceRunner --suite-dir=/path/to/json5 --test-runner=./build/GocciaTestRunner --output=tmp/json5-suite-results.json ``` -This runner extracts parser cases from the upstream `test/parse.js` and `test/errors.js` files, evaluates them with the reference implementation under Node.js, then compares Goccia's Pascal harness output against the canonical tagged values for valid cases. Invalid cases must fail to parse. The same command also runs `tests/built-ins/JSON5/stringify.js`, which mirrors the upstream stringify surface inside Goccia's JavaScript test harness. The runner exits non-zero when either the upstream parser corpus or the JSON5 stringify suite fails. +The runner verifies both the checkout and manifest against `tests/compliance/json5.pin`. It converts each manifest entry into a valid JavaScript test file and launches `GocciaTestRunner` in a separate process for that case, so invalid JSON5 remains parser input rather than invalid outer JavaScript. The same command runs `tests/built-ins/JSON5/stringify.js`. Normal compliance execution requires neither Python nor Node.js. + +Maintainers regenerate the executable manifest from an already-prepared checkout: + +```bash +./build/GocciaJSON5ComplianceRunner --regenerate-manifest --suite-dir=/path/to/json5 --manifest=tests/compliance/json5-manifest.json +``` + +Regeneration uses Node.js to execute the upstream reference tests, records the verified commit, and is intentionally separate from normal compliance runs. ### Pinned es-toolkit library probes @@ -671,9 +676,9 @@ build → test → artifacts **`test`** (needs build, all platforms) — Downloads pre-built binaries, runs all JavaScript tests and Pascal unit tests. Outputs JSON files via `--output=` for CI timing comparison. -**`toml-compliance`** (all platforms) — Downloads the prebuilt `GocciaTOMLCheck` harness from the matrix build artifacts, resolves `python3` or `python`, runs `scripts/run_toml_test_suite.py --harness=... --output=toml-test-results-.json`, checks that the JSON summary reports zero failures, and uploads the per-platform TOML conformance report as a workflow artifact. +**`toml-compliance`** (all platforms) — Downloads the prebuilt `GocciaTOMLComplianceRunner`, prepares the exact pinned `toml-test` checkout, and relies on the runner exit status. CI performs only lightweight validation of the report envelope before uploading it. -**`json5-compliance`** (all platforms) — Downloads the prebuilt `GocciaJSON5Check` harness and `GocciaTestRunner` binary from the matrix build artifacts, resolves `python3` or `python`, runs `scripts/run_json5_test_suite.py --harness=... --test-runner=... --output=json5-test-results-.json`, checks that both the parser and stringify summaries report zero failures, and uploads the per-platform JSON5 conformance report as a workflow artifact. +**`json5-compliance`** (all platforms) — Downloads `GocciaJSON5ComplianceRunner` and `GocciaTestRunner`, prepares the exact pinned JSON5 checkout, installs its matching committed manifest, and relies on the native runner exit status for parser and stringify compliance. CI performs only lightweight report-envelope validation before upload. **`test262`** (needs build, ubuntu-latest x64 only, **non-blocking**) — Runs the official conformance suite in bytecode mode from the shared pin in `scripts/test262-suite-sha.txt`, uploads the JSON report, and saves a `main` baseline cache for PR deltas. The run step is `continue-on-error: true` so known steady-state conformance failures do not block unrelated work; the downstream PR comment still gates regressions against the cached main baseline. **See [test262.md](test262.md) for the harness contract** and [Build System](build-system.md#ciyml--push-to-main--tags) for the workflow wiring. @@ -720,7 +725,8 @@ Weekly crons open (or update) a single PR every Monday with the latest upstream | Suite | Workflow | Schedule | Pin location | |-------|----------|----------|--------------| | test262 | `test262-bump.yml` | 06:00 UTC | `scripts/test262-suite-sha.txt` | -| toml-test | `toml-test-bump.yml` | 06:30 UTC | `scripts/run_toml_test_suite.py` | +| toml-test | `toml-test-bump.yml` | 06:30 UTC | `tests/compliance/toml-test.pin` | +| json5 | `json5-test-bump.yml` | 06:45 UTC | `tests/compliance/json5.pin` and `json5-manifest.json` | | yaml-test-suite | `yaml-test-bump.yml` | 07:00 UTC | `scripts/run_yaml_test_suite.py` | Each workflow reuses a fixed branch (`chore/-bump`), so an unmerged PR is updated in place rather than replaced. See [test262.md](test262.md) for details on the test262 harness contract. diff --git a/scripts/GocciaJSON5Check.dpr b/scripts/GocciaJSON5Check.dpr deleted file mode 100644 index 1564d2d14..000000000 --- a/scripts/GocciaJSON5Check.dpr +++ /dev/null @@ -1,210 +0,0 @@ -program GocciaJSON5Check; - -{$I Goccia.inc} - -uses - Classes, - Math, - SysUtils, - - StringBuffer, - - Goccia.GarbageCollector, - Goccia.JSON.Utils, - Goccia.JSON5, - Goccia.TextFiles, - Goccia.Values.ArrayValue, - Goccia.Values.ObjectValue, - Goccia.Values.Primitives; - -function SameDoubleBits(const ALeft, ARight: Double): Boolean; -var - LeftValue, RightValue: Double; - LeftBits: Int64 absolute LeftValue; - RightBits: Int64 absolute RightValue; -begin - LeftValue := ALeft; - RightValue := ARight; - Result := LeftBits = RightBits; -end; - -function TrimTrailingFractionalZeros(const AValue: string): string; -begin - Result := AValue; - while (Pos('.', Result) > 0) and (Length(Result) > 0) and - (Result[Length(Result)] = '0') do - Delete(Result, Length(Result), 1); - if (Length(Result) > 0) and (Result[Length(Result)] = '.') then - Delete(Result, Length(Result), 1); -end; - -function NormalizeExponentNumber(const AValue: string): string; -var - ExponentIndex, SignIndex: Integer; - ExponentPart, Mantissa: string; -begin - ExponentIndex := Pos('E', AValue); - if ExponentIndex = 0 then - begin - Result := TrimTrailingFractionalZeros(AValue); - Exit; - end; - - Mantissa := TrimTrailingFractionalZeros(Copy(AValue, 1, ExponentIndex - 1)); - ExponentPart := Copy(AValue, ExponentIndex + 1, MaxInt); - - if (Length(ExponentPart) > 0) and - ((ExponentPart[1] = '+') or (ExponentPart[1] = '-')) then - SignIndex := 2 - else - SignIndex := 1; - - while (Length(ExponentPart) > SignIndex) and - (ExponentPart[SignIndex] = '0') do - Delete(ExponentPart, SignIndex, 1); - - Result := Mantissa + 'e' + ExponentPart; -end; - -function NumericStringRoundTrips(const ASerialized: string; - const AValue: Double): Boolean; -var - FloatFormat: TFormatSettings; - ParsedValue: Double; -begin - FloatFormat := DefaultFormatSettings; - FloatFormat.DecimalSeparator := '.'; - if not TryStrToFloat(ASerialized, ParsedValue, FloatFormat) then - Exit(False); - Result := SameDoubleBits(ParsedValue, AValue); -end; - -function SerializeJSONNumber(const AValue: Double): string; -const - JSON_DOUBLE_ROUNDTRIP_SCIENTIFIC_FORMAT = '0.################E+00'; -var - FloatFormat: TFormatSettings; -begin - FloatFormat := DefaultFormatSettings; - FloatFormat.DecimalSeparator := '.'; - if AValue = 0 then - Exit('0'); - - Result := FloatToStr(AValue, FloatFormat); - if NumericStringRoundTrips(Result, AValue) then - Exit; - - Result := NormalizeExponentNumber( - FormatFloat(JSON_DOUBLE_ROUNDTRIP_SCIENTIFIC_FORMAT, AValue, - FloatFormat)); -end; - -function EncodeNumber(const AValue: TGocciaNumberLiteralValue): string; -begin - if AValue.IsNaN then - Exit('{"type":"number","value":"NaN"}'); - if AValue.IsInfinity then - Exit('{"type":"number","value":"Infinity"}'); - if AValue.IsNegativeInfinity then - Exit('{"type":"number","value":"-Infinity"}'); - if AValue.IsNegativeZero then - Exit('{"type":"number","value":"-0"}'); - Result := '{"type":"number","value":' + - QuoteJSONString(SerializeJSONNumber(AValue.Value)) + '}'; -end; - -function EncodeValue(const AValue: TGocciaValue): string; -var - Buffer: TStringBuffer; - Element: TGocciaValue; - I: Integer; - Key: string; - NumberLiteral: TGocciaNumberLiteralValue; -begin - if AValue is TGocciaNullLiteralValue then - Exit('{"type":"null"}'); - if AValue is TGocciaBooleanLiteralValue then - begin - if AValue.ToBooleanLiteral.Value then - Exit('{"type":"boolean","value":true}'); - Exit('{"type":"boolean","value":false}'); - end; - if AValue is TGocciaStringLiteralValue then - Exit('{"type":"string","value":' + - QuoteJSONString(AValue.ToStringLiteral.Value) + '}'); - if AValue is TGocciaNumberLiteralValue then - begin - NumberLiteral := AValue.ToNumberLiteral; - Exit(EncodeNumber(NumberLiteral)); - end; - if AValue is TGocciaArrayValue then - begin - Buffer := TStringBuffer.Create; - Buffer.Append('{"type":"array","items":['); - for I := 0 to TGocciaArrayValue(AValue).Elements.Count - 1 do - begin - if I > 0 then - Buffer.Append(','); - Element := TGocciaArrayValue(AValue).Elements[I]; - Buffer.Append(EncodeValue(Element)); - end; - Buffer.Append(']}'); - Exit(Buffer.ToString); - end; - if AValue is TGocciaObjectValue then - begin - Buffer := TStringBuffer.Create; - Buffer.Append('{"type":"object","entries":['); - I := 0; - for Key in TGocciaObjectValue(AValue).GetOwnPropertyKeys do - begin - if I > 0 then - Buffer.Append(','); - Buffer.Append('{"key":'); - Buffer.Append(QuoteJSONString(Key)); - Buffer.Append(',"value":'); - Buffer.Append(EncodeValue(TGocciaObjectValue(AValue).GetProperty(Key))); - Buffer.Append('}'); - Inc(I); - end; - Buffer.Append(']}'); - Exit(Buffer.ToString); - end; - - raise Exception.Create('Unsupported JSON5 value type in compliance harness'); -end; - -var - ParsedValue: TGocciaValue; - Parser: TGocciaJSON5Parser; - SourceText: UTF8String; -begin - if ParamCount <> 1 then - Halt(2); - - TGarbageCollector.Initialize; - PinPrimitiveSingletons; - - Parser := TGocciaJSON5Parser.Create; - try - try - SourceText := ReadUTF8FileText(ParamStr(1)); - ParsedValue := Parser.Parse(SourceText); - TGarbageCollector.Instance.AddTempRoot(ParsedValue); - try - WriteLn(EncodeValue(ParsedValue)); - finally - TGarbageCollector.Instance.RemoveTempRoot(ParsedValue); - end; - except - on E: Exception do - begin - WriteLn(E.Message); - ExitCode := 1; - end; - end; - finally - Parser.Free; - TGarbageCollector.Shutdown; - end; -end. diff --git a/scripts/GocciaTOMLCheck.dpr b/scripts/GocciaTOMLCheck.dpr deleted file mode 100644 index e9fed37fe..000000000 --- a/scripts/GocciaTOMLCheck.dpr +++ /dev/null @@ -1,138 +0,0 @@ -program GocciaTOMLCheck; - -{$I Goccia.inc} - -uses - Classes, - SysUtils, - - StringBuffer, - - Goccia.GarbageCollector, - Goccia.JSON.Utils, - Goccia.TextFiles, - Goccia.TOML, - Goccia.Values.Primitives; - -function ScalarKindName(const AKind: TGocciaTOMLScalarKind): string; -begin - case AKind of - tskString: - Result := 'string'; - tskInteger: - Result := 'integer'; - tskFloat: - Result := 'float'; - tskBool: - Result := 'bool'; - tskDateTime: - Result := 'datetime'; - tskDateTimeLocal: - Result := 'datetime-local'; - tskDateLocal: - Result := 'date-local'; - tskTimeLocal: - Result := 'time-local'; - else - Result := 'string'; - end; -end; - -function SerializeNode(const ANode: TGocciaTOMLNode): string; -var - Buffer: TStringBuffer; - I: Integer; - Pair: TGocciaTOMLNodeMap.TKeyValuePair; -begin - case ANode.Kind of - tnkScalar: - Result := '{"type":' + QuoteJSONString(ScalarKindName(ANode.ScalarKind)) + - ',"value":' + QuoteJSONString(ANode.CanonicalValue) + '}'; - tnkArray, - tnkArrayOfTables: - begin - Buffer := TStringBuffer.Create; - Buffer.AppendChar('['); - for I := 0 to ANode.Items.Count - 1 do - begin - if I > 0 then - Buffer.AppendChar(','); - Buffer.Append(SerializeNode(ANode.Items[I])); - end; - Buffer.AppendChar(']'); - Result := Buffer.ToString; - end; - tnkTable: - begin - Buffer := TStringBuffer.Create; - Buffer.AppendChar('{'); - I := 0; - for Pair in ANode.Children do - begin - if I > 0 then - Buffer.AppendChar(','); - Buffer.Append(QuoteJSONString(Pair.Key)); - Buffer.AppendChar(':'); - Buffer.Append(SerializeNode(Pair.Value)); - Inc(I); - end; - Buffer.AppendChar('}'); - Result := Buffer.ToString; - end; - else - Result := 'null'; - end; -end; - -procedure WriteUTF8Line(const AText: string); -var - OutputText: UTF8String; - Stream: THandleStream; -begin - OutputText := UTF8String(AText + #10); - Stream := THandleStream.Create(TTextRec(Output).Handle); - try - if Length(OutputText) > 0 then - Stream.WriteBuffer(Pointer(OutputText)^, Length(OutputText)); - finally - Stream.Free; - end; -end; - -var - ExitCode: Integer; - Parser: TGocciaTOMLParser; - Root: TGocciaTOMLNode; - SourceText: string; -begin - if ParamCount <> 1 then - Halt(2); - - TGarbageCollector.Initialize; - PinPrimitiveSingletons; - - Parser := TGocciaTOMLParser.Create; - try - try - SourceText := ReadUTF8FileText(ParamStr(1)); - Root := Parser.ParseDocument(SourceText); - try - WriteUTF8Line(SerializeNode(Root)); - ExitCode := 0; - finally - Root.Free; - end; - except - on E: Exception do - begin - WriteUTF8Line(E.Message); - ExitCode := 1; - end; - end; - finally - Parser.Free; - TGarbageCollector.Shutdown; - end; - - Halt(ExitCode); -end. diff --git a/scripts/extract_json5_cases.js b/scripts/regenerate-json5-manifest.js similarity index 88% rename from scripts/extract_json5_cases.js rename to scripts/regenerate-json5-manifest.js index ca7144fe7..3e42efe8f 100644 --- a/scripts/extract_json5_cases.js +++ b/scripts/regenerate-json5-manifest.js @@ -1,14 +1,21 @@ #!/usr/bin/env node +// Maintainer-only adapter: normal compliance runs consume the committed +// revision-tagged manifest and do not require Node.js. const Module = require("module"); +const fs = require("fs"); const path = require("path"); -if (process.argv.length !== 3) { - console.error("Usage: extract_json5_cases.js "); +if (process.argv.length !== 5) { + console.error( + "Usage: regenerate-json5-manifest.js ", + ); process.exit(2); } const suiteDir = path.resolve(process.argv[2]); +const revision = process.argv[3]; +const outputPath = path.resolve(process.argv[4]); const testDir = path.join(suiteDir, "test"); const realJSON5 = require(path.join(suiteDir, "lib")); @@ -199,4 +206,11 @@ Module._load = function patchedLoad(request, parent, isMain) { require(path.join(testDir, "parse.js")); require(path.join(testDir, "errors.js")); -process.stdout.write(`${JSON.stringify({ cases }, null, 2)}\n`); +const manifest = { + manifestVersion: 1, + suite: "json5", + revision, + cases, +}; +fs.writeFileSync(outputPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); +process.stdout.write(`${outputPath}\n`); diff --git a/scripts/run_json5_test_suite.py b/scripts/run_json5_test_suite.py deleted file mode 100644 index 20f339552..000000000 --- a/scripts/run_json5_test_suite.py +++ /dev/null @@ -1,466 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import argparse -import json -import math -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path - - -SUITE_REPO_URL = "https://github.com/json5/json5.git" -SUITE_BRANCH = "main" -DEFAULT_TIMEOUT_SECONDS = 5 -HARNESS_SOURCE_PATH = Path("scripts/GocciaJSON5Check.dpr") -CASE_EXTRACTOR_PATH = Path("scripts/extract_json5_cases.js") -STRINGIFY_SUITE_PATH = Path("tests/built-ins/JSON5/stringify.js") -TEST_RUNNER_SOURCE_PATH = Path("source/app/GocciaTestRunner.dpr") -SUBPROCESS_ENCODING = "utf-8" - - -def run(command: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess[str]: - return subprocess.run( - command, - cwd=str(cwd) if cwd else None, - text=True, - encoding=SUBPROCESS_ENCODING, - capture_output=True, - check=True, - ) - - -def ensure_suite_checkout(suite_dir: Path | None) -> tuple[Path, tempfile.TemporaryDirectory[str] | None]: - if suite_dir is not None: - return suite_dir.resolve(), None - - temp_dir = tempfile.TemporaryDirectory(prefix="json5-test-suite.") - checkout_dir = Path(temp_dir.name) / "repo" - run( - [ - "git", - "clone", - "--depth", - "1", - "--branch", - SUITE_BRANCH, - SUITE_REPO_URL, - str(checkout_dir), - ] - ) - return checkout_dir, temp_dir - - -def ensure_build_dir(build_dir: Path | None) -> tuple[Path, tempfile.TemporaryDirectory[str] | None]: - if build_dir is not None: - resolved_build_dir = build_dir.resolve() - resolved_build_dir.mkdir(parents=True, exist_ok=True) - return resolved_build_dir, None - - temp_dir = tempfile.TemporaryDirectory(prefix="goccia_json5_suite_build_") - return Path(temp_dir.name), temp_dir - - -def find_node_executable(explicit: str | None) -> str: - if explicit: - return explicit - - for candidate in ["node", "nodejs"]: - resolved = shutil.which(candidate) - if resolved: - return resolved - - raise FileNotFoundError("Node.js executable not found. Install `node` or pass --node.") - - -def compile_program(repo_root: Path, build_dir: Path, source_path: Path) -> Path: - # Each program gets its own unit-output directory, mirroring build.pas: - # FPC 3.2.2 aborts with internal error 200611011 when a second program is - # compiled against the .ppu files another program left in a shared -FU dir. - # FPC names the binary after the .dpr stem, so the stem also names the dir. - program_name = source_path.stem - target_dir = build_dir / program_name - target_dir.mkdir(parents=True, exist_ok=True) - - run( - [ - "fpc", - "@config.cfg", - f"-FU{target_dir}", - f"-FE{target_dir}", - str(repo_root / source_path), - ], - cwd=repo_root, - ) - - binary_name = f"{program_name}.exe" if sys.platform.startswith("win") else program_name - return target_dir / binary_name - - -def compile_harness(repo_root: Path, build_dir: Path) -> Path: - return compile_program(repo_root, build_dir, HARNESS_SOURCE_PATH) - - -def compile_test_runner(repo_root: Path, build_dir: Path) -> Path: - return compile_program(repo_root, build_dir, TEST_RUNNER_SOURCE_PATH) - - -def load_cases(repo_root: Path, suite_dir: Path, node_executable: str) -> list[dict]: - extractor = repo_root / CASE_EXTRACTOR_PATH - process = run([node_executable, str(extractor), str(suite_dir)]) - payload = json.loads(process.stdout) - return payload["cases"] - - -def compare_values(want: dict, have: dict, path: str = "") -> tuple[bool, str]: - if want.get("type") != have.get("type"): - return False, f"{path}: expected type {want.get('type')}, got {have.get('type')}" - - value_type = want["type"] - if value_type == "null": - return True, "" - - if value_type in {"boolean", "string"}: - if want.get("value") != have.get("value"): - return False, f"{path}: expected {want.get('value')!r}, got {have.get('value')!r}" - return True, "" - - if value_type == "number": - want_value = want["value"] - have_value = have["value"] - if want_value in {"NaN", "Infinity", "-Infinity", "-0"} or have_value in {"NaN", "Infinity", "-Infinity", "-0"}: - if want_value != have_value: - return False, f"{path}: expected number {want_value}, got {have_value}" - return True, "" - - if float(want_value) != float(have_value): - return False, f"{path}: expected number {want_value}, got {have_value}" - if math.copysign(1.0, float(want_value)) != math.copysign(1.0, float(have_value)): - return False, f"{path}: expected signed number {want_value}, got {have_value}" - return True, "" - - if value_type == "array": - want_items = want["items"] - have_items = have["items"] - if len(want_items) != len(have_items): - return False, f"{path}: expected array length {len(want_items)}, got {len(have_items)}" - for index, (want_item, have_item) in enumerate(zip(want_items, have_items, strict=True)): - ok, message = compare_values(want_item, have_item, f"{path}[{index}]") - if not ok: - return False, message - return True, "" - - if value_type == "object": - want_entries = want["entries"] - have_entries = have["entries"] - if len(want_entries) != len(have_entries): - return False, f"{path}: expected {len(want_entries)} object entries, got {len(have_entries)}" - for index, (want_entry, have_entry) in enumerate(zip(want_entries, have_entries, strict=True)): - if want_entry["key"] != have_entry["key"]: - return False, ( - f"{path}: expected key {want_entry['key']!r} at index {index}, " - f"got {have_entry['key']!r}" - ) - ok, message = compare_values( - want_entry["value"], - have_entry["value"], - f"{path}.{want_entry['key']}", - ) - if not ok: - return False, message - return True, "" - - return False, f"{path}: unsupported tagged value type {value_type}" - - -def evaluate_suite(harness_path: Path, cases: list[dict], timeout_seconds: int) -> dict: - results = [] - summary = { - "total": 0, - "passed": 0, - "failed": 0, - "false_accepts": 0, - "false_rejects": 0, - "valid_mismatches": 0, - "timeouts": 0, - } - - with tempfile.TemporaryDirectory(prefix="json5-suite-cases.") as case_dir_name: - case_dir = Path(case_dir_name) - - for index, case in enumerate(cases): - summary["total"] += 1 - case_path = case_dir / f"case_{index}.json5" - case_path.write_bytes(case["source"].encode(SUBPROCESS_ENCODING)) - - try: - process = subprocess.run( - [str(harness_path), str(case_path)], - text=True, - encoding=SUBPROCESS_ENCODING, - capture_output=True, - timeout=timeout_seconds, - ) - timed_out = False - except subprocess.TimeoutExpired: - timed_out = True - process = None - - if timed_out: - summary["failed"] += 1 - summary["timeouts"] += 1 - results.append( - { - "id": case["id"], - "valid": case["valid"], - "ok": False, - "timed_out": True, - "message": "timeout", - } - ) - continue - - stdout = process.stdout.strip() - stderr = process.stderr.strip() - actual_fail = process.returncode != 0 - message = "\n".join(part for part in [stdout, stderr] if part) - - if not case["valid"]: - ok = actual_fail - if ok: - summary["passed"] += 1 - else: - summary["failed"] += 1 - summary["false_accepts"] += 1 - results.append( - { - "id": case["id"], - "valid": False, - "ok": ok, - "timed_out": False, - "message": message, - } - ) - continue - - if actual_fail: - summary["failed"] += 1 - summary["false_rejects"] += 1 - results.append( - { - "id": case["id"], - "valid": True, - "ok": False, - "timed_out": False, - "message": message, - } - ) - continue - - try: - actual_value = json.loads(stdout) - except json.JSONDecodeError as error: - summary["failed"] += 1 - summary["valid_mismatches"] += 1 - results.append( - { - "id": case["id"], - "valid": True, - "ok": False, - "timed_out": False, - "message": f"invalid harness JSON: {error}", - } - ) - continue - - ok, mismatch_message = compare_values(case["expected"], actual_value) - if ok: - summary["passed"] += 1 - else: - summary["failed"] += 1 - summary["valid_mismatches"] += 1 - - results.append( - { - "id": case["id"], - "valid": True, - "ok": ok, - "timed_out": False, - "message": mismatch_message, - } - ) - - return {"summary": summary, "results": results} - - -def evaluate_stringify_suite(test_runner_path: Path, repo_root: Path, timeout_seconds: int) -> dict: - stringify_suite = repo_root / STRINGIFY_SUITE_PATH - - with tempfile.TemporaryDirectory(prefix="json5-stringify-suite.") as temp_dir_name: - output_path = Path(temp_dir_name) / "stringify-results.json" - try: - process = subprocess.run( - [ - str(test_runner_path), - str(stringify_suite), - "--output=" + str(output_path), - "--silent", - ], - cwd=repo_root, - text=True, - encoding=SUBPROCESS_ENCODING, - capture_output=True, - timeout=timeout_seconds, - ) - timed_out = False - except subprocess.TimeoutExpired: - timed_out = True - process = None - - if timed_out: - return { - "totalFiles": 1, - "totalTests": 0, - "passed": 0, - "failed": 1, - "skipped": 0, - "assertions": 0, - "ok": False, - "message": "timeout", - } - - if not output_path.exists(): - return { - "totalFiles": 1, - "totalTests": 0, - "passed": 0, - "failed": 1, - "skipped": 0, - "assertions": 0, - "ok": False, - "message": "\n".join( - part for part in [process.stdout.strip(), process.stderr.strip()] if part - ), - } - - report = json.loads(output_path.read_text(encoding="utf-8")) - report["ok"] = process.returncode == 0 and report.get("failed", 1) == 0 - report["message"] = "\n".join( - part for part in [process.stdout.strip(), process.stderr.strip()] if part - ) - return report - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=( - "Run Goccia's JSON5 compliance checks: the official json5/json5 parser corpus " - "plus the local upstream-aligned JSON5 stringify suite." - ) - ) - parser.add_argument( - "--suite-dir", - type=Path, - help="Existing checkout of json5/json5. If omitted, the script clones it to a temporary directory.", - ) - parser.add_argument( - "--build-dir", - type=Path, - help="Directory for the temporary JSON5 harness build artifacts. If omitted, a per-run temporary directory is used.", - ) - parser.add_argument( - "--harness", - type=Path, - help="Optional prebuilt GocciaJSON5Check harness. If omitted, the script compiles scripts/GocciaJSON5Check.dpr.", - ) - parser.add_argument( - "--test-runner", - type=Path, - help="Optional prebuilt GocciaTestRunner binary. If omitted, the script compiles source/app/GocciaTestRunner.dpr.", - ) - parser.add_argument( - "--node", - help="Optional Node.js executable path. Defaults to `node`/`nodejs` from PATH.", - ) - parser.add_argument( - "--output", - type=Path, - help="Optional JSON output path for the full summary and per-case results.", - ) - parser.add_argument( - "--timeout", - type=int, - default=DEFAULT_TIMEOUT_SECONDS, - help=f"Per-case timeout in seconds. Default: {DEFAULT_TIMEOUT_SECONDS}.", - ) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - repo_root = Path(__file__).resolve().parent.parent - node_executable = find_node_executable(args.node) - - suite_dir, temp_checkout = ensure_suite_checkout(args.suite_dir) - build_dir, temp_build_dir = ensure_build_dir(args.build_dir) - if args.harness is not None: - harness_path = args.harness.resolve() - else: - harness_path = compile_harness(repo_root, build_dir) - if args.test_runner is not None: - test_runner_path = args.test_runner.resolve() - else: - test_runner_path = compile_test_runner(repo_root, build_dir) - - cases = load_cases(repo_root, suite_dir, node_executable) - parse_report = evaluate_suite(harness_path, cases, args.timeout) - stringify_report = evaluate_stringify_suite(test_runner_path, repo_root, args.timeout) - report = { - "parse": parse_report, - "stringify": stringify_report, - "summary": { - "parse": parse_report["summary"], - "stringify": { - "totalFiles": stringify_report.get("totalFiles", 0), - "totalTests": stringify_report.get("totalTests", 0), - "passed": stringify_report.get("passed", 0), - "failed": stringify_report.get("failed", 0), - "skipped": stringify_report.get("skipped", 0), - "assertions": stringify_report.get("assertions", 0), - }, - }, - } - - if args.output is not None: - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") - - print(json.dumps(report["summary"], indent=2)) - if args.output is not None: - print(args.output.resolve()) - - if temp_checkout is not None: - temp_checkout.cleanup() - if temp_build_dir is not None: - temp_build_dir.cleanup() - - if parse_report["summary"]["failed"] != 0: - return 1 - if not stringify_report.get("ok", False): - return 1 - - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(main()) - except subprocess.CalledProcessError as error: - if error.stdout: - sys.stdout.write(error.stdout) - if error.stderr: - sys.stderr.write(error.stderr) - raise diff --git a/scripts/run_toml_test_suite.py b/scripts/run_toml_test_suite.py deleted file mode 100644 index ffe81855b..000000000 --- a/scripts/run_toml_test_suite.py +++ /dev/null @@ -1,375 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import argparse -import json -import math -import re -import subprocess -import sys -import tempfile -from pathlib import Path - - -SUITE_REPO_URL = "https://github.com/toml-lang/toml-test.git" -SUITE_VERSION = "1.1.0" -SUITE_SHA = "9eef1b959e0449d41a31d4e4e0a839faee534b36" -SUITE_FILE_LIST = f"tests/files-toml-{SUITE_VERSION}" -DEFAULT_TIMEOUT_SECONDS = 5 -HARNESS_SOURCE_PATH = Path("scripts/GocciaTOMLCheck.dpr") -TIME_TEXT_PATTERN = re.compile(r"^(?P\d{2}:\d{2}:\d{2})(?P\.\d+)?(?P)$") -DATETIME_TEXT_PATTERN = re.compile( - r"^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?P\.\d+)?(?P[+-]\d{2}:\d{2})?$" -) - - -def run(command: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess[str]: - return subprocess.run( - command, - cwd=str(cwd) if cwd else None, - text=True, - capture_output=True, - check=True, - ) - - -def ensure_suite_checkout(suite_dir: Path | None) -> tuple[Path, tempfile.TemporaryDirectory[str] | None]: - if suite_dir is not None: - return suite_dir.resolve(), None - - temp_dir = tempfile.TemporaryDirectory(prefix="toml-test-suite.") - checkout_dir = Path(temp_dir.name) / "repo" - run(["git", "init", str(checkout_dir)]) - run(["git", "remote", "add", "origin", SUITE_REPO_URL], cwd=checkout_dir) - run(["git", "fetch", "--depth", "1", "origin", SUITE_SHA], cwd=checkout_dir) - run(["git", "checkout", SUITE_SHA], cwd=checkout_dir) - return checkout_dir, temp_dir - - -def compile_harness(repo_root: Path, build_dir: Path) -> Path: - build_dir.mkdir(parents=True, exist_ok=True) - harness_source = repo_root / HARNESS_SOURCE_PATH - - run( - [ - "fpc", - "@config.cfg", - f"-FU{build_dir}", - f"-FE{build_dir}", - str(harness_source), - ], - cwd=repo_root, - ) - harness_name = "GocciaTOMLCheck.exe" if sys.platform.startswith("win") else "GocciaTOMLCheck" - return build_dir / harness_name - - -def normalize_datetime_text(kind: str, value: str) -> str: - value = value.replace(" ", "T").replace("t", "T").replace("z", "Z") - if value.endswith("Z"): - value = value[:-1] + "+00:00" - - if kind == "date-local": - return value - - pattern = TIME_TEXT_PATTERN if kind == "time-local" else DATETIME_TEXT_PATTERN - match = pattern.match(value) - if match is None: - return value - - fraction = match.group("fraction") or "" - if fraction: - fraction = fraction.rstrip("0") - if fraction == ".": - fraction = "" - - return match.group("prefix") + fraction + (match.group("suffix") or "") - - -def is_tagged_scalar(node) -> bool: - return ( - isinstance(node, dict) - and set(node.keys()) == {"type", "value"} - and isinstance(node.get("type"), str) - and isinstance(node.get("value"), str) - ) - - -def compare_tagged_json(want, have, path="") -> tuple[bool, str]: - if isinstance(want, dict): - if not isinstance(have, dict): - return False, f"{path or ''}: expected object, got {type(have).__name__}" - - if is_tagged_scalar(want): - if not is_tagged_scalar(have): - return False, f"{path or ''}: malformed tagged value" - - want_type = want["type"] - have_type = have["type"] - if want_type != have_type: - return False, f"{path or ''}: expected type {want_type}, got {have_type}" - - want_value = want["value"] - have_value = have["value"] - - if want_type == "float": - want_lower = want_value.lower() - have_lower = have_value.lower() - if want_lower.endswith("nan") or have_lower.endswith("nan"): - if want_lower.lstrip("+-") != have_lower.lstrip("+-"): - return False, f"{path or ''}: expected float {want_value}, got {have_value}" - return True, "" - want_float = float(want_lower) - have_float = float(have_lower) - if want_float != have_float: - return False, f"{path or ''}: expected float {want_value}, got {have_value}" - if want_float == 0.0 and math.copysign(1.0, want_float) != math.copysign(1.0, have_float): - return False, f"{path or ''}: expected float {want_value}, got {have_value}" - return True, "" - - if want_type in {"datetime", "datetime-local", "date-local", "time-local"}: - if normalize_datetime_text(want_type, want_value) != normalize_datetime_text(want_type, have_value): - return False, f"{path or ''}: expected datetime {want_value}, got {have_value}" - return True, "" - - if want_type == "bool": - if want_value.lower() != str(have_value).lower(): - return False, f"{path or ''}: expected bool {want_value}, got {have_value}" - return True, "" - - if want_value != have_value: - return False, f"{path or ''}: expected {want_value}, got {have_value}" - return True, "" - - if set(want.keys()) != set(have.keys()): - return False, f"{path or ''}: object keys differ" - - for key in sorted(want.keys()): - ok, message = compare_tagged_json(want[key], have[key], f"{path}.{key}" if path else key) - if not ok: - return False, message - return True, "" - - if isinstance(want, list): - if not isinstance(have, list): - return False, f"{path or ''}: expected array, got {type(have).__name__}" - if len(want) != len(have): - return False, f"{path or ''}: array lengths differ" - for index, (want_item, have_item) in enumerate(zip(want, have)): - ok, message = compare_tagged_json(want_item, have_item, f"{path}[{index}]") - if not ok: - return False, message - return True, "" - - if want != have: - return False, f"{path or ''}: expected {want}, got {have}" - return True, "" - - -def decode_toml_input(case_path: Path) -> tuple[bool, str]: - raw = case_path.read_bytes() - try: - text = raw.decode("utf-8") - except UnicodeDecodeError as error: - return False, f"invalid UTF-8: {error}" - - if "\r" in text.replace("\r\n", ""): - return False, "bare carriage return is not allowed" - - return True, text - - -def evaluate_suite(harness_path: Path, suite_dir: Path, timeout_seconds: int) -> dict: - file_list_path = suite_dir / SUITE_FILE_LIST - rel_paths = [ - line.strip() - for line in file_list_path.read_text(encoding="utf-8").splitlines() - if line.strip() and not line.startswith("#") and line.endswith(".toml") - ] - - results = [] - summary = { - "suite_version": SUITE_VERSION, - "total": 0, - "passed": 0, - "failed": 0, - "false_accepts": 0, - "false_rejects": 0, - "valid_mismatches": 0, - "timeouts": 0, - } - - for rel_path in rel_paths: - case_path = suite_dir / "tests" / rel_path - is_valid = rel_path.startswith("valid/") - expected_json_path = case_path.with_suffix(".json") - summary["total"] += 1 - - utf8_ok, utf8_message = decode_toml_input(case_path) - if not utf8_ok: - actual_fail = True - mismatch = False - ok = not is_valid - if ok: - summary["passed"] += 1 - else: - summary["failed"] += 1 - summary["false_rejects"] += 1 - results.append( - { - "id": rel_path, - "is_valid": is_valid, - "actual_fail": actual_fail, - "ok": ok, - "timed_out": False, - "mismatch": mismatch, - "message": utf8_message, - } - ) - continue - - try: - process = subprocess.run( - [str(harness_path), str(case_path)], - capture_output=True, - timeout=timeout_seconds, - ) - timed_out = False - actual_fail = process.returncode != 0 - stdout = process.stdout.decode("utf-8", errors="replace").strip() - stderr = process.stderr.decode("utf-8", errors="replace").strip() - message = "\n".join(part for part in [stdout, stderr] if part) - except subprocess.TimeoutExpired: - timed_out = True - actual_fail = True - stdout = "" - message = "timeout" - - ok = False - mismatch = False - if timed_out: - summary["timeouts"] += 1 - elif is_valid: - if not actual_fail: - try: - actual_json = json.loads(stdout or "null") - except json.JSONDecodeError as error: - ok = False - message = f"invalid decoder JSON: {error}" - else: - try: - expected_json = json.loads(expected_json_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: - ok = False - message = f"invalid expected JSON fixture: {error}" - else: - ok, message = compare_tagged_json(expected_json, actual_json) - mismatch = not ok - else: - ok = False - if not ok: - summary["failed"] += 1 - summary["false_rejects"] += 1 - if mismatch: - summary["valid_mismatches"] += 1 - if not message: - message = "decoded JSON mismatch" - else: - ok = actual_fail - if not ok: - summary["failed"] += 1 - summary["false_accepts"] += 1 - - if ok: - summary["passed"] += 1 - - results.append( - { - "id": rel_path, - "is_valid": is_valid, - "actual_fail": actual_fail, - "ok": ok, - "timed_out": timed_out, - "mismatch": mismatch, - "message": message, - } - ) - - return {"summary": summary, "results": results} - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Run Goccia's TOML parser against the official toml-test TOML 1.1.0 suite." - ) - parser.add_argument( - "--suite-dir", - type=Path, - help="Existing checkout of toml-test. If omitted, the script clones it to a temporary directory.", - ) - parser.add_argument( - "--build-dir", - type=Path, - default=Path(tempfile.gettempdir()) / "goccia_toml_suite_build", - help="Directory for the temporary harness source and binary.", - ) - parser.add_argument( - "--harness", - type=Path, - help="Existing TOML decoder harness binary. If omitted, the script compiles scripts/GocciaTOMLCheck.dpr.", - ) - parser.add_argument( - "--output", - type=Path, - help="Optional JSON output path for the full summary and per-case results.", - ) - parser.add_argument( - "--timeout", - type=int, - default=DEFAULT_TIMEOUT_SECONDS, - help=f"Per-case timeout in seconds. Default: {DEFAULT_TIMEOUT_SECONDS}.", - ) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - repo_root = Path(__file__).resolve().parent.parent - - suite_dir, temp_checkout = ensure_suite_checkout(args.suite_dir) - if args.harness is not None: - harness_path = args.harness.resolve() - if not harness_path.is_file(): - raise FileNotFoundError(f"TOML harness not found: {harness_path}") - else: - harness_path = compile_harness(repo_root, args.build_dir.resolve()) - report = evaluate_suite(harness_path, suite_dir.resolve(), args.timeout) - - if args.output is not None: - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") - - print(json.dumps(report["summary"], indent=2)) - if args.output is not None: - print(args.output.resolve()) - - if temp_checkout is not None: - temp_checkout.cleanup() - - summary = report["summary"] - if summary["failed"] > 0 or summary["timeouts"] > 0: - return 1 - - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(main()) - except subprocess.CalledProcessError as error: - if error.stdout: - sys.stdout.write(error.stdout) - if error.stderr: - sys.stderr.write(error.stderr) - raise diff --git a/scripts/suite-bump-pin.ts b/scripts/suite-bump-pin.ts index 7acb9a727..26d5bc3b9 100644 --- a/scripts/suite-bump-pin.ts +++ b/scripts/suite-bump-pin.ts @@ -2,8 +2,8 @@ /** * suite-bump-pin.ts * - * Update a SUITE_SHA pin in a compliance suite runner script. - * Invoked by weekly bump workflows (toml-test-bump.yml, yaml-test-bump.yml). + * Update either a dedicated one-line SHA pin or a SUITE_SHA constant. + * Invoked by weekly bump workflows (TOML, JSON5, and YAML). * * Usage: * bun scripts/suite-bump-pin.ts @@ -33,15 +33,25 @@ function main(argv: string[]): number { let seen = 0; let changed = 0; - const updated = text.replace(SUITE_SHA_RE, (full, prefix, oldSha, suffix) => { - seen++; - if (oldSha === newSha) return full; - changed++; - return `${prefix}${newSha}${suffix}`; - }); + let updated: string; + if (SHA_RE.test(text.trim())) { + seen = 1; + changed = text.trim() === newSha ? 0 : 1; + updated = `${newSha}\n`; + } else { + updated = text.replace( + SUITE_SHA_RE, + (full, prefix, oldSha, suffix) => { + seen++; + if (oldSha === newSha) return full; + changed++; + return `${prefix}${newSha}${suffix}`; + }, + ); + } if (seen === 0) { - console.error(`${target}: no SUITE_SHA constant found`); + console.error(`${target}: no one-line SHA or SUITE_SHA constant found`); return 1; } diff --git a/source/app/compliance/Goccia.Compliance.pas b/source/app/compliance/Goccia.Compliance.pas new file mode 100644 index 000000000..f0d89cc6c --- /dev/null +++ b/source/app/compliance/Goccia.Compliance.pas @@ -0,0 +1,477 @@ +unit Goccia.Compliance; + +{$I Goccia.inc} + +interface + +uses + Classes, + + Goccia.Threading; + +const + COMPLIANCE_REPORT_VERSION = 1; + COMPLIANCE_RUNNER_VERSION = 1; + DEFAULT_COMPLIANCE_TIMEOUT_MS = 5000; + +type + TComplianceOutcome = ( + coPassed, + coFalseAccept, + coFalseReject, + coMismatch, + coTimeout, + coCrash, + coInfrastructure + ); + + TComplianceProcessResult = record + Started: Boolean; + TimedOut: Boolean; + ExitCode: Integer; + DurationMilliseconds: QWord; + StdOutText: string; + StdErrText: string; + ErrorMessage: string; + end; + + TComplianceCaseResult = record + ID: string; + Valid: Boolean; + Outcome: TComplianceOutcome; + DurationMilliseconds: QWord; + MessageText: string; + end; + + TComplianceCaseResultArray = array of TComplianceCaseResult; + + TComplianceSummary = record + Total: Integer; + Passed: Integer; + Failed: Integer; + FalseAccepts: Integer; + FalseRejects: Integer; + Mismatches: Integer; + Timeouts: Integer; + Crashes: Integer; + InfrastructureFailures: Integer; + end; + + TComplianceCaseExecutor = class + public + procedure ExecuteCase(const ACaseID: string; const AIndex: Integer; + out AResult: TComplianceCaseResult); virtual; abstract; + end; + + TComplianceCoordinator = class + private + FExecutor: TComplianceCaseExecutor; + FResults: TComplianceCaseResultArray; + procedure WorkerProc(const AFileName: string; const AIndex: Integer; + out AConsoleOutput: string; out AErrorMessage: string; AData: Pointer); + public + constructor Create(const AExecutor: TComplianceCaseExecutor); + function Run(const ACases: TStringList; + const AJobCount: Integer): TComplianceCaseResultArray; + end; + +function ComplianceOutcomeName(const AOutcome: TComplianceOutcome): string; +function SummarizeComplianceResults( + const AResults: TComplianceCaseResultArray): TComplianceSummary; +function ComplianceSummaryJSON(const ASummary: TComplianceSummary): string; +function ComplianceCasesJSON( + const AResults: TComplianceCaseResultArray): string; +function RunComplianceProcess(const AExecutable: string; + const AArguments: TStrings; const ATimeoutMilliseconds: Integer; + const ACurrentDirectory: string = ''): TComplianceProcessResult; +function ReadRequiredPin(const APinPath: string): string; +procedure VerifySuiteRevision(const ASuiteDirectory, + AExpectedRevision: string); +function ReadOptionValue(const AName: string; out AValue: string): Boolean; +function HasCommandLineFlag(const AName: string): Boolean; +function PositiveIntegerOption(const AName: string; + const ADefaultValue: Integer): Integer; +function DefaultComplianceJobs: Integer; +procedure WriteJSONFile(const APath, AJSON: string); +function CombinedProcessMessage( + const AResult: TComplianceProcessResult): string; + +implementation + +uses + Math, + Process, + StrUtils, + SysUtils, + + FileUtils, + Pipes, + StringBuffer, + TextEncoding, + + Goccia.JSON.Utils; + +procedure DrainPipe(const APipe: TInputPipeStream; const ATarget: TStream); +var + Available, Count: LongInt; + Buffer: array[0..4095] of Byte; +begin + repeat + Available := APipe.NumBytesAvailable; + if Available <= 0 then + Exit; + Count := Min(Available, SizeOf(Buffer)); + Count := APipe.Read(Buffer, Count); + if Count > 0 then + ATarget.WriteBuffer(Buffer, Count); + until Count <= 0; +end; + +function StreamAsUTF8Text(const AStream: TMemoryStream): string; +var + Bytes: TBytes; +begin + SetLength(Bytes, AStream.Size); + AStream.Position := 0; + if AStream.Size > 0 then + AStream.ReadBuffer(Bytes[0], AStream.Size); + Result := DecodeUTF8WithReplacement(Bytes); +end; + +function RunComplianceProcess(const AExecutable: string; + const AArguments: TStrings; const ATimeoutMilliseconds: Integer; + const ACurrentDirectory: string): TComplianceProcessResult; +var + I: Integer; + ProcessRunner: TProcess; + StartedAt: QWord; + StdOutStream, StdErrStream: TMemoryStream; +begin + Result.Started := False; + Result.TimedOut := False; + Result.ExitCode := -1; + Result.DurationMilliseconds := 0; + Result.StdOutText := ''; + Result.StdErrText := ''; + Result.ErrorMessage := ''; + + ProcessRunner := TProcess.Create(nil); + StdOutStream := TMemoryStream.Create; + StdErrStream := TMemoryStream.Create; + try + ProcessRunner.Executable := AExecutable; + for I := 0 to AArguments.Count - 1 do + ProcessRunner.Parameters.Add(AArguments[I]); + if ACurrentDirectory <> '' then + ProcessRunner.CurrentDirectory := ACurrentDirectory; + ProcessRunner.Options := [poUsePipes, poStderrToOutPut]; + StartedAt := GetTickCount64; + try + ProcessRunner.Execute; + Result.Started := True; + while ProcessRunner.Running do + begin + DrainPipe(ProcessRunner.Output, StdOutStream); + if (ATimeoutMilliseconds > 0) and + (GetTickCount64 - StartedAt >= QWord(ATimeoutMilliseconds)) then + begin + Result.TimedOut := True; + ProcessRunner.Terminate(124); + Break; + end; + Sleep(5); + end; + ProcessRunner.WaitOnExit; + DrainPipe(ProcessRunner.Output, StdOutStream); + Result.ExitCode := ProcessRunner.ExitCode; + except + on E: Exception do + Result.ErrorMessage := E.Message; + end; + Result.DurationMilliseconds := GetTickCount64 - StartedAt; + Result.StdOutText := StreamAsUTF8Text(StdOutStream); + Result.StdErrText := StreamAsUTF8Text(StdErrStream); + finally + StdErrStream.Free; + StdOutStream.Free; + ProcessRunner.Free; + end; +end; + +function ComplianceOutcomeName(const AOutcome: TComplianceOutcome): string; +begin + case AOutcome of + coPassed: + Result := 'passed'; + coFalseAccept: + Result := 'false_accept'; + coFalseReject: + Result := 'false_reject'; + coMismatch: + Result := 'mismatch'; + coTimeout: + Result := 'timeout'; + coCrash: + Result := 'crash'; + coInfrastructure: + Result := 'infrastructure'; + else + Result := 'infrastructure'; + end; +end; + +function SummarizeComplianceResults( + const AResults: TComplianceCaseResultArray): TComplianceSummary; +var + I: Integer; +begin + FillChar(Result, SizeOf(Result), 0); + Result.Total := Length(AResults); + for I := 0 to High(AResults) do + begin + case AResults[I].Outcome of + coPassed: + Inc(Result.Passed); + coFalseAccept: + Inc(Result.FalseAccepts); + coFalseReject: + Inc(Result.FalseRejects); + coMismatch: + Inc(Result.Mismatches); + coTimeout: + Inc(Result.Timeouts); + coCrash: + Inc(Result.Crashes); + coInfrastructure: + Inc(Result.InfrastructureFailures); + end; + end; + Result.Failed := Result.Total - Result.Passed; +end; + +function ComplianceSummaryJSON( + const ASummary: TComplianceSummary): string; +begin + Result := Format( + '{"total":%d,"passed":%d,"failed":%d,"failureClasses":' + + '{"falseAccepts":%d,"falseRejects":%d,"mismatches":%d,' + + '"timeouts":%d,"crashes":%d,"infrastructure":%d}}', + [ASummary.Total, ASummary.Passed, ASummary.Failed, + ASummary.FalseAccepts, ASummary.FalseRejects, ASummary.Mismatches, + ASummary.Timeouts, ASummary.Crashes, ASummary.InfrastructureFailures]); +end; + +function ComplianceCasesJSON( + const AResults: TComplianceCaseResultArray): string; +var + I: Integer; + Buffer: TStringBuffer; +begin + Buffer := TStringBuffer.Create; + Buffer.AppendChar('['); + for I := 0 to High(AResults) do + begin + if I > 0 then + Buffer.AppendChar(','); + Buffer.Append('{"id":'); + Buffer.Append(QuoteJSONString(AResults[I].ID)); + Buffer.Append(',"valid":'); + if AResults[I].Valid then + Buffer.Append('true') + else + Buffer.Append('false'); + Buffer.Append(',"ok":'); + if AResults[I].Outcome = coPassed then + Buffer.Append('true') + else + Buffer.Append('false'); + Buffer.Append(',"outcome":'); + Buffer.Append(QuoteJSONString( + ComplianceOutcomeName(AResults[I].Outcome))); + Buffer.Append(',"durationMilliseconds":'); + Buffer.Append(IntToStr(AResults[I].DurationMilliseconds)); + Buffer.Append(',"message":'); + Buffer.Append(QuoteJSONString(AResults[I].MessageText)); + Buffer.AppendChar('}'); + end; + Buffer.AppendChar(']'); + Result := Buffer.ToString; +end; + +function ReadRequiredPin(const APinPath: string): string; +var + I: Integer; +begin + if not FileExists(APinPath) then + raise Exception.CreateFmt('Pin file not found: %s', [APinPath]); + Result := Trim(ReadUTF8FileText(APinPath)); + if (Length(Result) <> 40) then + raise Exception.CreateFmt('Pin file must contain one 40-character SHA: %s', + [APinPath]); + for I := 1 to Length(Result) do + if not CharInSet(Result[I], ['0'..'9', 'a'..'f', 'A'..'F']) then + raise Exception.CreateFmt( + 'Pin file must contain one hexadecimal SHA: %s', [APinPath]); +end; + +procedure VerifySuiteRevision(const ASuiteDirectory, + AExpectedRevision: string); +var + Arguments: TStringList; + ProcessResult: TComplianceProcessResult; + ActualRevision: string; +begin + if not DirectoryExists(ASuiteDirectory) then + raise Exception.CreateFmt('Suite directory not found: %s', + [ASuiteDirectory]); + Arguments := TStringList.Create; + try + Arguments.Add('-C'); + Arguments.Add(ASuiteDirectory); + Arguments.Add('rev-parse'); + Arguments.Add('HEAD'); + ProcessResult := RunComplianceProcess('git', Arguments, 10000); + finally + Arguments.Free; + end; + if (not ProcessResult.Started) or (ProcessResult.ExitCode <> 0) then + raise Exception.CreateFmt('Cannot read suite revision: %s', + [CombinedProcessMessage(ProcessResult)]); + ActualRevision := Trim(ProcessResult.StdOutText); + if not SameText(ActualRevision, AExpectedRevision) then + raise Exception.CreateFmt('Suite revision mismatch: expected %s, got %s', + [AExpectedRevision, ActualRevision]); +end; + +function ReadOptionValue(const AName: string; out AValue: string): Boolean; +var + I: Integer; + Prefix: string; +begin + Prefix := '--' + AName + '='; + for I := 1 to ParamCount do + begin + if StartsText(Prefix, ParamStr(I)) then + begin + AValue := Copy(ParamStr(I), Length(Prefix) + 1, MaxInt); + Exit(True); + end; + if SameText(ParamStr(I), '--' + AName) and (I < ParamCount) then + begin + AValue := ParamStr(I + 1); + Exit(True); + end; + end; + AValue := ''; + Result := False; +end; + +function HasCommandLineFlag(const AName: string): Boolean; +var + I: Integer; +begin + for I := 1 to ParamCount do + if SameText(ParamStr(I), '--' + AName) then + Exit(True); + Result := False; +end; + +function PositiveIntegerOption(const AName: string; + const ADefaultValue: Integer): Integer; +var + Value: string; +begin + if not ReadOptionValue(AName, Value) then + Exit(ADefaultValue); + if (not TryStrToInt(Value, Result)) or (Result <= 0) then + raise Exception.CreateFmt('--%s must be a positive integer', [AName]); +end; + +function DefaultComplianceJobs: Integer; +begin + Result := TThread.ProcessorCount; + if Result < 1 then + Result := 1; +end; + +procedure WriteJSONFile(const APath, AJSON: string); +var + Directory: string; +begin + Directory := ExtractFileDir(ExpandFileName(APath)); + if (Directory <> '') and (not DirectoryExists(Directory)) and + (not ForceDirectories(Directory)) then + raise Exception.CreateFmt('Cannot create report directory: %s', + [Directory]); + WriteUTF8FileText(APath, AJSON + LineEnding); +end; + +function CombinedProcessMessage( + const AResult: TComplianceProcessResult): string; +begin + Result := Trim(AResult.StdOutText); + if Trim(AResult.StdErrText) <> '' then + begin + if Result <> '' then + Result := Result + LineEnding; + Result := Result + Trim(AResult.StdErrText); + end; + if AResult.ErrorMessage <> '' then + begin + if Result <> '' then + Result := Result + LineEnding; + Result := Result + AResult.ErrorMessage; + end; +end; + +constructor TComplianceCoordinator.Create( + const AExecutor: TComplianceCaseExecutor); +begin + inherited Create; + FExecutor := AExecutor; +end; + +procedure TComplianceCoordinator.WorkerProc(const AFileName: string; + const AIndex: Integer; out AConsoleOutput: string; + out AErrorMessage: string; AData: Pointer); +begin + AConsoleOutput := ''; + AErrorMessage := ''; + try + FExecutor.ExecuteCase(AFileName, AIndex, FResults[AIndex]); + except + on E: Exception do + begin + FResults[AIndex].ID := AFileName; + FResults[AIndex].Outcome := coInfrastructure; + FResults[AIndex].MessageText := E.ClassName + ': ' + E.Message; + end; + end; +end; + +function TComplianceCoordinator.Run(const ACases: TStringList; + const AJobCount: Integer): TComplianceCaseResultArray; +var + I, JobCount: Integer; + Pool: TGocciaThreadPool; +begin + SetLength(FResults, ACases.Count); + for I := 0 to High(FResults) do + begin + FResults[I].ID := ACases[I]; + FResults[I].Outcome := coInfrastructure; + FResults[I].MessageText := 'case was not executed'; + end; + JobCount := Min(Max(1, AJobCount), Max(1, ACases.Count)); + Pool := TGocciaThreadPool.Create(JobCount); + try + Pool.RunAll(ACases, WorkerProc); + finally + Pool.Free; + end; + SetLength(Result, Length(FResults)); + for I := 0 to High(FResults) do + Result[I] := FResults[I]; +end; + +end. diff --git a/source/app/compliance/GocciaJSON5ComplianceRunner.dpr b/source/app/compliance/GocciaJSON5ComplianceRunner.dpr new file mode 100644 index 000000000..03f591c25 --- /dev/null +++ b/source/app/compliance/GocciaJSON5ComplianceRunner.dpr @@ -0,0 +1,487 @@ +program GocciaJSON5ComplianceRunner; + +{$I Goccia.inc} + +uses + {$IFDEF UNIX}cthreads,{$ENDIF} + Classes, + Math, + StrUtils, + SysUtils, + + Goccia.Compliance, + Goccia.GarbageCollector, + Goccia.JSON, + Goccia.JSON.Utils, + Goccia.TextFiles, + Goccia.Values.ArrayValue, + Goccia.Values.ObjectValue, + Goccia.Values.Primitives, + + FileUtils; + +const + SUITE_NAME = 'json5'; + DEFAULT_PIN_PATH = 'tests/compliance/json5.pin'; + DEFAULT_MANIFEST_NAME = 'goccia-json5-cases.json'; + DEFAULT_REGENERATOR = 'scripts/regenerate-json5-manifest.js'; + DEFAULT_STRINGIFY_SUITE = 'tests/built-ins/JSON5/stringify.js'; + +type + TJSON5CaseData = record + ID: string; + Valid: Boolean; + TestPath: string; + end; + + TJSON5CaseDataArray = array of TJSON5CaseData; + + TJSON5ComplianceExecutor = class(TComplianceCaseExecutor) + private + FCases: TJSON5CaseDataArray; + FTestRunner: string; + FTimeoutMilliseconds: Integer; + public + constructor Create(const ACases: TJSON5CaseDataArray; + const ATestRunner: string; const ATimeoutMilliseconds: Integer); + procedure ExecuteCase(const ACaseID: string; const AIndex: Integer; + out AResult: TComplianceCaseResult); override; + end; + +function GeneratedCaseSource(const AID, ASource, AExpectedJSON: string; + const AValid: Boolean): string; +const + ENCODER_SOURCE = + 'const encode = (value) => {' + LineEnding + + ' if (value === null) return { type: "null" };' + LineEnding + + ' if (typeof value === "boolean") return { type: "boolean", value };' + + LineEnding + + ' if (typeof value === "string") return { type: "string", value };' + + LineEnding + + ' if (typeof value === "number") {' + LineEnding + + ' if (Number.isNaN(value)) return { type: "number", value: "NaN" };' + + LineEnding + + ' if (value === Infinity) return { type: "number", value: "Infinity" };' + + LineEnding + + ' if (value === -Infinity) return { type: "number", value: "-Infinity" };' + + LineEnding + + ' if (Object.is(value, -0)) return { type: "number", value: "-0" };' + + LineEnding + + ' return { type: "number", value: String(value) };' + LineEnding + + ' }' + LineEnding + + ' if (Array.isArray(value)) {' + LineEnding + + ' return { type: "array", items: value.map(item => encode(item)) };' + + LineEnding + + ' }' + LineEnding + + ' return {' + LineEnding + + ' type: "object",' + LineEnding + + ' entries: Object.getOwnPropertyNames(value).map(key => ({' + LineEnding + + ' key, value: encode(value[key]),' + LineEnding + + ' })),' + LineEnding + + ' };' + LineEnding + + '};' + LineEnding; +var + Body: string; +begin + if AValid then + Body := + ' let actual;' + LineEnding + + ' try {' + LineEnding + + ' actual = JSON5.parse(source);' + LineEnding + + ' } catch (error) {' + LineEnding + + ' throw new Error("__GOCCIA_FALSE_REJECT__ " + error);' + LineEnding + + ' }' + LineEnding + + ' expect(JSON.stringify(encode(actual))).toBe(' + + 'JSON.stringify(expected));' + LineEnding + else + Body := ' expect(() => JSON5.parse(source)).toThrow(SyntaxError);' + + LineEnding; + Result := + 'import * as JSON5 from "goccia:json5";' + LineEnding + LineEnding + + ENCODER_SOURCE + LineEnding + + 'test(' + QuoteJSONString(AID) + ', () => {' + LineEnding + + ' const source = ' + QuoteJSONString(ASource) + ';' + LineEnding; + if AValid then + Result := Result + ' const expected = ' + AExpectedJSON + ';' + LineEnding; + Result := Result + Body + '});' + LineEnding; +end; + +constructor TJSON5ComplianceExecutor.Create(const ACases: TJSON5CaseDataArray; + const ATestRunner: string; const ATimeoutMilliseconds: Integer); +var + I: Integer; +begin + inherited Create; + SetLength(FCases, Length(ACases)); + for I := 0 to High(ACases) do + FCases[I] := ACases[I]; + FTestRunner := ATestRunner; + FTimeoutMilliseconds := ATimeoutMilliseconds; +end; + +procedure TJSON5ComplianceExecutor.ExecuteCase(const ACaseID: string; + const AIndex: Integer; out AResult: TComplianceCaseResult); +var + Arguments: TStringList; + ProcessResult: TComplianceProcessResult; +begin + AResult.ID := ACaseID; + AResult.Valid := FCases[AIndex].Valid; + AResult.Outcome := coInfrastructure; + Arguments := TStringList.Create; + try + Arguments.Add(FCases[AIndex].TestPath); + Arguments.Add('--silent'); + Arguments.Add('--no-progress'); + Arguments.Add('--no-results'); + ProcessResult := RunComplianceProcess(FTestRunner, Arguments, + FTimeoutMilliseconds); + finally + Arguments.Free; + end; + AResult.DurationMilliseconds := ProcessResult.DurationMilliseconds; + AResult.MessageText := CombinedProcessMessage(ProcessResult); + if not ProcessResult.Started then + Exit; + if ProcessResult.TimedOut then + begin + AResult.Outcome := coTimeout; + AResult.MessageText := 'timeout'; + Exit; + end; + if ProcessResult.ExitCode = 0 then + begin + AResult.Outcome := coPassed; + AResult.MessageText := ''; + end + else if ProcessResult.ExitCode = 1 then + begin + if not AResult.Valid then + AResult.Outcome := coFalseAccept + else if Pos('__GOCCIA_FALSE_REJECT__', AResult.MessageText) > 0 then + AResult.Outcome := coFalseReject + else + AResult.Outcome := coMismatch; + end + else + AResult.Outcome := coCrash; +end; + +function DefaultTestRunnerPath: string; +begin + {$IFDEF WINDOWS} + Result := 'build' + DirectorySeparator + 'GocciaTestRunner.exe'; + {$ELSE} + Result := 'build' + DirectorySeparator + 'GocciaTestRunner'; + {$ENDIF} +end; + +function CreateCaseDirectory: string; +begin + Result := IncludeTrailingPathDelimiter(GetTempDir(False)) + + 'goccia-json5-compliance-' + IntToStr(GetProcessID) + '-' + + IntToStr(GetTickCount64); + if not ForceDirectories(Result) then + raise Exception.CreateFmt('Cannot create temporary case directory: %s', + [Result]); +end; + +procedure RemoveCaseDirectory(const ADirectory: string; + const ACases: TJSON5CaseDataArray); +var + I: Integer; +begin + for I := 0 to High(ACases) do + if StartsText(IncludeTrailingPathDelimiter(ADirectory), + ExpandFileName(ACases[I].TestPath)) then + DeleteFile(ACases[I].TestPath); + RemoveDir(ADirectory); +end; + +function LoadCases(const AManifestPath, AExpectedRevision, + ACaseDirectory: string; out ACases: TJSON5CaseDataArray): TStringList; +var + CaseArray: TGocciaArrayValue; + CaseObject, ManifestObject: TGocciaObjectValue; + ExpectedValue: TGocciaValue; + I: Integer; + JSONParser: TGocciaJSONParser; + JSONStringifier: TGocciaJSONStringifier; + SourceText, ExpectedJSON: string; +begin + if not FileExists(AManifestPath) then + raise Exception.CreateFmt( + 'JSON5 manifest not found: %s (run --regenerate-manifest first)', + [AManifestPath]); + JSONParser := TGocciaJSONParser.Create; + JSONStringifier := TGocciaJSONStringifier.Create; + try + ManifestObject := TGocciaObjectValue(JSONParser.Parse( + ReadUTF8FileText(AManifestPath))); + if ManifestObject.GetProperty('manifestVersion').ToNumberLiteral.Value <> + 1 then + raise Exception.Create('Unsupported JSON5 manifest version'); + if ManifestObject.GetProperty('suite').ToStringLiteral.Value <> + SUITE_NAME then + raise Exception.Create('JSON5 manifest names the wrong suite'); + if ManifestObject.GetProperty('revision').ToStringLiteral.Value <> + AExpectedRevision then + raise Exception.CreateFmt( + 'JSON5 manifest revision mismatch: expected %s, got %s', + [AExpectedRevision, + ManifestObject.GetProperty('revision').ToStringLiteral.Value]); + CaseArray := TGocciaArrayValue(ManifestObject.GetProperty('cases')); + SetLength(ACases, CaseArray.Elements.Count); + Result := TStringList.Create; + for I := 0 to CaseArray.Elements.Count - 1 do + begin + CaseObject := TGocciaObjectValue(CaseArray.Elements[I]); + ACases[I].ID := CaseObject.GetProperty('id').ToStringLiteral.Value; + ACases[I].Valid := CaseObject.GetProperty('valid').ToBooleanLiteral.Value; + ACases[I].TestPath := IncludeTrailingPathDelimiter(ACaseDirectory) + + Format('case_%.4d.js', [I]); + SourceText := CaseObject.GetProperty('source').ToStringLiteral.Value; + ExpectedJSON := ''; + if ACases[I].Valid then + begin + ExpectedValue := CaseObject.GetProperty('expected'); + ExpectedJSON := JSONStringifier.Stringify(ExpectedValue); + end; + WriteUTF8FileText(ACases[I].TestPath, + GeneratedCaseSource(ACases[I].ID, SourceText, ExpectedJSON, + ACases[I].Valid)); + Result.Add(ACases[I].ID); + end; + finally + JSONStringifier.Free; + JSONParser.Free; + end; +end; + +function RunStringifySuite(const ATestRunner, ASuitePath, + AReportPath: string; const ATimeoutMilliseconds: Integer; + out AReportJSON: string): TComplianceCaseResult; +var + Arguments: TStringList; + ProcessResult: TComplianceProcessResult; +begin + Result.ID := 'JSON5.stringify'; + Result.Valid := True; + Result.Outcome := coInfrastructure; + Arguments := TStringList.Create; + try + Arguments.Add(ASuitePath); + Arguments.Add('--silent'); + Arguments.Add('--no-progress'); + Arguments.Add('--output=' + AReportPath); + ProcessResult := RunComplianceProcess(ATestRunner, Arguments, + ATimeoutMilliseconds); + finally + Arguments.Free; + end; + Result.DurationMilliseconds := ProcessResult.DurationMilliseconds; + Result.MessageText := CombinedProcessMessage(ProcessResult); + if ProcessResult.TimedOut then + begin + Result.Outcome := coTimeout; + Result.MessageText := 'timeout'; + end + else if not ProcessResult.Started then + Result.Outcome := coInfrastructure + else if ProcessResult.ExitCode = 0 then + begin + Result.Outcome := coPassed; + Result.MessageText := ''; + end + else if ProcessResult.ExitCode = 1 then + Result.Outcome := coMismatch + else + Result.Outcome := coCrash; + if FileExists(AReportPath) then + AReportJSON := ReadUTF8FileText(AReportPath) + else + AReportJSON := 'null'; +end; + +function RegenerateManifest: Integer; +var + Arguments: TStringList; + ManifestPath, NodeExecutable, PinPath, RegeneratorPath, Revision, + SuiteDirectory: string; + ProcessResult: TComplianceProcessResult; +begin + if not ReadOptionValue('suite-dir', SuiteDirectory) then + raise Exception.Create('--suite-dir is required'); + SuiteDirectory := ExpandFileName(SuiteDirectory); + if not ReadOptionValue('pin-file', PinPath) then + PinPath := DEFAULT_PIN_PATH; + Revision := ReadRequiredPin(PinPath); + VerifySuiteRevision(SuiteDirectory, Revision); + if not ReadOptionValue('manifest', ManifestPath) then + ManifestPath := IncludeTrailingPathDelimiter(SuiteDirectory) + + DEFAULT_MANIFEST_NAME; + if not ReadOptionValue('node', NodeExecutable) then + NodeExecutable := 'node'; + if not ReadOptionValue('regenerator', RegeneratorPath) then + RegeneratorPath := DEFAULT_REGENERATOR; + Arguments := TStringList.Create; + try + Arguments.Add(ExpandFileName(RegeneratorPath)); + Arguments.Add(SuiteDirectory); + Arguments.Add(Revision); + Arguments.Add(ExpandFileName(ManifestPath)); + ProcessResult := RunComplianceProcess(NodeExecutable, Arguments, 60000); + finally + Arguments.Free; + end; + if (not ProcessResult.Started) or (ProcessResult.ExitCode <> 0) then + raise Exception.Create('Manifest regeneration failed: ' + + CombinedProcessMessage(ProcessResult)); + WriteLn(Trim(ProcessResult.StdOutText)); + Result := 0; +end; + +procedure PrintUsage; +begin + WriteLn('Usage: GocciaJSON5ComplianceRunner --suite-dir=PATH [options]'); + WriteLn(' --manifest=PATH Prepared, revision-tagged case manifest'); + WriteLn(' --test-runner=PATH GocciaTestRunner executable'); + WriteLn(' --stringify-suite=PATH Local JSON5 stringify test file'); + WriteLn(' --pin-file=PATH Expected suite revision pin'); + WriteLn(' --jobs=N Maximum concurrent TestRunner processes'); + WriteLn(' --timeout=N Per-process timeout in milliseconds'); + WriteLn(' --output=PATH Write the complete JSON report'); + WriteLn(' --regenerate-manifest Maintainer-only manifest regeneration'); +end; + +function RunCoordinator: Integer; +var + CaseDirectory, ManifestPath, OutputPath, PinPath, Revision, + StringifyReportJSON, StringifySuite, SuiteDirectory, TestRunner: string; + Cases: TJSON5CaseDataArray; + StringifyResults: TComplianceCaseResultArray; + CaseIDs: TStringList; + Coordinator: TComplianceCoordinator; + Executor: TJSON5ComplianceExecutor; + I, Jobs, TimeoutMilliseconds: Integer; + AllResults: TComplianceCaseResultArray; + ParseResults: TComplianceCaseResultArray; + AllSummary, ParseSummary: TComplianceSummary; + ReportJSON, StringifyReportPath: string; + StartedAt, DurationMilliseconds: QWord; + StringifyResult: TComplianceCaseResult; +begin + if not ReadOptionValue('suite-dir', SuiteDirectory) then + begin + PrintUsage; + Exit(2); + end; + SuiteDirectory := ExpandFileName(SuiteDirectory); + if not ReadOptionValue('pin-file', PinPath) then + PinPath := DEFAULT_PIN_PATH; + Revision := ReadRequiredPin(PinPath); + VerifySuiteRevision(SuiteDirectory, Revision); + if not ReadOptionValue('manifest', ManifestPath) then + ManifestPath := IncludeTrailingPathDelimiter(SuiteDirectory) + + DEFAULT_MANIFEST_NAME; + if not ReadOptionValue('test-runner', TestRunner) then + TestRunner := DefaultTestRunnerPath; + TestRunner := ExpandFileName(TestRunner); + if not FileExists(TestRunner) then + raise Exception.CreateFmt('GocciaTestRunner not found: %s', [TestRunner]); + if not ReadOptionValue('stringify-suite', StringifySuite) then + StringifySuite := DEFAULT_STRINGIFY_SUITE; + StringifySuite := ExpandFileName(StringifySuite); + if not FileExists(StringifySuite) then + raise Exception.CreateFmt('JSON5 stringify suite not found: %s', + [StringifySuite]); + Jobs := PositiveIntegerOption('jobs', DefaultComplianceJobs); + TimeoutMilliseconds := PositiveIntegerOption('timeout', + DEFAULT_COMPLIANCE_TIMEOUT_MS); + ReadOptionValue('output', OutputPath); + + CaseDirectory := CreateCaseDirectory; + StringifyReportPath := IncludeTrailingPathDelimiter(CaseDirectory) + + 'stringify-report.json'; + TGarbageCollector.Initialize; + PinPrimitiveSingletons; + try + CaseIDs := LoadCases(ExpandFileName(ManifestPath), Revision, + CaseDirectory, Cases); + Executor := TJSON5ComplianceExecutor.Create(Cases, TestRunner, + TimeoutMilliseconds); + Coordinator := TComplianceCoordinator.Create(Executor); + StartedAt := GetTickCount64; + try + ParseResults := Coordinator.Run(CaseIDs, Jobs); + finally + Coordinator.Free; + Executor.Free; + CaseIDs.Free; + end; + StringifyResult := RunStringifySuite(TestRunner, StringifySuite, + StringifyReportPath, TimeoutMilliseconds, StringifyReportJSON); + SetLength(StringifyResults, 1); + StringifyResults[0] := StringifyResult; + SetLength(AllResults, Length(ParseResults) + 1); + for I := 0 to High(ParseResults) do + AllResults[I] := ParseResults[I]; + AllResults[High(AllResults)] := StringifyResult; + DurationMilliseconds := GetTickCount64 - StartedAt; + ParseSummary := SummarizeComplianceResults(ParseResults); + AllSummary := SummarizeComplianceResults(AllResults); + ReportJSON := Format( + '{"reportVersion":%d,"runner":{"name":%s,"version":%d},' + + '"suite":{"name":%s,"revision":%s},' + + '"mode":"testrunner","jobs":%d,' + + '"timing":{"durationMilliseconds":%d,"timeoutMilliseconds":%d},' + + '"summary":%s,"cases":%s,' + + '"sections":{"parse":{"summary":%s},' + + '"stringify":{"result":%s,"testRunnerReport":%s}}}', + [COMPLIANCE_REPORT_VERSION, + QuoteJSONString('GocciaJSON5ComplianceRunner'), + COMPLIANCE_RUNNER_VERSION, QuoteJSONString(SUITE_NAME), + QuoteJSONString(Revision), Min(Jobs, Max(1, ParseSummary.Total)), + DurationMilliseconds, TimeoutMilliseconds, + ComplianceSummaryJSON(AllSummary), ComplianceCasesJSON(AllResults), + ComplianceSummaryJSON(ParseSummary), + Copy(ComplianceCasesJSON(StringifyResults), 2, + Length(ComplianceCasesJSON(StringifyResults)) - 2), + Trim(StringifyReportJSON)]); + if OutputPath <> '' then + WriteJSONFile(OutputPath, ReportJSON); + WriteLn('JSON5 parse: ', ParseSummary.Passed, '/', ParseSummary.Total, + ' passed; stringify: ', + ComplianceOutcomeName(StringifyResult.Outcome)); + WriteLn(ComplianceSummaryJSON(AllSummary)); + finally + TGarbageCollector.Shutdown; + DeleteFile(StringifyReportPath); + RemoveCaseDirectory(CaseDirectory, Cases); + end; + if (ParseSummary.Failed > 0) or + (StringifyResult.Outcome <> coPassed) then + Result := 1 + else + Result := 0; +end; + +begin + try + if HasCommandLineFlag('help') then + begin + PrintUsage; + ExitCode := 0; + end + else if HasCommandLineFlag('regenerate-manifest') then + ExitCode := RegenerateManifest + else + ExitCode := RunCoordinator; + except + on E: Exception do + begin + WriteLn('Error: ', E.Message); + ExitCode := 2; + end; + end; +end. diff --git a/source/app/compliance/GocciaTOMLComplianceRunner.dpr b/source/app/compliance/GocciaTOMLComplianceRunner.dpr new file mode 100644 index 000000000..b06c3a3b8 --- /dev/null +++ b/source/app/compliance/GocciaTOMLComplianceRunner.dpr @@ -0,0 +1,544 @@ +program GocciaTOMLComplianceRunner; + +{$I Goccia.inc} + +uses + {$IFDEF UNIX}cthreads,{$ENDIF} + Classes, + Math, + StrUtils, + SysUtils, + + Goccia.Compliance, + Goccia.GarbageCollector, + Goccia.JSON, + Goccia.JSON.Utils, + Goccia.TextFiles, + Goccia.Threading, + Goccia.TOML, + Goccia.Values.ArrayValue, + Goccia.Values.ObjectValue, + Goccia.Values.Primitives, + + FileUtils, + StringBuffer; + +const + SUITE_NAME = 'toml-test'; + SUITE_VERSION = '1.1.0'; + DEFAULT_PIN_PATH = 'tests/compliance/toml-test.pin'; + SUITE_FILE_LIST = 'tests/files-toml-1.1.0'; + +type + TTOMLComplianceExecutor = class(TComplianceCaseExecutor) + private + FExecutable: string; + FSuiteDirectory: string; + FTimeoutMilliseconds: Integer; + public + constructor Create(const AExecutable, ASuiteDirectory: string; + const ATimeoutMilliseconds: Integer); + procedure ExecuteCase(const ACaseID: string; const AIndex: Integer; + out AResult: TComplianceCaseResult); override; + end; + +function ScalarKindName(const AKind: TGocciaTOMLScalarKind): string; +begin + case AKind of + tskString: + Result := 'string'; + tskInteger: + Result := 'integer'; + tskFloat: + Result := 'float'; + tskBool: + Result := 'bool'; + tskDateTime: + Result := 'datetime'; + tskDateTimeLocal: + Result := 'datetime-local'; + tskDateLocal: + Result := 'date-local'; + tskTimeLocal: + Result := 'time-local'; + else + Result := 'string'; + end; +end; + +function SerializeNode(const ANode: TGocciaTOMLNode): string; +var + Buffer: TStringBuffer; + I: Integer; + Pair: TGocciaTOMLNodeMap.TKeyValuePair; +begin + case ANode.Kind of + tnkScalar: + Result := '{"type":' + QuoteJSONString(ScalarKindName(ANode.ScalarKind)) + + ',"value":' + QuoteJSONString(ANode.CanonicalValue) + '}'; + tnkArray, + tnkArrayOfTables: + begin + Buffer := TStringBuffer.Create; + Buffer.AppendChar('['); + for I := 0 to ANode.Items.Count - 1 do + begin + if I > 0 then + Buffer.AppendChar(','); + Buffer.Append(SerializeNode(ANode.Items[I])); + end; + Buffer.AppendChar(']'); + Result := Buffer.ToString; + end; + tnkTable: + begin + Buffer := TStringBuffer.Create; + Buffer.AppendChar('{'); + I := 0; + for Pair in ANode.Children do + begin + if I > 0 then + Buffer.AppendChar(','); + Buffer.Append(QuoteJSONString(Pair.Key)); + Buffer.AppendChar(':'); + Buffer.Append(SerializeNode(Pair.Value)); + Inc(I); + end; + Buffer.AppendChar('}'); + Result := Buffer.ToString; + end; + else + Result := 'null'; + end; +end; + +function NormalizeDateTimeText(const AKind, AValue: string): string; +var + DotIndex, EndIndex: Integer; +begin + Result := StringReplace(AValue, ' ', 'T', []); + Result := StringReplace(Result, 't', 'T', []); + Result := StringReplace(Result, 'z', 'Z', []); + if EndsText('Z', Result) then + Result := Copy(Result, 1, Length(Result) - 1) + '+00:00'; + if AKind = 'date-local' then + Exit; + DotIndex := Pos('.', Result); + if DotIndex = 0 then + Exit; + EndIndex := DotIndex + 1; + while (EndIndex <= Length(Result)) and CharInSet(Result[EndIndex], ['0'..'9']) do + Inc(EndIndex); + while (EndIndex > DotIndex + 1) and (Result[EndIndex - 1] = '0') do + begin + Delete(Result, EndIndex - 1, 1); + Dec(EndIndex); + end; + if EndIndex = DotIndex + 1 then + Delete(Result, DotIndex, 1); +end; + +function SameDoubleBits(const ALeft, ARight: Double): Boolean; +var + LeftValue, RightValue: Double; + LeftBits: Int64 absolute LeftValue; + RightBits: Int64 absolute RightValue; +begin + LeftValue := ALeft; + RightValue := ARight; + Result := LeftBits = RightBits; +end; + +function WithoutSign(const AValue: string): string; +begin + Result := AValue; + if (Result <> '') and CharInSet(Result[1], ['+', '-']) then + Delete(Result, 1, 1); +end; + +function CompareTaggedJSON(const AExpected, AActual: TGocciaValue; + const APath: string; out AMessage: string): Boolean; +var + ExpectedObject, ActualObject: TGocciaObjectValue; + ExpectedArray, ActualArray: TGocciaArrayValue; + ExpectedType, ActualType, ExpectedValue, ActualValue, ChildPath: string; + ExpectedFloat, ActualFloat: Double; + FloatFormat: TFormatSettings; + I: Integer; + ExpectedKeys: TArray; +begin + if ((AExpected is TGocciaObjectValue) <> + (AActual is TGocciaObjectValue)) or + ((AExpected is TGocciaArrayValue) <> + (AActual is TGocciaArrayValue)) then + begin + AMessage := APath + ': JSON kinds differ'; + Exit(False); + end; + if AExpected is TGocciaArrayValue then + begin + ExpectedArray := TGocciaArrayValue(AExpected); + ActualArray := TGocciaArrayValue(AActual); + if ExpectedArray.Elements.Count <> ActualArray.Elements.Count then + begin + AMessage := APath + ': array lengths differ'; + Exit(False); + end; + for I := 0 to ExpectedArray.Elements.Count - 1 do + if not CompareTaggedJSON(ExpectedArray.Elements[I], + ActualArray.Elements[I], Format('%s[%d]', [APath, I]), + AMessage) then + Exit(False); + Exit(True); + end; + if AExpected is TGocciaObjectValue then + begin + ExpectedObject := TGocciaObjectValue(AExpected); + ActualObject := TGocciaObjectValue(AActual); + ExpectedKeys := ExpectedObject.GetOwnPropertyKeys; + if Length(ExpectedKeys) <> Length(ActualObject.GetOwnPropertyKeys) then + begin + AMessage := APath + ': object keys differ'; + Exit(False); + end; + if ExpectedObject.HasOwnProperty('type') and + ExpectedObject.HasOwnProperty('value') and + (Length(ExpectedKeys) = 2) then + begin + if not ActualObject.HasOwnProperty('type') or + not ActualObject.HasOwnProperty('value') then + begin + AMessage := APath + ': malformed tagged value'; + Exit(False); + end; + ExpectedType := ExpectedObject.GetProperty('type').ToStringLiteral.Value; + ActualType := ActualObject.GetProperty('type').ToStringLiteral.Value; + if ExpectedType <> ActualType then + begin + AMessage := Format('%s: expected type %s, got %s', + [APath, ExpectedType, ActualType]); + Exit(False); + end; + ExpectedValue := + ExpectedObject.GetProperty('value').ToStringLiteral.Value; + ActualValue := ActualObject.GetProperty('value').ToStringLiteral.Value; + if ExpectedType = 'float' then + begin + if SameText(WithoutSign(ExpectedValue), 'nan') or + SameText(WithoutSign(ActualValue), 'nan') then + begin + Result := SameText(WithoutSign(ExpectedValue), + WithoutSign(ActualValue)); + if not Result then + AMessage := Format('%s: expected float %s, got %s', + [APath, ExpectedValue, ActualValue]); + Exit; + end; + FloatFormat := DefaultFormatSettings; + FloatFormat.DecimalSeparator := '.'; + Result := TryStrToFloat(ExpectedValue, ExpectedFloat, FloatFormat) and + TryStrToFloat(ActualValue, ActualFloat, FloatFormat) and + SameDoubleBits(ExpectedFloat, ActualFloat); + if not Result then + AMessage := Format('%s: expected float %s, got %s', + [APath, ExpectedValue, ActualValue]); + Exit; + end; + if AnsiMatchText(ExpectedType, ['datetime', 'datetime-local', + 'date-local', 'time-local']) then + Result := NormalizeDateTimeText(ExpectedType, ExpectedValue) = + NormalizeDateTimeText(ActualType, ActualValue) + else if ExpectedType = 'bool' then + Result := SameText(ExpectedValue, ActualValue) + else + Result := ExpectedValue = ActualValue; + if not Result then + AMessage := Format('%s: expected %s, got %s', + [APath, ExpectedValue, ActualValue]); + Exit; + end; + for I := 0 to High(ExpectedKeys) do + begin + if not ActualObject.HasOwnProperty(ExpectedKeys[I]) then + begin + AMessage := APath + ': object keys differ'; + Exit(False); + end; + if APath = '' then + ChildPath := ExpectedKeys[I] + else + ChildPath := APath + '.' + ExpectedKeys[I]; + if not CompareTaggedJSON(ExpectedObject.GetProperty(ExpectedKeys[I]), + ActualObject.GetProperty(ExpectedKeys[I]), ChildPath, AMessage) then + Exit(False); + end; + Exit(True); + end; + Result := AExpected.ToStringLiteral.Value = + AActual.ToStringLiteral.Value; + if not Result then + AMessage := APath + ': values differ'; +end; + +constructor TTOMLComplianceExecutor.Create(const AExecutable, + ASuiteDirectory: string; const ATimeoutMilliseconds: Integer); +begin + inherited Create; + FExecutable := AExecutable; + FSuiteDirectory := ASuiteDirectory; + FTimeoutMilliseconds := ATimeoutMilliseconds; +end; + +procedure TTOMLComplianceExecutor.ExecuteCase(const ACaseID: string; + const AIndex: Integer; out AResult: TComplianceCaseResult); +var + ActualJSON, ExpectedJSON: TGocciaValue; + JSONParser: TGocciaJSONParser; + Arguments: TStringList; + CasePath, ExpectedPath, MessageText: string; + ProcessResult: TComplianceProcessResult; +begin + AResult.ID := ACaseID; + AResult.Valid := StartsText('valid/', ACaseID); + AResult.Outcome := coInfrastructure; + CasePath := IncludeTrailingPathDelimiter(FSuiteDirectory) + 'tests' + + DirectorySeparator + StringReplace(ACaseID, '/', DirectorySeparator, + [rfReplaceAll]); + Arguments := TStringList.Create; + try + Arguments.Add('--worker'); + Arguments.Add('--case=' + CasePath); + ProcessResult := RunComplianceProcess(FExecutable, Arguments, + FTimeoutMilliseconds); + finally + Arguments.Free; + end; + AResult.DurationMilliseconds := ProcessResult.DurationMilliseconds; + AResult.MessageText := CombinedProcessMessage(ProcessResult); + if not ProcessResult.Started then + Exit; + if ProcessResult.TimedOut then + begin + AResult.Outcome := coTimeout; + AResult.MessageText := 'timeout'; + Exit; + end; + if AResult.Valid then + begin + if ProcessResult.ExitCode = 1 then + begin + AResult.Outcome := coFalseReject; + Exit; + end; + if ProcessResult.ExitCode <> 0 then + begin + if ProcessResult.ExitCode = 2 then + AResult.Outcome := coInfrastructure + else + AResult.Outcome := coCrash; + Exit; + end; + ExpectedPath := ChangeFileExt(CasePath, '.json'); + try + JSONParser := TGocciaJSONParser.Create; + try + ExpectedJSON := JSONParser.Parse(ReadUTF8FileText(ExpectedPath)); + ActualJSON := JSONParser.Parse(Trim(ProcessResult.StdOutText)); + if CompareTaggedJSON(ExpectedJSON, ActualJSON, '', + MessageText) then + begin + AResult.Outcome := coPassed; + AResult.MessageText := ''; + end + else + begin + AResult.Outcome := coMismatch; + AResult.MessageText := MessageText; + end; + finally + JSONParser.Free; + end; + except + on E: Exception do + begin + AResult.Outcome := coInfrastructure; + AResult.MessageText := E.Message; + end; + end; + end + else if ProcessResult.ExitCode = 1 then + begin + AResult.Outcome := coPassed; + AResult.MessageText := ''; + end + else if ProcessResult.ExitCode = 0 then + AResult.Outcome := coFalseAccept + else if ProcessResult.ExitCode = 2 then + AResult.Outcome := coInfrastructure + else + AResult.Outcome := coCrash; +end; + +function DiscoverCases(const ASuiteDirectory: string): TStringList; +var + I: Integer; + Lines: TStringList; + LineText, ListPath: string; +begin + Result := TStringList.Create; + Lines := TStringList.Create; + try + ListPath := IncludeTrailingPathDelimiter(ASuiteDirectory) + + StringReplace(SUITE_FILE_LIST, '/', DirectorySeparator, [rfReplaceAll]); + if not FileExists(ListPath) then + raise Exception.CreateFmt('Suite file list not found: %s', [ListPath]); + Lines.Text := ReadUTF8FileText(ListPath); + for I := 0 to Lines.Count - 1 do + begin + LineText := Trim(Lines[I]); + if (LineText <> '') and (not StartsText('#', LineText)) and + EndsText('.toml', LineText) then + Result.Add(LineText); + end; + finally + Lines.Free; + end; +end; + +procedure RunWorker; +var + CasePath, SourceText: string; + Parser: TGocciaTOMLParser; + Root: TGocciaTOMLNode; +begin + if not ReadOptionValue('case', CasePath) then + Halt(2); + TGarbageCollector.Initialize; + PinPrimitiveSingletons; + Parser := TGocciaTOMLParser.Create; + try + try + SourceText := ReadUTF8FileText(CasePath); + Root := Parser.ParseDocument(SourceText); + try + WriteLn(SerializeNode(Root)); + finally + Root.Free; + end; + except + on E: EGocciaTOMLParseError do + begin + WriteLn(E.Message); + Halt(1); + end; + on E: Exception do + begin + WriteLn(E.Message); + Halt(1); + end; + end; + finally + Parser.Free; + TGarbageCollector.Shutdown; + end; +end; + +procedure PrintUsage; +begin + WriteLn('Usage: GocciaTOMLComplianceRunner --suite-dir=PATH [options]'); + WriteLn(' --pin-file=PATH Expected suite revision pin'); + WriteLn(' --jobs=N Maximum concurrent case processes'); + WriteLn(' --timeout=N Per-case timeout in milliseconds'); + WriteLn(' --output=PATH Write the complete JSON report'); +end; + +function RunCoordinator: Integer; +var + Cases: TStringList; + Coordinator: TComplianceCoordinator; + Executor: TTOMLComplianceExecutor; + Jobs, TimeoutMilliseconds: Integer; + OutputPath, PinPath, Revision, SuiteDirectory: string; + ReportJSON: string; + Results: TComplianceCaseResultArray; + StartedAt, DurationMilliseconds: QWord; + Summary: TComplianceSummary; +begin + if not ReadOptionValue('suite-dir', SuiteDirectory) then + begin + PrintUsage; + Exit(2); + end; + SuiteDirectory := ExpandFileName(SuiteDirectory); + if not ReadOptionValue('pin-file', PinPath) then + PinPath := DEFAULT_PIN_PATH; + Revision := ReadRequiredPin(PinPath); + VerifySuiteRevision(SuiteDirectory, Revision); + Jobs := PositiveIntegerOption('jobs', DefaultComplianceJobs); + TimeoutMilliseconds := PositiveIntegerOption('timeout', + DEFAULT_COMPLIANCE_TIMEOUT_MS); + ReadOptionValue('output', OutputPath); + + Cases := DiscoverCases(SuiteDirectory); + Executor := TTOMLComplianceExecutor.Create(ExpandFileName(ParamStr(0)), + SuiteDirectory, TimeoutMilliseconds); + Coordinator := TComplianceCoordinator.Create(Executor); + StartedAt := GetTickCount64; + TGarbageCollector.Initialize; + PinPrimitiveSingletons; + try + Results := Coordinator.Run(Cases, Jobs); + finally + TGarbageCollector.Shutdown; + Coordinator.Free; + Executor.Free; + Cases.Free; + end; + DurationMilliseconds := GetTickCount64 - StartedAt; + Summary := SummarizeComplianceResults(Results); + + ReportJSON := Format( + '{"reportVersion":%d,"runner":{"name":%s,"version":%d},' + + '"suite":{"name":%s,"version":%s,"revision":%s},' + + '"mode":"native-tagged-ast","jobs":%d,' + + '"timing":{"durationMilliseconds":%d,"timeoutMilliseconds":%d},' + + '"summary":%s,"cases":%s,' + + '"sections":{"decoder":{"mode":"tagged-ast"}}}', + [COMPLIANCE_REPORT_VERSION, + QuoteJSONString('GocciaTOMLComplianceRunner'), + COMPLIANCE_RUNNER_VERSION, QuoteJSONString(SUITE_NAME), + QuoteJSONString(SUITE_VERSION), QuoteJSONString(Revision), + Min(Jobs, Max(1, Summary.Total)), DurationMilliseconds, + TimeoutMilliseconds, ComplianceSummaryJSON(Summary), + ComplianceCasesJSON(Results)]); + if OutputPath <> '' then + WriteJSONFile(OutputPath, ReportJSON); + WriteLn('TOML compliance: ', Summary.Passed, '/', Summary.Total, ' passed'); + WriteLn(ComplianceSummaryJSON(Summary)); + if Summary.Failed > 0 then + Result := 1 + else + Result := 0; +end; + +begin + if HasCommandLineFlag('worker') then + RunWorker + else if HasCommandLineFlag('help') then + begin + PrintUsage; + ExitCode := 0; + end + else + begin + try + ExitCode := RunCoordinator; + except + on E: Exception do + begin + WriteLn('Error: ', E.Message); + ExitCode := 2; + end; + end; + end; +end. diff --git a/tests/compliance/json5-manifest.json b/tests/compliance/json5-manifest.json new file mode 100644 index 000000000..a62c36a02 --- /dev/null +++ b/tests/compliance/json5-manifest.json @@ -0,0 +1 @@ +{"manifestVersion":1,"suite":"json5","revision":"b935d4a280eafa8835e6182551b63809e61243b0","cases":[{"id":"parses empty objects","valid":true,"source":"{}","expected":{"type":"object","entries":[]}},{"id":"parses double string property names","valid":true,"source":"{\"a\":1}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"number","value":"1"}}]}},{"id":"parses single string property names","valid":true,"source":"{'a':1}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"number","value":"1"}}]}},{"id":"parses unquoted property names","valid":true,"source":"{a:1}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"number","value":"1"}}]}},{"id":"parses special character property names","valid":true,"source":"{$_:1,_$:2,a‌:3}","expected":{"type":"object","entries":[{"key":"$_","value":{"type":"number","value":"1"}},{"key":"_$","value":{"type":"number","value":"2"}},{"key":"a‌","value":{"type":"number","value":"3"}}]}},{"id":"parses unicode property names","valid":true,"source":"{ùńîċõďë:9}","expected":{"type":"object","entries":[{"key":"ùńîċõďë","value":{"type":"number","value":"9"}}]}},{"id":"parses escaped property names","valid":true,"source":"{\\u0061\\u0062:1,\\u0024\\u005F:2,\\u005F\\u0024:3}","expected":{"type":"object","entries":[{"key":"ab","value":{"type":"number","value":"1"}},{"key":"$_","value":{"type":"number","value":"2"}},{"key":"_$","value":{"type":"number","value":"3"}}]}},{"id":"preserves __proto__ property names","valid":true,"source":"{\"__proto__\":1}","expected":{"type":"object","entries":[{"key":"__proto__","value":{"type":"number","value":"1"}}]}},{"id":"parses multiple properties","valid":true,"source":"{abc:1,def:2}","expected":{"type":"object","entries":[{"key":"abc","value":{"type":"number","value":"1"}},{"key":"def","value":{"type":"number","value":"2"}}]}},{"id":"parses nested objects","valid":true,"source":"{a:{b:2}}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"object","entries":[{"key":"b","value":{"type":"number","value":"2"}}]}}]}},{"id":"parses empty arrays","valid":true,"source":"[]","expected":{"type":"array","items":[]}},{"id":"parses array values","valid":true,"source":"[1]","expected":{"type":"array","items":[{"type":"number","value":"1"}]}},{"id":"parses multiple array values","valid":true,"source":"[1,2]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"2"}]}},{"id":"parses nested arrays","valid":true,"source":"[1,[2,3]]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"array","items":[{"type":"number","value":"2"},{"type":"number","value":"3"}]}]}},{"id":"parses nulls","valid":true,"source":"null","expected":{"type":"null"}},{"id":"parses true","valid":true,"source":"true","expected":{"type":"boolean","value":true}},{"id":"parses false","valid":true,"source":"false","expected":{"type":"boolean","value":false}},{"id":"parses leading zeroes","valid":true,"source":"[0,0.,0e0]","expected":{"type":"array","items":[{"type":"number","value":"0"},{"type":"number","value":"0"},{"type":"number","value":"0"}]}},{"id":"parses integers","valid":true,"source":"[1,23,456,7890]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"23"},{"type":"number","value":"456"},{"type":"number","value":"7890"}]}},{"id":"parses signed numbers","valid":true,"source":"[-1,+2,-.1,-0]","expected":{"type":"array","items":[{"type":"number","value":"-1"},{"type":"number","value":"2"},{"type":"number","value":"-0.1"},{"type":"number","value":"-0"}]}},{"id":"parses leading decimal points","valid":true,"source":"[.1,.23]","expected":{"type":"array","items":[{"type":"number","value":"0.1"},{"type":"number","value":"0.23"}]}},{"id":"parses fractional numbers","valid":true,"source":"[1.0,1.23]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"1.23"}]}},{"id":"parses exponents","valid":true,"source":"[1e0,1e1,1e01,1.e0,1.1e0,1e-1,1e+1]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"10"},{"type":"number","value":"10"},{"type":"number","value":"1"},{"type":"number","value":"1.1"},{"type":"number","value":"0.1"},{"type":"number","value":"10"}]}},{"id":"parses hexadecimal numbers","valid":true,"source":"[0x1,0x10,0xff,0xFF]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"16"},{"type":"number","value":"255"},{"type":"number","value":"255"}]}},{"id":"parses signed and unsigned Infinity","valid":true,"source":"[Infinity,-Infinity]","expected":{"type":"array","items":[{"type":"number","value":"Infinity"},{"type":"number","value":"-Infinity"}]}},{"id":"parses NaN","valid":true,"source":"NaN","expected":{"type":"number","value":"NaN"}},{"id":"parses signed NaN","valid":true,"source":"-NaN","expected":{"type":"number","value":"NaN"}},{"id":"parses 1","valid":true,"source":"1","expected":{"type":"number","value":"1"}},{"id":"parses +1.23e100","valid":true,"source":"+1.23e100","expected":{"type":"number","value":"1.23e+100"}},{"id":"parses bare hexadecimal number","valid":true,"source":"0x1","expected":{"type":"number","value":"1"}},{"id":"parses bare long hexadecimal number","valid":true,"source":"-0x0123456789abcdefABCDEF","expected":{"type":"number","value":"-1.3754889325393114e+24"}},{"id":"parses double quoted strings","valid":true,"source":"\"abc\"","expected":{"type":"string","value":"abc"}},{"id":"parses single quoted strings","valid":true,"source":"'abc'","expected":{"type":"string","value":"abc"}},{"id":"parses quotes in strings","valid":true,"source":"['\"',\"'\"]","expected":{"type":"array","items":[{"type":"string","value":"\""},{"type":"string","value":"'"}]}},{"id":"parses escaped characters","valid":true,"source":"'\\b\\f\\n\\r\\t\\v\\0\\x0f\\u01fF\\\n\\\r\n\\\r\\
\\
\\a\\'\\\"'","expected":{"type":"string","value":"\b\f\n\r\t\u000b\u0000\u000fǿa'\""}},{"id":"assert.deepStrictEqual","valid":true,"source":"'

'","expected":{"type":"string","value":"

"}},{"id":"parses single-line comments","valid":true,"source":"{//comment\n}","expected":{"type":"object","entries":[]}},{"id":"parses single-line comments at end of input","valid":true,"source":"{}//comment","expected":{"type":"object","entries":[]}},{"id":"parses multi-line comments","valid":true,"source":"{/*comment\n** */}","expected":{"type":"object","entries":[]}},{"id":"parses whitespace","valid":true,"source":"{\t\u000b\f  \n\r

 }","expected":{"type":"object","entries":[]}},{"id":"throws on empty documents","valid":false,"source":""},{"id":"throws on documents with only comments","valid":false,"source":"//a"},{"id":"throws on incomplete single line comments","valid":false,"source":"/a"},{"id":"throws on unterminated multiline comments","valid":false,"source":"/*"},{"id":"throws on unterminated multiline comment closings","valid":false,"source":"/**"},{"id":"throws on invalid characters in values","valid":false,"source":"a"},{"id":"throws on invalid characters in identifier start escapes","valid":false,"source":"{\\a:1}"},{"id":"throws on invalid identifier start characters","valid":false,"source":"{\\u0021:1}"},{"id":"throws on invalid characters in identifier continue escapes","valid":false,"source":"{a\\a:1}"},{"id":"throws on invalid identifier continue characters","valid":false,"source":"{a\\u0021:1}"},{"id":"throws on invalid characters following a sign","valid":false,"source":"-a"},{"id":"throws on invalid characters following a leading decimal point","valid":false,"source":".a"},{"id":"throws on invalid characters following an exponent indicator","valid":false,"source":"1ea"},{"id":"throws on invalid characters following an exponent sign","valid":false,"source":"1e-a"},{"id":"throws on invalid characters following a hexadecimal indicator","valid":false,"source":"0xg"},{"id":"throws on invalid new lines in strings","valid":false,"source":"\"\n\""},{"id":"throws on unterminated strings","valid":false,"source":"\""},{"id":"throws on invalid identifier start characters in property names","valid":false,"source":"{!:1}"},{"id":"throws on invalid characters following a property name","valid":false,"source":"{a!1}"},{"id":"throws on invalid characters following a property value","valid":false,"source":"{a:1!}"},{"id":"throws on invalid characters following an array value","valid":false,"source":"[1!]"},{"id":"throws on invalid characters in literals","valid":false,"source":"tru!"},{"id":"throws on unterminated escapes","valid":false,"source":"\"\\"},{"id":"throws on invalid first digits in hexadecimal escapes","valid":false,"source":"\"\\xg\""},{"id":"throws on invalid second digits in hexadecimal escapes","valid":false,"source":"\"\\x0g\""},{"id":"throws on invalid unicode escapes","valid":false,"source":"\"\\u000g\""},{"id":"throws on escaped digit 1","valid":false,"source":"'\\1'"},{"id":"throws on escaped digit 2","valid":false,"source":"'\\2'"},{"id":"throws on escaped digit 3","valid":false,"source":"'\\3'"},{"id":"throws on escaped digit 4","valid":false,"source":"'\\4'"},{"id":"throws on escaped digit 5","valid":false,"source":"'\\5'"},{"id":"throws on escaped digit 6","valid":false,"source":"'\\6'"},{"id":"throws on escaped digit 7","valid":false,"source":"'\\7'"},{"id":"throws on escaped digit 8","valid":false,"source":"'\\8'"},{"id":"throws on escaped digit 9","valid":false,"source":"'\\9'"},{"id":"throws on octal escapes","valid":false,"source":"'\\01'"},{"id":"throws on multiple values","valid":false,"source":"1 2"},{"id":"throws with control characters escaped in the message","valid":false,"source":"\u0001"},{"id":"throws on unclosed objects before property names","valid":false,"source":"{"},{"id":"throws on unclosed objects after property names","valid":false,"source":"{a"},{"id":"throws on unclosed objects before property values","valid":false,"source":"{a:"},{"id":"throws on unclosed objects after property values","valid":false,"source":"{a:1"},{"id":"throws on unclosed arrays before values","valid":false,"source":"["},{"id":"throws on unclosed arrays after values","valid":false,"source":"[1"}]} diff --git a/tests/compliance/json5.pin b/tests/compliance/json5.pin new file mode 100644 index 000000000..0975e7841 --- /dev/null +++ b/tests/compliance/json5.pin @@ -0,0 +1 @@ +b935d4a280eafa8835e6182551b63809e61243b0 diff --git a/tests/compliance/toml-test.pin b/tests/compliance/toml-test.pin new file mode 100644 index 000000000..43540c7e1 --- /dev/null +++ b/tests/compliance/toml-test.pin @@ -0,0 +1 @@ +9eef1b959e0449d41a31d4e4e0a839faee534b36 From cb2341d79d17c34dd9aa7181efb645836144c5eb Mon Sep 17 00:00:00 2001 From: Johannes Stein Date: Fri, 24 Jul 2026 18:56:07 +0100 Subject: [PATCH 2/5] refactor(compliance): delegate JSON5 suite to TestRunner --- .github/workflows/ci.yml | 11 +- docs/build-system.md | 2 +- docs/built-ins-data-formats.md | 3 +- docs/testing.md | 4 +- .../GocciaJSON5ComplianceRunner.dpr | 325 +++++------------- 5 files changed, 88 insertions(+), 257 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdb091e33..3026f41cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -525,13 +525,10 @@ jobs: } const report = JSON.parse(fs.readFileSync(path, "utf8")); if ( - report.reportVersion !== 1 || - report.runner?.name !== "GocciaJSON5ComplianceRunner" || - report.suite?.name !== "json5" || - typeof report.summary?.failed !== "number" || - !Array.isArray(report.cases) || - typeof report.sections?.parse?.summary?.failed !== "number" || - typeof report.sections?.stringify?.testRunnerReport !== "object" + report.ok !== true || + report.failed !== 0 || + report.totalFiles !== report.results?.length || + !Array.isArray(report.results) ) { throw new Error(`Unexpected JSON5 report shape: ${path}`); } diff --git a/docs/build-system.md b/docs/build-system.md index 8471d24db..ed8c06717 100644 --- a/docs/build-system.md +++ b/docs/build-system.md @@ -658,7 +658,7 @@ Runs on the full platform matrix: **`test`** (needs build) — Runs all JavaScript tests and Pascal unit tests on all platforms. **`toml-compliance`** — Downloads the prebuilt `GocciaTOMLComplianceRunner`, prepares the pinned `toml-test` checkout, trusts the runner exit status, validates the report envelope, and uploads the per-platform JSON report. The pin is bumped weekly by `.github/workflows/toml-test-bump.yml`. -**`json5-compliance`** — Downloads `GocciaJSON5ComplianceRunner` and `GocciaTestRunner`, prepares the pinned JSON5 checkout with its matching manifest, trusts the runner exit status, validates the shared report envelope, and uploads the per-platform report. +**`json5-compliance`** — Downloads `GocciaJSON5ComplianceRunner` and `GocciaTestRunner`, prepares the pinned JSON5 checkout with its matching manifest, and delegates the generated parser cases plus local stringify suite to one TestRunner invocation. The workflow trusts TestRunner's exit status, validates its JSON report shape, and uploads that report. **`test262`** (needs build, ubuntu-latest x64 only, **non-blocking**) — Downloads the `gocciascript-x86_64-linux` build, checks out [`tc39/test262`](https://github.com/tc39/test262) at the SHA pinned in `scripts/test262-suite-sha.txt`, runs `bun scripts/run_test262_suite.ts --suite-dir test262-suite --mode=bytecode --jobs=4 --timeout-ms=20000 --output=test262-results.json`, and uploads the report as a 30-day workflow artifact. The run step uses `continue-on-error: true` because the conformance lane is allowed to carry known steady-state failures while the engine closes compatibility gaps. On main, the JSON is also stashed via `actions/cache/save` under `test262-baseline-` so the PR workflow can compute Δ vs main, and `cd website && bun run publish-test262 ../test262-results.json` publishes the compressed run report plus its UTC daily dashboard pointer to Vercel Blob when `BLOB_READ_WRITE_TOKEN` is configured. Main runs also upload the `test262-profile` artifact and publish matching aggregate/detail profile payloads under the separate `test262-profiles/` Blob namespace for weekly performance review. The one-off `cd website && bun run backfill-test262` command seeds retained artifact reports and reruns expired historical days directly into Blob. The pin is bumped weekly by `.github/workflows/test262-bump.yml`. See [docs/test262.md](test262.md) for the harness and profile report contracts. diff --git a/docs/built-ins-data-formats.md b/docs/built-ins-data-formats.md index 466ac6965..1c9969c63 100644 --- a/docs/built-ins-data-formats.md +++ b/docs/built-ins-data-formats.md @@ -74,7 +74,8 @@ The `stringify` export delegates to `TGocciaJSON5Stringifier`, which reuses the Compatibility goal: GocciaScript is targeting full JSON5 parser compatibility plus upstream-aligned stringify behavior. The native `GocciaJSON5ComplianceRunner` runs the pinned upstream parser manifest through -`GocciaTestRunner` and then runs the local upstream-aligned stringify suite. +one `GocciaTestRunner` invocation together with the local upstream-aligned +stringify suite. See [Testing](testing.md#running-tests) for checkout preparation and commands. diff --git a/docs/testing.md b/docs/testing.md index d94f0a3a5..c1046df4f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -448,7 +448,7 @@ cp tests/compliance/json5-manifest.json /path/to/json5/goccia-json5-cases.json ./build/GocciaJSON5ComplianceRunner --suite-dir=/path/to/json5 --test-runner=./build/GocciaTestRunner --output=tmp/json5-suite-results.json ``` -The runner verifies both the checkout and manifest against `tests/compliance/json5.pin`. It converts each manifest entry into a valid JavaScript test file and launches `GocciaTestRunner` in a separate process for that case, so invalid JSON5 remains parser input rather than invalid outer JavaScript. The same command runs `tests/built-ins/JSON5/stringify.js`. Normal compliance execution requires neither Python nor Node.js. +The runner verifies both the checkout and manifest against `tests/compliance/json5.pin`, converts every manifest entry into a valid JavaScript test file, and invokes `GocciaTestRunner` once for the generated parser suite plus `tests/built-ins/JSON5/stringify.js`. TestRunner owns concurrency, per-file timeouts, aggregation, exit status, and the JSON report. Invalid JSON5 remains input to `JSON5.parse(...)` inside valid outer JavaScript. Normal compliance execution requires neither Python nor Node.js. Maintainers regenerate the executable manifest from an already-prepared checkout: @@ -678,7 +678,7 @@ build → test → artifacts **`toml-compliance`** (all platforms) — Downloads the prebuilt `GocciaTOMLComplianceRunner`, prepares the exact pinned `toml-test` checkout, and relies on the runner exit status. CI performs only lightweight validation of the report envelope before uploading it. -**`json5-compliance`** (all platforms) — Downloads `GocciaJSON5ComplianceRunner` and `GocciaTestRunner`, prepares the exact pinned JSON5 checkout, installs its matching committed manifest, and relies on the native runner exit status for parser and stringify compliance. CI performs only lightweight report-envelope validation before upload. +**`json5-compliance`** (all platforms) — Downloads `GocciaJSON5ComplianceRunner` and `GocciaTestRunner`, prepares the exact pinned JSON5 checkout, installs its matching committed manifest, and runs the generated parser cases and local stringify suite together through TestRunner. CI relies on TestRunner's exit status and performs only lightweight validation of its JSON report before upload. **`test262`** (needs build, ubuntu-latest x64 only, **non-blocking**) — Runs the official conformance suite in bytecode mode from the shared pin in `scripts/test262-suite-sha.txt`, uploads the JSON report, and saves a `main` baseline cache for PR deltas. The run step is `continue-on-error: true` so known steady-state conformance failures do not block unrelated work; the downstream PR comment still gates regressions against the cached main baseline. **See [test262.md](test262.md) for the harness contract** and [Build System](build-system.md#ciyml--push-to-main--tags) for the workflow wiring. diff --git a/source/app/compliance/GocciaJSON5ComplianceRunner.dpr b/source/app/compliance/GocciaJSON5ComplianceRunner.dpr index 03f591c25..2f3c5bba6 100644 --- a/source/app/compliance/GocciaJSON5ComplianceRunner.dpr +++ b/source/app/compliance/GocciaJSON5ComplianceRunner.dpr @@ -5,8 +5,6 @@ program GocciaJSON5ComplianceRunner; uses {$IFDEF UNIX}cthreads,{$ENDIF} Classes, - Math, - StrUtils, SysUtils, Goccia.Compliance, @@ -27,27 +25,6 @@ const DEFAULT_REGENERATOR = 'scripts/regenerate-json5-manifest.js'; DEFAULT_STRINGIFY_SUITE = 'tests/built-ins/JSON5/stringify.js'; -type - TJSON5CaseData = record - ID: string; - Valid: Boolean; - TestPath: string; - end; - - TJSON5CaseDataArray = array of TJSON5CaseData; - - TJSON5ComplianceExecutor = class(TComplianceCaseExecutor) - private - FCases: TJSON5CaseDataArray; - FTestRunner: string; - FTimeoutMilliseconds: Integer; - public - constructor Create(const ACases: TJSON5CaseDataArray; - const ATestRunner: string; const ATimeoutMilliseconds: Integer); - procedure ExecuteCase(const ACaseID: string; const AIndex: Integer; - out AResult: TComplianceCaseResult); override; - end; - function GeneratedCaseSource(const AID, ASource, AExpectedJSON: string; const AValid: Boolean): string; const @@ -85,12 +62,7 @@ var begin if AValid then Body := - ' let actual;' + LineEnding + - ' try {' + LineEnding + - ' actual = JSON5.parse(source);' + LineEnding + - ' } catch (error) {' + LineEnding + - ' throw new Error("__GOCCIA_FALSE_REJECT__ " + error);' + LineEnding + - ' }' + LineEnding + + ' const actual = JSON5.parse(source);' + LineEnding + ' expect(JSON.stringify(encode(actual))).toBe(' + 'JSON.stringify(expected));' + LineEnding else @@ -106,67 +78,6 @@ begin Result := Result + Body + '});' + LineEnding; end; -constructor TJSON5ComplianceExecutor.Create(const ACases: TJSON5CaseDataArray; - const ATestRunner: string; const ATimeoutMilliseconds: Integer); -var - I: Integer; -begin - inherited Create; - SetLength(FCases, Length(ACases)); - for I := 0 to High(ACases) do - FCases[I] := ACases[I]; - FTestRunner := ATestRunner; - FTimeoutMilliseconds := ATimeoutMilliseconds; -end; - -procedure TJSON5ComplianceExecutor.ExecuteCase(const ACaseID: string; - const AIndex: Integer; out AResult: TComplianceCaseResult); -var - Arguments: TStringList; - ProcessResult: TComplianceProcessResult; -begin - AResult.ID := ACaseID; - AResult.Valid := FCases[AIndex].Valid; - AResult.Outcome := coInfrastructure; - Arguments := TStringList.Create; - try - Arguments.Add(FCases[AIndex].TestPath); - Arguments.Add('--silent'); - Arguments.Add('--no-progress'); - Arguments.Add('--no-results'); - ProcessResult := RunComplianceProcess(FTestRunner, Arguments, - FTimeoutMilliseconds); - finally - Arguments.Free; - end; - AResult.DurationMilliseconds := ProcessResult.DurationMilliseconds; - AResult.MessageText := CombinedProcessMessage(ProcessResult); - if not ProcessResult.Started then - Exit; - if ProcessResult.TimedOut then - begin - AResult.Outcome := coTimeout; - AResult.MessageText := 'timeout'; - Exit; - end; - if ProcessResult.ExitCode = 0 then - begin - AResult.Outcome := coPassed; - AResult.MessageText := ''; - end - else if ProcessResult.ExitCode = 1 then - begin - if not AResult.Valid then - AResult.Outcome := coFalseAccept - else if Pos('__GOCCIA_FALSE_REJECT__', AResult.MessageText) > 0 then - AResult.Outcome := coFalseReject - else - AResult.Outcome := coMismatch; - end - else - AResult.Outcome := coCrash; -end; - function DefaultTestRunnerPath: string; begin {$IFDEF WINDOWS} @@ -187,19 +98,18 @@ begin end; procedure RemoveCaseDirectory(const ADirectory: string; - const ACases: TJSON5CaseDataArray); + const ACasePaths: TStrings); var I: Integer; begin - for I := 0 to High(ACases) do - if StartsText(IncludeTrailingPathDelimiter(ADirectory), - ExpandFileName(ACases[I].TestPath)) then - DeleteFile(ACases[I].TestPath); + if ACasePaths <> nil then + for I := 0 to ACasePaths.Count - 1 do + DeleteFile(ACasePaths[I]); RemoveDir(ADirectory); end; -function LoadCases(const AManifestPath, AExpectedRevision, - ACaseDirectory: string; out ACases: TJSON5CaseDataArray): TStringList; +function PrepareCases(const AManifestPath, AExpectedRevision, + ACaseDirectory: string): TStringList; var CaseArray: TGocciaArrayValue; CaseObject, ManifestObject: TGocciaObjectValue; @@ -207,50 +117,53 @@ var I: Integer; JSONParser: TGocciaJSONParser; JSONStringifier: TGocciaJSONStringifier; - SourceText, ExpectedJSON: string; + CasePath, ExpectedJSON: string; begin if not FileExists(AManifestPath) then raise Exception.CreateFmt( 'JSON5 manifest not found: %s (run --regenerate-manifest first)', [AManifestPath]); + Result := TStringList.Create; JSONParser := TGocciaJSONParser.Create; JSONStringifier := TGocciaJSONStringifier.Create; try - ManifestObject := TGocciaObjectValue(JSONParser.Parse( - ReadUTF8FileText(AManifestPath))); - if ManifestObject.GetProperty('manifestVersion').ToNumberLiteral.Value <> - 1 then - raise Exception.Create('Unsupported JSON5 manifest version'); - if ManifestObject.GetProperty('suite').ToStringLiteral.Value <> - SUITE_NAME then - raise Exception.Create('JSON5 manifest names the wrong suite'); - if ManifestObject.GetProperty('revision').ToStringLiteral.Value <> - AExpectedRevision then - raise Exception.CreateFmt( - 'JSON5 manifest revision mismatch: expected %s, got %s', - [AExpectedRevision, - ManifestObject.GetProperty('revision').ToStringLiteral.Value]); - CaseArray := TGocciaArrayValue(ManifestObject.GetProperty('cases')); - SetLength(ACases, CaseArray.Elements.Count); - Result := TStringList.Create; - for I := 0 to CaseArray.Elements.Count - 1 do - begin - CaseObject := TGocciaObjectValue(CaseArray.Elements[I]); - ACases[I].ID := CaseObject.GetProperty('id').ToStringLiteral.Value; - ACases[I].Valid := CaseObject.GetProperty('valid').ToBooleanLiteral.Value; - ACases[I].TestPath := IncludeTrailingPathDelimiter(ACaseDirectory) + - Format('case_%.4d.js', [I]); - SourceText := CaseObject.GetProperty('source').ToStringLiteral.Value; - ExpectedJSON := ''; - if ACases[I].Valid then + try + ManifestObject := TGocciaObjectValue(JSONParser.Parse( + ReadUTF8FileText(AManifestPath))); + if ManifestObject.GetProperty('manifestVersion').ToNumberLiteral.Value <> + 1 then + raise Exception.Create('Unsupported JSON5 manifest version'); + if ManifestObject.GetProperty('suite').ToStringLiteral.Value <> + SUITE_NAME then + raise Exception.Create('JSON5 manifest names the wrong suite'); + if ManifestObject.GetProperty('revision').ToStringLiteral.Value <> + AExpectedRevision then + raise Exception.CreateFmt( + 'JSON5 manifest revision mismatch: expected %s, got %s', + [AExpectedRevision, + ManifestObject.GetProperty('revision').ToStringLiteral.Value]); + CaseArray := TGocciaArrayValue(ManifestObject.GetProperty('cases')); + for I := 0 to CaseArray.Elements.Count - 1 do begin - ExpectedValue := CaseObject.GetProperty('expected'); - ExpectedJSON := JSONStringifier.Stringify(ExpectedValue); + CaseObject := TGocciaObjectValue(CaseArray.Elements[I]); + CasePath := IncludeTrailingPathDelimiter(ACaseDirectory) + + Format('case_%.4d.js', [I]); + ExpectedJSON := ''; + if CaseObject.GetProperty('valid').ToBooleanLiteral.Value then + begin + ExpectedValue := CaseObject.GetProperty('expected'); + ExpectedJSON := JSONStringifier.Stringify(ExpectedValue); + end; + WriteUTF8FileText(CasePath, GeneratedCaseSource( + CaseObject.GetProperty('id').ToStringLiteral.Value, + CaseObject.GetProperty('source').ToStringLiteral.Value, + ExpectedJSON, + CaseObject.GetProperty('valid').ToBooleanLiteral.Value)); + Result.Add(CasePath); end; - WriteUTF8FileText(ACases[I].TestPath, - GeneratedCaseSource(ACases[I].ID, SourceText, ExpectedJSON, - ACases[I].Valid)); - Result.Add(ACases[I].ID); + except + Result.Free; + raise; end; finally JSONStringifier.Free; @@ -258,51 +171,6 @@ begin end; end; -function RunStringifySuite(const ATestRunner, ASuitePath, - AReportPath: string; const ATimeoutMilliseconds: Integer; - out AReportJSON: string): TComplianceCaseResult; -var - Arguments: TStringList; - ProcessResult: TComplianceProcessResult; -begin - Result.ID := 'JSON5.stringify'; - Result.Valid := True; - Result.Outcome := coInfrastructure; - Arguments := TStringList.Create; - try - Arguments.Add(ASuitePath); - Arguments.Add('--silent'); - Arguments.Add('--no-progress'); - Arguments.Add('--output=' + AReportPath); - ProcessResult := RunComplianceProcess(ATestRunner, Arguments, - ATimeoutMilliseconds); - finally - Arguments.Free; - end; - Result.DurationMilliseconds := ProcessResult.DurationMilliseconds; - Result.MessageText := CombinedProcessMessage(ProcessResult); - if ProcessResult.TimedOut then - begin - Result.Outcome := coTimeout; - Result.MessageText := 'timeout'; - end - else if not ProcessResult.Started then - Result.Outcome := coInfrastructure - else if ProcessResult.ExitCode = 0 then - begin - Result.Outcome := coPassed; - Result.MessageText := ''; - end - else if ProcessResult.ExitCode = 1 then - Result.Outcome := coMismatch - else - Result.Outcome := coCrash; - if FileExists(AReportPath) then - AReportJSON := ReadUTF8FileText(AReportPath) - else - AReportJSON := 'null'; -end; - function RegenerateManifest: Integer; var Arguments: TStringList; @@ -348,28 +216,19 @@ begin WriteLn(' --test-runner=PATH GocciaTestRunner executable'); WriteLn(' --stringify-suite=PATH Local JSON5 stringify test file'); WriteLn(' --pin-file=PATH Expected suite revision pin'); - WriteLn(' --jobs=N Maximum concurrent TestRunner processes'); - WriteLn(' --timeout=N Per-process timeout in milliseconds'); - WriteLn(' --output=PATH Write the complete JSON report'); + WriteLn(' --jobs=N TestRunner worker count'); + WriteLn(' --timeout=N TestRunner per-file timeout in milliseconds'); + WriteLn(' --output=PATH Write the TestRunner JSON report'); WriteLn(' --regenerate-manifest Maintainer-only manifest regeneration'); end; function RunCoordinator: Integer; var + Arguments, CasePaths: TStringList; CaseDirectory, ManifestPath, OutputPath, PinPath, Revision, - StringifyReportJSON, StringifySuite, SuiteDirectory, TestRunner: string; - Cases: TJSON5CaseDataArray; - StringifyResults: TComplianceCaseResultArray; - CaseIDs: TStringList; - Coordinator: TComplianceCoordinator; - Executor: TJSON5ComplianceExecutor; - I, Jobs, TimeoutMilliseconds: Integer; - AllResults: TComplianceCaseResultArray; - ParseResults: TComplianceCaseResultArray; - AllSummary, ParseSummary: TComplianceSummary; - ReportJSON, StringifyReportPath: string; - StartedAt, DurationMilliseconds: QWord; - StringifyResult: TComplianceCaseResult; + StringifySuite, SuiteDirectory, TestRunner: string; + Jobs, TimeoutMilliseconds: Integer; + ProcessResult: TComplianceProcessResult; begin if not ReadOptionValue('suite-dir', SuiteDirectory) then begin @@ -399,71 +258,45 @@ begin TimeoutMilliseconds := PositiveIntegerOption('timeout', DEFAULT_COMPLIANCE_TIMEOUT_MS); ReadOptionValue('output', OutputPath); + if (OutputPath <> '') and + (not ForceDirectories(ExtractFileDir(ExpandFileName(OutputPath)))) then + raise Exception.CreateFmt('Cannot create report directory: %s', + [ExtractFileDir(ExpandFileName(OutputPath))]); CaseDirectory := CreateCaseDirectory; - StringifyReportPath := IncludeTrailingPathDelimiter(CaseDirectory) + - 'stringify-report.json'; + CasePaths := nil; TGarbageCollector.Initialize; PinPrimitiveSingletons; try - CaseIDs := LoadCases(ExpandFileName(ManifestPath), Revision, - CaseDirectory, Cases); - Executor := TJSON5ComplianceExecutor.Create(Cases, TestRunner, - TimeoutMilliseconds); - Coordinator := TComplianceCoordinator.Create(Executor); - StartedAt := GetTickCount64; + CasePaths := PrepareCases(ExpandFileName(ManifestPath), Revision, + CaseDirectory); + Arguments := TStringList.Create; try - ParseResults := Coordinator.Run(CaseIDs, Jobs); + Arguments.Add(CaseDirectory); + Arguments.Add(StringifySuite); + Arguments.Add('--jobs=' + IntToStr(Jobs)); + Arguments.Add('--timeout=' + IntToStr(TimeoutMilliseconds)); + Arguments.Add('--silent'); + Arguments.Add('--no-progress'); + if OutputPath <> '' then + Arguments.Add('--output=' + ExpandFileName(OutputPath)); + ProcessResult := RunComplianceProcess(TestRunner, Arguments, 0); finally - Coordinator.Free; - Executor.Free; - CaseIDs.Free; + Arguments.Free; end; - StringifyResult := RunStringifySuite(TestRunner, StringifySuite, - StringifyReportPath, TimeoutMilliseconds, StringifyReportJSON); - SetLength(StringifyResults, 1); - StringifyResults[0] := StringifyResult; - SetLength(AllResults, Length(ParseResults) + 1); - for I := 0 to High(ParseResults) do - AllResults[I] := ParseResults[I]; - AllResults[High(AllResults)] := StringifyResult; - DurationMilliseconds := GetTickCount64 - StartedAt; - ParseSummary := SummarizeComplianceResults(ParseResults); - AllSummary := SummarizeComplianceResults(AllResults); - ReportJSON := Format( - '{"reportVersion":%d,"runner":{"name":%s,"version":%d},' + - '"suite":{"name":%s,"revision":%s},' + - '"mode":"testrunner","jobs":%d,' + - '"timing":{"durationMilliseconds":%d,"timeoutMilliseconds":%d},' + - '"summary":%s,"cases":%s,' + - '"sections":{"parse":{"summary":%s},' + - '"stringify":{"result":%s,"testRunnerReport":%s}}}', - [COMPLIANCE_REPORT_VERSION, - QuoteJSONString('GocciaJSON5ComplianceRunner'), - COMPLIANCE_RUNNER_VERSION, QuoteJSONString(SUITE_NAME), - QuoteJSONString(Revision), Min(Jobs, Max(1, ParseSummary.Total)), - DurationMilliseconds, TimeoutMilliseconds, - ComplianceSummaryJSON(AllSummary), ComplianceCasesJSON(AllResults), - ComplianceSummaryJSON(ParseSummary), - Copy(ComplianceCasesJSON(StringifyResults), 2, - Length(ComplianceCasesJSON(StringifyResults)) - 2), - Trim(StringifyReportJSON)]); - if OutputPath <> '' then - WriteJSONFile(OutputPath, ReportJSON); - WriteLn('JSON5 parse: ', ParseSummary.Passed, '/', ParseSummary.Total, - ' passed; stringify: ', - ComplianceOutcomeName(StringifyResult.Outcome)); - WriteLn(ComplianceSummaryJSON(AllSummary)); finally TGarbageCollector.Shutdown; - DeleteFile(StringifyReportPath); - RemoveCaseDirectory(CaseDirectory, Cases); + RemoveCaseDirectory(CaseDirectory, CasePaths); + CasePaths.Free; end; - if (ParseSummary.Failed > 0) or - (StringifyResult.Outcome <> coPassed) then - Result := 1 - else - Result := 0; + if not ProcessResult.Started then + raise Exception.Create('Cannot start GocciaTestRunner: ' + + ProcessResult.ErrorMessage); + if ProcessResult.StdOutText <> '' then + Write(TrimRight(ProcessResult.StdOutText), LineEnding); + if ProcessResult.StdErrText <> '' then + Write(TrimRight(ProcessResult.StdErrText), LineEnding); + Result := ProcessResult.ExitCode; end; begin From 6690045458e6d089cb42d370a2d037b5a40c1304 Mon Sep 17 00:00:00 2001 From: Johannes Stein Date: Fri, 24 Jul 2026 19:23:54 +0100 Subject: [PATCH 3/5] refactor(compliance): run JSON5 as generated tests --- .github/workflows/ci.yml | 42 +-- .github/workflows/json5-test-bump.yml | 23 +- build.pas | 18 - docs/build-system.md | 7 +- docs/built-ins-data-formats.md | 5 +- docs/contributing/tooling.md | 2 +- docs/interpreter.md | 2 +- docs/testing.md | 19 +- ...-manifest.js => regenerate-json5-tests.js} | 70 +++- .../GocciaJSON5ComplianceRunner.dpr | 320 ------------------ tests/built-ins/JSON5/upstream-parse.js | 41 +++ tests/compliance/json5-manifest.json | 1 - 12 files changed, 145 insertions(+), 405 deletions(-) rename scripts/{regenerate-json5-manifest.js => regenerate-json5-tests.js} (68%) delete mode 100644 source/app/compliance/GocciaJSON5ComplianceRunner.dpr create mode 100644 tests/built-ins/JSON5/upstream-parse.js delete mode 100644 tests/compliance/json5-manifest.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3026f41cb..9ce857e84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,21 +225,19 @@ jobs: - name: Stage build artifacts run: bash ./.github/scripts/stage-build-artifacts.sh build ci-artifacts --include-tests --strip - - name: Stage compliance runners + - name: Stage TOML compliance runner run: | - for name in GocciaTOMLComplianceRunner GocciaJSON5ComplianceRunner; do - copied=0 - for candidate in "build/$name" "build/$name.exe"; do - if [ -f "$candidate" ]; then - cp "$candidate" ci-artifacts/ - copied=1 - fi - done - if [ "$copied" -eq 0 ]; then - echo "::error::Missing compiled $name binary in build/" - exit 1 + copied=0 + for candidate in build/GocciaTOMLComplianceRunner build/GocciaTOMLComplianceRunner.exe; do + if [ -f "$candidate" ]; then + cp "$candidate" ci-artifacts/ + copied=1 fi done + if [ "$copied" -eq 0 ]; then + echo "::error::Missing compiled GocciaTOMLComplianceRunner binary in build/" + exit 1 + fi - name: Upload build artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -496,23 +494,17 @@ jobs: - name: Run JSON5 compliance suite run: | pin="$(tr -d '\r\n' < tests/compliance/json5.pin)" - suite_dir="$RUNNER_TEMP/json5" - git init "$suite_dir" - git -C "$suite_dir" remote add origin https://github.com/json5/json5.git - git -C "$suite_dir" fetch --depth 1 origin "$pin" - git -C "$suite_dir" checkout --detach "$pin" - cp tests/compliance/json5-manifest.json \ - "$suite_dir/goccia-json5-cases.json" - runner="./build/GocciaJSON5ComplianceRunner" + grep -Fq "Upstream json5/json5 revision: $pin" \ + tests/built-ins/JSON5/upstream-parse.js test_runner="./build/GocciaTestRunner" - if [ -f "$runner.exe" ]; then - runner="$runner.exe" - fi if [ -f "$test_runner.exe" ]; then test_runner="$test_runner.exe" fi - "$runner" --suite-dir="$suite_dir" --test-runner="$test_runner" \ - --jobs=4 --output=json5-test-results-${{ matrix.target }}.json + "$test_runner" \ + tests/built-ins/JSON5/upstream-parse.js \ + tests/built-ins/JSON5/stringify.js \ + --jobs=4 --no-progress \ + --output=json5-test-results-${{ matrix.target }}.json - name: Check JSON5 compliance summary if: always() diff --git a/.github/workflows/json5-test-bump.yml b/.github/workflows/json5-test-bump.yml index 49cd0b445..d97dfc2bc 100644 --- a/.github/workflows/json5-test-bump.yml +++ b/.github/workflows/json5-test-bump.yml @@ -1,7 +1,7 @@ name: JSON5 suite weekly bump # Updates the pinned json5/json5 revision and deterministically regenerates -# the executable parser-case manifest from a prepared checkout. +# the executable JavaScript parser suite from a prepared checkout. on: schedule: @@ -22,9 +22,6 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - - name: Install FPC - run: sudo apt-get update && sudo apt-get install fpc -qq > /dev/null - - id: latest run: | sha=$(gh api repos/json5/json5/commits/main --jq .sha) @@ -43,17 +40,16 @@ jobs: git -C "$RUNNER_TEMP/json5" fetch --depth 1 origin "${{ steps.latest.outputs.sha }}" git -C "$RUNNER_TEMP/json5" checkout --detach "${{ steps.latest.outputs.sha }}" - - name: Regenerate manifest + - name: Regenerate parser tests run: | - ./build.pas json5compliancerunner - ./build/GocciaJSON5ComplianceRunner \ - --regenerate-manifest \ - --suite-dir="$RUNNER_TEMP/json5" \ - --manifest=tests/compliance/json5-manifest.json + node scripts/regenerate-json5-tests.js \ + "$RUNNER_TEMP/json5" \ + "${{ steps.latest.outputs.sha }}" \ + tests/built-ins/JSON5/upstream-parse.js - id: changed run: | - if git diff --quiet -- tests/compliance/json5.pin tests/compliance/json5-manifest.json; then + if git diff --quiet -- tests/compliance/json5.pin tests/built-ins/JSON5/upstream-parse.js; then echo "changed=false" >> "$GITHUB_OUTPUT" echo "JSON5 suite already pinned to ${{ steps.latest.outputs.short }} — no PR to open." else @@ -70,8 +66,9 @@ jobs: body: | Automated bump of the json5/json5 SHA pin to `${{ steps.latest.outputs.sha }}`. - The executable parser-case manifest was regenerated from that - exact checkout. CI will run parser and stringify compliance. + The executable JavaScript parser suite was regenerated from that + exact checkout. CI will run it with the stringify suite through + `GocciaTestRunner`. Cron: weekly, Mondays 06:45 UTC. labels: | diff --git a/build.pas b/build.pas index df700e7c7..b1fd3e3a3 100755 --- a/build.pas +++ b/build.pas @@ -463,21 +463,6 @@ procedure BuildTOMLComplianceRunner; WriteLn('GocciaTOMLComplianceRunner built successfully'); end; -procedure BuildJSON5ComplianceRunner; -var - Output: string; -begin - WriteLn('Building GocciaJSON5ComplianceRunner...'); - if not RunCommand('fpc', FPCArgs( - 'source/app/compliance/GocciaJSON5ComplianceRunner.dpr', - EnsureUnitOutputDirectory( - TargetUnitOutputDirectory('json5compliancerunner'))), Output) then - PrintBuildFailureAndExit(Output, - 'GocciaJSON5ComplianceRunner build failed', 'json5compliancerunner'); - WriteLn(Output); - WriteLn('GocciaJSON5ComplianceRunner built successfully'); -end; - procedure BuildWasmTestRunner; var Output: string; @@ -581,8 +566,6 @@ procedure Build(const ATrigger: string); end else if ATrigger = 'tomlcompliancerunner' then BuildTOMLComplianceRunner - else if ATrigger = 'json5compliancerunner' then - BuildJSON5ComplianceRunner else if ATrigger = 'wasmtestrunner' then BuildWasmTestRunner else if ATrigger = 'benchmarkrunner' then @@ -642,7 +625,6 @@ procedure Build(const ATrigger: string); BuildTriggers.Add('sandboxrunner'); BuildTriggers.Add('testrunner'); BuildTriggers.Add('tomlcompliancerunner'); - BuildTriggers.Add('json5compliancerunner'); BuildTriggers.Add('wasmtestrunner'); BuildTriggers.Add('benchmarkrunner'); BuildTriggers.Add('bundler'); diff --git a/docs/build-system.md b/docs/build-system.md index ed8c06717..ab8d40496 100644 --- a/docs/build-system.md +++ b/docs/build-system.md @@ -47,7 +47,7 @@ The build script supports two modes via `--dev` (default) and `--prod` flags: ./build.pas --prod # Production build of all components ``` -Builds all components in order: tests, loader, loaderbare, sandboxrunner, testrunner, TOML and JSON5 compliance runners, benchmarkrunner, bundler, and repl. The default full build does not clean first; pass `--clean` explicitly when you need to remove stale build artifacts. +Builds all components in order: tests, loader, loaderbare, sandboxrunner, testrunner, the TOML compliance runner, benchmarkrunner, bundler, and repl. The default full build does not clean first; pass `--clean` explicitly when you need to remove stale build artifacts. ### Build Specific Components @@ -57,7 +57,7 @@ Builds all components in order: tests, loader, loaderbare, sandboxrunner, testru ./build.pas loaderbare # Bare Script Loader (core engine only) ./build.pas sandboxrunner # Sandbox Runner with virtual filesystem ./build.pas testrunner # JavaScript test runner + native FFI fixture -./build.pas tomlcompliancerunner json5compliancerunner # Compliance runners +./build.pas tomlcompliancerunner # Native TOML compliance runner ./build.pas benchmarkrunner # Performance benchmark runner ./build.pas bundler # Bundler (compile to .gbc) ./build.pas tests # Pascal unit tests @@ -484,7 +484,6 @@ All compiled binaries go to the `build/` directory: | `build/GocciaSandboxRunner` | `source/app/GocciaSandboxRunner.dpr` | Execute sandbox entry paths inside a seeded virtual filesystem | | `build/GocciaTestRunner` | `source/app/GocciaTestRunner.dpr` | JavaScript test runner | | `build/GocciaTOMLComplianceRunner` | `source/app/compliance/GocciaTOMLComplianceRunner.dpr` | Native pinned `toml-test` runner | -| `build/GocciaJSON5ComplianceRunner` | `source/app/compliance/GocciaJSON5ComplianceRunner.dpr` | Pinned JSON5 parser/stringify runner | | `build/GocciaBenchmarkRunner` | `source/app/GocciaBenchmarkRunner.dpr` | Performance benchmark runner for files or stdin input | | `build/GocciaBundler` | `source/app/GocciaBundler.dpr` | Bundler (source to `.gbc`) | | `build/Goccia.Values.Primitives.Test` | `*.Test.pas` | Pascal unit test binaries | @@ -658,7 +657,7 @@ Runs on the full platform matrix: **`test`** (needs build) — Runs all JavaScript tests and Pascal unit tests on all platforms. **`toml-compliance`** — Downloads the prebuilt `GocciaTOMLComplianceRunner`, prepares the pinned `toml-test` checkout, trusts the runner exit status, validates the report envelope, and uploads the per-platform JSON report. The pin is bumped weekly by `.github/workflows/toml-test-bump.yml`. -**`json5-compliance`** — Downloads `GocciaJSON5ComplianceRunner` and `GocciaTestRunner`, prepares the pinned JSON5 checkout with its matching manifest, and delegates the generated parser cases plus local stringify suite to one TestRunner invocation. The workflow trusts TestRunner's exit status, validates its JSON report shape, and uploads that report. +**`json5-compliance`** — Downloads `GocciaTestRunner`, verifies that the committed generated parser suite names the SHA in `tests/compliance/json5.pin`, and runs that suite together with the local stringify suite. The workflow trusts TestRunner's exit status, validates its JSON report shape, and uploads that report. **`test262`** (needs build, ubuntu-latest x64 only, **non-blocking**) — Downloads the `gocciascript-x86_64-linux` build, checks out [`tc39/test262`](https://github.com/tc39/test262) at the SHA pinned in `scripts/test262-suite-sha.txt`, runs `bun scripts/run_test262_suite.ts --suite-dir test262-suite --mode=bytecode --jobs=4 --timeout-ms=20000 --output=test262-results.json`, and uploads the report as a 30-day workflow artifact. The run step uses `continue-on-error: true` because the conformance lane is allowed to carry known steady-state failures while the engine closes compatibility gaps. On main, the JSON is also stashed via `actions/cache/save` under `test262-baseline-` so the PR workflow can compute Δ vs main, and `cd website && bun run publish-test262 ../test262-results.json` publishes the compressed run report plus its UTC daily dashboard pointer to Vercel Blob when `BLOB_READ_WRITE_TOKEN` is configured. Main runs also upload the `test262-profile` artifact and publish matching aggregate/detail profile payloads under the separate `test262-profiles/` Blob namespace for weekly performance review. The one-off `cd website && bun run backfill-test262` command seeds retained artifact reports and reruns expired historical days directly into Blob. The pin is bumped weekly by `.github/workflows/test262-bump.yml`. See [docs/test262.md](test262.md) for the harness and profile report contracts. diff --git a/docs/built-ins-data-formats.md b/docs/built-ins-data-formats.md index 1c9969c63..23abd9b7c 100644 --- a/docs/built-ins-data-formats.md +++ b/docs/built-ins-data-formats.md @@ -72,9 +72,8 @@ After `import * as JSON5 from "goccia:json5"`, `JSON5.parse` delegates to the st The `stringify` export delegates to `TGocciaJSON5Stringifier`, which reuses the same shared serialization core as strict JSON but switches to JSON5 formatting rules. That means unquoted identifier keys, single- or double-quoted strings (with optional `{ quote: "'" | '"' }` override), preserved `Infinity` / `-Infinity` / `NaN`, trailing commas when pretty-printing, `toJSON5()` preference over `toJSON()`, and the same replacer / space semantics as JSON plus the upstream JSON5 options-object form `{ replacer, space, quote }`. Compatibility goal: GocciaScript is targeting full JSON5 parser compatibility -plus upstream-aligned stringify behavior. The native -`GocciaJSON5ComplianceRunner` runs the pinned upstream parser manifest through -one `GocciaTestRunner` invocation together with the local upstream-aligned +plus upstream-aligned stringify behavior. `GocciaTestRunner` runs the committed +generated upstream parser suite together with the local upstream-aligned stringify suite. See [Testing](testing.md#running-tests) for checkout preparation and commands. diff --git a/docs/contributing/tooling.md b/docs/contributing/tooling.md index 83cd3caaa..54c2d55fc 100644 --- a/docs/contributing/tooling.md +++ b/docs/contributing/tooling.md @@ -111,7 +111,7 @@ FPC 3.2.2 aborts with `Fatal: Internal error 200611011` when a second program is compiled against the `.ppu` files another program left in a shared `-FU` unit-output directory (the inliner trips while recompiling a unit it loaded from the first program's build). This is why `build.pas` compiles every target, -including the TOML and JSON5 compliance runners, into its own +including the TOML compliance runner, into its own `build/compiled/targets/` directory. Any script that compiles more than one program with `fpc @config.cfg` must use a separate `-FU` directory per program. diff --git a/docs/interpreter.md b/docs/interpreter.md index d5c88d653..6c20dbf04 100644 --- a/docs/interpreter.md +++ b/docs/interpreter.md @@ -103,7 +103,7 @@ Scopes form a tree with parent pointers, implementing lexical scoping: - **Standalone JSON utilities** — `Goccia.JSON` provides `TGocciaJSONParser` and `TGocciaJSONStringifier` as dependency-free utility classes that convert between JSON text and `TGocciaValue` types. `Goccia.Builtins.JSON` (the `JSON.parse`/`JSON.stringify` built-in) delegates to these, keeping the built-in a thin adapter. This separation allows the interpreter and any other component to parse JSON without instantiating a built-in. - **Capability-driven JSON parsing** — `JSONParser.pas` now owns one event-driven parser core plus a `TJSONParserCapabilities` set. Strict JSON uses the empty capability set, while JSON5 opts into comments, trailing commas, single-quoted strings, identifier keys, hexadecimal numbers, signed numbers, `Infinity` / `NaN`, line continuations, and ECMAScript whitespace extensions. This keeps JSON and JSON5 behavior aligned on the shared grammar machinery instead of maintaining two diverging parser implementations. - **JSON5 parser/stringifier split** — `Goccia.JSON5` provides standalone `TGocciaJSON5Parser` and `TGocciaJSON5Stringifier` utilities. The parser reuses the same core capability-driven parser engine as strict JSON but enables the JSON5 capability set, while the stringifier reuses the shared JSON serialization engine in JSON5 mode instead of maintaining a second formatter. `Goccia.Builtins.JSON5` backs the named exports of `goccia:json5` (`parse`, `stringify`), the module loader reuses the parser for `.json5` imports, and globals injection reuses it for `--globals=file.json5` and embedding helpers. -- **JSON5 compatibility target** — The project goal for JSON5 is full parser compatibility with the reference `json5/json5` implementation plus upstream-aligned stringify behavior. `GocciaJSON5ComplianceRunner` verifies the pinned checkout and generated manifest, runs each parser case through `GocciaTestRunner`, and then runs the local stringify suite. A rerun on 2026-07-21 against upstream commit `b935d4a280eafa8835e6182551b63809e61243b0` matched 84 of 84 extracted parser cases; the local stringify suite passed all 42 upstream-aligned tests covering special numeric values, quote handling, replacers, boxed primitives, options objects, and pretty-print trailing commas. +- **JSON5 compatibility target** — The project goal for JSON5 is full parser compatibility with the reference `json5/json5` implementation plus upstream-aligned stringify behavior. The pinned upstream parser cases are generated as an ordinary JavaScript suite and run directly through `GocciaTestRunner` with the local stringify suite. A rerun on 2026-07-21 against upstream commit `b935d4a280eafa8835e6182551b63809e61243b0` matched 84 of 84 extracted parser cases; the local stringify suite passed all 42 upstream-aligned tests covering special numeric values, quote handling, replacers, boxed primitives, options objects, and pretty-print trailing commas. - **JSONL parser split** — `Goccia.JSONL` provides a standalone `TGocciaJSONLParser` utility that builds on `TGocciaJSONParser` one line at a time, preserving JSONL source line numbers in parse errors and supporting Bun-style chunked parsing through `ParseChunk(...)`. `Goccia.Builtins.JSONL` backs the named exports of `goccia:jsonl` (`parse`, `parseChunk`), while the module loader reuses the same parser for `.jsonl` imports. - **JSONL module imports** — `.jsonl` modules intentionally expose each non-empty line as a zero-based string-indexed named export (`"0"`, `"1"`, ...). This keeps the structured-data import surface consistent with the existing string-literal named import/export work and means namespace imports can reuse the same export table without introducing a JSONL-specific synthetic wrapper object just for modules. - **Text asset module imports** — `.txt` and `.md` modules bypass the script parser and expose a small named-export surface: `content` is the UTF-8 file text with source newlines canonicalized to LF (`\n`), and `metadata` is a frozen object containing `kind`, `path`, `fileName`, `extension`, and `byteLength`. Text assets do not invent a default export wrapper just for plain text files, which keeps imported text stable across Windows and non-Windows hosts. diff --git a/docs/testing.md b/docs/testing.md index c1046df4f..7809a4c74 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -440,23 +440,22 @@ identical on Windows and non-Windows hosts and prevent mojibake such as **Official JSON5 Suite** -For JSON5 parser compatibility checks against a prepared checkout of the official `json5/json5` parser test corpus, copy the committed manifest into the checkout and run: +The upstream JSON5 parser cases are committed as an ordinary generated JavaScript test suite. Run parser and stringify compliance directly through TestRunner: ```bash -./build.pas json5compliancerunner testrunner -cp tests/compliance/json5-manifest.json /path/to/json5/goccia-json5-cases.json -./build/GocciaJSON5ComplianceRunner --suite-dir=/path/to/json5 --test-runner=./build/GocciaTestRunner --output=tmp/json5-suite-results.json +./build.pas testrunner +./build/GocciaTestRunner tests/built-ins/JSON5/upstream-parse.js tests/built-ins/JSON5/stringify.js --output=tmp/json5-suite-results.json ``` -The runner verifies both the checkout and manifest against `tests/compliance/json5.pin`, converts every manifest entry into a valid JavaScript test file, and invokes `GocciaTestRunner` once for the generated parser suite plus `tests/built-ins/JSON5/stringify.js`. TestRunner owns concurrency, per-file timeouts, aggregation, exit status, and the JSON report. Invalid JSON5 remains input to `JSON5.parse(...)` inside valid outer JavaScript. Normal compliance execution requires neither Python nor Node.js. +The generated suite records the revision in `tests/compliance/json5.pin`. TestRunner owns concurrency, per-file timeouts, aggregation, exit status, and the JSON report. Invalid JSON5 remains input to `JSON5.parse(...)` inside valid outer JavaScript. Normal compliance execution requires neither an upstream checkout, Python, nor Node.js. -Maintainers regenerate the executable manifest from an already-prepared checkout: +Maintainers regenerate the JavaScript suite from an already-prepared checkout: ```bash -./build/GocciaJSON5ComplianceRunner --regenerate-manifest --suite-dir=/path/to/json5 --manifest=tests/compliance/json5-manifest.json +node scripts/regenerate-json5-tests.js /path/to/json5 tests/built-ins/JSON5/upstream-parse.js ``` -Regeneration uses Node.js to execute the upstream reference tests, records the verified commit, and is intentionally separate from normal compliance runs. +Regeneration uses Node.js to execute the upstream reference tests, verifies and records the checkout commit, and is intentionally separate from normal compliance runs. ### Pinned es-toolkit library probes @@ -678,7 +677,7 @@ build → test → artifacts **`toml-compliance`** (all platforms) — Downloads the prebuilt `GocciaTOMLComplianceRunner`, prepares the exact pinned `toml-test` checkout, and relies on the runner exit status. CI performs only lightweight validation of the report envelope before uploading it. -**`json5-compliance`** (all platforms) — Downloads `GocciaJSON5ComplianceRunner` and `GocciaTestRunner`, prepares the exact pinned JSON5 checkout, installs its matching committed manifest, and runs the generated parser cases and local stringify suite together through TestRunner. CI relies on TestRunner's exit status and performs only lightweight validation of its JSON report before upload. +**`json5-compliance`** (all platforms) — Downloads `GocciaTestRunner`, verifies that the committed generated parser suite names the pinned JSON5 revision, and runs it together with the local stringify suite. CI relies on TestRunner's exit status and performs only lightweight validation of its JSON report before upload. **`test262`** (needs build, ubuntu-latest x64 only, **non-blocking**) — Runs the official conformance suite in bytecode mode from the shared pin in `scripts/test262-suite-sha.txt`, uploads the JSON report, and saves a `main` baseline cache for PR deltas. The run step is `continue-on-error: true` so known steady-state conformance failures do not block unrelated work; the downstream PR comment still gates regressions against the cached main baseline. **See [test262.md](test262.md) for the harness contract** and [Build System](build-system.md#ciyml--push-to-main--tags) for the workflow wiring. @@ -726,7 +725,7 @@ Weekly crons open (or update) a single PR every Monday with the latest upstream |-------|----------|----------|--------------| | test262 | `test262-bump.yml` | 06:00 UTC | `scripts/test262-suite-sha.txt` | | toml-test | `toml-test-bump.yml` | 06:30 UTC | `tests/compliance/toml-test.pin` | -| json5 | `json5-test-bump.yml` | 06:45 UTC | `tests/compliance/json5.pin` and `json5-manifest.json` | +| json5 | `json5-test-bump.yml` | 06:45 UTC | `tests/compliance/json5.pin` and `tests/built-ins/JSON5/upstream-parse.js` | | yaml-test-suite | `yaml-test-bump.yml` | 07:00 UTC | `scripts/run_yaml_test_suite.py` | Each workflow reuses a fixed branch (`chore/-bump`), so an unmerged PR is updated in place rather than replaced. See [test262.md](test262.md) for details on the test262 harness contract. diff --git a/scripts/regenerate-json5-manifest.js b/scripts/regenerate-json5-tests.js similarity index 68% rename from scripts/regenerate-json5-manifest.js rename to scripts/regenerate-json5-tests.js index 3e42efe8f..ce1d180ad 100644 --- a/scripts/regenerate-json5-manifest.js +++ b/scripts/regenerate-json5-tests.js @@ -1,14 +1,15 @@ #!/usr/bin/env node -// Maintainer-only adapter: normal compliance runs consume the committed -// revision-tagged manifest and do not require Node.js. +// Maintainer-only adapter: normal compliance runs execute the committed +// generated JavaScript suite and do not require Node.js. +const { execFileSync } = require("child_process"); const Module = require("module"); const fs = require("fs"); const path = require("path"); if (process.argv.length !== 5) { console.error( - "Usage: regenerate-json5-manifest.js ", + "Usage: regenerate-json5-tests.js ", ); process.exit(2); } @@ -18,6 +19,17 @@ const revision = process.argv[3]; const outputPath = path.resolve(process.argv[4]); const testDir = path.join(suiteDir, "test"); const realJSON5 = require(path.join(suiteDir, "lib")); +const checkoutRevision = execFileSync( + "git", + ["-C", suiteDir, "rev-parse", "HEAD"], + { encoding: "utf8" }, +).trim(); + +if (checkoutRevision !== revision) { + throw new Error( + `Suite revision mismatch: expected ${revision}, got ${checkoutRevision}`, + ); +} const cases = []; const parseQueue = []; @@ -206,11 +218,51 @@ Module._load = function patchedLoad(request, parent, isMain) { require(path.join(testDir, "parse.js")); require(path.join(testDir, "errors.js")); -const manifest = { - manifestVersion: 1, - suite: "json5", - revision, - cases, +const serializedCases = JSON.stringify(cases) + .replaceAll("\u2028", "\\u2028") + .replaceAll("\u2029", "\\u2029"); +const generatedSuite = `// Generated by scripts/regenerate-json5-tests.js. +// Upstream json5/json5 revision: ${revision} +import * as JSON5 from "goccia:json5"; + +const cases = ${serializedCases}; + +const encode = (value) => { + if (value === null) return { type: "null" }; + if (typeof value === "boolean") return { type: "boolean", value }; + if (typeof value === "string") return { type: "string", value }; + if (typeof value === "number") { + if (Number.isNaN(value)) return { type: "number", value: "NaN" }; + if (value === Infinity) return { type: "number", value: "Infinity" }; + if (value === -Infinity) return { type: "number", value: "-Infinity" }; + if (Object.is(value, -0)) return { type: "number", value: "-0" }; + return { type: "number", value: String(value) }; + } + if (Array.isArray(value)) { + return { type: "array", items: value.map((item) => encode(item)) }; + } + return { + type: "object", + entries: Object.getOwnPropertyNames(value).map((key) => ({ + key, + value: encode(value[key]), + })), + }; }; -fs.writeFileSync(outputPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); + +for (const testCase of cases) { + test(testCase.id, () => { + if (!testCase.valid) { + expect(() => JSON5.parse(testCase.source)).toThrow(SyntaxError); + return; + } + const actual = JSON5.parse(testCase.source); + expect(JSON.stringify(encode(actual))).toBe( + JSON.stringify(testCase.expected), + ); + }); +} +`; +fs.mkdirSync(path.dirname(outputPath), { recursive: true }); +fs.writeFileSync(outputPath, generatedSuite, "utf8"); process.stdout.write(`${outputPath}\n`); diff --git a/source/app/compliance/GocciaJSON5ComplianceRunner.dpr b/source/app/compliance/GocciaJSON5ComplianceRunner.dpr deleted file mode 100644 index 2f3c5bba6..000000000 --- a/source/app/compliance/GocciaJSON5ComplianceRunner.dpr +++ /dev/null @@ -1,320 +0,0 @@ -program GocciaJSON5ComplianceRunner; - -{$I Goccia.inc} - -uses - {$IFDEF UNIX}cthreads,{$ENDIF} - Classes, - SysUtils, - - Goccia.Compliance, - Goccia.GarbageCollector, - Goccia.JSON, - Goccia.JSON.Utils, - Goccia.TextFiles, - Goccia.Values.ArrayValue, - Goccia.Values.ObjectValue, - Goccia.Values.Primitives, - - FileUtils; - -const - SUITE_NAME = 'json5'; - DEFAULT_PIN_PATH = 'tests/compliance/json5.pin'; - DEFAULT_MANIFEST_NAME = 'goccia-json5-cases.json'; - DEFAULT_REGENERATOR = 'scripts/regenerate-json5-manifest.js'; - DEFAULT_STRINGIFY_SUITE = 'tests/built-ins/JSON5/stringify.js'; - -function GeneratedCaseSource(const AID, ASource, AExpectedJSON: string; - const AValid: Boolean): string; -const - ENCODER_SOURCE = - 'const encode = (value) => {' + LineEnding + - ' if (value === null) return { type: "null" };' + LineEnding + - ' if (typeof value === "boolean") return { type: "boolean", value };' + - LineEnding + - ' if (typeof value === "string") return { type: "string", value };' + - LineEnding + - ' if (typeof value === "number") {' + LineEnding + - ' if (Number.isNaN(value)) return { type: "number", value: "NaN" };' + - LineEnding + - ' if (value === Infinity) return { type: "number", value: "Infinity" };' + - LineEnding + - ' if (value === -Infinity) return { type: "number", value: "-Infinity" };' + - LineEnding + - ' if (Object.is(value, -0)) return { type: "number", value: "-0" };' + - LineEnding + - ' return { type: "number", value: String(value) };' + LineEnding + - ' }' + LineEnding + - ' if (Array.isArray(value)) {' + LineEnding + - ' return { type: "array", items: value.map(item => encode(item)) };' + - LineEnding + - ' }' + LineEnding + - ' return {' + LineEnding + - ' type: "object",' + LineEnding + - ' entries: Object.getOwnPropertyNames(value).map(key => ({' + LineEnding + - ' key, value: encode(value[key]),' + LineEnding + - ' })),' + LineEnding + - ' };' + LineEnding + - '};' + LineEnding; -var - Body: string; -begin - if AValid then - Body := - ' const actual = JSON5.parse(source);' + LineEnding + - ' expect(JSON.stringify(encode(actual))).toBe(' + - 'JSON.stringify(expected));' + LineEnding - else - Body := ' expect(() => JSON5.parse(source)).toThrow(SyntaxError);' + - LineEnding; - Result := - 'import * as JSON5 from "goccia:json5";' + LineEnding + LineEnding + - ENCODER_SOURCE + LineEnding + - 'test(' + QuoteJSONString(AID) + ', () => {' + LineEnding + - ' const source = ' + QuoteJSONString(ASource) + ';' + LineEnding; - if AValid then - Result := Result + ' const expected = ' + AExpectedJSON + ';' + LineEnding; - Result := Result + Body + '});' + LineEnding; -end; - -function DefaultTestRunnerPath: string; -begin - {$IFDEF WINDOWS} - Result := 'build' + DirectorySeparator + 'GocciaTestRunner.exe'; - {$ELSE} - Result := 'build' + DirectorySeparator + 'GocciaTestRunner'; - {$ENDIF} -end; - -function CreateCaseDirectory: string; -begin - Result := IncludeTrailingPathDelimiter(GetTempDir(False)) + - 'goccia-json5-compliance-' + IntToStr(GetProcessID) + '-' + - IntToStr(GetTickCount64); - if not ForceDirectories(Result) then - raise Exception.CreateFmt('Cannot create temporary case directory: %s', - [Result]); -end; - -procedure RemoveCaseDirectory(const ADirectory: string; - const ACasePaths: TStrings); -var - I: Integer; -begin - if ACasePaths <> nil then - for I := 0 to ACasePaths.Count - 1 do - DeleteFile(ACasePaths[I]); - RemoveDir(ADirectory); -end; - -function PrepareCases(const AManifestPath, AExpectedRevision, - ACaseDirectory: string): TStringList; -var - CaseArray: TGocciaArrayValue; - CaseObject, ManifestObject: TGocciaObjectValue; - ExpectedValue: TGocciaValue; - I: Integer; - JSONParser: TGocciaJSONParser; - JSONStringifier: TGocciaJSONStringifier; - CasePath, ExpectedJSON: string; -begin - if not FileExists(AManifestPath) then - raise Exception.CreateFmt( - 'JSON5 manifest not found: %s (run --regenerate-manifest first)', - [AManifestPath]); - Result := TStringList.Create; - JSONParser := TGocciaJSONParser.Create; - JSONStringifier := TGocciaJSONStringifier.Create; - try - try - ManifestObject := TGocciaObjectValue(JSONParser.Parse( - ReadUTF8FileText(AManifestPath))); - if ManifestObject.GetProperty('manifestVersion').ToNumberLiteral.Value <> - 1 then - raise Exception.Create('Unsupported JSON5 manifest version'); - if ManifestObject.GetProperty('suite').ToStringLiteral.Value <> - SUITE_NAME then - raise Exception.Create('JSON5 manifest names the wrong suite'); - if ManifestObject.GetProperty('revision').ToStringLiteral.Value <> - AExpectedRevision then - raise Exception.CreateFmt( - 'JSON5 manifest revision mismatch: expected %s, got %s', - [AExpectedRevision, - ManifestObject.GetProperty('revision').ToStringLiteral.Value]); - CaseArray := TGocciaArrayValue(ManifestObject.GetProperty('cases')); - for I := 0 to CaseArray.Elements.Count - 1 do - begin - CaseObject := TGocciaObjectValue(CaseArray.Elements[I]); - CasePath := IncludeTrailingPathDelimiter(ACaseDirectory) + - Format('case_%.4d.js', [I]); - ExpectedJSON := ''; - if CaseObject.GetProperty('valid').ToBooleanLiteral.Value then - begin - ExpectedValue := CaseObject.GetProperty('expected'); - ExpectedJSON := JSONStringifier.Stringify(ExpectedValue); - end; - WriteUTF8FileText(CasePath, GeneratedCaseSource( - CaseObject.GetProperty('id').ToStringLiteral.Value, - CaseObject.GetProperty('source').ToStringLiteral.Value, - ExpectedJSON, - CaseObject.GetProperty('valid').ToBooleanLiteral.Value)); - Result.Add(CasePath); - end; - except - Result.Free; - raise; - end; - finally - JSONStringifier.Free; - JSONParser.Free; - end; -end; - -function RegenerateManifest: Integer; -var - Arguments: TStringList; - ManifestPath, NodeExecutable, PinPath, RegeneratorPath, Revision, - SuiteDirectory: string; - ProcessResult: TComplianceProcessResult; -begin - if not ReadOptionValue('suite-dir', SuiteDirectory) then - raise Exception.Create('--suite-dir is required'); - SuiteDirectory := ExpandFileName(SuiteDirectory); - if not ReadOptionValue('pin-file', PinPath) then - PinPath := DEFAULT_PIN_PATH; - Revision := ReadRequiredPin(PinPath); - VerifySuiteRevision(SuiteDirectory, Revision); - if not ReadOptionValue('manifest', ManifestPath) then - ManifestPath := IncludeTrailingPathDelimiter(SuiteDirectory) + - DEFAULT_MANIFEST_NAME; - if not ReadOptionValue('node', NodeExecutable) then - NodeExecutable := 'node'; - if not ReadOptionValue('regenerator', RegeneratorPath) then - RegeneratorPath := DEFAULT_REGENERATOR; - Arguments := TStringList.Create; - try - Arguments.Add(ExpandFileName(RegeneratorPath)); - Arguments.Add(SuiteDirectory); - Arguments.Add(Revision); - Arguments.Add(ExpandFileName(ManifestPath)); - ProcessResult := RunComplianceProcess(NodeExecutable, Arguments, 60000); - finally - Arguments.Free; - end; - if (not ProcessResult.Started) or (ProcessResult.ExitCode <> 0) then - raise Exception.Create('Manifest regeneration failed: ' + - CombinedProcessMessage(ProcessResult)); - WriteLn(Trim(ProcessResult.StdOutText)); - Result := 0; -end; - -procedure PrintUsage; -begin - WriteLn('Usage: GocciaJSON5ComplianceRunner --suite-dir=PATH [options]'); - WriteLn(' --manifest=PATH Prepared, revision-tagged case manifest'); - WriteLn(' --test-runner=PATH GocciaTestRunner executable'); - WriteLn(' --stringify-suite=PATH Local JSON5 stringify test file'); - WriteLn(' --pin-file=PATH Expected suite revision pin'); - WriteLn(' --jobs=N TestRunner worker count'); - WriteLn(' --timeout=N TestRunner per-file timeout in milliseconds'); - WriteLn(' --output=PATH Write the TestRunner JSON report'); - WriteLn(' --regenerate-manifest Maintainer-only manifest regeneration'); -end; - -function RunCoordinator: Integer; -var - Arguments, CasePaths: TStringList; - CaseDirectory, ManifestPath, OutputPath, PinPath, Revision, - StringifySuite, SuiteDirectory, TestRunner: string; - Jobs, TimeoutMilliseconds: Integer; - ProcessResult: TComplianceProcessResult; -begin - if not ReadOptionValue('suite-dir', SuiteDirectory) then - begin - PrintUsage; - Exit(2); - end; - SuiteDirectory := ExpandFileName(SuiteDirectory); - if not ReadOptionValue('pin-file', PinPath) then - PinPath := DEFAULT_PIN_PATH; - Revision := ReadRequiredPin(PinPath); - VerifySuiteRevision(SuiteDirectory, Revision); - if not ReadOptionValue('manifest', ManifestPath) then - ManifestPath := IncludeTrailingPathDelimiter(SuiteDirectory) + - DEFAULT_MANIFEST_NAME; - if not ReadOptionValue('test-runner', TestRunner) then - TestRunner := DefaultTestRunnerPath; - TestRunner := ExpandFileName(TestRunner); - if not FileExists(TestRunner) then - raise Exception.CreateFmt('GocciaTestRunner not found: %s', [TestRunner]); - if not ReadOptionValue('stringify-suite', StringifySuite) then - StringifySuite := DEFAULT_STRINGIFY_SUITE; - StringifySuite := ExpandFileName(StringifySuite); - if not FileExists(StringifySuite) then - raise Exception.CreateFmt('JSON5 stringify suite not found: %s', - [StringifySuite]); - Jobs := PositiveIntegerOption('jobs', DefaultComplianceJobs); - TimeoutMilliseconds := PositiveIntegerOption('timeout', - DEFAULT_COMPLIANCE_TIMEOUT_MS); - ReadOptionValue('output', OutputPath); - if (OutputPath <> '') and - (not ForceDirectories(ExtractFileDir(ExpandFileName(OutputPath)))) then - raise Exception.CreateFmt('Cannot create report directory: %s', - [ExtractFileDir(ExpandFileName(OutputPath))]); - - CaseDirectory := CreateCaseDirectory; - CasePaths := nil; - TGarbageCollector.Initialize; - PinPrimitiveSingletons; - try - CasePaths := PrepareCases(ExpandFileName(ManifestPath), Revision, - CaseDirectory); - Arguments := TStringList.Create; - try - Arguments.Add(CaseDirectory); - Arguments.Add(StringifySuite); - Arguments.Add('--jobs=' + IntToStr(Jobs)); - Arguments.Add('--timeout=' + IntToStr(TimeoutMilliseconds)); - Arguments.Add('--silent'); - Arguments.Add('--no-progress'); - if OutputPath <> '' then - Arguments.Add('--output=' + ExpandFileName(OutputPath)); - ProcessResult := RunComplianceProcess(TestRunner, Arguments, 0); - finally - Arguments.Free; - end; - finally - TGarbageCollector.Shutdown; - RemoveCaseDirectory(CaseDirectory, CasePaths); - CasePaths.Free; - end; - if not ProcessResult.Started then - raise Exception.Create('Cannot start GocciaTestRunner: ' + - ProcessResult.ErrorMessage); - if ProcessResult.StdOutText <> '' then - Write(TrimRight(ProcessResult.StdOutText), LineEnding); - if ProcessResult.StdErrText <> '' then - Write(TrimRight(ProcessResult.StdErrText), LineEnding); - Result := ProcessResult.ExitCode; -end; - -begin - try - if HasCommandLineFlag('help') then - begin - PrintUsage; - ExitCode := 0; - end - else if HasCommandLineFlag('regenerate-manifest') then - ExitCode := RegenerateManifest - else - ExitCode := RunCoordinator; - except - on E: Exception do - begin - WriteLn('Error: ', E.Message); - ExitCode := 2; - end; - end; -end. diff --git a/tests/built-ins/JSON5/upstream-parse.js b/tests/built-ins/JSON5/upstream-parse.js new file mode 100644 index 000000000..c1de9504f --- /dev/null +++ b/tests/built-ins/JSON5/upstream-parse.js @@ -0,0 +1,41 @@ +// Generated by scripts/regenerate-json5-tests.js. +// Upstream json5/json5 revision: b935d4a280eafa8835e6182551b63809e61243b0 +import * as JSON5 from "goccia:json5"; + +const cases = [{"id":"parses empty objects","valid":true,"source":"{}","expected":{"type":"object","entries":[]}},{"id":"parses double string property names","valid":true,"source":"{\"a\":1}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"number","value":"1"}}]}},{"id":"parses single string property names","valid":true,"source":"{'a':1}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"number","value":"1"}}]}},{"id":"parses unquoted property names","valid":true,"source":"{a:1}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"number","value":"1"}}]}},{"id":"parses special character property names","valid":true,"source":"{$_:1,_$:2,a‌:3}","expected":{"type":"object","entries":[{"key":"$_","value":{"type":"number","value":"1"}},{"key":"_$","value":{"type":"number","value":"2"}},{"key":"a‌","value":{"type":"number","value":"3"}}]}},{"id":"parses unicode property names","valid":true,"source":"{ùńîċõďë:9}","expected":{"type":"object","entries":[{"key":"ùńîċõďë","value":{"type":"number","value":"9"}}]}},{"id":"parses escaped property names","valid":true,"source":"{\\u0061\\u0062:1,\\u0024\\u005F:2,\\u005F\\u0024:3}","expected":{"type":"object","entries":[{"key":"ab","value":{"type":"number","value":"1"}},{"key":"$_","value":{"type":"number","value":"2"}},{"key":"_$","value":{"type":"number","value":"3"}}]}},{"id":"preserves __proto__ property names","valid":true,"source":"{\"__proto__\":1}","expected":{"type":"object","entries":[{"key":"__proto__","value":{"type":"number","value":"1"}}]}},{"id":"parses multiple properties","valid":true,"source":"{abc:1,def:2}","expected":{"type":"object","entries":[{"key":"abc","value":{"type":"number","value":"1"}},{"key":"def","value":{"type":"number","value":"2"}}]}},{"id":"parses nested objects","valid":true,"source":"{a:{b:2}}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"object","entries":[{"key":"b","value":{"type":"number","value":"2"}}]}}]}},{"id":"parses empty arrays","valid":true,"source":"[]","expected":{"type":"array","items":[]}},{"id":"parses array values","valid":true,"source":"[1]","expected":{"type":"array","items":[{"type":"number","value":"1"}]}},{"id":"parses multiple array values","valid":true,"source":"[1,2]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"2"}]}},{"id":"parses nested arrays","valid":true,"source":"[1,[2,3]]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"array","items":[{"type":"number","value":"2"},{"type":"number","value":"3"}]}]}},{"id":"parses nulls","valid":true,"source":"null","expected":{"type":"null"}},{"id":"parses true","valid":true,"source":"true","expected":{"type":"boolean","value":true}},{"id":"parses false","valid":true,"source":"false","expected":{"type":"boolean","value":false}},{"id":"parses leading zeroes","valid":true,"source":"[0,0.,0e0]","expected":{"type":"array","items":[{"type":"number","value":"0"},{"type":"number","value":"0"},{"type":"number","value":"0"}]}},{"id":"parses integers","valid":true,"source":"[1,23,456,7890]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"23"},{"type":"number","value":"456"},{"type":"number","value":"7890"}]}},{"id":"parses signed numbers","valid":true,"source":"[-1,+2,-.1,-0]","expected":{"type":"array","items":[{"type":"number","value":"-1"},{"type":"number","value":"2"},{"type":"number","value":"-0.1"},{"type":"number","value":"-0"}]}},{"id":"parses leading decimal points","valid":true,"source":"[.1,.23]","expected":{"type":"array","items":[{"type":"number","value":"0.1"},{"type":"number","value":"0.23"}]}},{"id":"parses fractional numbers","valid":true,"source":"[1.0,1.23]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"1.23"}]}},{"id":"parses exponents","valid":true,"source":"[1e0,1e1,1e01,1.e0,1.1e0,1e-1,1e+1]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"10"},{"type":"number","value":"10"},{"type":"number","value":"1"},{"type":"number","value":"1.1"},{"type":"number","value":"0.1"},{"type":"number","value":"10"}]}},{"id":"parses hexadecimal numbers","valid":true,"source":"[0x1,0x10,0xff,0xFF]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"16"},{"type":"number","value":"255"},{"type":"number","value":"255"}]}},{"id":"parses signed and unsigned Infinity","valid":true,"source":"[Infinity,-Infinity]","expected":{"type":"array","items":[{"type":"number","value":"Infinity"},{"type":"number","value":"-Infinity"}]}},{"id":"parses NaN","valid":true,"source":"NaN","expected":{"type":"number","value":"NaN"}},{"id":"parses signed NaN","valid":true,"source":"-NaN","expected":{"type":"number","value":"NaN"}},{"id":"parses 1","valid":true,"source":"1","expected":{"type":"number","value":"1"}},{"id":"parses +1.23e100","valid":true,"source":"+1.23e100","expected":{"type":"number","value":"1.23e+100"}},{"id":"parses bare hexadecimal number","valid":true,"source":"0x1","expected":{"type":"number","value":"1"}},{"id":"parses bare long hexadecimal number","valid":true,"source":"-0x0123456789abcdefABCDEF","expected":{"type":"number","value":"-1.3754889325393114e+24"}},{"id":"parses double quoted strings","valid":true,"source":"\"abc\"","expected":{"type":"string","value":"abc"}},{"id":"parses single quoted strings","valid":true,"source":"'abc'","expected":{"type":"string","value":"abc"}},{"id":"parses quotes in strings","valid":true,"source":"['\"',\"'\"]","expected":{"type":"array","items":[{"type":"string","value":"\""},{"type":"string","value":"'"}]}},{"id":"parses escaped characters","valid":true,"source":"'\\b\\f\\n\\r\\t\\v\\0\\x0f\\u01fF\\\n\\\r\n\\\r\\\u2028\\\u2029\\a\\'\\\"'","expected":{"type":"string","value":"\b\f\n\r\t\u000b\u0000\u000fǿa'\""}},{"id":"assert.deepStrictEqual","valid":true,"source":"'\u2028\u2029'","expected":{"type":"string","value":"\u2028\u2029"}},{"id":"parses single-line comments","valid":true,"source":"{//comment\n}","expected":{"type":"object","entries":[]}},{"id":"parses single-line comments at end of input","valid":true,"source":"{}//comment","expected":{"type":"object","entries":[]}},{"id":"parses multi-line comments","valid":true,"source":"{/*comment\n** */}","expected":{"type":"object","entries":[]}},{"id":"parses whitespace","valid":true,"source":"{\t\u000b\f  \n\r\u2028\u2029 }","expected":{"type":"object","entries":[]}},{"id":"throws on empty documents","valid":false,"source":""},{"id":"throws on documents with only comments","valid":false,"source":"//a"},{"id":"throws on incomplete single line comments","valid":false,"source":"/a"},{"id":"throws on unterminated multiline comments","valid":false,"source":"/*"},{"id":"throws on unterminated multiline comment closings","valid":false,"source":"/**"},{"id":"throws on invalid characters in values","valid":false,"source":"a"},{"id":"throws on invalid characters in identifier start escapes","valid":false,"source":"{\\a:1}"},{"id":"throws on invalid identifier start characters","valid":false,"source":"{\\u0021:1}"},{"id":"throws on invalid characters in identifier continue escapes","valid":false,"source":"{a\\a:1}"},{"id":"throws on invalid identifier continue characters","valid":false,"source":"{a\\u0021:1}"},{"id":"throws on invalid characters following a sign","valid":false,"source":"-a"},{"id":"throws on invalid characters following a leading decimal point","valid":false,"source":".a"},{"id":"throws on invalid characters following an exponent indicator","valid":false,"source":"1ea"},{"id":"throws on invalid characters following an exponent sign","valid":false,"source":"1e-a"},{"id":"throws on invalid characters following a hexadecimal indicator","valid":false,"source":"0xg"},{"id":"throws on invalid new lines in strings","valid":false,"source":"\"\n\""},{"id":"throws on unterminated strings","valid":false,"source":"\""},{"id":"throws on invalid identifier start characters in property names","valid":false,"source":"{!:1}"},{"id":"throws on invalid characters following a property name","valid":false,"source":"{a!1}"},{"id":"throws on invalid characters following a property value","valid":false,"source":"{a:1!}"},{"id":"throws on invalid characters following an array value","valid":false,"source":"[1!]"},{"id":"throws on invalid characters in literals","valid":false,"source":"tru!"},{"id":"throws on unterminated escapes","valid":false,"source":"\"\\"},{"id":"throws on invalid first digits in hexadecimal escapes","valid":false,"source":"\"\\xg\""},{"id":"throws on invalid second digits in hexadecimal escapes","valid":false,"source":"\"\\x0g\""},{"id":"throws on invalid unicode escapes","valid":false,"source":"\"\\u000g\""},{"id":"throws on escaped digit 1","valid":false,"source":"'\\1'"},{"id":"throws on escaped digit 2","valid":false,"source":"'\\2'"},{"id":"throws on escaped digit 3","valid":false,"source":"'\\3'"},{"id":"throws on escaped digit 4","valid":false,"source":"'\\4'"},{"id":"throws on escaped digit 5","valid":false,"source":"'\\5'"},{"id":"throws on escaped digit 6","valid":false,"source":"'\\6'"},{"id":"throws on escaped digit 7","valid":false,"source":"'\\7'"},{"id":"throws on escaped digit 8","valid":false,"source":"'\\8'"},{"id":"throws on escaped digit 9","valid":false,"source":"'\\9'"},{"id":"throws on octal escapes","valid":false,"source":"'\\01'"},{"id":"throws on multiple values","valid":false,"source":"1 2"},{"id":"throws with control characters escaped in the message","valid":false,"source":"\u0001"},{"id":"throws on unclosed objects before property names","valid":false,"source":"{"},{"id":"throws on unclosed objects after property names","valid":false,"source":"{a"},{"id":"throws on unclosed objects before property values","valid":false,"source":"{a:"},{"id":"throws on unclosed objects after property values","valid":false,"source":"{a:1"},{"id":"throws on unclosed arrays before values","valid":false,"source":"["},{"id":"throws on unclosed arrays after values","valid":false,"source":"[1"}]; + +const encode = (value) => { + if (value === null) return { type: "null" }; + if (typeof value === "boolean") return { type: "boolean", value }; + if (typeof value === "string") return { type: "string", value }; + if (typeof value === "number") { + if (Number.isNaN(value)) return { type: "number", value: "NaN" }; + if (value === Infinity) return { type: "number", value: "Infinity" }; + if (value === -Infinity) return { type: "number", value: "-Infinity" }; + if (Object.is(value, -0)) return { type: "number", value: "-0" }; + return { type: "number", value: String(value) }; + } + if (Array.isArray(value)) { + return { type: "array", items: value.map((item) => encode(item)) }; + } + return { + type: "object", + entries: Object.getOwnPropertyNames(value).map((key) => ({ + key, + value: encode(value[key]), + })), + }; +}; + +for (const testCase of cases) { + test(testCase.id, () => { + if (!testCase.valid) { + expect(() => JSON5.parse(testCase.source)).toThrow(SyntaxError); + return; + } + const actual = JSON5.parse(testCase.source); + expect(JSON.stringify(encode(actual))).toBe( + JSON.stringify(testCase.expected), + ); + }); +} diff --git a/tests/compliance/json5-manifest.json b/tests/compliance/json5-manifest.json deleted file mode 100644 index a62c36a02..000000000 --- a/tests/compliance/json5-manifest.json +++ /dev/null @@ -1 +0,0 @@ -{"manifestVersion":1,"suite":"json5","revision":"b935d4a280eafa8835e6182551b63809e61243b0","cases":[{"id":"parses empty objects","valid":true,"source":"{}","expected":{"type":"object","entries":[]}},{"id":"parses double string property names","valid":true,"source":"{\"a\":1}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"number","value":"1"}}]}},{"id":"parses single string property names","valid":true,"source":"{'a':1}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"number","value":"1"}}]}},{"id":"parses unquoted property names","valid":true,"source":"{a:1}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"number","value":"1"}}]}},{"id":"parses special character property names","valid":true,"source":"{$_:1,_$:2,a‌:3}","expected":{"type":"object","entries":[{"key":"$_","value":{"type":"number","value":"1"}},{"key":"_$","value":{"type":"number","value":"2"}},{"key":"a‌","value":{"type":"number","value":"3"}}]}},{"id":"parses unicode property names","valid":true,"source":"{ùńîċõďë:9}","expected":{"type":"object","entries":[{"key":"ùńîċõďë","value":{"type":"number","value":"9"}}]}},{"id":"parses escaped property names","valid":true,"source":"{\\u0061\\u0062:1,\\u0024\\u005F:2,\\u005F\\u0024:3}","expected":{"type":"object","entries":[{"key":"ab","value":{"type":"number","value":"1"}},{"key":"$_","value":{"type":"number","value":"2"}},{"key":"_$","value":{"type":"number","value":"3"}}]}},{"id":"preserves __proto__ property names","valid":true,"source":"{\"__proto__\":1}","expected":{"type":"object","entries":[{"key":"__proto__","value":{"type":"number","value":"1"}}]}},{"id":"parses multiple properties","valid":true,"source":"{abc:1,def:2}","expected":{"type":"object","entries":[{"key":"abc","value":{"type":"number","value":"1"}},{"key":"def","value":{"type":"number","value":"2"}}]}},{"id":"parses nested objects","valid":true,"source":"{a:{b:2}}","expected":{"type":"object","entries":[{"key":"a","value":{"type":"object","entries":[{"key":"b","value":{"type":"number","value":"2"}}]}}]}},{"id":"parses empty arrays","valid":true,"source":"[]","expected":{"type":"array","items":[]}},{"id":"parses array values","valid":true,"source":"[1]","expected":{"type":"array","items":[{"type":"number","value":"1"}]}},{"id":"parses multiple array values","valid":true,"source":"[1,2]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"2"}]}},{"id":"parses nested arrays","valid":true,"source":"[1,[2,3]]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"array","items":[{"type":"number","value":"2"},{"type":"number","value":"3"}]}]}},{"id":"parses nulls","valid":true,"source":"null","expected":{"type":"null"}},{"id":"parses true","valid":true,"source":"true","expected":{"type":"boolean","value":true}},{"id":"parses false","valid":true,"source":"false","expected":{"type":"boolean","value":false}},{"id":"parses leading zeroes","valid":true,"source":"[0,0.,0e0]","expected":{"type":"array","items":[{"type":"number","value":"0"},{"type":"number","value":"0"},{"type":"number","value":"0"}]}},{"id":"parses integers","valid":true,"source":"[1,23,456,7890]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"23"},{"type":"number","value":"456"},{"type":"number","value":"7890"}]}},{"id":"parses signed numbers","valid":true,"source":"[-1,+2,-.1,-0]","expected":{"type":"array","items":[{"type":"number","value":"-1"},{"type":"number","value":"2"},{"type":"number","value":"-0.1"},{"type":"number","value":"-0"}]}},{"id":"parses leading decimal points","valid":true,"source":"[.1,.23]","expected":{"type":"array","items":[{"type":"number","value":"0.1"},{"type":"number","value":"0.23"}]}},{"id":"parses fractional numbers","valid":true,"source":"[1.0,1.23]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"1.23"}]}},{"id":"parses exponents","valid":true,"source":"[1e0,1e1,1e01,1.e0,1.1e0,1e-1,1e+1]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"10"},{"type":"number","value":"10"},{"type":"number","value":"1"},{"type":"number","value":"1.1"},{"type":"number","value":"0.1"},{"type":"number","value":"10"}]}},{"id":"parses hexadecimal numbers","valid":true,"source":"[0x1,0x10,0xff,0xFF]","expected":{"type":"array","items":[{"type":"number","value":"1"},{"type":"number","value":"16"},{"type":"number","value":"255"},{"type":"number","value":"255"}]}},{"id":"parses signed and unsigned Infinity","valid":true,"source":"[Infinity,-Infinity]","expected":{"type":"array","items":[{"type":"number","value":"Infinity"},{"type":"number","value":"-Infinity"}]}},{"id":"parses NaN","valid":true,"source":"NaN","expected":{"type":"number","value":"NaN"}},{"id":"parses signed NaN","valid":true,"source":"-NaN","expected":{"type":"number","value":"NaN"}},{"id":"parses 1","valid":true,"source":"1","expected":{"type":"number","value":"1"}},{"id":"parses +1.23e100","valid":true,"source":"+1.23e100","expected":{"type":"number","value":"1.23e+100"}},{"id":"parses bare hexadecimal number","valid":true,"source":"0x1","expected":{"type":"number","value":"1"}},{"id":"parses bare long hexadecimal number","valid":true,"source":"-0x0123456789abcdefABCDEF","expected":{"type":"number","value":"-1.3754889325393114e+24"}},{"id":"parses double quoted strings","valid":true,"source":"\"abc\"","expected":{"type":"string","value":"abc"}},{"id":"parses single quoted strings","valid":true,"source":"'abc'","expected":{"type":"string","value":"abc"}},{"id":"parses quotes in strings","valid":true,"source":"['\"',\"'\"]","expected":{"type":"array","items":[{"type":"string","value":"\""},{"type":"string","value":"'"}]}},{"id":"parses escaped characters","valid":true,"source":"'\\b\\f\\n\\r\\t\\v\\0\\x0f\\u01fF\\\n\\\r\n\\\r\\
\\
\\a\\'\\\"'","expected":{"type":"string","value":"\b\f\n\r\t\u000b\u0000\u000fǿa'\""}},{"id":"assert.deepStrictEqual","valid":true,"source":"'

'","expected":{"type":"string","value":"

"}},{"id":"parses single-line comments","valid":true,"source":"{//comment\n}","expected":{"type":"object","entries":[]}},{"id":"parses single-line comments at end of input","valid":true,"source":"{}//comment","expected":{"type":"object","entries":[]}},{"id":"parses multi-line comments","valid":true,"source":"{/*comment\n** */}","expected":{"type":"object","entries":[]}},{"id":"parses whitespace","valid":true,"source":"{\t\u000b\f  \n\r

 }","expected":{"type":"object","entries":[]}},{"id":"throws on empty documents","valid":false,"source":""},{"id":"throws on documents with only comments","valid":false,"source":"//a"},{"id":"throws on incomplete single line comments","valid":false,"source":"/a"},{"id":"throws on unterminated multiline comments","valid":false,"source":"/*"},{"id":"throws on unterminated multiline comment closings","valid":false,"source":"/**"},{"id":"throws on invalid characters in values","valid":false,"source":"a"},{"id":"throws on invalid characters in identifier start escapes","valid":false,"source":"{\\a:1}"},{"id":"throws on invalid identifier start characters","valid":false,"source":"{\\u0021:1}"},{"id":"throws on invalid characters in identifier continue escapes","valid":false,"source":"{a\\a:1}"},{"id":"throws on invalid identifier continue characters","valid":false,"source":"{a\\u0021:1}"},{"id":"throws on invalid characters following a sign","valid":false,"source":"-a"},{"id":"throws on invalid characters following a leading decimal point","valid":false,"source":".a"},{"id":"throws on invalid characters following an exponent indicator","valid":false,"source":"1ea"},{"id":"throws on invalid characters following an exponent sign","valid":false,"source":"1e-a"},{"id":"throws on invalid characters following a hexadecimal indicator","valid":false,"source":"0xg"},{"id":"throws on invalid new lines in strings","valid":false,"source":"\"\n\""},{"id":"throws on unterminated strings","valid":false,"source":"\""},{"id":"throws on invalid identifier start characters in property names","valid":false,"source":"{!:1}"},{"id":"throws on invalid characters following a property name","valid":false,"source":"{a!1}"},{"id":"throws on invalid characters following a property value","valid":false,"source":"{a:1!}"},{"id":"throws on invalid characters following an array value","valid":false,"source":"[1!]"},{"id":"throws on invalid characters in literals","valid":false,"source":"tru!"},{"id":"throws on unterminated escapes","valid":false,"source":"\"\\"},{"id":"throws on invalid first digits in hexadecimal escapes","valid":false,"source":"\"\\xg\""},{"id":"throws on invalid second digits in hexadecimal escapes","valid":false,"source":"\"\\x0g\""},{"id":"throws on invalid unicode escapes","valid":false,"source":"\"\\u000g\""},{"id":"throws on escaped digit 1","valid":false,"source":"'\\1'"},{"id":"throws on escaped digit 2","valid":false,"source":"'\\2'"},{"id":"throws on escaped digit 3","valid":false,"source":"'\\3'"},{"id":"throws on escaped digit 4","valid":false,"source":"'\\4'"},{"id":"throws on escaped digit 5","valid":false,"source":"'\\5'"},{"id":"throws on escaped digit 6","valid":false,"source":"'\\6'"},{"id":"throws on escaped digit 7","valid":false,"source":"'\\7'"},{"id":"throws on escaped digit 8","valid":false,"source":"'\\8'"},{"id":"throws on escaped digit 9","valid":false,"source":"'\\9'"},{"id":"throws on octal escapes","valid":false,"source":"'\\01'"},{"id":"throws on multiple values","valid":false,"source":"1 2"},{"id":"throws with control characters escaped in the message","valid":false,"source":"\u0001"},{"id":"throws on unclosed objects before property names","valid":false,"source":"{"},{"id":"throws on unclosed objects after property names","valid":false,"source":"{a"},{"id":"throws on unclosed objects before property values","valid":false,"source":"{a:"},{"id":"throws on unclosed objects after property values","valid":false,"source":"{a:1"},{"id":"throws on unclosed arrays before values","valid":false,"source":"["},{"id":"throws on unclosed arrays after values","valid":false,"source":"[1"}]} From 4b3cd2e608b3e1d5ba4a41f870934f580edf1c34 Mon Sep 17 00:00:00 2001 From: Johannes Stein Date: Fri, 24 Jul 2026 20:24:33 +0100 Subject: [PATCH 4/5] fix(compliance): classify TOML runner failures Fail CI immediately on suite preparation errors, distinguish parser rejections from infrastructure failures, and reject array extension headers without crashing the TOML parser. --- .github/workflows/ci.yml | 2 ++ .../app/compliance/GocciaTOMLComplianceRunner.dpr | 15 +++++++++++++-- source/units/Goccia.TOML.pas | 7 ++++--- tests/built-ins/TOML/parse.js | 15 +++++++++++++++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ce857e84..b52a53d06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -411,6 +411,7 @@ jobs: - name: Run TOML 1.1.0 compliance suite run: | + set -euo pipefail pin="$(tr -d '\r\n' < tests/compliance/toml-test.pin)" suite_dir="$RUNNER_TEMP/toml-test" git init "$suite_dir" @@ -493,6 +494,7 @@ jobs: - name: Run JSON5 compliance suite run: | + set -euo pipefail pin="$(tr -d '\r\n' < tests/compliance/json5.pin)" grep -Fq "Upstream json5/json5 revision: $pin" \ tests/built-ins/JSON5/upstream-parse.js diff --git a/source/app/compliance/GocciaTOMLComplianceRunner.dpr b/source/app/compliance/GocciaTOMLComplianceRunner.dpr index b06c3a3b8..53c941273 100644 --- a/source/app/compliance/GocciaTOMLComplianceRunner.dpr +++ b/source/app/compliance/GocciaTOMLComplianceRunner.dpr @@ -426,16 +426,21 @@ begin Root.Free; end; except - on E: EGocciaTOMLParseError do + on E: EConvertError do begin WriteLn(E.Message); Halt(1); end; - on E: Exception do + on E: EGocciaTOMLParseError do begin WriteLn(E.Message); Halt(1); end; + on E: Exception do + begin + WriteLn(E.ClassName, ': ', E.Message); + Halt(2); + end; end; finally Parser.Free; @@ -480,6 +485,12 @@ begin ReadOptionValue('output', OutputPath); Cases := DiscoverCases(SuiteDirectory); + if Cases.Count = 0 then + begin + Cases.Free; + raise Exception.CreateFmt('No TOML cases discovered in %s', + [SuiteDirectory]); + end; Executor := TTOMLComplianceExecutor.Create(ExpandFileName(ParamStr(0)), SuiteDirectory, TimeoutMilliseconds); Coordinator := TComplianceCoordinator.Create(Executor); diff --git a/source/units/Goccia.TOML.pas b/source/units/Goccia.TOML.pas index eefbbea31..4fe8ea7cf 100644 --- a/source/units/Goccia.TOML.pas +++ b/source/units/Goccia.TOML.pas @@ -932,7 +932,8 @@ function TGocciaTOMLParser.EnsureDottedKeyTable( end; case Existing.Kind of - tnkScalar: + tnkScalar, + tnkArray: RaiseParseError(Format( 'Cannot redefine "%s" as a table after assigning it a value.', [AKey])); @@ -1012,7 +1013,7 @@ function TGocciaTOMLParser.ParseRegularTable( I = Length(APath) - 1, I <> Length(APath) - 1, False, False); AttachChild(Context, APath[I], Existing); end - else if Existing.Kind = tnkScalar then + else if Existing.Kind in [tnkScalar, tnkArray] then RaiseParseError(Format( 'Cannot redefine "%s" as a table after assigning it a value.', [JoinPathPrefix(APath, I + 1)])) @@ -1071,7 +1072,7 @@ function TGocciaTOMLParser.ParseTableArray( False, False); AttachChild(Context, APath[I], Existing); end - else if Existing.Kind = tnkScalar then + else if Existing.Kind in [tnkScalar, tnkArray] then RaiseParseError(Format( 'Cannot redefine "%s" as a table after assigning it a value.', [JoinPathPrefix(APath, I + 1)])) diff --git a/tests/built-ins/TOML/parse.js b/tests/built-ins/TOML/parse.js index 6b9d28e57..39f599555 100644 --- a/tests/built-ins/TOML/parse.js +++ b/tests/built-ins/TOML/parse.js @@ -132,6 +132,21 @@ fruit.apple.smooth = true TOML.parse(` point = { x = 1 } point.y = 2 +`), + ).toThrow(SyntaxError); + + expect(() => + TOML.parse(` +items = [{ name = "first" }] +[items.metadata] +enabled = true +`), + ).toThrow(SyntaxError); + + expect(() => + TOML.parse(` +items = [] +[[items.metadata]] `), ).toThrow(SyntaxError); }); From 820f3c3fd40a1cb490ecf2abf23d41d121fb4224 Mon Sep 17 00:00:00 2001 From: Johannes Stein Date: Fri, 24 Jul 2026 23:45:25 +0100 Subject: [PATCH 5/5] test(toml): cover dotted keys after arrays --- tests/built-ins/TOML/parse.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/built-ins/TOML/parse.js b/tests/built-ins/TOML/parse.js index 39f599555..24dcd4050 100644 --- a/tests/built-ins/TOML/parse.js +++ b/tests/built-ins/TOML/parse.js @@ -147,6 +147,13 @@ enabled = true TOML.parse(` items = [] [[items.metadata]] +`), + ).toThrow(SyntaxError); + + expect(() => + TOML.parse(` +items = [] +items.metadata = true `), ).toThrow(SyntaxError); });