feat(gate): Pyodide differential parity gate (Phase 2)

.github/workflows/gate.yml

@@ -0,0 +1,60 @@
+name: Pyodide Gate
+
+# Differential parity gate: the transpiler in real wasm32 Pyodide must match
+# native CPython on every corpus input (success + failure). Heavier than the
+# unit matrix (loads Pyodide), so: on demand, nightly, and on PRs that touch the
+# gate itself. It does NOT block releases yet — that wiring is Phase 3.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "27 5 * * *"   # nightly 05:27 UTC
+  pull_request:
+    paths:
+      - "gate/**"
+      - "tests/gate-corpus/**"
+      - "package.json"
+      - "package-lock.json"
+      - ".github/workflows/gate.yml"
+      - "pineforge_codegen/**"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: gate-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  gate:
+    runs-on: ubuntu-latest
+    env:
+      PYTHONHASHSEED: "0"
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.14"
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4
+        with:
+          node-version: "22"
+
+      - name: Install codegen (native oracle) + node deps
+        run: |
+          python -m pip install -e .
+          npm ci
+
+      - name: Selftest (comparator catches divergences)
+        run: npm run gate:selftest
+
+      - name: Run differential gate (full corpus)
+        run: npm run gate:full
+
+      - name: Upload release.json (derived versions, for Phase 3 handoff)
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
+        with:
+          name: release-json
+          path: release.json
+          if-no-files-found: ignore

.gitignore

@@ -56,8 +56,6 @@ pineforge_codegen/**/*.cpp
 # ─── Node / TypeScript ─────────────────────────────────────────────────────
 node_modules/
 **/node_modules/
-package-lock.json
-**/package-lock.json
 yarn.lock
 pnpm-lock.yaml
 npm-debug.log*
@@ -120,3 +118,6 @@ temp/
 # cloud/mcp-local/ was extracted to a public repo:
 #   https://github.com/fullpass-4pass/pineforge-codegen-mcp
 # Don't bring it back here.
+
+# ─── Gate output ───────────────────────────────────────────────────────────
+/release.json

PYODIDE_TARGET

@@ -0,0 +1 @@
+314.0.0

gate/compare.mjs

@@ -0,0 +1,29 @@
+// Pure, side-effect-free comparator for the gate. Both run-gate.mjs (the runner)
+// and selftest.mjs (the canary) import this — so importing it never triggers a
+// gate run. Each side is {json: "<transpile_json string>"} (normal return) or
+// {unexpected: "Type: msg"} (non-CompileError exception). Returns a mismatch
+// string, or null if the two sides agree.
+export function compareResults(name, native, browser) {
+  if (!native) return `${name}: oracle produced no result`;
+  if (!browser) return `${name}: pyodide produced no result`;
+  if (native.unexpected || browser.unexpected) {
+    if (native.unexpected !== browser.unexpected) {
+      return `${name}: unexpected-exception mismatch\n  native : ${native.unexpected ?? "<none>"}\n  pyodide: ${browser.unexpected ?? "<none>"}`;
+    }
+    return null;
+  }
+  if (native.json !== browser.json) {
+    let detail = "";
+    try {
+      const n = JSON.parse(native.json);
+      const b = JSON.parse(browser.json);
+      if (n.ok !== b.ok) detail = `verdict ${b.ok} (pyodide) != ${n.ok} (native)`;
+      else if (n.ok) detail = "C++ output differs";
+      else detail = `error/diagnostics differ\n  native : ${native.json}\n  pyodide: ${browser.json}`;
+    } catch {
+      detail = `raw json differs\n  native : ${native.json}\n  pyodide: ${browser.json}`;
+    }
+    return `${name}: ${detail}`;
+  }
+  return null;
+}

gate/glue.py

@@ -0,0 +1,35 @@
+# CANONICAL GLUE — runtime-equivalent to the body of PY_GLUE in
+# pineforge-app/apps/web/lib/pyodide-transpiler/glue.ts (produces identical
+# transpile_json output). The browser worker and this gate run the same logic,
+# so the gate's parity guarantee reflects shipped behavior.
+# (Phase 3: ship this from the npm package so there is one source of truth.)
+import json
+import sys
+
+if "/codegen" not in sys.path:
+    sys.path.insert(0, "/codegen")
+
+from pineforge_codegen import transpile
+from pineforge_codegen.errors import CompileError
+
+
+def transpile_json(source: str) -> str:
+    try:
+        cpp = transpile(source)
+    except CompileError as e:
+        diags = []
+        for d in e.diagnostics:
+            loc = d.location
+            message = d.message + " — " + d.hint if getattr(d, "hint", None) else d.message
+            entry = {
+                "line": loc.line if loc else 1,
+                "col": loc.col if loc else 1,
+                "message": message,
+                "severity": getattr(d.level, "value", "error"),
+            }
+            end_col = getattr(loc, "end_col", None) if loc else None
+            if end_col is not None:
+                entry["endCol"] = end_col
+            diags.append(entry)
+        return json.dumps({"ok": False, "error": str(e), "diagnostics": diags})
+    return json.dumps({"ok": True, "cpp": cpp})

gate/oracle.py

@@ -0,0 +1,34 @@
+"""Native-CPython oracle for the conformance gate.
+
+Reads a JSON array of {"name","src"} from stdin, runs the CANONICAL glue's
+transpile_json on each, and writes a JSON object {name: result} to stdout.
+Each result is {"json": <transpile_json string>} on a normal return, or
+{"unexpected": "<ExcType>: <msg>"} if transpile_json raised something other
+than CompileError (CompileError is already encoded inside the json string).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+GLUE = os.path.join(os.path.dirname(__file__), "glue.py")
+exec(compile(open(GLUE).read(), GLUE, "exec"))  # defines transpile_json  # noqa: S102
+
+
+def main() -> None:
+    items = json.load(sys.stdin)
+    out: dict[str, dict] = {}
+    for item in items:
+        name = item["name"]
+        src = item["src"]
+        try:
+            out[name] = {"json": transpile_json(src)}  # noqa: F821 — from exec
+        except Exception as exc:  # noqa: BLE001 — capture for parity, don't throw
+            out[name] = {"unexpected": f"{type(exc).__name__}: {exc}"}
+    json.dump(out, sys.stdout)
+
+
+if __name__ == "__main__":
+    main()

gate/run-gate.mjs

@@ -0,0 +1,119 @@
+// Differential parity gate: run the transpiler in real wasm32 Pyodide and assert
+// it behaves identically to native CPython on EVERY corpus input — success
+// (byte-identical cpp) AND failure (same verdict/error/diagnostics + same
+// unexpected-exception type+message). Also writes release.json.
+//
+//   node gate/run-gate.mjs              # smoke: all err + first N ok fixtures
+//   GATE_FULL=1 node gate/run-gate.mjs  # entire corpus
+//
+// Exit 0 = parity holds; exit 1 = mismatch(es) or setup failure.
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { compareResults } from "./compare.mjs";
+
+const require = createRequire(import.meta.url);
+const HERE = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(HERE, "..");
+const CORPUS = join(ROOT, "tests", "gate-corpus");
+const SCRATCH = join(HERE, ".scratch");
+const GLUE = readFileSync(join(HERE, "glue.py"), "utf8");
+const PYTHON = process.env.GATE_PYTHON ?? "python3";
+const OK_CAP = process.env.GATE_FULL ? Infinity : 60;
+
+function branchItems(branch) {
+  const dir = join(CORPUS, branch);
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((x) => x.endsWith(".pine"))
+    .sort()
+    .map((f) => ({ name: `${branch}/${f}`, src: readFileSync(join(dir, f), "utf8") }));
+}
+
+async function main() {
+  const okItems = branchItems("ok");
+  const errItems = branchItems("err");
+  if (okItems.length + errItems.length === 0) {
+    console.error("gate: empty corpus at tests/gate-corpus/{ok,err}");
+    process.exit(1);
+  }
+  // ALWAYS include the (small) err branch so the smoke run exercises the failure
+  // path; cap only the large ok branch.
+  const items = [
+    ...(Number.isFinite(OK_CAP) ? okItems.slice(0, OK_CAP) : okItems),
+    ...errItems,
+  ];
+  console.log(
+    `gate: ${items.length} fixtures (ok=${Math.min(okItems.length, OK_CAP)}/${okItems.length}, err=${errItems.length}, GATE_FULL=${process.env.GATE_FULL ? "1" : "0"})`,
+  );
+
+  // 1. Pack the in-repo source into an archive (excluding __pycache__).
+  mkdirSync(SCRATCH, { recursive: true });
+  const archive = join(SCRATCH, "pineforge_codegen.tar.gz");
+  const tar = require("tar");
+  await tar.create(
+    { gzip: true, file: archive, cwd: ROOT, portable: true, filter: (p) => !p.includes("__pycache__") },
+    ["pineforge_codegen"],
+  );
+  const archiveBytes = readFileSync(archive);
+
+  // 2. Load Pyodide, unpack, run glue.
+  const { loadPyodide } = await import("pyodide");
+  const indexURL = dirname(require.resolve("pyodide/package.json"));
+  const pyodide = await loadPyodide({ indexURL });
+  const u8 = new Uint8Array(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength);
+  pyodide.unpackArchive(u8, "gztar", { extractDir: "/codegen" });
+  pyodide.runPython(GLUE);
+  const transpileJson = pyodide.globals.get("transpile_json");
+
+  // 3. Native oracle (one process, whole corpus). PYTHONHASHSEED pinned for
+  //    determinism per spec §6.1.
+  const oracleOut = execFileSync(PYTHON, [join(HERE, "oracle.py")], {
+    input: JSON.stringify(items),
+    env: { ...process.env, PYTHONPATH: ROOT, PYTHONHASHSEED: "0" },
+    maxBuffer: 256 * 1024 * 1024,
+    encoding: "utf8",
+  });
+  const native = JSON.parse(oracleOut);
+
+  // 4. Pyodide side + compare.
+  const mismatches = [];
+  for (const { name, src } of items) {
+    let browser;
+    try {
+      browser = { json: transpileJson(src) };
+    } catch (err) {
+      browser = { unexpected: `${err?.constructor?.name ?? "Error"}: ${err?.message ?? String(err)}` };
+    }
+    const m = compareResults(name, native[name], browser);
+    if (m) mismatches.push(m);
+  }
+
+  // 5. release.json (versions derived from the loaded Pyodide lock).
+  const lock = require("pyodide/pyodide-lock.json");
+  const codegen = readFileSync(join(ROOT, "VERSION"), "utf8").trim();
+  const pyodideVer = readFileSync(join(ROOT, "PYODIDE_TARGET"), "utf8").trim();
+  const release = {
+    codegen,
+    pyodide: pyodideVer,
+    python: lock.info.python,
+    emscripten: lock.info.platform,
+    sha256: createHash("sha256").update(archiveBytes).digest("hex"),
+  };
+  writeFileSync(join(ROOT, "release.json"), JSON.stringify(release, null, 2) + "\n");
+  console.log("gate: release.json ->", JSON.stringify(release));
+
+  if (mismatches.length) {
+    console.error(`gate: ${mismatches.length} MISMATCH(es):\n` + mismatches.join("\n"));
+    process.exit(1);
+  }
+  console.log(`gate: PARITY OK over ${items.length} fixtures`);
+}
+
+main().catch((e) => {
+  console.error("gate: fatal", e);
+  process.exit(1);
+});

gate/selftest.mjs

@@ -0,0 +1,29 @@
+// Canary: prove the gate's comparator actually catches a divergence. Imports the
+// PURE comparator (gate/compare.mjs) so it runs in <1s without loading Pyodide.
+import { compareResults } from "./compare.mjs";
+
+const cases = [
+  // [name, native, browser, mustFlag]
+  ["same-ok", { json: '{"ok":true,"cpp":"X"}' }, { json: '{"ok":true,"cpp":"X"}' }, false],
+  ["cpp-differs", { json: '{"ok":true,"cpp":"X"}' }, { json: '{"ok":true,"cpp":"Y"}' }, true],
+  ["verdict-differs", { json: '{"ok":true,"cpp":"X"}' }, { json: '{"ok":false,"error":"e","diagnostics":[]}' }, true],
+  ["error-differs", { json: '{"ok":false,"error":"a","diagnostics":[]}' }, { json: '{"ok":false,"error":"b","diagnostics":[]}' }, true],
+  ["unexpected-one-side", { json: '{"ok":true,"cpp":"X"}' }, { unexpected: "TypeError: boom" }, true],
+  ["unexpected-both-same", { unexpected: "TypeError: boom" }, { unexpected: "TypeError: boom" }, false],
+  ["unexpected-both-diff", { unexpected: "TypeError: a" }, { unexpected: "ValueError: b" }, true],
+  ["missing-native", undefined, { json: '{"ok":true,"cpp":"X"}' }, true],
+];
+
+let failed = 0;
+for (const [name, n, b, mustFlag] of cases) {
+  const flagged = compareResults(name, n, b) !== null;
+  if (flagged !== mustFlag) {
+    console.error(`selftest FAIL: ${name} expected mustFlag=${mustFlag} got ${flagged}`);
+    failed++;
+  }
+}
+if (failed) {
+  console.error(`gate selftest: ${failed} case(s) failed`);
+  process.exit(1);
+}
+console.log(`gate selftest: ${cases.length} comparator cases OK`);