From 61dd07d5a20bbebfd9055a9562a2d3037a3b3846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Fri, 14 Aug 2026 23:44:10 +0800 Subject: [PATCH 01/13] docs: design WebGPU FP32 model support --- .../2026-08-14-webgpu-fp32-model-design.md | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-14-webgpu-fp32-model-design.md diff --git a/docs/superpowers/specs/2026-08-14-webgpu-fp32-model-design.md b/docs/superpowers/specs/2026-08-14-webgpu-fp32-model-design.md new file mode 100644 index 0000000..6a8c3c3 --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-webgpu-fp32-model-design.md @@ -0,0 +1,160 @@ +# WebGPU FP32 Model Design + +## Problem + +The published FP32 ONNX model works through WASM but cannot create an ONNX Runtime Web WebGPU session. A strict physical-browser run with the current model fails before inference with: + +```text +ONNX session-create failed for webgpu: Can't create a session. +transformer_memcpy.cc:254 ... Provider type for Cast node with name +'node__to_copy_4' is not set. +``` + +The model is not an FP64 model. Its weights, graph inputs, graph outputs, and intended execution precision are FP32. The exported graph contains four precomputed positional-encoding initializers named `sin`, `cos`, `sin_1`, and `cos_1`. Each initializer is DOUBLE with shape `[625, 64]`. `node_cat_7` concatenates them into a DOUBLE `[625, 256]` tensor, and `node__to_copy_4` immediately casts that tensor to FLOAT. + +ONNX Runtime Web 1.27.0 has no WebGPU DOUBLE tensor mapping. The DOUBLE-to-FLOAT Cast therefore receives no execution provider and triggers the internal assertion. WASM can execute the DOUBLE path. The FP16 conversion already rewrites the path to FP16/FLOAT and therefore creates a WebGPU session successfully. + +## Goals + +- Produce an immutable PP-DocLayoutV3 model version `1.0.1` whose FP32 graph contains no DOUBLE data path. +- Preserve the accepted FP32 numerical and detection behavior on all seven licensed fixtures. +- Validate the new FP32 artifact through both browser WASM and physical WebGPU without runtime fallback. +- Keep WebGPU FP16 as the automatic default when `shader-f16` is available. +- Allow explicit WebGPU FP32 selection after validation succeeds. +- Release the model assets independently before SDK `1.0.5` adopts the new default manifest. + +## Non-Goals + +- Do not overwrite or delete any `v1.0.0-models` asset. +- Do not change the existing `1.0.0` manifest or the behavior of SDK `1.0.4`. +- Do not claim FP64 inference support. +- Do not retrain or change learned model weights. +- Do not weaken existing numerical, detection, integrity, or browser acceptance thresholds. + +## Versioned Assets + +Create `models/pp-doclayoutv3/1.0.1/` with: + +- `model-fp32.onnx`: a deterministically sanitized FP32 graph; +- `model-fp16.onnx`: byte-identical to the accepted `1.0.0` FP16 artifact; +- `manifest.json`: a generated schema-version-1 manifest for model version `1.0.1`. + +Publish the files through an immutable `v1.0.1-models` GitHub Release. The new manifest uses release URLs under that tag and retains `minSdkVersion: "1.0.0"` because its schema and runtime contract remain compatible with existing SDK parsing. SDK `1.0.5` is the first release that selects this manifest by default. + +The manifest contains two variants: + +| Variant | Precision | Backends | Priority | +| --- | --- | --- | --- | +| `fp16` | FP16 | WebGPU | 1 | +| `fp32` | FP32 | WebGPU, WASM | 2 | + +`variantPriority` remains `["fp16", "fp32"]`. + +## Deterministic FP32 Sanitization + +Add a focused model-pipeline transform that reads the accepted `1.0.0/model-fp32.onnx` and writes the `1.0.1` FP32 artifact. The transform must: + +1. Run ONNX validation before changing the graph. +2. Require exactly four DOUBLE initializers named `sin`, `cos`, `sin_1`, and `cos_1`, each with shape `[625, 64]`. +3. Require those values to feed `node_cat_7`, followed by the known `node__to_copy_4` Cast to FLOAT. +4. Convert the four initializer payloads to FLOAT without changing their names or shapes. +5. Preserve the Cast unless ONNX optimization removes it deterministically; FLOAT-to-FLOAT is semantically redundant. +6. Reject any additional DOUBLE initializer, graph input, graph output, or inferred intermediate value. +7. Run ONNX checker and shape inference on the result. +8. Write the output atomically and report its byte size and SHA-256. + +Converting the constants early preserves the intended value: the existing graph already rounds each DOUBLE value to FLOAT immediately after concatenation. The transform remains guarded by numerical and browser validation rather than relying on this reasoning alone. + +## Validation Gates + +### Structural Gate + +- ONNX checker passes at opset 18. +- Input remains `pixel_values`, FLOAT `[1, 3, 800, 800]`. +- The four public outputs retain their current names, FLOAT types, and shapes. +- No DOUBLE initializer or inferred DOUBLE tensor remains. +- Learned parameter tensors are unchanged. + +### CPU and Detection Gate + +Run the existing seven-fixture FP32 validation against both the accepted `1.0.0` FP32 model and the sanitized `1.0.1` FP32 model. + +- Detection counts, label sequences, and reading order must match. +- No current coordinate, polygon, or score threshold may be relaxed. +- Raw outputs should be bit-identical. If execution-graph optimization prevents bit identity, the existing FP32 raw-output and detection thresholds remain the maximum accepted difference. +- The validation report records both source and candidate SHA-256 values. + +### Browser WASM Gate + +- Create a strict WASM FP32 session from the new artifact. +- Run all seven fixtures without fallback. +- Match the accepted FP32 detection behavior. +- Record browser, ORT version, model hash, session-creation time, inference time, and output evidence. + +### Physical WebGPU Gate + +- Create a strict WebGPU FP32 session with fallback disabled. +- Run all seven fixtures on a physical adapter. +- Match the accepted FP32 detection behavior. +- Record browser version, operating system, adapter identity and features, ORT version, model hash, session-creation time, per-fixture inference timing, output hashes, and detection comparisons. + +Any failed gate prevents the manifest from advertising WebGPU FP32 and prevents SDK adoption. + +## SDK Selection Behavior + +SDK `1.0.5` changes `DEFAULT_MANIFEST_URL` to the versioned `1.0.1` manifest. The generic selector already orders automatic candidates as: + +1. WebGPU FP16; +2. WebGPU FP32; +3. WASM INT8 when a validated variant exists; +4. WASM FP32. + +With the new manifest: + +- devices with `shader-f16` continue to select WebGPU FP16 automatically; +- WebGPU devices without `shader-f16` may select WebGPU FP32; +- explicit WebGPU FP32 requests are valid and remain strict by default; +- automatic mode may fall back to WASM FP32 after a WebGPU runtime failure; +- manual Demo selections never silently change precision or backend. + +## Demo and Documentation + +The Demo continues to derive enabled combinations from the active manifest. After SDK `1.0.5` adopts model `1.0.1`, GPU mode enables FP32. FP16 remains the recommended default because it has a smaller download and normally uses less memory. + +Update the root READMEs, packaged SDK README, compatibility, API, conversion, model, and benchmark documentation in both languages. Documentation must state that: + +- the new FP32 artifact is validated for WebGPU and WASM; +- FP64 inference is not supported; +- explicit selections are strict; +- FP32 is larger and may be slower or use more GPU memory than FP16; +- historical `1.0.0` model assets remain immutable. + +## Pipeline and Release Boundaries + +Parameterize model-version and release-tag constants that are currently fixed to `1.0.0`. Update manifest generation, model validation, Pages staging, release verification, and model-asset workflow tests to bind every URL, hash, report, and local path to `1.0.1`. + +Use two integration phases: + +1. **Model asset phase**: add the transform, tests, `1.0.1` artifacts, reports, generated manifest, documentation evidence, and the `v1.0.1-models` upload workflow. Merge and publish the immutable model Release after all model gates pass. +2. **SDK adoption phase**: after the new asset URLs are publicly fetchable, switch the SDK default manifest, enable the Demo matrix, update consumer documentation and release contracts, then prepare SDK `1.0.5`. + +This ordering prevents CI or published SDKs from referring to model URLs that do not yet exist. + +## Error Handling + +- The sanitizer fails closed on unexpected names, shapes, topology, or DOUBLE values. +- Manifest generation fails if model hashes or browser evidence do not match the artifacts. +- Release verification fetches staged assets and checks byte size and SHA-256. +- Explicit WebGPU FP32 session failures remain visible with the detailed ONNX Runtime cause and never fall back silently. +- Automatic fallback history records provider, precision, failure stage, code, and cause. + +## Acceptance Criteria + +- Historical `v1.0.0-models` assets and SDK `1.0.4` remain unchanged. +- The sanitizer has red-green regression coverage and produces a checked graph with no DOUBLE values. +- The new FP32 artifact passes current FP32 parity and detection gates on seven fixtures. +- The new FP32 artifact passes strict browser WASM and strict physical WebGPU execution. +- The `1.0.1` manifest is generated from verified reports and advertises FP32 for both WebGPU and WASM. +- SDK selector and Demo tests cover automatic and explicit WebGPU FP32 behavior without weakening strict manual semantics. +- Bilingual documentation and release contracts agree with the validated matrix. +- Full workspace verification, package smoke tests, Pages staging, and production builds pass before SDK release preparation. From 77dae7eba8f91f350cb6c74e040828fc35698675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 00:03:38 +0800 Subject: [PATCH 02/13] docs: plan WebGPU FP32 model support --- .gitignore | 1 + .../plans/2026-08-14-webgpu-fp32-model.md | 1590 +++++++++++++++++ 2 files changed, 1591 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-14-webgpu-fp32-model.md diff --git a/.gitignore b/.gitignore index 61927ba..7d9261b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .superpowers/ +.worktrees/ outputs/ work/ node_modules/ diff --git a/docs/superpowers/plans/2026-08-14-webgpu-fp32-model.md b/docs/superpowers/plans/2026-08-14-webgpu-fp32-model.md new file mode 100644 index 0000000..cbb64dd --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-webgpu-fp32-model.md @@ -0,0 +1,1590 @@ +# WebGPU FP32 Model Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish an immutable PP-DocLayoutV3 `1.0.1` model whose FP32 graph runs strictly on WebGPU and WASM, then adopt it as the default model in SDK `1.0.5` and the Demo. + +**Architecture:** Phase 1 adds a fail-closed ONNX sanitizer, versioned validation evidence, a generated `1.0.1` manifest, and an immutable `v1.0.1-models` release without changing SDK `1.0.4`. Phase 2 starts only after those URLs are public, switches the SDK and Pages defaults to model `1.0.1`, enables strict GPU + FP32 selection in the Demo, synchronizes documentation, and prepares SDK `1.0.5`. + +**Tech Stack:** Python 3.11, ONNX, ONNX Runtime CPU, TypeScript, ONNX Runtime Web 1.27.0, React, Vitest, Playwright, Node.js test runner, GitHub Actions, Git LFS, pnpm 11.16.0 + +--- + +## File Map + +Phase 1 creates or changes these ownership units: + +- `tools/model-pipeline/ppdoclayout/sanitize_fp32.py`: guarded, deterministic DOUBLE-to-FLOAT transform for the four positional constants only. +- `tools/model-pipeline/tests/test_sanitize_fp32.py`: synthetic-graph unit tests plus real-model structural and reproducibility checks. +- `tools/model-pipeline/ppdoclayout/validate.py`: accepted-versus-candidate FP32 raw-output and seven-fixture parity evidence. +- `tools/model-pipeline/tests/test_parity_fp32.py`: report-contract and slow real-model parity tests. +- `tests/browser/benchmark.spec.ts`: strict seven-fixture browser runner for `wasm-fp32`, `webgpu-fp16`, and `webgpu-fp32`. +- `.github/workflows/benchmark.yml`: hosted WASM and physical-adapter WebGPU validation jobs. +- `tools/model-pipeline/ppdoclayout/build_manifest.py`: version/release parameters and browser-evidence gates. +- `tools/model-pipeline/tests/test_manifest.py`: versioned generation and fail-closed evidence tests. +- `models/pp-doclayoutv3/1.0.1/`: sanitized FP32, byte-identical accepted FP16, and generated manifest. +- `tools/model-pipeline/reports/1.0.1/`: candidate FP32, variant, and browser evidence without rewriting historical reports. +- `scripts/verify-release.mjs`, `scripts/verify-release.test.mjs`: version-aware local/release asset verification. +- `.github/workflows/model-validation.yml`: explicit immutable `v1.0.1-models` publication workflow. +- `models/README.md`, `docs/en/conversion.md`, `docs/zh-CN/conversion.md`: transform provenance and model-phase release instructions. + +Phase 2 changes these consumer units only after `v1.0.1-models` is public: + +- `packages/sdk/src/detector.ts`: default Pages manifest URL `models/v1.0.1/manifest.json`. +- `packages/sdk/src/model/manifest.ts`, `packages/sdk/package.json`, `pnpm-lock.yaml`: SDK version `1.0.5`. +- `packages/sdk/tests/detector.test.ts`, `packages/sdk/tests/manifest.test.ts`, `packages/sdk/tests/runtime-selector.test.ts`: default manifest and strict WebGPU FP32 selection contracts. +- `apps/demo/src/execution-preferences.ts`, `apps/demo/src/i18n/en.ts`, `apps/demo/src/i18n/zh-CN.ts`, `apps/demo/tests/demo.spec.ts`: enabled GPU + FP32 control with strict manual semantics. +- `scripts/stage-pages-models.mjs`, `scripts/verify-release.mjs`, `scripts/verify-release.test.mjs`: Pages `v1.0.1` staging and SDK release contract. +- `README.md`, `README.en.md`, `packages/sdk/README.md`, `docs/en/api.md`, `docs/en/compatibility.md`, `docs/en/models.md`, `docs/en/performance.md`, `docs/zh-CN/api.md`, `docs/zh-CN/compatibility.md`, `docs/zh-CN/models.md`, `docs/zh-CN/performance.md`, `CHANGELOG.md`: bilingual support and release documentation. + +## Phase 1: Model Asset `1.0.1` + +### Task 1: Specify the guarded FP32 sanitizer + +**Files:** +- Create: `tools/model-pipeline/tests/test_sanitize_fp32.py` +- Create later: `tools/model-pipeline/ppdoclayout/sanitize_fp32.py` + +- [ ] **Step 1: Add synthetic graph fixtures and failing transform tests** + +Create `tools/model-pipeline/tests/test_sanitize_fp32.py` with helpers that build the exact known topology and assertions for valid and invalid inputs: + +```python +from __future__ import annotations + +import hashlib +from pathlib import Path + +import numpy as np +import onnx +import pytest +from onnx import TensorProto, helper, numpy_helper + +from ppdoclayout.sanitize_fp32 import ( + POSITIONAL_NAMES, + _double_names, + sanitize_webgpu_fp32, +) + + +POSITIONAL_NAMES = ("sin", "cos", "sin_1", "cos_1") + + +def sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def source_model(*, extra_double: bool = False, cast_to: int = TensorProto.FLOAT) -> onnx.ModelProto: + initializers = [ + numpy_helper.from_array( + np.arange(625 * 64, dtype=np.float64).reshape(625, 64), name=name + ) + for name in POSITIONAL_NAMES + ] + initializers.append( + numpy_helper.from_array(np.asarray([3.25], dtype=np.float32), name="learned_weight") + ) + if extra_double: + initializers.append( + numpy_helper.from_array(np.asarray([1.0], dtype=np.float64), name="unexpected") + ) + nodes = [ + helper.make_node( + "Concat", POSITIONAL_NAMES, ["cat_7"], axis=1, name="node_cat_7" + ), + helper.make_node( + "Cast", ["cat_7"], ["_to_copy_4"], to=cast_to, name="node__to_copy_4" + ), + helper.make_node( + "Add", ["input", "learned_weight"], ["output"], name="learned_add" + ), + ] + graph = helper.make_graph( + nodes, + "sanitize-test", + [helper.make_tensor_value_info("input", TensorProto.FLOAT, [1])], + [helper.make_tensor_value_info("output", TensorProto.FLOAT, [1])], + initializers, + ) + return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 18)]) + + +def write_source(path: Path, model: onnx.ModelProto) -> None: + path.write_bytes(model.SerializeToString(deterministic=True)) + + +def replace_first_initializer_with_wrong_shape(model: onnx.ModelProto) -> None: + model.graph.initializer[0].CopyFrom( + numpy_helper.from_array( + np.zeros((624, 64), dtype=np.float64), name=POSITIONAL_NAMES[0] + ) + ) + + +def test_converts_only_known_positional_constants(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + write_source(source, source_model()) + + result = sanitize_webgpu_fp32(source, output) + + model = onnx.load(output, load_external_data=False) + by_name = {value.name: value for value in model.graph.initializer} + assert all(by_name[name].data_type == TensorProto.FLOAT for name in POSITIONAL_NAMES) + assert by_name["learned_weight"].raw_data == np.asarray([3.25], dtype=np.float32).tobytes() + assert result == {"bytes": output.stat().st_size, "sha256": sha256_file(output)} + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda model: setattr(model.graph.initializer[0], "name", "renamed"), "exactly"), + (replace_first_initializer_with_wrong_shape, "shape"), + (lambda model: setattr(model.graph.node[0], "name", "other_concat"), "node_cat_7"), + (lambda model: setattr(model.graph.node[1].attribute[0], "i", TensorProto.FLOAT16), "FLOAT"), + ], +) +def test_rejects_unexpected_source_contract(tmp_path: Path, mutation, message: str) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + model = source_model() + mutation(model) + write_source(source, model) + + with pytest.raises(ValueError, match=message): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + +def test_rejects_any_additional_double_value(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + write_source(source, source_model(extra_double=True)) + + with pytest.raises(ValueError, match="unexpected DOUBLE initializer"): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + +def test_is_byte_reproducible(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + first = tmp_path / "first.onnx" + second = tmp_path / "second.onnx" + write_source(source, source_model()) + + sanitize_webgpu_fp32(source, first) + sanitize_webgpu_fp32(source, second) + + assert first.read_bytes() == second.read_bytes() +``` + +- [ ] **Step 2: Run the sanitizer tests and verify the RED state** + +Run: + +```powershell +.\.venv-model\Scripts\python.exe -m pytest tools/model-pipeline/tests/test_sanitize_fp32.py -q +``` + +Expected: collection fails with `ModuleNotFoundError: No module named 'ppdoclayout.sanitize_fp32'`. + +- [ ] **Step 3: Commit the executable specification** + +```powershell +git add -- tools/model-pipeline/tests/test_sanitize_fp32.py +git commit -m "test(models): specify WebGPU FP32 sanitizer" +``` + +### Task 2: Implement and materialize the deterministic sanitizer + +**Files:** +- Create: `tools/model-pipeline/ppdoclayout/sanitize_fp32.py` +- Modify: `tools/model-pipeline/tests/test_sanitize_fp32.py` +- Create: `models/pp-doclayoutv3/1.0.1/model-fp32.onnx` +- Create: `models/pp-doclayoutv3/1.0.1/model-fp16.onnx` + +- [ ] **Step 1: Implement the fail-closed transform** + +Create `tools/model-pipeline/ppdoclayout/sanitize_fp32.py` with this public contract and guarded transform: + +```python +from __future__ import annotations + +import argparse +import hashlib +import os +import tempfile +from pathlib import Path + +import numpy as np +import onnx +from onnx import TensorProto, numpy_helper + + +POSITIONAL_NAMES = ("sin", "cos", "sin_1", "cos_1") +POSITIONAL_SHAPE = [625, 64] + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _node(model: onnx.ModelProto, name: str) -> onnx.NodeProto: + matches = [node for node in model.graph.node if node.name == name] + if len(matches) != 1: + raise ValueError(f"Expected exactly one {name} node") + return matches[0] + + +def _require_source_contract(model: onnx.ModelProto) -> None: + doubles = { + value.name: value + for value in model.graph.initializer + if value.data_type == TensorProto.DOUBLE + } + if set(doubles) != set(POSITIONAL_NAMES): + unexpected = sorted(set(doubles) - set(POSITIONAL_NAMES)) + raise ValueError( + f"Expected exactly {list(POSITIONAL_NAMES)} DOUBLE initializers; " + f"unexpected DOUBLE initializer(s): {unexpected}" + ) + for name in POSITIONAL_NAMES: + if list(doubles[name].dims) != POSITIONAL_SHAPE: + raise ValueError(f"{name} must have shape {POSITIONAL_SHAPE}") + + concat = _node(model, "node_cat_7") + if concat.op_type != "Concat" or list(concat.input) != list(POSITIONAL_NAMES): + raise ValueError("node_cat_7 must concatenate the four positional constants") + if list(concat.output) != ["cat_7"]: + raise ValueError("node_cat_7 must produce cat_7") + axis = next((item.i for item in concat.attribute if item.name == "axis"), None) + if axis != 1: + raise ValueError("node_cat_7 must concatenate on axis 1") + + cast = _node(model, "node__to_copy_4") + cast_to = next((item.i for item in cast.attribute if item.name == "to"), None) + if ( + cast.op_type != "Cast" + or list(cast.input) != ["cat_7"] + or list(cast.output) != ["_to_copy_4"] + or cast_to != TensorProto.FLOAT + ): + raise ValueError("node__to_copy_4 must Cast cat_7 to FLOAT") + + +def _double_names(model: onnx.ModelProto) -> list[str]: + values = [*model.graph.input, *model.graph.output, *model.graph.value_info] + names = [ + value.name + for value in values + if value.type.tensor_type.elem_type == TensorProto.DOUBLE + ] + names.extend( + value.name + for value in model.graph.initializer + if value.data_type == TensorProto.DOUBLE + ) + return sorted(set(names)) + + +def sanitize_webgpu_fp32(source: Path, output: Path) -> dict[str, int | str]: + source = source.resolve() + output = output.resolve() + model = onnx.load(source, load_external_data=False) + onnx.checker.check_model(model) + if any(value.external_data for value in model.graph.initializer): + raise ValueError("Source model must be self-contained") + _require_source_contract(model) + + for index, value in enumerate(model.graph.initializer): + if value.name not in POSITIONAL_NAMES: + continue + converted = numpy_helper.from_array( + numpy_helper.to_array(value).astype(np.float32), name=value.name + ) + model.graph.initializer[index].CopyFrom(converted) + + inferred = onnx.shape_inference.infer_shapes(model, strict_mode=True) + onnx.checker.check_model(inferred) + remaining = _double_names(inferred) + if remaining: + raise ValueError(f"Sanitized graph still contains DOUBLE values: {remaining}") + + output.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{output.name}.", suffix=".tmp", dir=output.parent + ) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(inferred.SerializeToString(deterministic=True)) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary_name, output) + except BaseException: + Path(temporary_name).unlink(missing_ok=True) + raise + return {"bytes": output.stat().st_size, "sha256": sha256_file(output)} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Sanitize PP-DocLayoutV3 FP32 for WebGPU") + parser.add_argument("--source", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + result = sanitize_webgpu_fp32(args.source, args.output) + print(f"{args.output}: {result['bytes']} bytes sha256={result['sha256']}") + + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 2: Run unit tests and verify the GREEN state** + +Run: + +```powershell +.\.venv-model\Scripts\python.exe -m pytest tools/model-pipeline/tests/test_sanitize_fp32.py -q +``` + +Expected: all synthetic sanitizer tests pass, including atomic failure behavior and byte reproducibility. + +- [ ] **Step 3: Add real-model invariants before generating the artifact** + +Append a slow-free structural test that reads the checked-in source graph, sanitizes it in a temporary directory, and asserts the exact graph boundary and unchanged learned initializer hashes: + +```python +ROOT = Path(__file__).parents[3] +SOURCE_FP32 = ROOT / "models" / "pp-doclayoutv3" / "1.0.0" / "model-fp32.onnx" + + +def initializer_hashes(model: onnx.ModelProto, excluded: set[str]) -> dict[str, str]: + return { + value.name: hashlib.sha256(value.SerializeToString(deterministic=True)).hexdigest() + for value in model.graph.initializer + if value.name not in excluded + } + + +def test_real_model_preserves_contract_and_learned_parameters(tmp_path: Path) -> None: + output = tmp_path / "model-fp32.onnx" + sanitize_webgpu_fp32(SOURCE_FP32, output) + source = onnx.load(SOURCE_FP32, load_external_data=False) + candidate = onnx.load(output, load_external_data=False) + + assert [(item.domain, item.version) for item in candidate.opset_import] == [("", 18)] + assert [value.SerializeToString() for value in candidate.graph.input] == [ + value.SerializeToString() for value in source.graph.input + ] + assert [value.SerializeToString() for value in candidate.graph.output] == [ + value.SerializeToString() for value in source.graph.output + ] + assert initializer_hashes(candidate, set(POSITIONAL_NAMES)) == initializer_hashes( + source, set(POSITIONAL_NAMES) + ) + assert not _double_names(onnx.shape_inference.infer_shapes(candidate, strict_mode=True)) +``` + +- [ ] **Step 4: Materialize both `1.0.1` models** + +Run: + +```powershell +.\.venv-model\Scripts\python.exe -m ppdoclayout.sanitize_fp32 --source models/pp-doclayoutv3/1.0.0/model-fp32.onnx --output models/pp-doclayoutv3/1.0.1/model-fp32.onnx +New-Item -ItemType Directory -Force models/pp-doclayoutv3/1.0.1 | Out-Null +Copy-Item -LiteralPath models/pp-doclayoutv3/1.0.0/model-fp16.onnx -Destination models/pp-doclayoutv3/1.0.1/model-fp16.onnx +``` + +Expected: the sanitizer prints the candidate byte count and SHA-256; the FP16 copy succeeds. + +- [ ] **Step 5: Verify reproducibility and immutable source hashes** + +Run: + +```powershell +$regen = Join-Path $env:TEMP 'ppdoclayout-model-fp32-1.0.1.onnx' +.\.venv-model\Scripts\python.exe -m ppdoclayout.sanitize_fp32 --source models/pp-doclayoutv3/1.0.0/model-fp32.onnx --output $regen +Get-FileHash -Algorithm SHA256 models/pp-doclayoutv3/1.0.1/model-fp32.onnx,$regen +Get-FileHash -Algorithm SHA256 models/pp-doclayoutv3/1.0.0/model-fp16.onnx,models/pp-doclayoutv3/1.0.1/model-fp16.onnx +Remove-Item -LiteralPath $regen +``` + +Expected: both FP32 hashes match each other; both FP16 hashes equal `463ba56faa555baf84271b4002b33b0c5fcc50776fe4f39344235eccb72073f2`; the source FP32 remains `fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b`. + +- [ ] **Step 6: Commit the sanitizer and versioned binaries** + +```powershell +git add -- tools/model-pipeline/ppdoclayout/sanitize_fp32.py tools/model-pipeline/tests/test_sanitize_fp32.py models/pp-doclayoutv3/1.0.1/model-fp32.onnx models/pp-doclayoutv3/1.0.1/model-fp16.onnx +git commit -m "feat(models): sanitize FP32 graph for WebGPU" +``` + +### Task 3: Bind seven-fixture FP32 parity to accepted and candidate models + +**Files:** +- Modify: `tools/model-pipeline/ppdoclayout/validate.py` +- Modify: `tools/model-pipeline/tests/test_parity_fp32.py` +- Create after validation: `tools/model-pipeline/reports/1.0.1/fp32-validation.json` + +- [ ] **Step 1: Write failing report-contract tests** + +Extend the import in `tools/model-pipeline/tests/test_parity_fp32.py` to include `sha256_file`, then make `validate_fp32` receive `accepted_onnx_path` and bind both ONNX files: + +```python +from ppdoclayout.validate import canonical_json, sha256_file, validate_fp32, write_report +``` + +Replace the existing slow test with: + +```python +@pytest.mark.slow +def test_sanitized_fp32_matches_accepted_fp32_and_official_transformers() -> None: + accepted = ROOT / "models" / "pp-doclayoutv3" / "1.0.0" / "model-fp32.onnx" + candidate = ROOT / "models" / "pp-doclayoutv3" / "1.0.1" / "model-fp32.onnx" + report = validate_fp32( + model_path=Path(r"E:\models\PP-DocLayoutV3_safetensors"), + accepted_onnx_path=accepted, + onnx_path=candidate, + fixtures_lock=ROOT / "tools" / "model-pipeline" / "fixtures" / "fixtures.lock.json", + ) + + assert report["overallPass"] is True + assert report["sourceHashes"] == { + "acceptedOnnx": sha256_file(accepted), + "modelSafetensors": sha256_file( + Path(r"E:\models\PP-DocLayoutV3_safetensors\model.safetensors") + ), + "onnx": sha256_file(candidate), + } + assert len(report["fixtures"]) == 7 + for fixture in report["fixtures"]: + assert fixture["acceptedDetectionCount"] == fixture["onnxDetectionCount"] + assert fixture["acceptedLabelSequenceEqual"] is True + assert fixture["acceptedReadingOrderEqual"] is True + assert fixture["rawOutputs"]["allBitIdentical"] is True + assert fixture["pass"] is True +``` + +- [ ] **Step 2: Run the slow test and verify the RED state** + +Run: + +```powershell +.\.venv-model\Scripts\python.exe -m pytest tools/model-pipeline/tests/test_parity_fp32.py -m slow -q +``` + +Expected: FAIL because `validate_fp32()` does not accept `accepted_onnx_path` and the report has no accepted-model or raw-output fields. + +- [ ] **Step 3: Extend validation without relaxing existing thresholds** + +In `tools/model-pipeline/ppdoclayout/validate.py`: + +1. Add `accepted_onnx_path: Path` to `validate_fp32`. +2. Create an ONNX Runtime CPU session for both accepted and candidate files using `providers=["CPUExecutionProvider"]`. +3. Feed the identical preprocessed tensor to both sessions for every locked fixture. +4. For each output name, record shape, dtype, byte SHA-256, `bitIdentical`, and maximum absolute delta. +5. Postprocess accepted and candidate results with the same existing processor and thresholds. +6. Add `acceptedDetectionCount`, `acceptedLabelSequenceEqual`, and `acceptedReadingOrderEqual`. +7. Keep `PARITY_THRESHOLDS` exactly `scoreDelta=0.001`, `boxCoordinateDeltaPixels=1.0`, and `polygonCoordinateDeltaPixels=1.5`. +8. Require accepted detection count, label sequence, and reading order equality in `_fixture_passes`. +9. Prefer bit identity by recording it; if a platform produces non-bit-identical outputs, the existing numerical thresholds remain the only permitted tolerance. +10. Add CLI option `--accepted-onnx` and bind `sourceHashes.acceptedOnnx`. + +Use this exact raw-output summary shape: + +```python +raw_outputs = { + name: { + "acceptedSha256": hashlib.sha256(accepted_value.tobytes()).hexdigest(), + "candidateSha256": hashlib.sha256(candidate_value.tobytes()).hexdigest(), + "bitIdentical": bool(np.array_equal(accepted_value, candidate_value)), + "dtype": str(candidate_value.dtype), + "maxAbsoluteDelta": float(np.max(np.abs(accepted_value - candidate_value))), + "shape": list(candidate_value.shape), + } + for name, accepted_value, candidate_value in zip( + output_names, accepted_outputs, candidate_outputs, strict=True + ) +} +fixture_report["rawOutputs"] = { + "allBitIdentical": all(item["bitIdentical"] for item in raw_outputs.values()), + "outputs": raw_outputs, +} +``` + +- [ ] **Step 4: Run focused unit and slow parity tests** + +Run: + +```powershell +.\.venv-model\Scripts\python.exe -m pytest tools/model-pipeline/tests/test_parity_fp32.py -q +``` + +Expected: all seven fixtures pass; counts, label sequences, and reading order match; raw output evidence is populated. If `allBitIdentical` is false, inspect the per-output deltas and stop if any existing threshold is exceeded. + +- [ ] **Step 5: Generate the versioned FP32 report** + +Run: + +```powershell +.\.venv-model\Scripts\python.exe -m ppdoclayout.validate --model E:\models\PP-DocLayoutV3_safetensors --accepted-onnx models/pp-doclayoutv3/1.0.0/model-fp32.onnx --onnx models/pp-doclayoutv3/1.0.1/model-fp32.onnx --fixtures-lock tools/model-pipeline/fixtures/fixtures.lock.json --output tools/model-pipeline/reports/1.0.1/fp32-validation.json +``` + +Expected: exit zero and `overallPass: true`; `sourceHashes.acceptedOnnx` is the historical FP32 hash and `sourceHashes.onnx` is the sanitized artifact hash. + +- [ ] **Step 6: Commit candidate parity code and evidence** + +```powershell +git add -- tools/model-pipeline/ppdoclayout/validate.py tools/model-pipeline/tests/test_parity_fp32.py tools/model-pipeline/reports/1.0.1/fp32-validation.json +git commit -m "test(models): validate sanitized FP32 parity" +``` + +### Task 4: Run strict seven-fixture browser WASM and physical WebGPU validation + +**Files:** +- Modify: `tests/browser/benchmark.spec.ts` +- Modify: `.github/workflows/benchmark.yml` +- Modify: `scripts/benchmark-contract.test.mjs` +- Create after browser runs: `tools/model-pipeline/reports/1.0.1/browser-evidence.json` + +- [ ] **Step 1: Write failing benchmark contract assertions** + +Update `scripts/benchmark-contract.test.mjs` to require a third hardware job and all seven fixture records: + +```js +assert.match(workflow, /PPDOCLAYOUT_BENCHMARK_MODE:\s*["']?webgpu-fp32/); +assert.match(workflow, /name:\s*benchmark-webgpu-fp32/); +assert.match(workflow, /runs-on:\s*\[self-hosted, windows, x64, webgpu-hardware\]/); + +for (const name of ["wasm-fp32.json", "webgpu-fp32.json"]) { + const report = readJson(name, "1.0.1"); + assert.equal(report.status, "passed"); + assert.equal(report.fallbacks.length, 0); + assert.equal(report.fixtures.length, 7); + assert.ok(report.fixtures.every((fixture) => fixture.parity === "passed")); +} +``` + +Change `readJson` to accept the model version: + +```js +function readJson(name, version = "1.0.0") { + const path = join(repositoryRoot, "benchmarks", version, name); + assert.ok(existsSync(path), `missing benchmark artifact: benchmarks/${version}/${name}`); + return JSON.parse(readFileSync(path, "utf8")); +} +``` + +- [ ] **Step 2: Run the benchmark contract and verify the RED state** + +Run: + +```powershell +pnpm benchmark:test +``` + +Expected: FAIL because `webgpu-fp32` is not an accepted mode/job and `benchmarks/1.0.1` evidence is absent. + +- [ ] **Step 3: Generalize the browser benchmark to model `1.0.1` and seven fixtures** + +In `tests/browser/benchmark.spec.ts`: + +- set `modelRoot` to `models/pp-doclayoutv3/1.0.1` when the mode is `wasm-fp32` or `webgpu-fp32`; +- accept `wasm-fp32`, `webgpu-fp16`, and `webgpu-fp32`; +- use Chrome for both WebGPU modes; +- derive precision with `mode.endsWith("fp32") ? "fp32" : "fp16"`; +- keep `allowFallback: false` and assert `runtime.fallbacks` is empty; +- read `fixtures.lock.json`, verify each fixture SHA-256 in Node, run all seven images, and record one detection/parity/timing/output evidence object per fixture; +- record `browser.version()`, user agent, `platform()`/`release()`, adapter identity, sorted adapter features, ORT `1.27.0`, model size/hash, session creation/load timings, and SDK commit; +- write `test-results/benchmark/${mode}.json`. + +Keep these fields at the report top level so manifest generation can validate evidence without interpreting presentation-specific nesting: + +```ts +const report = { + schemaVersion: 1, + status: "passed", + executionProvider: backend, + precision, + fallbacks: result.runtime.fallbacks, + modelBytes: result.model.bytes, + modelSha256: result.model.sha256, + onnxruntimeWebVersion: "1.27.0", + adapter: result.adapter, + adapterFeatures: result.adapterFeatures, + browser: { name: "Chromium", version: browser.version(), userAgent: result.browser }, + operatingSystem: `${platform()} ${release()}`, + fixtures: result.fixtures, + timingsMs: result.timings, + sdkCommit +}; +``` + +The strict browser-side assertion must be: + +```ts +expect(result.runtime).toMatchObject({ backend, fallbacks: [], precision }); +expect(result.model.sha256).toBe(manifestVariant.sha256); +expect(result.fixtures).toHaveLength(fixturesLock.fixtures.length); +for (const fixture of result.fixtures) { + expect(fixture.detectionCount).toBe(fixture.expectedDetectionCount); + expect(fixture.labelSequenceEqual).toBe(true); + expect(fixture.readingOrderEqual).toBe(true); + expect(fixture.parity).toBe("passed"); +} +``` + +The report must include a stable hash of every complete detection result as output evidence, not only counts and timings: + +```ts +async function sha256(bytes: Uint8Array): Promise { + const digest = await crypto.subtle.digest("SHA-256", bytes); + return [...new Uint8Array(digest)] + .map((value) => value.toString(16).padStart(2, "0")) + .join(""); +} + +const detectionJson = JSON.stringify(detection.detections); +const outputSha256 = await sha256(new TextEncoder().encode(detectionJson)); +return { + detectionCount: detection.detections.length, + labelSequenceEqual, + outputSha256, + parity: "passed", + readingOrderEqual, + timings: detection.timings +}; +``` + +- [ ] **Step 4: Add hosted WASM and physical WebGPU FP32 jobs** + +In `.github/workflows/benchmark.yml`, keep the existing jobs and add the physical adapter job: + +```yaml + webgpu-fp32: + runs-on: [self-hosted, windows, x64, webgpu-hardware] + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + with: + lfs: true + - uses: pnpm/action-setup@v6 + with: + version: 11.16.0 + - uses: actions/setup-node@v7 + with: + node-version-file: .nvmrc + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm exec playwright install chromium + - run: pnpm exec playwright test tests/browser/benchmark.spec.ts + env: + PPDOCLAYOUT_BENCHMARK_MODE: webgpu-fp32 + - uses: actions/upload-artifact@v7 + with: + name: benchmark-webgpu-fp32 + path: test-results/benchmark/webgpu-fp32.json +``` + +Change the hosted `wasm-fp32` job to run the generalized seven-fixture test against `1.0.1`. Preserve `allowFallback: false` in the test; do not emulate WebGPU with a software adapter. + +- [ ] **Step 5: Run strict WASM FP32 locally** + +Run: + +```powershell +$env:PPDOCLAYOUT_BENCHMARK_MODE = 'wasm-fp32' +pnpm exec playwright test tests/browser/benchmark.spec.ts --project=chromium +Remove-Item Env:PPDOCLAYOUT_BENCHMARK_MODE +``` + +Expected: exit zero, seven fixture entries, `runtime.backend: "wasm"`, `runtime.precision: "fp32"`, and `fallbacks: []` in `test-results/benchmark/wasm-fp32.json`. + +- [ ] **Step 6: Run strict FP32 on the physical WebGPU adapter** + +Run on the hardware-tagged Windows machine or dispatch the benchmark workflow: + +```powershell +$env:PPDOCLAYOUT_BENCHMARK_MODE = 'webgpu-fp32' +pnpm exec playwright test tests/browser/benchmark.spec.ts --project=chromium +Remove-Item Env:PPDOCLAYOUT_BENCHMARK_MODE +``` + +Expected: session creation succeeds without `node__to_copy_4` provider errors; all seven fixtures pass; `runtime.backend: "webgpu"`, `runtime.precision: "fp32"`, and `fallbacks: []`; adapter identity/features and per-fixture timings/output hashes are present. Any failure stops Task 4 and prevents WebGPU from being added to the manifest. + +- [ ] **Step 7: Persist versioned browser evidence** + +Create `tools/model-pipeline/reports/1.0.1/browser-evidence.json` from the two successful reports with this stable envelope: + +```json +{ + "schemaVersion": 1, + "fp16Webgpu": {}, + "fp32Wasm": {}, + "fp32Webgpu": {} +} +``` + +Copy `fp16Webgpu` byte-for-byte as a JSON value from `tools/model-pipeline/reports/browser-evidence.json`; the FP16 artifact is byte-identical and its accepted hardware evidence remains valid. The FP32 values are the complete generated `wasm-fp32.json` and `webgpu-fp32.json` objects. Copy the same two FP32 reports to `benchmarks/1.0.1/wasm-fp32.json` and `benchmarks/1.0.1/webgpu-fp32.json`; do not hand-edit measured values. + +- [ ] **Step 8: Run the benchmark contract and commit evidence** + +Run: + +```powershell +pnpm benchmark:test +``` + +Expected: PASS with both version `1.0.1` strict FP32 reports accepted. + +```powershell +git add -- tests/browser/benchmark.spec.ts .github/workflows/benchmark.yml scripts/benchmark-contract.test.mjs benchmarks/1.0.1 tools/model-pipeline/reports/1.0.1/browser-evidence.json +git commit -m "test(models): validate FP32 in browser runtimes" +``` + +### Task 5: Generate a gated model `1.0.1` manifest + +**Files:** +- Modify: `tools/model-pipeline/ppdoclayout/build_manifest.py` +- Modify: `tools/model-pipeline/tests/test_manifest.py` +- Modify: `tools/model-pipeline/ppdoclayout/variant_validation.py` +- Modify: `tools/model-pipeline/tests/test_variants.py` +- Create after generation: `tools/model-pipeline/reports/1.0.1/variant-validation.json` +- Create after generation: `models/pp-doclayoutv3/1.0.1/manifest.json` + +- [ ] **Step 1: Write failing version and evidence-gate tests** + +Refactor `tools/model-pipeline/tests/test_manifest.py` so paths and expected hashes are derived from generated files under `1.0.1`, then add: + +```python +MODEL_VERSION = "1.0.1" +RELEASE_TAG = "v1.0.1-models" +MODEL_DIR = ROOT / "models" / "pp-doclayoutv3" / MODEL_VERSION +BROWSER_REPORT_PATH = PIPELINE_DIR / "reports" / MODEL_VERSION / "browser-evidence.json" + + +def test_fp32_requires_strict_wasm_and_webgpu_evidence(tmp_path: Path) -> None: + evidence = json.loads(BROWSER_REPORT_PATH.read_text(encoding="utf-8")) + evidence["fp32Webgpu"]["fallbacks"] = [{"provider": "wasm"}] + path = tmp_path / "browser-evidence.json" + path.write_text(json.dumps(evidence), encoding="utf-8") + + with pytest.raises(ValueError, match="fallback"): + build_from_paths(browser_report_path=path) + + +def test_manifest_advertises_validated_fp32_for_both_backends() -> None: + manifest = build_from_paths() + fp32 = next(item for item in manifest["variants"] if item["id"] == "fp32") + + assert manifest["model"]["version"] == MODEL_VERSION + assert manifest["variantPriority"] == ["fp16", "fp32"] + assert fp32["backendCompatibility"] == ["wasm", "webgpu"] + assert fp32["url"].endswith(f"/{RELEASE_TAG}/model-fp32.onnx") +``` + +- [ ] **Step 2: Run focused tests and verify the RED state** + +Run: + +```powershell +.\.venv-model\Scripts\python.exe -m pytest tools/model-pipeline/tests/test_manifest.py tools/model-pipeline/tests/test_variants.py -q +``` + +Expected: FAIL because manifest generation is fixed to `1.0.0` and does not consume FP32 browser evidence. + +- [ ] **Step 3: Parameterize manifest identity and enforce evidence** + +Change `build_manifest` and `write_manifest` to require `model_version`, `release_tag`, and `browser_report_path`. Remove `MODEL_VERSION` and `RELEASE_BASE_URL` module constants; retain `MODEL_ID` and `MIN_SDK_VERSION`. + +Build the release URL with validated values: + +```python +SEMVER = re.compile(r"^\d+\.\d+\.\d+$") + + +def release_base_url(model_version: str, release_tag: str) -> str: + if not SEMVER.fullmatch(model_version): + raise ValueError(f"Invalid model version: {model_version}") + if release_tag != f"v{model_version}-models": + raise ValueError("Release tag must match model version") + return ( + "https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/" + f"releases/download/{release_tag}/" + ) +``` + +Require both browser entries to have: + +- `status == "passed"`; +- `executionProvider` equal to `wasm` or `webgpu` respectively; +- `precision == "fp32"`; +- `fallbacks == []`; +- model byte size and SHA-256 equal to the candidate; +- ORT version `1.27.0`; +- exactly seven passing fixtures with valid output hashes; +- WebGPU adapter identity and feature list. + +Only after both checks pass, emit: + +```python +"backendCompatibility": ["wasm", "webgpu"] +``` + +Use versioned validation links in the generated manifest: + +```python +"validation": { + "included": True, + "pass": True, + "report": f"tools/model-pipeline/reports/{model_version}/fp32-validation.json", +} +``` + +Apply the equivalent versioned path to FP16 variant validation. Default `--model-dir`, `--fp32-report`, `--variant-report`, `--browser-report`, and `--output` paths must all derive from `--model-version`; no `1.0.0` path remains embedded in the generator. + +Add CLI flags with fixed safe defaults: + +```python +parser.add_argument("--model-version", default="1.0.1") +parser.add_argument("--release-tag", default="v1.0.1-models") +parser.add_argument( + "--browser-report", + type=Path, + default=pipeline_dir / "reports" / "1.0.1" / "browser-evidence.json", +) +``` + +- [ ] **Step 4: Version variant validation and preserve FP16 bytes** + +Update `variant_validation.py` so the accepted FP16 artifact is re-evaluated from `models/pp-doclayoutv3/1.0.1/model-fp16.onnx` while `source.fp32Sha256` binds the candidate. Replace the required `--int8` argument with required `--accepted-variant-report`: load the historical report, require its INT8 entry to have `pass: false` and `included: false`, and carry that exclusion evidence forward without requiring the intentionally unpublished INT8 binary. Read `fp16Webgpu` from the versioned browser evidence and keep every existing FP16/INT8 numerical threshold unchanged. In `test_variants.py`, assert: + +```python +OLD_FP16 = ROOT / "models" / "pp-doclayoutv3" / "1.0.0" / "model-fp16.onnx" +NEW_FP16 = ROOT / "models" / "pp-doclayoutv3" / "1.0.1" / "model-fp16.onnx" + + +def test_model_1_0_1_reuses_accepted_fp16_bytes() -> None: + assert NEW_FP16.read_bytes() == OLD_FP16.read_bytes() + + +def test_rejected_int8_evidence_is_carried_forward_without_binary() -> None: + accepted = json.loads( + (ROOT / "tools" / "model-pipeline" / "reports" / "variant-validation.json") + .read_text(encoding="utf-8") + )["variants"]["int8"] + assert accepted["pass"] is False + assert accepted["included"] is False +``` + +- [ ] **Step 5: Generate candidate variant evidence and manifest** + +Run the versioned variant command with the accepted FP16 artifact and historical rejected-INT8 evidence, then generate the manifest: + +```powershell +.\.venv-model\Scripts\python.exe -m ppdoclayout.variant_validation --model E:\models\PP-DocLayoutV3_safetensors --fp32 models/pp-doclayoutv3/1.0.1/model-fp32.onnx --fp16 models/pp-doclayoutv3/1.0.1/model-fp16.onnx --accepted-variant-report tools/model-pipeline/reports/variant-validation.json --fixtures-lock tools/model-pipeline/fixtures/fixtures.lock.json --browser-evidence tools/model-pipeline/reports/1.0.1/browser-evidence.json --output tools/model-pipeline/reports/1.0.1/variant-validation.json +.\.venv-model\Scripts\python.exe -m ppdoclayout.build_manifest --model-version 1.0.1 --release-tag v1.0.1-models --model-dir models/pp-doclayoutv3/1.0.1 --fp32-report tools/model-pipeline/reports/1.0.1/fp32-validation.json --variant-report tools/model-pipeline/reports/1.0.1/variant-validation.json --browser-report tools/model-pipeline/reports/1.0.1/browser-evidence.json --output models/pp-doclayoutv3/1.0.1/manifest.json +``` + +Expected manifest: version `1.0.1`, `minSdkVersion: "1.0.0"`, priority `fp16` then `fp32`, FP16 WebGPU only, FP32 WASM and WebGPU. The carried INT8 record stays excluded and no INT8 file is published. + +- [ ] **Step 6: Run model generator tests and commit** + +```powershell +.\.venv-model\Scripts\python.exe -m pytest tools/model-pipeline/tests/test_manifest.py tools/model-pipeline/tests/test_variants.py -q +git add -- tools/model-pipeline/ppdoclayout/build_manifest.py tools/model-pipeline/ppdoclayout/variant_validation.py tools/model-pipeline/tests/test_manifest.py tools/model-pipeline/tests/test_variants.py tools/model-pipeline/reports/1.0.1/variant-validation.json models/pp-doclayoutv3/1.0.1/manifest.json +git commit -m "feat(models): generate validated model 1.0.1 manifest" +``` + +Expected: all tests pass and the checked-in manifest is byte-identical to `canonical_json(build_manifest(...))`. + +### Task 6: Make model verification and publication version-aware and immutable + +**Files:** +- Modify: `scripts/verify-release.mjs` +- Modify: `scripts/verify-release.test.mjs` +- Modify: `.github/workflows/model-validation.yml` + +- [ ] **Step 1: Write failing release-contract tests** + +Add tests in `scripts/verify-release.test.mjs` for versioned model verification and immutable workflow behavior: + +```js +test("verifies model 1.0.1 without changing the SDK 1.0.4 default", () => { + const output = execFileSync( + process.execPath, + [resolve(repositoryRoot, "scripts/verify-release.mjs"), "--models", "1.0.1"], + { cwd: repositoryRoot, encoding: "utf8" } + ); + assert.match(output, /model 1\.0\.1/); +}); + +test("creates the immutable model release without clobber", () => { + const workflow = readFileSync( + resolve(repositoryRoot, ".github/workflows/model-validation.yml"), + "utf8" + ); + assert.match(workflow, /model_version:[\s\S]*default:\s*["']?1\.0\.1/); + assert.match(workflow, /release_tag:[\s\S]*default:\s*["']?v1\.0\.1-models/); + assert.match(workflow, /gh release create/); + assert.doesNotMatch(workflow, /--clobber/); +}); +``` + +- [ ] **Step 2: Run release tests and verify the RED state** + +Run: + +```powershell +pnpm release:test +``` + +Expected: FAIL because `--models` accepts no version and the workflow uploads to `v1.0.0-models` with `--clobber`. + +- [ ] **Step 3: Parameterize local model verification** + +In `scripts/verify-release.mjs`, parse `--models `, validate `/^\d+\.\d+\.\d+$/`, load `models/pp-doclayoutv3//manifest.json`, and derive reports from `tools/model-pipeline/reports//`. For `1.0.1`, additionally require: + +```js +if (browser.fp32Wasm?.status !== "passed") fail("strict FP32 WASM evidence is missing"); +if (browser.fp32Webgpu?.status !== "passed") fail("strict FP32 WebGPU evidence is missing"); +if (browser.fp32Wasm?.fallbacks?.length !== 0) fail("FP32 WASM evidence contains fallback"); +if (browser.fp32Webgpu?.fallbacks?.length !== 0) fail("FP32 WebGPU evidence contains fallback"); +if (manifestVariants.fp32?.backendCompatibility.join(",") !== "wasm,webgpu") { + fail("FP32 manifest compatibility must be wasm,webgpu"); +} +``` + +Keep `node scripts/verify-release.mjs --models 1.0.0` able to verify historical assets. During Phase 1, `--static` must continue treating `1.0.0` as the SDK default. + +- [ ] **Step 4: Replace mutable upload with explicit release creation** + +Change `.github/workflows/model-validation.yml` inputs to `model_version`, `release_tag`, and `upload_assets`. Validate `release_tag == v${model_version}-models`, pass the version to the verifier, and upload versioned reports/artifacts. Use a creation step that refuses an existing release: + +```yaml + - name: Create immutable model release + shell: bash + run: | + set -euo pipefail + if gh release view "${RELEASE_TAG}" >/dev/null 2>&1; then + echo "Release ${RELEASE_TAG} already exists; immutable assets will not be overwritten." >&2 + exit 1 + fi + gh release create "${RELEASE_TAG}" \ + "models/pp-doclayoutv3/${MODEL_VERSION}/manifest.json" \ + "models/pp-doclayoutv3/${MODEL_VERSION}/model-fp16.onnx" \ + "models/pp-doclayoutv3/${MODEL_VERSION}/model-fp32.onnx" \ + "tools/model-pipeline/reports/${MODEL_VERSION}/browser-evidence.json" \ + "tools/model-pipeline/reports/${MODEL_VERSION}/fp32-validation.json" \ + "tools/model-pipeline/reports/${MODEL_VERSION}/variant-validation.json" \ + --target main \ + --title "PP-DocLayoutV3 model ${MODEL_VERSION}" \ + --notes "Immutable PP-DocLayoutV3 ${MODEL_VERSION} browser model assets." + env: + GH_TOKEN: ${{ github.token }} + MODEL_VERSION: ${{ inputs.model_version }} + RELEASE_TAG: ${{ inputs.release_tag }} +``` + +The upload job remains gated by `inputs.upload_assets`; default workflow permissions stay read-only and only the upload job gets `contents: write`. + +- [ ] **Step 5: Run release and action syntax checks** + +```powershell +pnpm release:test +pnpm exec prettier --check .github/workflows/model-validation.yml +node scripts/verify-release.mjs --models 1.0.0 +node scripts/verify-release.mjs --models 1.0.1 +``` + +Expected: all commands pass; historical and new model assets verify independently. + +- [ ] **Step 6: Commit publication safeguards** + +```powershell +git add -- scripts/verify-release.mjs scripts/verify-release.test.mjs .github/workflows/model-validation.yml +git commit -m "ci(models): publish versioned immutable assets" +``` + +### Task 7: Document and verify the model asset phase + +**Files:** +- Modify: `models/README.md` +- Modify: `docs/en/conversion.md` +- Modify: `docs/zh-CN/conversion.md` +- Modify: `scripts/check-doc-parity.test.mjs` + +- [ ] **Step 1: Add failing documentation provenance checks** + +In `scripts/check-doc-parity.test.mjs`, require both language documents and the model README to contain: + +```js +for (const document of [modelReadme, englishConversion, chineseConversion]) { + assert.match(document, /1\.0\.1/); + assert.match(document, /v1\.0\.1-models/); + assert.match(document, /sin.*cos.*sin_1.*cos_1/s); + assert.match(document, /625.*64/s); + assert.match(document, /FP64.*不支持|FP64.*not supported/is); +} +``` + +- [ ] **Step 2: Run docs tests and verify the RED state** + +```powershell +pnpm docs:test +``` + +Expected: FAIL because model `1.0.1` and its sanitation provenance are not documented. + +- [ ] **Step 3: Add exact provenance and reproduction commands** + +Document these facts in Chinese and English: + +- upstream `torch_dtype` is float32; this is not FP64 inference; +- source FP32 hash is `fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b`; +- only `sin`, `cos`, `sin_1`, `cos_1`, each DOUBLE `[625, 64]`, are converted to FLOAT before the existing FLOAT Cast; +- learned initializers and graph input/output contract are unchanged; +- model `1.0.1` is validated on seven licensed fixtures in strict browser WASM and physical WebGPU; +- FP16 is byte-identical to the accepted `1.0.0` FP16 hash; +- the reproduction command is the sanitizer invocation from Task 2; +- historical `v1.0.0-models` assets remain immutable. + +- [ ] **Step 4: Run the complete Phase 1 verification** + +```powershell +.\.venv-model\Scripts\python.exe -m pytest tools/model-pipeline -q +pnpm docs:test +pnpm benchmark:test +pnpm release:test +pnpm lint +pnpm typecheck +pnpm build +git diff --check +node scripts/verify-release.mjs --models 1.0.1 +``` + +Expected: every command exits zero. Confirm `packages/sdk/src/detector.ts` still points to `models/v1.0.0/manifest.json` and `packages/sdk/package.json` is still `1.0.4`. + +- [ ] **Step 5: Commit model-phase documentation** + +```powershell +git add -- models/README.md docs/en/conversion.md docs/zh-CN/conversion.md scripts/check-doc-parity.test.mjs +git commit -m "docs(models): record FP32 sanitation evidence" +``` + +- [ ] **Step 6: Merge the model asset phase only after review** + +Open a PR containing Tasks 1-7. Review the artifact hashes, all seven fixture reports, physical adapter identity, exact manifest URLs, and the absence of SDK default changes. Do not merge the SDK adoption changes in this PR. + +### Checkpoint A: Explicit confirmation before publishing `v1.0.1-models` + +- [ ] **Wait for the user to explicitly confirm model release publication** + +After the Phase 1 PR is merged to `main`, show the user: + +- the merged commit; +- sanitized FP32 byte count and SHA-256; +- FP16 byte count and SHA-256; +- WASM and physical WebGPU seven-fixture summaries; +- exact release tag `v1.0.1-models`; +- confirmation that the workflow creates a new release and cannot clobber it. + +Only after an explicit confirmation, dispatch `.github/workflows/model-validation.yml` with: + +```text +model_version = 1.0.1 +release_tag = v1.0.1-models +upload_assets = true +``` + +- [ ] **Verify every public immutable URL before Phase 2** + +Run: + +```powershell +gh release view v1.0.1-models --json tagName,isDraft,isPrerelease,assets,url +$urls = @( + 'https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/releases/download/v1.0.1-models/manifest.json', + 'https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/releases/download/v1.0.1-models/model-fp16.onnx', + 'https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/releases/download/v1.0.1-models/model-fp32.onnx' +) +foreach ($url in $urls) { (Invoke-WebRequest -Method Head -Uri $url).StatusCode } +``` + +Expected: release is public and not draft; every URL returns HTTP 200. Download the three assets to a temporary directory and compare byte size/SHA-256 with the checked-in manifest before starting Task 8. + +## Phase 2: SDK `1.0.5` Adoption + +### Task 8: Switch the SDK default and lock WebGPU FP32 selection behavior + +**Files:** +- Modify: `packages/sdk/tests/detector.test.ts` +- Modify: `packages/sdk/tests/manifest.test.ts` +- Modify: `packages/sdk/tests/runtime-selector.test.ts` +- Modify: `packages/sdk/src/detector.ts` + +- [ ] **Step 1: Write failing SDK adoption tests** + +Change SDK test fixtures to load `models/pp-doclayoutv3/1.0.1/manifest.json`. In `detector.test.ts`, expect: + +```ts +expect(DEFAULT_MANIFEST_URL).toBe( + "https://chenmohan123.github.io/web-sdk-PP-DocLayoutV3/models/v1.0.1/manifest.json" +); +``` + +In `manifest.test.ts`, require FP32 compatibility and release URL: + +```ts +expect(manifest.model.version).toBe("1.0.1"); +expect(manifest.minSdkVersion).toBe("1.0.0"); +expect(manifest.variants.find(({ id }) => id === "fp32")).toMatchObject({ + backendCompatibility: ["wasm", "webgpu"], + precision: "fp32", + url: "https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/releases/download/v1.0.1-models/model-fp32.onnx" +}); +``` + +In `runtime-selector.test.ts`, add explicit and automatic contracts: + +```ts +it("uses WebGPU FP32 when WebGPU exists without shader-f16", () => { + const plan = selectExecutionPlan( + {}, + capabilities({ webgpu: true, webgpuFp16: false }), + manifest.variants + ); + expect(plan.selected).toMatchObject({ provider: "webgpu", precision: "fp32" }); +}); + +it("keeps explicit WebGPU FP32 strict", () => { + const plan = selectExecutionPlan( + { allowFallback: false, backend: "webgpu", precision: "fp32" }, + capabilities({ webgpu: true, webgpuFp16: false }), + manifest.variants + ); + expect(plan.candidates.filter(({ status }) => status === "selected")).toEqual([ + expect.objectContaining({ provider: "webgpu", precision: "fp32", variantId: "fp32" }) + ]); +}); +``` + +Retain the automatic priority assertion: WebGPU FP16, WebGPU FP32, accepted WASM INT8 if present, WASM FP32. + +- [ ] **Step 2: Run SDK tests and verify the RED state** + +```powershell +pnpm --filter web-sdk-pp-doclayoutv3 exec vitest run tests/detector.test.ts tests/manifest.test.ts tests/runtime-selector.test.ts +``` + +Expected: FAIL because `DEFAULT_MANIFEST_URL` still targets Pages model `1.0.0`. + +- [ ] **Step 3: Change only the default manifest URL** + +In `packages/sdk/src/detector.ts`: + +```ts +export const DEFAULT_MANIFEST_URL = + "https://chenmohan123.github.io/web-sdk-PP-DocLayoutV3/models/v1.0.1/manifest.json"; +``` + +No new selection algorithm is needed: the generic selector already has the approved candidate order and reads backend compatibility from the manifest. + +- [ ] **Step 4: Run SDK tests and commit** + +```powershell +pnpm --filter web-sdk-pp-doclayoutv3 exec vitest run tests/detector.test.ts tests/manifest.test.ts tests/runtime-selector.test.ts +git add -- packages/sdk/src/detector.ts packages/sdk/tests/detector.test.ts packages/sdk/tests/manifest.test.ts packages/sdk/tests/runtime-selector.test.ts +git commit -m "feat(sdk): adopt model manifest 1.0.1" +``` + +Expected: focused tests pass, including strict explicit WebGPU FP32 and automatic FP16 priority. + +### Task 9: Enable GPU FP32 in the Demo without weakening manual strictness + +**Files:** +- Modify: `apps/demo/src/execution-preferences.ts` +- Modify: `apps/demo/src/i18n/en.ts` +- Modify: `apps/demo/src/i18n/zh-CN.ts` +- Modify: `apps/demo/tests/demo.spec.ts` + +- [ ] **Step 1: Change Demo tests first** + +In `apps/demo/tests/demo.spec.ts`, update the pure matrix expectation: + +```ts +expect(behavior).toMatchObject({ + autoFallback: true, + backendFallback: false, + precisionFallback: false, + gpuFp16: true, + gpuFp32: true, + wasmFp16: false, + wasmFp32: true, + gpuCorrection: "fp32", + wasmCorrection: "fp32" +}); +``` + +Replace the UI test that expected GPU FP32 to be disabled with: + +```ts +await precision.getByRole("button", { name: "FP32" }).click(); +await backend.getByRole("button", { name: "GPU" }).click(); +await expect(precision.getByRole("button", { name: "FP32" })).toBeEnabled(); +await expect(precision.getByRole("button", { name: "FP32" })).toHaveAttribute( + "aria-pressed", + "true" +); +await expect(page.getByTestId("notice")).not.toContainText("已为你切换模型精度"); +``` + +Keep CPU + FP16 disabled/corrected and keep `allowFallbackForSelection` true only for auto + auto. + +- [ ] **Step 2: Run Demo tests and verify the RED state** + +```powershell +pnpm --filter demo exec playwright test tests/demo.spec.ts --grep "manual choices strict|validated default model matrix" +``` + +Expected: FAIL because `DEFAULT_SUPPORT.webgpu` contains only `fp16`. + +- [ ] **Step 3: Enable the validated pair and update obsolete messages** + +In `apps/demo/src/execution-preferences.ts`: + +```ts +const DEFAULT_SUPPORT = { + webgpu: ["fp16", "fp32"], + wasm: ["fp32"] +} as const; +``` + +Remove the obsolete default-GPU-FP32-unvalidated notice from both locale files if no remaining call site uses it. Do not change `allowFallbackForSelection`: manual GPU + FP32 must still set `allowFallback: false`. + +- [ ] **Step 4: Run focused and full Demo tests** + +```powershell +pnpm --filter demo exec playwright test tests/demo.spec.ts --grep "manual choices strict|validated default model matrix" +pnpm --filter demo test +``` + +Expected: all tests pass; CPU still resolves to WASM FP32; manual GPU FP32 remains selected and never silently falls back. + +- [ ] **Step 5: Commit the Demo matrix** + +```powershell +git add -- apps/demo/src/execution-preferences.ts apps/demo/src/i18n/en.ts apps/demo/src/i18n/zh-CN.ts apps/demo/tests/demo.spec.ts +git commit -m "feat(demo): enable validated WebGPU FP32" +``` + +### Task 10: Stage model `1.0.1` for Pages and update release contracts + +**Files:** +- Modify: `scripts/stage-pages-models.mjs` +- Modify: `scripts/verify-release.mjs` +- Modify: `scripts/verify-release.test.mjs` +- Modify: `scripts/benchmark-contract.test.mjs` + +- [ ] **Step 1: Write failing Pages staging tests** + +Change the expected roots in `scripts/verify-release.test.mjs`: + +```js +assert.equal(staged.model.version, "1.0.1"); +assert.equal(staged.variants[0].url, "https://pages.test/models/v1.0.1/model-fp16.onnx"); +assert.equal(staged.variants[1].url, "https://pages.test/models/v1.0.1/model-fp32.onnx"); +``` + +Add a static contract assertion: + +```js +assert.match( + readFileSync(resolve(repositoryRoot, "scripts/stage-pages-models.mjs"), "utf8"), + /releases\/download\/v1\.0\.1-models/ +); +``` + +- [ ] **Step 2: Run release tests and verify the RED state** + +```powershell +pnpm release:test +``` + +Expected: FAIL because Pages staging still downloads `v1.0.0-models` into `models/v1.0.0`. + +- [ ] **Step 3: Switch Pages staging constants** + +In `scripts/stage-pages-models.mjs`: + +```js +export const MODEL_RELEASE_ROOT = + "https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/releases/download/v1.0.1-models"; +export const MODEL_PUBLIC_ROOT = + "https://chenmohan123.github.io/web-sdk-PP-DocLayoutV3/models/v1.0.1"; +``` + +Change the executable output directory to: + +```js +outputRoot: resolve(repositoryRoot, "apps/demo/dist/models/v1.0.1") +``` + +Keep manifest integrity verification before every write. + +- [ ] **Step 4: Change the static SDK release contract to model `1.0.1`** + +Update `scripts/verify-release.mjs` static verification to load model `1.0.1`, require FP32 `wasm,webgpu`, and require the Pages staging URL `v1.0.1-models`. Retain `node scripts/verify-release.mjs --models 1.0.0` as an explicit historical verification path. + +- [ ] **Step 5: Run staging and release tests** + +```powershell +pnpm release:test +pnpm --filter web-sdk-pp-doclayoutv3 build +pnpm --filter demo exec vite build --base /web-sdk-PP-DocLayoutV3/ +node scripts/stage-pages-models.mjs +node scripts/verify-release.mjs --static +``` + +Expected: tests pass; `apps/demo/dist/models/v1.0.1/manifest.json` references Pages URLs and both staged models match release byte size/SHA-256. + +- [ ] **Step 6: Commit Pages adoption** + +```powershell +git add -- scripts/stage-pages-models.mjs scripts/verify-release.mjs scripts/verify-release.test.mjs scripts/benchmark-contract.test.mjs +git commit -m "build(pages): stage model assets 1.0.1" +``` + +Do not commit `apps/demo/dist` unless the repository's existing release process explicitly tracks the regenerated build output. + +### Task 11: Synchronize bilingual SDK and model documentation + +**Files:** +- Modify: `README.md` +- Modify: `README.en.md` +- Modify: `packages/sdk/README.md` +- Modify: `models/README.md` +- Modify: `docs/en/api.md` +- Modify: `docs/en/compatibility.md` +- Modify: `docs/en/models.md` +- Modify: `docs/en/performance.md` +- Modify: `docs/zh-CN/api.md` +- Modify: `docs/zh-CN/compatibility.md` +- Modify: `docs/zh-CN/models.md` +- Modify: `docs/zh-CN/performance.md` +- Modify: `scripts/check-doc-parity.test.mjs` +- Modify: `CHANGELOG.md` + +- [ ] **Step 1: Write failing documentation matrix assertions** + +In `scripts/check-doc-parity.test.mjs`, require all public docs to agree on: + +```js +for (const document of [rootReadme, englishReadme, packageReadme, englishModels, chineseModels]) { + assert.match(document, /1\.0\.1/); + assert.match(document, /WebGPU.*FP16.*FP32|FP16.*FP32.*WebGPU/is); + assert.match(document, /WASM.*FP32|FP32.*WASM/is); + assert.match(document, /FP64.*not supported|不支持.*FP64/is); +} +assert.match(packageReadme, /explicit.*strict|手动.*严格/is); +assert.match(packageReadme, /FP32.*larger.*memory|FP32.*更大.*显存/is); +``` + +- [ ] **Step 2: Run docs tests and verify the RED state** + +```powershell +pnpm docs:test +``` + +Expected: FAIL because consumer docs still describe WebGPU FP32 as unvalidated or model `1.0.0` as the default. + +- [ ] **Step 3: Apply the approved support matrix everywhere** + +Use these exact facts in both languages: + +- default model version is `1.0.1`, released at `v1.0.1-models`; +- FP16 supports WebGPU and remains the recommended automatic default when `shader-f16` exists; +- sanitized FP32 supports WebGPU and WASM; +- a WebGPU adapter without `shader-f16` can automatically select WebGPU FP32; +- automatic runtime order is WebGPU FP16, WebGPU FP32, validated WASM INT8 if present, WASM FP32; +- explicit selections are strict and expose any session error without silently changing backend/precision; +- FP64 inference is not supported and the original model is float32; +- FP32 is approximately twice the download size of FP16 and may be slower or consume more GPU memory; +- model `1.0.0` and SDK `1.0.4` remain immutable historical releases. + +In `CHANGELOG.md`, add an SDK `1.0.5` section: + +```markdown +## 1.0.5 + +- Adopted immutable PP-DocLayoutV3 model `1.0.1`, enabling validated strict WebGPU FP32 execution while retaining WebGPU FP16 as the preferred automatic path. +- Versioned model validation evidence and Pages staging so historical `1.0.0` assets remain unchanged. +``` + +- [ ] **Step 4: Run docs parity and commit** + +```powershell +pnpm docs:test +git add -- README.md README.en.md packages/sdk/README.md models/README.md docs/en/api.md docs/en/compatibility.md docs/en/models.md docs/en/performance.md docs/zh-CN/api.md docs/zh-CN/compatibility.md docs/zh-CN/models.md docs/zh-CN/performance.md scripts/check-doc-parity.test.mjs CHANGELOG.md +git commit -m "docs: document WebGPU FP32 support" +``` + +### Task 12: Prepare SDK package version `1.0.5` + +**Files:** +- Modify: `packages/sdk/package.json` +- Modify: `packages/sdk/src/model/manifest.ts` +- Modify: `packages/sdk/tests/manifest.test.ts` +- Modify: `scripts/verify-release.test.mjs` +- Modify: `pnpm-lock.yaml` + +- [ ] **Step 1: Write failing version-alignment tests** + +Change assertions to `1.0.5`: + +```ts +expect(CURRENT_SDK_VERSION).toBe("1.0.5"); +``` + +```js +assert.equal(packageMetadata.version, "1.0.5"); +assert.match(runtime, /CURRENT_SDK_VERSION = "1\.0\.5"/); +assert.match(changelog, /^## 1\.0\.5$/m); +``` + +- [ ] **Step 2: Run focused tests and verify the RED state** + +```powershell +pnpm --filter web-sdk-pp-doclayoutv3 exec vitest run tests/manifest.test.ts +pnpm release:test +``` + +Expected: FAIL because package/runtime versions remain `1.0.4`. + +- [ ] **Step 3: Align package, runtime, and lockfile versions** + +Set: + +```ts +export const CURRENT_SDK_VERSION = "1.0.5"; +``` + +Set `packages/sdk/package.json` version to `1.0.5`, then refresh only workspace metadata: + +```powershell +pnpm install --lockfile-only +``` + +- [ ] **Step 4: Run focused tests and package build** + +```powershell +pnpm --filter web-sdk-pp-doclayoutv3 exec vitest run tests/manifest.test.ts +pnpm release:test +pnpm --filter web-sdk-pp-doclayoutv3 build +``` + +Expected: all commands pass; generated API declarations contain no unintended public API changes. + +- [ ] **Step 5: Commit SDK release preparation** + +```powershell +git add -- packages/sdk/package.json packages/sdk/src/model/manifest.ts packages/sdk/tests/manifest.test.ts scripts/verify-release.test.mjs pnpm-lock.yaml CHANGELOG.md +git commit -m "chore(release): prepare v1.0.5" +``` + +### Task 13: Complete adoption verification and visual QA + +**Files:** +- Verify all Phase 2 files + +- [ ] **Step 1: Run static and model quality gates** + +```powershell +pnpm exec prettier --check .github apps/demo/src apps/demo/tests packages/sdk/src packages/sdk/tests models docs scripts tests CHANGELOG.md README.md README.en.md +pnpm lint +pnpm typecheck +.\.venv-model\Scripts\python.exe -m pytest tools/model-pipeline -q +git diff --check +``` + +Expected: zero formatting, lint, type, model-pipeline, and whitespace failures. + +- [ ] **Step 2: Run workspace, browser, build, and release verification** + +```powershell +pnpm docs:test +pnpm benchmark:test +pnpm release:test +pnpm test +pnpm build +pnpm exec playwright test tests/browser/package.spec.ts +node scripts/verify-release.mjs --models 1.0.0 +node scripts/verify-release.mjs --models 1.0.1 +node scripts/verify-release.mjs --release v1.0.5 +``` + +Expected: all suites pass; release verification sees SDK `1.0.5` and model `1.0.1`; historical model verification remains green. + +- [ ] **Step 3: Verify the production Demo against public assets** + +Start the built Demo on an unused port and use a physical WebGPU browser at 1440x900 and 390x844. Verify: + +- GPU + FP32 is enabled; +- GPU + FP32 completes with actual runtime `webgpu + fp32`; +- the fallback list is empty for the strict manual run; +- auto mode still selects `webgpu + fp16` on an adapter with `shader-f16`; +- CPU mode remains `wasm + fp32`; +- progress distinguishes model download from model loading; +- result, model, timing, fallback, and action sections do not overlap or overflow. + +Capture screenshots and the exported JSON. Confirm the exported runtime, model version/hash, and fallback list match the UI. + +- [ ] **Step 4: Inspect package and final diff** + +```powershell +pnpm --filter web-sdk-pp-doclayoutv3 pack --pack-destination test-results/package +git status --short --branch +git diff origin/develop...HEAD --stat +git diff origin/develop...HEAD --check +``` + +Expected: the package contains built SDK files only; the diff contains the approved model pipeline, evidence, SDK, Demo, workflow, and documentation changes; generated local test output is not staged. + +### Checkpoint B: Explicit confirmation before merging SDK adoption + +- [ ] **Wait for the user to explicitly confirm the Phase 2 merge** + +Present the passing verification summary, production Demo WebGPU FP32 result, public model URL checks, package contents, and adoption PR diff. Merge the adoption PR only after the user explicitly confirms. + +### Checkpoint C: Explicit confirmation before tagging and publishing SDK `v1.0.5` + +- [ ] **Wait for the user to explicitly confirm SDK publication** + +After the adoption PR is merged to `main`, show the exact main commit and the dry-run result: + +```powershell +node scripts/verify-release.mjs --release v1.0.5 +pnpm --filter web-sdk-pp-doclayoutv3 publish --dry-run +``` + +Only after a separate explicit confirmation, create and push tag `v1.0.5`. The existing release workflow must publish `web-sdk-pp-doclayoutv3@1.0.5` through npm Trusted Publishing with provenance. + +- [ ] **Verify the published SDK and Demo** + +After the release workflow succeeds: + +```powershell +npm view web-sdk-pp-doclayoutv3@1.0.5 version dist.integrity dist.tarball --json +gh release view v1.0.5 --json tagName,url,assets +``` + +Open the production Demo, clear model cache, and repeat strict GPU + FP32 once. Confirm it fetches `models/v1.0.1/manifest.json`, reports `webgpu + fp32`, records no fallback, and uses the sanitized FP32 SHA-256. From 88e14e4cdaf9906ab3d9bd2b3332bf49cf89bd91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 00:28:33 +0800 Subject: [PATCH 03/13] test(models): specify WebGPU FP32 sanitizer --- .../tests/test_sanitize_fp32.py | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 tools/model-pipeline/tests/test_sanitize_fp32.py diff --git a/tools/model-pipeline/tests/test_sanitize_fp32.py b/tools/model-pipeline/tests/test_sanitize_fp32.py new file mode 100644 index 0000000..a1018fb --- /dev/null +++ b/tools/model-pipeline/tests/test_sanitize_fp32.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + +import numpy as np +import onnx +import pytest +from onnx import TensorProto, helper, numpy_helper + +from ppdoclayout.sanitize_fp32 import sanitize_webgpu_fp32 + + +POSITIONAL_NAMES = ("sin", "cos", "sin_1", "cos_1") + + +def sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def source_model(*, extra_double: bool = False, cast_to: int = TensorProto.FLOAT) -> onnx.ModelProto: + initializers = [ + numpy_helper.from_array( + np.arange(625 * 64, dtype=np.float64).reshape(625, 64), name=name + ) + for name in POSITIONAL_NAMES + ] + initializers.append( + numpy_helper.from_array(np.asarray([3.25], dtype=np.float32), name="learned_weight") + ) + if extra_double: + initializers.append( + numpy_helper.from_array(np.asarray([1.0], dtype=np.float64), name="unexpected") + ) + nodes = [ + helper.make_node( + "Concat", POSITIONAL_NAMES, ["cat_7"], axis=1, name="node_cat_7" + ), + helper.make_node( + "Cast", ["cat_7"], ["_to_copy_4"], to=cast_to, name="node__to_copy_4" + ), + helper.make_node( + "Add", ["input", "learned_weight"], ["output"], name="learned_add" + ), + ] + graph = helper.make_graph( + nodes, + "sanitize-test", + [helper.make_tensor_value_info("input", TensorProto.FLOAT, [1])], + [helper.make_tensor_value_info("output", TensorProto.FLOAT, [1])], + initializers, + ) + return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 18)]) + + +def write_source(path: Path, model: onnx.ModelProto) -> None: + path.write_bytes(model.SerializeToString(deterministic=True)) + + +def replace_first_initializer_with_wrong_shape(model: onnx.ModelProto) -> None: + model.graph.initializer[0].CopyFrom( + numpy_helper.from_array( + np.zeros((624, 64), dtype=np.float64), name=POSITIONAL_NAMES[0] + ) + ) + + +def test_converts_only_known_positional_constants(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + write_source(source, source_model()) + + result = sanitize_webgpu_fp32(source, output) + + model = onnx.load(output, load_external_data=False) + by_name = {value.name: value for value in model.graph.initializer} + assert all(by_name[name].data_type == TensorProto.FLOAT for name in POSITIONAL_NAMES) + assert by_name["learned_weight"].raw_data == np.asarray([3.25], dtype=np.float32).tobytes() + assert result == {"bytes": output.stat().st_size, "sha256": sha256_file(output)} + + +def test_rejects_missing_positional_initializer(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + model = source_model() + del model.graph.initializer[0] + write_source(source, model) + + with pytest.raises(ValueError, match="exactly"): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + +def test_rejects_renamed_positional_initializer(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + model = source_model() + model.graph.initializer[0].name = "renamed" + write_source(source, model) + + with pytest.raises(ValueError, match="exactly"): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + +def test_rejects_wrong_shape_positional_initializer(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + model = source_model() + replace_first_initializer_with_wrong_shape(model) + write_source(source, model) + + with pytest.raises(ValueError, match="shape"): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + +def test_rejects_renamed_concat_node(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + model = source_model() + model.graph.node[0].name = "other_concat" + write_source(source, model) + + with pytest.raises(ValueError, match="node_cat_7"): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + +def test_rejects_wrong_concat_topology(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + model = source_model() + model.graph.node[0].input[0], model.graph.node[0].input[1] = ( + model.graph.node[0].input[1], + model.graph.node[0].input[0], + ) + write_source(source, model) + + with pytest.raises(ValueError, match="Concat"): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + +def test_rejects_cast_target_other_than_float(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + write_source(source, source_model(cast_to=TensorProto.FLOAT16)) + + with pytest.raises(ValueError, match="FLOAT"): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + +def test_rejects_any_additional_double_initializer(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + write_source(source, source_model(extra_double=True)) + + with pytest.raises(ValueError, match="unexpected DOUBLE initializer"): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + +def test_is_byte_reproducible(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + first = tmp_path / "first.onnx" + second = tmp_path / "second.onnx" + write_source(source, source_model()) + + sanitize_webgpu_fp32(source, first) + sanitize_webgpu_fp32(source, second) + + assert first.read_bytes() == second.read_bytes() From 78608ea1387c1d2978b3bd5f7c41bb0cb05c859b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 00:33:17 +0800 Subject: [PATCH 04/13] test(models): guard Cast topology contract --- .../tests/test_sanitize_fp32.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tools/model-pipeline/tests/test_sanitize_fp32.py b/tools/model-pipeline/tests/test_sanitize_fp32.py index a1018fb..07e5c9e 100644 --- a/tools/model-pipeline/tests/test_sanitize_fp32.py +++ b/tools/model-pipeline/tests/test_sanitize_fp32.py @@ -147,6 +147,32 @@ def test_rejects_wrong_concat_topology(tmp_path: Path) -> None: assert not output.exists() +def test_rejects_renamed_cast_node(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + model = source_model() + model.graph.node[1].name = "other_cast" + write_source(source, model) + + with pytest.raises(ValueError, match="node__to_copy_4"): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + +def test_rejects_wrong_cast_topology(tmp_path: Path) -> None: + source = tmp_path / "source.onnx" + output = tmp_path / "output.onnx" + model = source_model() + model.graph.node[1].input[0] = "input" + write_source(source, model) + + with pytest.raises(ValueError, match="Cast"): + sanitize_webgpu_fp32(source, output) + + assert not output.exists() + + def test_rejects_cast_target_other_than_float(tmp_path: Path) -> None: source = tmp_path / "source.onnx" output = tmp_path / "output.onnx" From 0372f7426ed695112b4db4e0e0cf8a2eab4f59c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 00:40:05 +0800 Subject: [PATCH 05/13] test(models): keep sanitizer fixtures valid --- tools/model-pipeline/tests/test_sanitize_fp32.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/model-pipeline/tests/test_sanitize_fp32.py b/tools/model-pipeline/tests/test_sanitize_fp32.py index 07e5c9e..708cd56 100644 --- a/tools/model-pipeline/tests/test_sanitize_fp32.py +++ b/tools/model-pipeline/tests/test_sanitize_fp32.py @@ -84,6 +84,7 @@ def test_rejects_missing_positional_initializer(tmp_path: Path) -> None: output = tmp_path / "output.onnx" model = source_model() del model.graph.initializer[0] + del model.graph.node[0].input[0] write_source(source, model) with pytest.raises(ValueError, match="exactly"): @@ -97,6 +98,7 @@ def test_rejects_renamed_positional_initializer(tmp_path: Path) -> None: output = tmp_path / "output.onnx" model = source_model() model.graph.initializer[0].name = "renamed" + model.graph.node[0].input[0] = "renamed" write_source(source, model) with pytest.raises(ValueError, match="exactly"): From 7236002e630aac84d5b0614bcfd33a79968d6b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 00:47:21 +0800 Subject: [PATCH 06/13] feat(models): sanitize FP32 graph for WebGPU --- models/pp-doclayoutv3/1.0.1/model-fp16.onnx | 3 + models/pp-doclayoutv3/1.0.1/model-fp32.onnx | 3 + .../ppdoclayout/sanitize_fp32.py | 206 ++++++++++++++++++ .../tests/test_sanitize_fp32.py | 43 +++- 4 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 models/pp-doclayoutv3/1.0.1/model-fp16.onnx create mode 100644 models/pp-doclayoutv3/1.0.1/model-fp32.onnx create mode 100644 tools/model-pipeline/ppdoclayout/sanitize_fp32.py diff --git a/models/pp-doclayoutv3/1.0.1/model-fp16.onnx b/models/pp-doclayoutv3/1.0.1/model-fp16.onnx new file mode 100644 index 0000000..ba83a3b --- /dev/null +++ b/models/pp-doclayoutv3/1.0.1/model-fp16.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:463ba56faa555baf84271b4002b33b0c5fcc50776fe4f39344235eccb72073f2 +size 74279796 diff --git a/models/pp-doclayoutv3/1.0.1/model-fp32.onnx b/models/pp-doclayoutv3/1.0.1/model-fp32.onnx new file mode 100644 index 0000000..ec9e7b9 --- /dev/null +++ b/models/pp-doclayoutv3/1.0.1/model-fp32.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269 +size 142574928 diff --git a/tools/model-pipeline/ppdoclayout/sanitize_fp32.py b/tools/model-pipeline/ppdoclayout/sanitize_fp32.py new file mode 100644 index 0000000..571c372 --- /dev/null +++ b/tools/model-pipeline/ppdoclayout/sanitize_fp32.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +import argparse +import hashlib +import os +import tempfile +from pathlib import Path + +import numpy as np +import onnx +from onnx import TensorProto, numpy_helper + + +POSITIONAL_NAMES = ("sin", "cos", "sin_1", "cos_1") +POSITIONAL_SHAPE = [625, 64] +KNOWN_LIVE_DOUBLE_NAMES = {*POSITIONAL_NAMES, "cat_7"} +# The exporter left these unreferenced value_info entries behind. +KNOWN_ORPHAN_DOUBLE_VALUE_INFO = {"mul_241", "mul_242"} +RECOMPUTED_DOUBLE_VALUE_INFO = {*POSITIONAL_NAMES, "cat_7"} + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _node(model: onnx.ModelProto, name: str) -> onnx.NodeProto: + matches = [node for node in model.graph.node if node.name == name] + if len(matches) != 1: + raise ValueError(f"Expected exactly one {name} node") + return matches[0] + + +def _require_source_contract(model: onnx.ModelProto) -> None: + doubles = { + value.name: value + for value in model.graph.initializer + if value.data_type == TensorProto.DOUBLE + } + if set(doubles) != set(POSITIONAL_NAMES): + unexpected = sorted(set(doubles) - set(POSITIONAL_NAMES)) + raise ValueError( + f"Expected exactly {list(POSITIONAL_NAMES)} DOUBLE initializers; " + f"unexpected DOUBLE initializer(s): {unexpected}" + ) + for name in POSITIONAL_NAMES: + if list(doubles[name].dims) != POSITIONAL_SHAPE: + raise ValueError(f"{name} must have shape {POSITIONAL_SHAPE}") + + concat = _node(model, "node_cat_7") + axis = next((item.i for item in concat.attribute if item.name == "axis"), None) + if ( + concat.op_type != "Concat" + or list(concat.input) != list(POSITIONAL_NAMES) + or list(concat.output) != ["cat_7"] + or axis != 1 + ): + raise ValueError( + "node_cat_7 must Concat the four positional constants to cat_7 on axis 1" + ) + + cast = _node(model, "node__to_copy_4") + cast_to = next((item.i for item in cast.attribute if item.name == "to"), None) + if ( + cast.op_type != "Cast" + or list(cast.input) != ["cat_7"] + or list(cast.output) != ["_to_copy_4"] + or cast_to != TensorProto.FLOAT + ): + raise ValueError("node__to_copy_4 must Cast cat_7 to FLOAT as _to_copy_4") + + +def _double_names(model: onnx.ModelProto) -> list[str]: + values = [*model.graph.input, *model.graph.output, *model.graph.value_info] + names = [ + value.name + for value in values + if value.type.tensor_type.elem_type == TensorProto.DOUBLE + ] + names.extend( + value.name + for value in model.graph.initializer + if value.data_type == TensorProto.DOUBLE + ) + return sorted(set(names)) + + +def _validate_source(model: onnx.ModelProto) -> None: + _require_source_contract(model) + onnx.checker.check_model(model) + if any(value.external_data for value in model.graph.initializer): + raise ValueError("Source model must be self-contained") + inferred = onnx.shape_inference.infer_shapes(model, strict_mode=True) + doubles = set(_double_names(inferred)) + live_names = { + name + for node in model.graph.node + for name in [*node.input, *node.output] + if name + } + live_names.update(value.name for value in model.graph.initializer) + live_names.update(value.name for value in [*model.graph.input, *model.graph.output]) + live_doubles = doubles & live_names + orphan_doubles = doubles - live_names + if live_doubles != KNOWN_LIVE_DOUBLE_NAMES: + unexpected = sorted(live_doubles - KNOWN_LIVE_DOUBLE_NAMES) + missing = sorted(KNOWN_LIVE_DOUBLE_NAMES - live_doubles) + raise ValueError( + "Source graph live DOUBLE values do not match the known positional path; " + f"unexpected={unexpected}, missing={missing}" + ) + if not orphan_doubles <= KNOWN_ORPHAN_DOUBLE_VALUE_INFO: + unexpected = sorted(orphan_doubles - KNOWN_ORPHAN_DOUBLE_VALUE_INFO) + raise ValueError(f"Source graph contains unexpected orphan DOUBLE values: {unexpected}") + by_name = {value.name: value for value in inferred.graph.value_info} + for name in orphan_doubles: + shape = [ + dimension.dim_value + for dimension in by_name[name].type.tensor_type.shape.dim + ] + if shape != POSITIONAL_SHAPE: + raise ValueError(f"Orphan DOUBLE value {name} must have shape {POSITIONAL_SHAPE}") + + +def _remove_recomputed_double_metadata(model: onnx.ModelProto) -> None: + live_names = { + name + for node in model.graph.node + for name in [*node.input, *node.output] + if name + } + retained = [ + value + for value in model.graph.value_info + if not ( + value.type.tensor_type.elem_type == TensorProto.DOUBLE + and ( + value.name in RECOMPUTED_DOUBLE_VALUE_INFO + or ( + value.name in KNOWN_ORPHAN_DOUBLE_VALUE_INFO + and value.name not in live_names + ) + ) + ) + ] + del model.graph.value_info[:] + model.graph.value_info.extend(retained) + + +def sanitize_webgpu_fp32(source: Path, output: Path) -> dict[str, int | str]: + source = source.resolve() + output = output.resolve() + model = onnx.load(source, load_external_data=False) + _validate_source(model) + _remove_recomputed_double_metadata(model) + + for index, value in enumerate(model.graph.initializer): + if value.name not in POSITIONAL_NAMES: + continue + converted = numpy_helper.from_array( + numpy_helper.to_array(value).astype(np.float32), name=value.name + ) + model.graph.initializer[index].CopyFrom(converted) + + inferred = onnx.shape_inference.infer_shapes(model, strict_mode=True) + onnx.checker.check_model(inferred) + remaining = _double_names(inferred) + if remaining: + raise ValueError(f"Sanitized graph still contains DOUBLE values: {remaining}") + + output.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{output.name}.", suffix=".tmp", dir=output.parent + ) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(inferred.SerializeToString(deterministic=True)) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary_name, output) + except BaseException: + Path(temporary_name).unlink(missing_ok=True) + raise + return {"bytes": output.stat().st_size, "sha256": sha256_file(output)} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Sanitize PP-DocLayoutV3 FP32 for WebGPU" + ) + parser.add_argument("--source", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + result = sanitize_webgpu_fp32(args.source, args.output) + print(f"{args.output}: {result['bytes']} bytes sha256={result['sha256']}") + + +if __name__ == "__main__": + main() diff --git a/tools/model-pipeline/tests/test_sanitize_fp32.py b/tools/model-pipeline/tests/test_sanitize_fp32.py index 708cd56..0adf950 100644 --- a/tools/model-pipeline/tests/test_sanitize_fp32.py +++ b/tools/model-pipeline/tests/test_sanitize_fp32.py @@ -8,16 +8,33 @@ import pytest from onnx import TensorProto, helper, numpy_helper -from ppdoclayout.sanitize_fp32 import sanitize_webgpu_fp32 +from ppdoclayout.sanitize_fp32 import ( + POSITIONAL_NAMES, + _double_names, + sanitize_webgpu_fp32, +) -POSITIONAL_NAMES = ("sin", "cos", "sin_1", "cos_1") +ROOT = Path(__file__).parents[3] +SOURCE_FP32 = ROOT / "models" / "pp-doclayoutv3" / "1.0.0" / "model-fp32.onnx" def sha256_file(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def initializer_hashes( + model: onnx.ModelProto, excluded: set[str] +) -> dict[str, str]: + return { + value.name: hashlib.sha256( + value.SerializeToString(deterministic=True) + ).hexdigest() + for value in model.graph.initializer + if value.name not in excluded + } + + def source_model(*, extra_double: bool = False, cast_to: int = TensorProto.FLOAT) -> onnx.ModelProto: initializers = [ numpy_helper.from_array( @@ -207,3 +224,25 @@ def test_is_byte_reproducible(tmp_path: Path) -> None: sanitize_webgpu_fp32(source, second) assert first.read_bytes() == second.read_bytes() + + +def test_real_model_preserves_contract_and_learned_parameters(tmp_path: Path) -> None: + output = tmp_path / "model-fp32.onnx" + sanitize_webgpu_fp32(SOURCE_FP32, output) + source = onnx.load(SOURCE_FP32, load_external_data=False) + candidate = onnx.load(output, load_external_data=False) + + assert [(item.domain, item.version) for item in candidate.opset_import] == [ + ("", 18) + ] + assert [value.SerializeToString() for value in candidate.graph.input] == [ + value.SerializeToString() for value in source.graph.input + ] + assert [value.SerializeToString() for value in candidate.graph.output] == [ + value.SerializeToString() for value in source.graph.output + ] + assert initializer_hashes(candidate, set(POSITIONAL_NAMES)) == initializer_hashes( + source, set(POSITIONAL_NAMES) + ) + inferred = onnx.shape_inference.infer_shapes(candidate, strict_mode=True) + assert not _double_names(inferred) From 8c47754068ff4ec7b34451cb7d562e6e9a8b1c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 01:02:00 +0800 Subject: [PATCH 07/13] test(models): validate sanitized FP32 parity --- tools/model-pipeline/ppdoclayout/validate.py | 94 ++- .../reports/1.0.1/fp32-validation.json | 548 ++++++++++++++++++ .../model-pipeline/tests/test_parity_fp32.py | 32 +- 3 files changed, 653 insertions(+), 21 deletions(-) create mode 100644 tools/model-pipeline/reports/1.0.1/fp32-validation.json diff --git a/tools/model-pipeline/ppdoclayout/validate.py b/tools/model-pipeline/ppdoclayout/validate.py index b3e5071..a1ba685 100644 --- a/tools/model-pipeline/ppdoclayout/validate.py +++ b/tools/model-pipeline/ppdoclayout/validate.py @@ -20,7 +20,6 @@ OUTPUT_NAMES, as_transformers_output, compare_postprocessed, - tensor_metrics, ) @@ -50,7 +49,10 @@ def sha256_file(path: Path) -> str: def validate_fp32( - model_path: Path, onnx_path: Path, fixtures_lock: Path + model_path: Path, + accepted_onnx_path: Path, + onnx_path: Path, + fixtures_lock: Path, ) -> dict[str, Any]: lock = json.loads(fixtures_lock.read_text(encoding="utf-8")) fixtures_dir = fixtures_lock.parent / "images" @@ -62,7 +64,10 @@ def validate_fp32( model = AutoModelForObjectDetection.from_pretrained( model_path, local_files_only=True ).eval() - session = ort.InferenceSession( + accepted_session = ort.InferenceSession( + str(accepted_onnx_path), providers=["CPUExecutionProvider"] + ) + candidate_session = ort.InferenceSession( str(onnx_path), providers=["CPUExecutionProvider"] ) @@ -73,38 +78,85 @@ def validate_fp32( image = opened.convert("RGB") inputs = processor(images=image, return_tensors="pt") official_output = model(**inputs) - onnx_values = session.run( + pixel_values = inputs["pixel_values"].cpu().numpy() + accepted_values = accepted_session.run( + list(OUTPUT_NAMES), {"pixel_values": pixel_values} + ) + candidate_values = candidate_session.run( list(OUTPUT_NAMES), - {"pixel_values": inputs["pixel_values"].cpu().numpy()}, + {"pixel_values": pixel_values}, ) - onnx_outputs = dict(zip(OUTPUT_NAMES, onnx_values)) + accepted_outputs = dict(zip(OUTPUT_NAMES, accepted_values)) + candidate_outputs = dict(zip(OUTPUT_NAMES, candidate_values)) target_sizes = [image.size[::-1]] official_result = processor.post_process_object_detection( official_output, threshold=THRESHOLD, target_sizes=target_sizes, )[0] - onnx_result = processor.post_process_object_detection( - as_transformers_output(onnx_outputs), + accepted_result = processor.post_process_object_detection( + as_transformers_output(accepted_outputs), + threshold=THRESHOLD, + target_sizes=target_sizes, + )[0] + candidate_result = processor.post_process_object_detection( + as_transformers_output(candidate_outputs), threshold=THRESHOLD, target_sizes=target_sizes, )[0] - comparison = compare_postprocessed(official_result, onnx_result) - raw_metrics = { - name: tensor_metrics( - getattr(official_output, name).detach().cpu().numpy(), - onnx_outputs[name], + comparison = compare_postprocessed(official_result, candidate_result) + accepted_comparison = compare_postprocessed( + accepted_result, candidate_result + ) + raw_outputs = { + name: { + "acceptedSha256": hashlib.sha256( + accepted_value.tobytes() + ).hexdigest(), + "candidateSha256": hashlib.sha256( + candidate_value.tobytes() + ).hexdigest(), + "bitIdentical": bool( + np.array_equal(accepted_value, candidate_value) + ), + "dtype": str(candidate_value.dtype), + "maxAbsoluteDelta": float( + np.max(np.abs(accepted_value - candidate_value)) + ) + if accepted_value.size + else 0.0, + "shape": list(candidate_value.shape), + } + for name, accepted_value, candidate_value in zip( + OUTPUT_NAMES, accepted_values, candidate_values, strict=True ) - for name in OUTPUT_NAMES } comparison.update( { + "acceptedDetectionCount": accepted_comparison[ + "detectionCountOfficial" + ], + "onnxDetectionCount": accepted_comparison[ + "detectionCountOnnx" + ], + "acceptedLabelSequenceEqual": accepted_comparison[ + "labelSequenceEqual" + ], + "acceptedReadingOrderEqual": accepted_comparison[ + "readingOrderEqual" + ], "filename": fixture["filename"], "sha256": fixture["sha256"], "width": fixture["width"], "height": fixture["height"], - "rawOutputs": raw_metrics, + "rawOutputs": { + "allBitIdentical": all( + item["bitIdentical"] + for item in raw_outputs.values() + ), + "outputs": raw_outputs, + }, } ) comparison["pass"] = _fixture_passes(comparison) @@ -115,6 +167,7 @@ def validate_fp32( "threshold": THRESHOLD, "thresholds": PARITY_THRESHOLDS, "sourceHashes": { + "acceptedOnnx": sha256_file(accepted_onnx_path), "modelSafetensors": sha256_file(model_path / "model.safetensors"), "onnx": sha256_file(onnx_path), }, @@ -142,7 +195,10 @@ def _verified_fixtures( def _fixture_passes(report: dict[str, Any]) -> bool: return bool( - report["labelSequenceEqual"] + report["acceptedDetectionCount"] == report["onnxDetectionCount"] + and report["acceptedLabelSequenceEqual"] + and report["acceptedReadingOrderEqual"] + and report["labelSequenceEqual"] and report["readingOrderEqual"] and not report["unmatchedOfficial"] and not report["unmatchedOnnx"] @@ -178,6 +234,7 @@ def _paddle_status() -> dict[str, str]: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Validate FP32 ONNX parity") parser.add_argument("--model", required=True, type=Path) + parser.add_argument("--accepted-onnx", required=True, type=Path) parser.add_argument("--onnx", required=True, type=Path) parser.add_argument("--fixtures-lock", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) @@ -187,7 +244,10 @@ def parse_args() -> argparse.Namespace: def main() -> None: args = parse_args() report = validate_fp32( - args.model.resolve(), args.onnx.resolve(), args.fixtures_lock.resolve() + args.model.resolve(), + args.accepted_onnx.resolve(), + args.onnx.resolve(), + args.fixtures_lock.resolve(), ) args.output.parent.mkdir(parents=True, exist_ok=True) write_report(args.output, report) diff --git a/tools/model-pipeline/reports/1.0.1/fp32-validation.json b/tools/model-pipeline/reports/1.0.1/fp32-validation.json new file mode 100644 index 0000000..e40fc1e --- /dev/null +++ b/tools/model-pipeline/reports/1.0.1/fp32-validation.json @@ -0,0 +1,548 @@ +{ + "environment": { + "numpy": "2.4.6", + "onnxruntime": "1.28.0", + "opencv-python-headless": "5.0.0.93", + "platform": "Windows-10-10.0.26200-SP0", + "python": "3.11.15", + "torch": "2.13.0+cpu", + "transformers": "5.15.0" + }, + "fixtures": [ + { + "acceptedDetectionCount": 12, + "acceptedLabelSequenceEqual": true, + "acceptedReadingOrderEqual": true, + "detectionCountOfficial": 12, + "detectionCountOnnx": 12, + "filename": "curved-document.jpg", + "height": 1273, + "labelSequenceEqual": true, + "maxBoxCoordinateDeltaPixels": 0.00020837783813476562, + "maxPolygonCoordinateDeltaPixels": 0.0, + "maxScoreDelta": 1.6689300537109375e-6, + "onnxDetectionCount": 12, + "pass": true, + "rawOutputs": { + "allBitIdentical": true, + "outputs": { + "logits": { + "acceptedSha256": "e4133d98a74db6ccfe23940501bd0244be713af040a1465124529e0f8281142b", + "bitIdentical": true, + "candidateSha256": "e4133d98a74db6ccfe23940501bd0244be713af040a1465124529e0f8281142b", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 25 + ] + }, + "order_logits": { + "acceptedSha256": "ee1887bf86d95b06a7b2c55fa4c94e02776d7c16130cf6ce4270786cd33c12e5", + "bitIdentical": true, + "candidateSha256": "ee1887bf86d95b06a7b2c55fa4c94e02776d7c16130cf6ce4270786cd33c12e5", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 300 + ] + }, + "out_masks": { + "acceptedSha256": "494abcc172eccfc3427b7dfca80d906e1a4d019a7af399becc080f8503ed80ef", + "bitIdentical": true, + "candidateSha256": "494abcc172eccfc3427b7dfca80d906e1a4d019a7af399becc080f8503ed80ef", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 200, + 200 + ] + }, + "pred_boxes": { + "acceptedSha256": "4f72bfd2294edaf7f1b4141d9f6078b098b92cdf7be72eac947e356051d70d7b", + "bitIdentical": true, + "candidateSha256": "4f72bfd2294edaf7f1b4141d9f6078b098b92cdf7be72eac947e356051d70d7b", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 4 + ] + } + } + }, + "readingOrderEqual": true, + "sha256": "fce39d864ff7b0612f7073415c2a7c656f7790a26f96d539831f1bb1a994a069", + "unmatchedOfficial": [], + "unmatchedOnnx": [], + "width": 900 + }, + { + "acceptedDetectionCount": 59, + "acceptedLabelSequenceEqual": true, + "acceptedReadingOrderEqual": true, + "detectionCountOfficial": 59, + "detectionCountOnnx": 59, + "filename": "doc-formula.png", + "height": 1056, + "labelSequenceEqual": true, + "maxBoxCoordinateDeltaPixels": 0.00030517578125, + "maxPolygonCoordinateDeltaPixels": 0.0, + "maxScoreDelta": 1.8477439880371094e-6, + "onnxDetectionCount": 59, + "pass": true, + "rawOutputs": { + "allBitIdentical": true, + "outputs": { + "logits": { + "acceptedSha256": "5fce7480a5f6ecbe926bbed9b93aade165fedf9de47d39caabe41e2e623e5e55", + "bitIdentical": true, + "candidateSha256": "5fce7480a5f6ecbe926bbed9b93aade165fedf9de47d39caabe41e2e623e5e55", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 25 + ] + }, + "order_logits": { + "acceptedSha256": "fe70871e6e847638b70942ddd123ba8a219655e84fa4fbc722f97a963b27ef33", + "bitIdentical": true, + "candidateSha256": "fe70871e6e847638b70942ddd123ba8a219655e84fa4fbc722f97a963b27ef33", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 300 + ] + }, + "out_masks": { + "acceptedSha256": "c25be644690feb8ae9bb1f218edeb6b569b341134f9667763e5305e3513d9563", + "bitIdentical": true, + "candidateSha256": "c25be644690feb8ae9bb1f218edeb6b569b341134f9667763e5305e3513d9563", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 200, + 200 + ] + }, + "pred_boxes": { + "acceptedSha256": "66a9d7e1de1591d1666ff4b55394f4529d99681308bb9583bdb5dc4f32c27e4e", + "bitIdentical": true, + "candidateSha256": "66a9d7e1de1591d1666ff4b55394f4529d99681308bb9583bdb5dc4f32c27e4e", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 4 + ] + } + } + }, + "readingOrderEqual": true, + "sha256": "6b07d28527dc9e930804fa73df562f1a81599c6b8a1a8bbc2a80742fa9f26e80", + "unmatchedOfficial": [], + "unmatchedOnnx": [], + "width": 816 + }, + { + "acceptedDetectionCount": 44, + "acceptedLabelSequenceEqual": true, + "acceptedReadingOrderEqual": true, + "detectionCountOfficial": 44, + "detectionCountOnnx": 44, + "filename": "image-layout.jpg", + "height": 757, + "labelSequenceEqual": true, + "maxBoxCoordinateDeltaPixels": 0.00048828125, + "maxPolygonCoordinateDeltaPixels": 0.0, + "maxScoreDelta": 1.3113021850585938e-6, + "onnxDetectionCount": 44, + "pass": true, + "rawOutputs": { + "allBitIdentical": true, + "outputs": { + "logits": { + "acceptedSha256": "d69d92e4d0066136c71a99a7daa9968629c1668b21fbb357dd6133cffd212104", + "bitIdentical": true, + "candidateSha256": "d69d92e4d0066136c71a99a7daa9968629c1668b21fbb357dd6133cffd212104", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 25 + ] + }, + "order_logits": { + "acceptedSha256": "b05f2433ebf57aab16b1e58769fd13887a37b26b31e3309b34366da151bc8c95", + "bitIdentical": true, + "candidateSha256": "b05f2433ebf57aab16b1e58769fd13887a37b26b31e3309b34366da151bc8c95", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 300 + ] + }, + "out_masks": { + "acceptedSha256": "8f9a13d537b60255e4f654f0415ef3642874f3deb0ad9d03f448e77a7be2f7eb", + "bitIdentical": true, + "candidateSha256": "8f9a13d537b60255e4f654f0415ef3642874f3deb0ad9d03f448e77a7be2f7eb", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 200, + 200 + ] + }, + "pred_boxes": { + "acceptedSha256": "9795d8be3223d62eac914b4588b418177065102e4888c4cbd4e16b2b88b0fca3", + "bitIdentical": true, + "candidateSha256": "9795d8be3223d62eac914b4588b418177065102e4888c4cbd4e16b2b88b0fca3", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 4 + ] + } + } + }, + "readingOrderEqual": true, + "sha256": "cfebd4e0716da8ef01ad29c6f5bf7ed0dcc7d3a07bd38e32219c3b10645798de", + "unmatchedOfficial": [], + "unmatchedOnnx": [], + "width": 1659 + }, + { + "acceptedDetectionCount": 13, + "acceptedLabelSequenceEqual": true, + "acceptedReadingOrderEqual": true, + "detectionCountOfficial": 13, + "detectionCountOnnx": 13, + "filename": "layout-demo.jpg", + "height": 2339, + "labelSequenceEqual": true, + "maxBoxCoordinateDeltaPixels": 0.000244140625, + "maxPolygonCoordinateDeltaPixels": 0.0, + "maxScoreDelta": 1.7285346984863281e-6, + "onnxDetectionCount": 13, + "pass": true, + "rawOutputs": { + "allBitIdentical": true, + "outputs": { + "logits": { + "acceptedSha256": "ff761ba437202de2aabb28bd557bf30f07365e079d8e83e7157a72f890e2779e", + "bitIdentical": true, + "candidateSha256": "ff761ba437202de2aabb28bd557bf30f07365e079d8e83e7157a72f890e2779e", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 25 + ] + }, + "order_logits": { + "acceptedSha256": "9fa299dfa5f12f0425286cfaad7147887c0c184661c3971eb5731fae5c0e2274", + "bitIdentical": true, + "candidateSha256": "9fa299dfa5f12f0425286cfaad7147887c0c184661c3971eb5731fae5c0e2274", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 300 + ] + }, + "out_masks": { + "acceptedSha256": "18c48f9938beb0c3e527768630148d793f667e195bef16fe6d5f9f16018b5ad6", + "bitIdentical": true, + "candidateSha256": "18c48f9938beb0c3e527768630148d793f667e195bef16fe6d5f9f16018b5ad6", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 200, + 200 + ] + }, + "pred_boxes": { + "acceptedSha256": "3592393014e402b2bcaf7f89e07a9020e6d69913705c0095401c62c72a276a2e", + "bitIdentical": true, + "candidateSha256": "3592393014e402b2bcaf7f89e07a9020e6d69913705c0095401c62c72a276a2e", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 4 + ] + } + } + }, + "readingOrderEqual": true, + "sha256": "785b7d19f158dcb636342dd3378ed3a4cddb7333d2d71688f0baa5c25a88ad51", + "unmatchedOfficial": [], + "unmatchedOnnx": [], + "width": 1654 + }, + { + "acceptedDetectionCount": 13, + "acceptedLabelSequenceEqual": true, + "acceptedReadingOrderEqual": true, + "detectionCountOfficial": 13, + "detectionCountOnnx": 13, + "filename": "screen-photo.jpg", + "height": 1400, + "labelSequenceEqual": true, + "maxBoxCoordinateDeltaPixels": 0.000244140625, + "maxPolygonCoordinateDeltaPixels": 0.0, + "maxScoreDelta": 5.960464477539062e-7, + "onnxDetectionCount": 13, + "pass": true, + "rawOutputs": { + "allBitIdentical": true, + "outputs": { + "logits": { + "acceptedSha256": "dae5933ceecea367ac8b4c5778e471bd8f372bff91593ec054b8c5e5daba557b", + "bitIdentical": true, + "candidateSha256": "dae5933ceecea367ac8b4c5778e471bd8f372bff91593ec054b8c5e5daba557b", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 25 + ] + }, + "order_logits": { + "acceptedSha256": "0c600a04387f1f0ac88f279ecdde51d5315fdd105409ff7247b99d3e429dd624", + "bitIdentical": true, + "candidateSha256": "0c600a04387f1f0ac88f279ecdde51d5315fdd105409ff7247b99d3e429dd624", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 300 + ] + }, + "out_masks": { + "acceptedSha256": "4f166c99f61dfc69e53a0637543b4a5d7faf75f971203c041af059559fa9a71c", + "bitIdentical": true, + "candidateSha256": "4f166c99f61dfc69e53a0637543b4a5d7faf75f971203c041af059559fa9a71c", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 200, + 200 + ] + }, + "pred_boxes": { + "acceptedSha256": "32c2ee356e4f4780542ee2ec60373d38429a01f60090f52d6f35cbd5841456f2", + "bitIdentical": true, + "candidateSha256": "32c2ee356e4f4780542ee2ec60373d38429a01f60090f52d6f35cbd5841456f2", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 4 + ] + } + } + }, + "readingOrderEqual": true, + "sha256": "f27a8ad40192f2bff4bcc3605beaddf246bc35b07355e688defde1a2de333aa1", + "unmatchedOfficial": [], + "unmatchedOnnx": [], + "width": 1000 + }, + { + "acceptedDetectionCount": 13, + "acceptedLabelSequenceEqual": true, + "acceptedReadingOrderEqual": true, + "detectionCountOfficial": 13, + "detectionCountOnnx": 13, + "filename": "skew-document.jpg", + "height": 1386, + "labelSequenceEqual": true, + "maxBoxCoordinateDeltaPixels": 0.000244140625, + "maxPolygonCoordinateDeltaPixels": 0.0, + "maxScoreDelta": 1.1920928955078125e-6, + "onnxDetectionCount": 13, + "pass": true, + "rawOutputs": { + "allBitIdentical": true, + "outputs": { + "logits": { + "acceptedSha256": "a22dc8e94aa4e809480c29aedbeef057e2e7d0a3b694cc63231235a2ae0f17cb", + "bitIdentical": true, + "candidateSha256": "a22dc8e94aa4e809480c29aedbeef057e2e7d0a3b694cc63231235a2ae0f17cb", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 25 + ] + }, + "order_logits": { + "acceptedSha256": "4e4e6af629a74b8d84c3caa465fd5d84590c74c7e1033b2d3317a6c7d9952933", + "bitIdentical": true, + "candidateSha256": "4e4e6af629a74b8d84c3caa465fd5d84590c74c7e1033b2d3317a6c7d9952933", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 300 + ] + }, + "out_masks": { + "acceptedSha256": "73e2bd39e4ab3363d48106cb7aab38869c20f00dc51e582e24f493e9ee651aae", + "bitIdentical": true, + "candidateSha256": "73e2bd39e4ab3363d48106cb7aab38869c20f00dc51e582e24f493e9ee651aae", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 200, + 200 + ] + }, + "pred_boxes": { + "acceptedSha256": "e2a4d30dee1acc953e7637201c11501bb8dea51ed857bc88d93c627a502c9513", + "bitIdentical": true, + "candidateSha256": "e2a4d30dee1acc953e7637201c11501bb8dea51ed857bc88d93c627a502c9513", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 4 + ] + } + } + }, + "readingOrderEqual": true, + "sha256": "4ae0d5bebbe152a9cca8add806e376b3eb3314c3a55d6b7ccba70d9c4de97a1e", + "unmatchedOfficial": [], + "unmatchedOnnx": [], + "width": 1068 + }, + { + "acceptedDetectionCount": 1, + "acceptedLabelSequenceEqual": true, + "acceptedReadingOrderEqual": true, + "detectionCountOfficial": 1, + "detectionCountOnnx": 1, + "filename": "table.png", + "height": 345, + "labelSequenceEqual": true, + "maxBoxCoordinateDeltaPixels": 6.103515625e-5, + "maxPolygonCoordinateDeltaPixels": 0.0, + "maxScoreDelta": 1.1920928955078125e-7, + "onnxDetectionCount": 1, + "pass": true, + "rawOutputs": { + "allBitIdentical": true, + "outputs": { + "logits": { + "acceptedSha256": "9292025c692bf595631a317965035be42e574caf7c27f9f5a47b9b140b21ceca", + "bitIdentical": true, + "candidateSha256": "9292025c692bf595631a317965035be42e574caf7c27f9f5a47b9b140b21ceca", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 25 + ] + }, + "order_logits": { + "acceptedSha256": "83256b094d74df5e84153923390f1ee9c0b4a53dec38d961ce4578db537b1a6d", + "bitIdentical": true, + "candidateSha256": "83256b094d74df5e84153923390f1ee9c0b4a53dec38d961ce4578db537b1a6d", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 300 + ] + }, + "out_masks": { + "acceptedSha256": "5bfcfee296f00fb650ae37161d26518a1fbf2c0f1d6bed024b02a663a35e79c8", + "bitIdentical": true, + "candidateSha256": "5bfcfee296f00fb650ae37161d26518a1fbf2c0f1d6bed024b02a663a35e79c8", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 200, + 200 + ] + }, + "pred_boxes": { + "acceptedSha256": "8dffcef7a09ae707ec68239245bbcc8708532bac7e5b3bcc5fd269b11be907e5", + "bitIdentical": true, + "candidateSha256": "8dffcef7a09ae707ec68239245bbcc8708532bac7e5b3bcc5fd269b11be907e5", + "dtype": "float32", + "maxAbsoluteDelta": 0.0, + "shape": [ + 1, + 300, + 4 + ] + } + } + }, + "readingOrderEqual": true, + "sha256": "6d50148ceccb2d5cecc50b084b5105e3167f2d55a8899b29e04c3ebe46e88fa8", + "unmatchedOfficial": [], + "unmatchedOnnx": [], + "width": 550 + } + ], + "overallPass": true, + "paddleReference": { + "reason": "PaddlePaddle is not installed in the isolated Python 3.11 environment", + "status": "unavailable" + }, + "schemaVersion": 1, + "sourceHashes": { + "acceptedOnnx": "fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b", + "modelSafetensors": "5ea422c6cc5fe759a47e1357c35639b58173508e025a3131cbe4b6ac59e2b85e", + "onnx": "476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269" + }, + "threshold": 0.5, + "thresholds": { + "boxCoordinateDeltaPixels": 1.0, + "polygonCoordinateDeltaPixels": 1.5, + "scoreDelta": 0.001 + } +} diff --git a/tools/model-pipeline/tests/test_parity_fp32.py b/tools/model-pipeline/tests/test_parity_fp32.py index a388d60..e1579da 100644 --- a/tools/model-pipeline/tests/test_parity_fp32.py +++ b/tools/model-pipeline/tests/test_parity_fp32.py @@ -3,7 +3,12 @@ import pytest -from ppdoclayout.validate import canonical_json, validate_fp32, write_report +from ppdoclayout.validate import ( + canonical_json, + sha256_file, + validate_fp32, + write_report, +) ROOT = Path(__file__).parents[3] @@ -25,20 +30,38 @@ def test_write_report_uses_lf_line_endings(tmp_path: Path) -> None: @pytest.mark.slow -def test_fp32_matches_official_transformers() -> None: +def test_sanitized_fp32_matches_accepted_fp32_and_official_transformers() -> None: + accepted = ( + ROOT / "models" / "pp-doclayoutv3" / "1.0.0" / "model-fp32.onnx" + ) + candidate = ( + ROOT / "models" / "pp-doclayoutv3" / "1.0.1" / "model-fp32.onnx" + ) + model_path = Path(r"E:\models\PP-DocLayoutV3_safetensors") report = validate_fp32( - model_path=Path(r"E:\models\PP-DocLayoutV3_safetensors"), - onnx_path=ROOT / "models" / "pp-doclayoutv3" / "1.0.0" / "model-fp32.onnx", + model_path=model_path, + accepted_onnx_path=accepted, + onnx_path=candidate, fixtures_lock=ROOT / "tools" / "model-pipeline" / "fixtures" / "fixtures.lock.json", ) assert report["overallPass"] is True + assert report["sourceHashes"] == { + "acceptedOnnx": sha256_file(accepted), + "modelSafetensors": sha256_file(model_path / "model.safetensors"), + "onnx": sha256_file(candidate), + } assert report["thresholds"] == { "boxCoordinateDeltaPixels": 1.0, "polygonCoordinateDeltaPixels": 1.5, "scoreDelta": 0.001, } + assert len(report["fixtures"]) == 7 for fixture in report["fixtures"]: + assert fixture["acceptedDetectionCount"] == fixture["onnxDetectionCount"] + assert fixture["acceptedLabelSequenceEqual"] is True + assert fixture["acceptedReadingOrderEqual"] is True + assert fixture["rawOutputs"]["allBitIdentical"] is True assert fixture["unmatchedOfficial"] == [] assert fixture["unmatchedOnnx"] == [] assert fixture["labelSequenceEqual"] is True @@ -46,3 +69,4 @@ def test_fp32_matches_official_transformers() -> None: assert fixture["maxScoreDelta"] <= 0.001 assert fixture["maxBoxCoordinateDeltaPixels"] <= 1.0 assert fixture["maxPolygonCoordinateDeltaPixels"] <= 1.5 + assert fixture["pass"] is True From f8580ac3ef8294906658425357ba87995f100cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 01:32:20 +0800 Subject: [PATCH 08/13] test(models): validate FP32 in browser runtimes --- .github/workflows/benchmark.yml | 24 + benchmarks/1.0.1/wasm-fp32.json | 187 +++++++ benchmarks/1.0.1/webgpu-fp32.json | 212 ++++++++ scripts/benchmark-contract.test.mjs | 20 +- tests/browser/benchmark.spec.ts | 299 ++++++++--- .../reports/1.0.1/browser-evidence.json | 497 ++++++++++++++++++ 6 files changed, 1155 insertions(+), 84 deletions(-) create mode 100644 benchmarks/1.0.1/wasm-fp32.json create mode 100644 benchmarks/1.0.1/webgpu-fp32.json create mode 100644 tools/model-pipeline/reports/1.0.1/browser-evidence.json diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index be73aed..3c01fb6 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -60,6 +60,30 @@ jobs: name: benchmark-webgpu-fp16 path: test-results/benchmark/webgpu-fp16.json + webgpu-fp32: + runs-on: [self-hosted, windows, x64, webgpu-hardware] + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + with: + lfs: true + - uses: pnpm/action-setup@v6 + with: + version: 11.16.0 + - uses: actions/setup-node@v7 + with: + node-version-file: .nvmrc + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm exec playwright install chromium + - run: pnpm exec playwright test tests/browser/benchmark.spec.ts + env: + PPDOCLAYOUT_BENCHMARK_MODE: webgpu-fp32 + - uses: actions/upload-artifact@v7 + with: + name: benchmark-webgpu-fp32 + path: test-results/benchmark/webgpu-fp32.json + responsive-screenshots: runs-on: ubuntu-latest steps: diff --git a/benchmarks/1.0.1/wasm-fp32.json b/benchmarks/1.0.1/wasm-fp32.json new file mode 100644 index 0000000..f0760c8 --- /dev/null +++ b/benchmarks/1.0.1/wasm-fp32.json @@ -0,0 +1,187 @@ +{ + "schemaVersion": 1, + "status": "passed", + "executionProvider": "wasm", + "precision": "fp32", + "fallbacks": [], + "modelBytes": 142574928, + "modelSha256": "476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269", + "onnxruntimeWebVersion": "1.27.0", + "adapter": null, + "adapterFeatures": [], + "browser": { + "name": "Chromium", + "version": "151.0.7922.34", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.7922.34 Safari/537.36" + }, + "operatingSystem": "win32 10.0.26200", + "fixtures": [ + { + "detectionCount": 12, + "expectedDetectionCount": 12, + "filename": "curved-document.jpg", + "labelSequenceEqual": true, + "outputSha256": "98601e9b45ffca68a280c95604d67f40bca747aefa7f07ec502369969dfd4025", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 10.405000001192093, + "inferenceMs": 7859.410000003874, + "postprocessMs": 39.41499999910593, + "preprocessMs": 127.48499999940395, + "totalMs": 8045.539999999106 + } + }, + { + "detectionCount": 59, + "expectedDetectionCount": 59, + "filename": "doc-formula.png", + "labelSequenceEqual": true, + "outputSha256": "8bba1fb0a794a86ec0b2f4d1b1425d28e000a49beab399e4c55bbb04b4cf8cf4", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 10.439999997615814, + "inferenceMs": 7962.560000002384, + "postprocessMs": 35.59000000357628, + "preprocessMs": 115.08500000089407, + "totalMs": 8132.585000000894 + } + }, + { + "detectionCount": 44, + "expectedDetectionCount": 44, + "filename": "image-layout.jpg", + "labelSequenceEqual": true, + "outputSha256": "da3c2a041cc94c4b5616637b7acd9af116c0882befd434920ebe8aaf96be626d", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 27.730000004172325, + "inferenceMs": 7777.990000002086, + "postprocessMs": 59.399999998509884, + "preprocessMs": 105.34000000357628, + "totalMs": 7979.435000002384 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "layout-demo.jpg", + "labelSequenceEqual": true, + "outputSha256": "c6051575214356859bcb9f87be446f27f32dc405a22bdc99ef9f887866c5ddb7", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 31.45500000566244, + "inferenceMs": 7889.280000001192, + "postprocessMs": 99.87999999523163, + "preprocessMs": 172.27000000327826, + "totalMs": 8202.32500000298 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "screen-photo.jpg", + "labelSequenceEqual": true, + "outputSha256": "b036ef27908bd3b94406d9a3106cb5ef6ecca8030874f2fceacac7a0a6407a90", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 11.390000000596046, + "inferenceMs": 8091.3499999940395, + "postprocessMs": 44.645000003278255, + "preprocessMs": 122.03499999642372, + "totalMs": 8279.155000001192 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "skew-document.jpg", + "labelSequenceEqual": true, + "outputSha256": "07f0c613f0d87b91597984e01e83acdca5c5bf440d3efcc19873153b9250fa82", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 11.479999996721745, + "inferenceMs": 7944.47500000149, + "postprocessMs": 52.269999995827675, + "preprocessMs": 125.89500000327826, + "totalMs": 8142.655000001192 + } + }, + { + "detectionCount": 1, + "expectedDetectionCount": 1, + "filename": "table.png", + "labelSequenceEqual": true, + "outputSha256": "8ff528db91eb3893ec1fbf50d69ac23aac17bc2aefec79e70cf0c11f0602550a", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 3.0499999970197678, + "inferenceMs": 7915.270000003278, + "postprocessMs": 55.57499999552965, + "preprocessMs": 76.70500000566244, + "totalMs": 8060.77499999851 + }, + "parityMetrics": { + "iou": 0.9999961699392192, + "maxScoreDelta": 0.000013727193068024945, + "meanPolygonPointDistancePixels": 0 + }, + "parityThresholds": { + "iou": 0.95, + "maxScoreDelta": 0.02, + "meanPolygonPointDistancePixels": 2 + } + } + ], + "timingsMs": { + "coldLoad": { + "capabilitiesMs": 3.530000001192093, + "integrityMs": 366.4150000065565, + "manifestMs": 1.589999996125698, + "modelCacheMs": 0.8250000029802322, + "modelDownloadMs": 1204.2849999964237, + "modelMs": 1968.9499999955297, + "modelSource": "network", + "sessionMs": 1593.4200000017881, + "totalMs": 3568.1999999955297 + }, + "warmLoad": { + "capabilitiesMs": 1.2349999994039536, + "integrityMs": 361.66499999910593, + "manifestMs": 0.0949999988079071, + "modelCacheMs": 79.13000000268221, + "modelDownloadMs": 0, + "modelMs": 440.8799999952316, + "modelSource": "cache", + "sessionMs": 510.9950000047684, + "totalMs": 953.2649999931455 + } + }, + "sdkCommit": "8c47754068ff4ec7b34451cb7d562e6e9a8b1c8a", + "capabilities": { + "crossOriginIsolated": true, + "diagnostics": [ + "webgpu: unavailable because no adapter was returned", + "webgpu-fp16: shader-f16 is unavailable", + "wasm: supported", + "wasm-simd: supported", + "wasm-threads: supported", + "worker: unavailable in this environment" + ], + "wasm": true, + "wasmSimd": true, + "wasmThreads": true, + "webgpu": false, + "webgpuFp16": false, + "worker": false + }, + "cpu": "Intel(R) Core(TM) i5-10400F CPU @ 2.90GHz", + "generatedAt": "2026-08-14T17:26:16.905Z", + "id": "wasm-fp32" +} diff --git a/benchmarks/1.0.1/webgpu-fp32.json b/benchmarks/1.0.1/webgpu-fp32.json new file mode 100644 index 0000000..13e16a5 --- /dev/null +++ b/benchmarks/1.0.1/webgpu-fp32.json @@ -0,0 +1,212 @@ +{ + "schemaVersion": 1, + "status": "passed", + "executionProvider": "webgpu", + "precision": "fp32", + "fallbacks": [], + "modelBytes": 142574928, + "modelSha256": "476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269", + "onnxruntimeWebVersion": "1.27.0", + "adapter": { + "architecture": "blackwell", + "description": null, + "device": null, + "vendor": "nvidia" + }, + "adapterFeatures": [ + "bgra8unorm-storage", + "clip-distances", + "core-features-and-limits", + "depth-clip-control", + "depth32float-stencil8", + "dual-source-blending", + "float32-blendable", + "float32-filterable", + "indirect-first-instance", + "primitive-index", + "rg11b10ufloat-renderable", + "shader-f16", + "subgroups", + "texture-component-swizzle", + "texture-compression-bc", + "texture-compression-bc-sliced-3d", + "texture-formats-tier1", + "texture-formats-tier2", + "timestamp-query" + ], + "browser": { + "name": "Chromium", + "version": "151.0.7922.138", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" + }, + "operatingSystem": "win32 10.0.26200", + "fixtures": [ + { + "detectionCount": 12, + "expectedDetectionCount": 12, + "filename": "curved-document.jpg", + "labelSequenceEqual": true, + "outputSha256": "49d017eb3a0946b85dbf0076cf3fc8af88ad12faa1c46b2e5063b54f0b082393", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 15.869999997317791, + "inferenceMs": 5850.69999999553, + "postprocessMs": 68.39000000059605, + "preprocessMs": 161.90999999642372, + "totalMs": 6113.865000002086 + } + }, + { + "detectionCount": 59, + "expectedDetectionCount": 59, + "filename": "doc-formula.png", + "labelSequenceEqual": true, + "outputSha256": "70e57bf726376f2e3aad8fd35a6d492d679b6b38087df95de2648aee09babc51", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 8.570000000298023, + "inferenceMs": 315.6899999976158, + "postprocessMs": 50.16499999910593, + "preprocessMs": 119.58000000566244, + "totalMs": 504.45499999821186 + } + }, + { + "detectionCount": 44, + "expectedDetectionCount": 44, + "filename": "image-layout.jpg", + "labelSequenceEqual": true, + "outputSha256": "16320eaccc62ae6d9509d498ade7996febe9fa72cc7543f716c91d8577b06b00", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 24.344999998807907, + "inferenceMs": 278.85999999940395, + "postprocessMs": 72.42000000178814, + "preprocessMs": 121.79500000178814, + "totalMs": 508.01500000059605 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "layout-demo.jpg", + "labelSequenceEqual": true, + "outputSha256": "24d11a2ed203d8ee72ca4e14e803cab560ef57855976ead40efa65a07bd0fbb7", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 30.639999993145466, + "inferenceMs": 290.7799999937415, + "postprocessMs": 94.3399999961257, + "preprocessMs": 177.5300000011921, + "totalMs": 606.9549999982119 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "screen-photo.jpg", + "labelSequenceEqual": true, + "outputSha256": "6f0fe242016734de274a9ac9d002ee166694e8360cb52cc3cb1d0b4000d1d160", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 11.925000004470348, + "inferenceMs": 290.51500000059605, + "postprocessMs": 44.42000000178814, + "preprocessMs": 144, + "totalMs": 502.4949999973178 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "skew-document.jpg", + "labelSequenceEqual": true, + "outputSha256": "840d7c61343deee9fa966b541e17a19671d5987aa27280ecec8ca2943c133c47", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 11.689999997615814, + "inferenceMs": 230.11499999463558, + "postprocessMs": 57.30500000715256, + "preprocessMs": 138.96000000089407, + "totalMs": 451.71000000089407 + } + }, + { + "detectionCount": 1, + "expectedDetectionCount": 1, + "filename": "table.png", + "labelSequenceEqual": true, + "outputSha256": "7978ca0ec7143f98fc24ef2f4613c72d785a82ca4a1b31bcf97fc1952ffb9e8c", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 3.0450000017881393, + "inferenceMs": 355.1149999946356, + "postprocessMs": 53.979999996721745, + "preprocessMs": 83.14499999582767, + "totalMs": 504.5949999988079 + }, + "parityMetrics": { + "iou": 0.9999962574460338, + "maxScoreDelta": 0.00001386435552097609, + "meanPolygonPointDistancePixels": 0 + }, + "parityThresholds": { + "iou": 0.95, + "maxScoreDelta": 0.02, + "meanPolygonPointDistancePixels": 2 + } + } + ], + "timingsMs": { + "coldLoad": { + "capabilitiesMs": 168.0949999988079, + "integrityMs": 539.7800000011921, + "manifestMs": 2.3250000029802322, + "modelCacheMs": 1.089999996125698, + "modelDownloadMs": 3077.2300000041723, + "modelMs": 4335.57499999553, + "modelSource": "network", + "sessionMs": 3461.4400000050664, + "totalMs": 7968.234999999404 + }, + "warmLoad": { + "capabilitiesMs": 0.5600000023841858, + "integrityMs": 363.2800000011921, + "manifestMs": 0.11999999731779099, + "modelCacheMs": 102.08500000089407, + "modelDownloadMs": 0, + "modelMs": 465.5, + "modelSource": "cache", + "sessionMs": 1073.929999999702, + "totalMs": 1540.1750000044703 + } + }, + "sdkCommit": "8c47754068ff4ec7b34451cb7d562e6e9a8b1c8a", + "capabilities": { + "crossOriginIsolated": true, + "diagnostics": [ + "webgpu: supported by an acquired adapter", + "webgpu-fp16: shader-f16 is supported", + "wasm: supported", + "wasm-simd: supported", + "wasm-threads: supported", + "worker: supported" + ], + "wasm": true, + "wasmSimd": true, + "wasmThreads": true, + "webgpu": true, + "webgpuFp16": true, + "worker": true + }, + "cpu": "Intel(R) Core(TM) i5-10400F CPU @ 2.90GHz", + "generatedAt": "2026-08-14T17:22:41.632Z", + "id": "webgpu-fp32" +} diff --git a/scripts/benchmark-contract.test.mjs b/scripts/benchmark-contract.test.mjs index 8d697d1..50f5706 100644 --- a/scripts/benchmark-contract.test.mjs +++ b/scripts/benchmark-contract.test.mjs @@ -7,9 +7,9 @@ import { describe, test } from "node:test"; const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const benchmarkRoot = join(repositoryRoot, "benchmarks/1.0.0"); -function readJson(name) { - const path = join(benchmarkRoot, name); - assert.ok(existsSync(path), `missing benchmark artifact: benchmarks/1.0.0/${name}`); +function readJson(name, version = "1.0.0") { + const path = join(repositoryRoot, "benchmarks", version, name); + assert.ok(existsSync(path), `missing benchmark artifact: benchmarks/${version}/${name}`); return JSON.parse(readFileSync(path, "utf8")); } @@ -21,10 +21,12 @@ describe("1.0.0 benchmark release contract", () => { assert.match(workflow, /tests\/browser\/benchmark\.spec\.ts/); assert.match(workflow, /PPDOCLAYOUT_BENCHMARK_MODE:\s*["']?wasm-fp32/); assert.match(workflow, /PPDOCLAYOUT_BENCHMARK_MODE:\s*["']?webgpu-fp16/); + assert.match(workflow, /PPDOCLAYOUT_BENCHMARK_MODE:\s*["']?webgpu-fp32/); + assert.match(workflow, /name:\s*benchmark-webgpu-fp32/); assert.match(workflow, /runs-on:\s*\[self-hosted, windows, x64, webgpu-hardware\]/); assert.match(workflow, /benchmark\.spec\.ts/); const artifactWorkflows = [ - ["benchmark.yml", workflow, 3], + ["benchmark.yml", workflow, 4], ["ci.yml", readFileSync(join(repositoryRoot, ".github/workflows/ci.yml"), "utf8"), 1], [ "model-validation.yml", @@ -101,6 +103,16 @@ describe("1.0.0 benchmark release contract", () => { assert.deepEqual(report.responsiveScreenshots.viewports, [390, 768, 1440, 1920]); }); + test("publishes seven-fixture evidence for model 1.0.1 FP32 runtimes", () => { + for (const name of ["wasm-fp32.json", "webgpu-fp32.json"]) { + const report = readJson(name, "1.0.1"); + assert.equal(report.status, "passed"); + assert.equal(report.fallbacks.length, 0); + assert.equal(report.fixtures.length, 7); + assert.ok(report.fixtures.every((fixture) => fixture.parity === "passed")); + } + }); + test("documents evidence provenance and unsupported variants", () => { const readme = readFileSync(join(benchmarkRoot, "README.md"), "utf8"); assert.match(readme, /真实|real/i); diff --git a/tests/browser/benchmark.spec.ts b/tests/browser/benchmark.spec.ts index c274ce9..03a454f 100644 --- a/tests/browser/benchmark.spec.ts +++ b/tests/browser/benchmark.spec.ts @@ -1,4 +1,5 @@ import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { createReadStream, existsSync, @@ -15,17 +16,46 @@ import { expect, test } from "playwright/test"; import reference from "../../packages/sdk/tests/fixtures/model-output-reference.json"; -const mode = process.env.PPDOCLAYOUT_BENCHMARK_MODE; +type BenchmarkMode = "wasm-fp32" | "webgpu-fp16" | "webgpu-fp32"; + +interface BenchmarkManifest { + model: { version: string; [key: string]: unknown }; + variants: Array<{ + backendCompatibility: string[]; + bytes: number; + filename: string; + id: string; + precision: string; + sha256: string; + url: string; + [key: string]: unknown; + }>; + [key: string]: unknown; +} + +interface FixtureLock { + fixtures: Array<{ + filename: string; + height: number; + sha256: string; + width: number; + }>; +} + +const mode = process.env.PPDOCLAYOUT_BENCHMARK_MODE as BenchmarkMode | undefined; const repositoryRoot = resolve(__dirname, "../.."); const sdkRoot = join(repositoryRoot, "packages/sdk"); const ortRoot = join(sdkRoot, "node_modules/onnxruntime-web/dist"); -const modelRoot = join(repositoryRoot, "models/pp-doclayoutv3/1.0.0"); +const acceptedModelRoot = join(repositoryRoot, "models/pp-doclayoutv3/1.0.0"); +const candidateModelVersion = mode === "webgpu-fp16" ? "1.0.0" : "1.0.1"; +const candidateModelRoot = join(repositoryRoot, `models/pp-doclayoutv3/${candidateModelVersion}`); const fixtureRoot = join(repositoryRoot, "tools/model-pipeline/fixtures/images"); +const fixturesLockPath = join(repositoryRoot, "tools/model-pipeline/fixtures/fixtures.lock.json"); const outputRoot = join(repositoryRoot, "test-results/benchmark"); let origin = ""; let server: Server; -test.use(mode === "webgpu-fp16" ? { channel: "chrome" } : {}); +test.use(mode?.startsWith("webgpu-") ? { channel: "chrome" } : {}); const parityThresholds = { iou: 0.95, @@ -33,6 +63,44 @@ const parityThresholds = { meanPolygonPointDistancePixels: 2 } as const; +function sha256File(path: string): string { + return createHash("sha256").update(readFileSync(path)).digest("hex"); +} + +function loadManifest(): BenchmarkManifest { + return JSON.parse( + readFileSync(join(acceptedModelRoot, "manifest.json"), "utf8") + ) as BenchmarkManifest; +} + +function localManifest( + modelRoot: string, + urlPrefix: "accepted" | "candidate", + fp32Backends: readonly string[] +): BenchmarkManifest { + const manifest = structuredClone(loadManifest()); + manifest.model.version = basename(modelRoot); + for (const variant of manifest.variants) { + const path = join(modelRoot, variant.filename); + variant.bytes = statSync(path).size; + variant.sha256 = sha256File(path); + variant.url = `${origin}/models/${urlPrefix}/${variant.filename}`; + if (variant.precision === "fp32") { + variant.backendCompatibility = [...fp32Backends]; + } + } + return manifest; +} + +function verifiedFixtures(): FixtureLock { + const lock = JSON.parse(readFileSync(fixturesLockPath, "utf8")) as FixtureLock; + for (const fixture of lock.fixtures) { + const path = join(fixtureRoot, fixture.filename); + expect(sha256File(path), `fixture integrity: ${fixture.filename}`).toBe(fixture.sha256); + } + return lock; +} + function boxIou(actual: { xMin: number; xMax: number; yMin: number; yMax: number }): number { const [xMin, yMin, xMax, yMax] = reference.realImage.expected.boxes[0]!; const intersectionWidth = Math.max( @@ -70,7 +138,12 @@ function resolveAsset(url: string): string | undefined { const pathname = new URL(url, "http://localhost").pathname; if (pathname.startsWith("/dist/")) return join(sdkRoot, pathname.slice(1)); if (pathname.startsWith("/ort/")) return join(ortRoot, basename(pathname)); - if (pathname.startsWith("/models/")) return join(modelRoot, basename(pathname)); + if (pathname.startsWith("/models/accepted/")) { + return join(acceptedModelRoot, basename(pathname)); + } + if (pathname.startsWith("/models/candidate/")) { + return join(candidateModelRoot, basename(pathname)); + } if (pathname.startsWith("/fixtures/")) return join(fixtureRoot, basename(pathname)); return undefined; } @@ -78,6 +151,8 @@ function resolveAsset(url: string): string | undefined { function contentType(path: string): string { return ( { + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", ".js": "text/javascript; charset=utf-8", ".mjs": "text/javascript; charset=utf-8", ".onnx": "application/octet-stream", @@ -88,7 +163,10 @@ function contentType(path: string): string { } test.beforeAll(async () => { - test.skip(!["wasm-fp32", "webgpu-fp16"].includes(mode ?? ""), "Set benchmark mode"); + test.skip( + !["wasm-fp32", "webgpu-fp16", "webgpu-fp32"].includes(mode ?? ""), + "Set benchmark mode" + ); runPnpm(["--filter", "web-sdk-pp-doclayoutv3", "build"]); server = createServer((request, response) => { response.setHeader("Cross-Origin-Embedder-Policy", "require-corp"); @@ -128,30 +206,56 @@ test.afterAll(async () => { ); }); -test("records complete real-model timings", async ({ browser, page }) => { - const manifest = JSON.parse(readFileSync(join(modelRoot, "manifest.json"), "utf8")) as unknown; - await page.goto(origin); +test("records strict seven-fixture browser evidence", async ({ browser, page }) => { + test.setTimeout(20 * 60_000); const backend = mode === "wasm-fp32" ? "wasm" : "webgpu"; - const precision = mode === "wasm-fp32" ? "fp32" : "fp16"; + const precision = mode?.endsWith("fp32") ? "fp32" : "fp16"; + const fixturesLock = verifiedFixtures(); + const acceptedManifest = localManifest(acceptedModelRoot, "accepted", ["wasm"]); + const targetManifest = localManifest(candidateModelRoot, "candidate", ["wasm", "webgpu"]); + const manifestVariant = targetManifest.variants.find( + (variant) => variant.precision === precision && variant.backendCompatibility.includes(backend) + ); + expect(manifestVariant).toBeDefined(); + + await page.goto(origin); const result = await page.evaluate( - async ({ backend, manifest, origin: browserOrigin, precision }) => { - const configured = structuredClone(manifest) as { - variants: Array<{ filename: string; url: string }>; - }; - for (const variant of configured.variants) - variant.url = `${browserOrigin}/models/${variant.filename}`; - const options = { + async ({ + acceptedManifest, + backend, + fixtures, + origin: browserOrigin, + precision, + targetManifest + }) => { + async function sha256(bytes: Uint8Array): Promise { + const digest = await crypto.subtle.digest("SHA-256", bytes); + return [...new Uint8Array(digest)] + .map((value) => value.toString(16).padStart(2, "0")) + .join(""); + } + + const targetOptions = { allowFallback: false, backend, cache: true, - model: configured, + model: targetManifest, ort: { wasm: { numThreads: 1, paths: `${browserOrigin}/ort/` } }, precision } as const; + const acceptedOptions = { + allowFallback: false, + backend: "wasm", + cache: true, + model: acceptedManifest, + ort: { wasm: { numThreads: 1, paths: `${browserOrigin}/ort/` } }, + precision: "fp32" + } as const; + await window.PPDocLayout!.clearModelCache(); - let cold; + let target; try { - cold = await window.PPDocLayout!.createDocLayout(options); + target = await window.PPDocLayout!.createDocLayout(targetOptions); } catch (error) { const capabilities = await window.PPDocLayout!.probeDocLayoutCapabilities(); const failure = error as Error & { @@ -163,6 +267,7 @@ test("records complete real-model timings", async ({ browser, page }) => { JSON.stringify({ capabilities, cause: failure.cause instanceof Error ? failure.cause.message : failure.cause, + causeMessage: failure.details?.causeMessage, code: failure.code, details: failure.details, message: failure.message, @@ -170,16 +275,47 @@ test("records complete real-model timings", async ({ browser, page }) => { }) ); } - const image = await (await fetch(`${browserOrigin}/fixtures/table.png`)).blob(); - const detection = await cold.detect(image, { threshold: 0.5 }); - const coldLoad = cold.loadTimings; - const model = cold.model; - const runtime = cold.runtime; - await cold.dispose(); - const warm = await window.PPDocLayout!.createDocLayout(options); + const accepted = await window.PPDocLayout!.createDocLayout(acceptedOptions); + const fixtureResults = []; + for (const fixture of fixtures) { + const image = await (await fetch(`${browserOrigin}/fixtures/${fixture.filename}`)).blob(); + const acceptedDetection = await accepted.detect(image, { threshold: 0.5 }); + const detection = await target.detect(image, { threshold: 0.5 }); + const acceptedLabels = acceptedDetection.detections.map(({ labelId }) => labelId); + const labels = detection.detections.map(({ labelId }) => labelId); + const acceptedOrder = acceptedDetection.detections.map(({ readingOrder }) => readingOrder); + const readingOrder = detection.detections.map(({ readingOrder }) => readingOrder); + const labelSequenceEqual = JSON.stringify(labels) === JSON.stringify(acceptedLabels); + const readingOrderEqual = JSON.stringify(readingOrder) === JSON.stringify(acceptedOrder); + const detectionJson = JSON.stringify(detection.detections); + const outputSha256 = await sha256(new TextEncoder().encode(detectionJson)); + fixtureResults.push({ + detectionCount: detection.detections.length, + detections: detection.detections, + expectedDetectionCount: acceptedDetection.detections.length, + filename: fixture.filename, + labelSequenceEqual, + outputSha256, + parity: + detection.detections.length === acceptedDetection.detections.length && + labelSequenceEqual && + readingOrderEqual + ? "passed" + : "failed", + readingOrderEqual, + timings: detection.timings + }); + } + const coldLoad = target.loadTimings; + const model = target.model; + const runtime = target.runtime; + await accepted.dispose(); + await target.dispose(); + const warm = await window.PPDocLayout!.createDocLayout(targetOptions); const warmLoad = warm.loadTimings; await warm.dispose(); await window.PPDocLayout!.clearModelCache(); + const adapter = backend === "webgpu" ? await navigator.gpu?.requestAdapter({ powerPreference: "high-performance" }) @@ -191,75 +327,78 @@ test("records complete real-model timings", async ({ browser, page }) => { architecture: adapter.info.architecture || null, description: adapter.info.description || null, device: adapter.info.device || null, - shaderF16: adapter.features.has("shader-f16"), vendor: adapter.info.vendor || null }; return { adapter: adapterInfo, + adapterFeatures: adapter === undefined ? [] : [...adapter.features].sort(), browser: navigator.userAgent, - coldLoad, - detection, + fixtures: fixtureResults, model, runtime, - warmLoad + timings: { coldLoad, warmLoad } }; }, - { backend, manifest, origin, precision } - ); - expect(result.runtime).toMatchObject({ backend, precision }); - expect(result.detection.detections).toHaveLength(reference.realImage.expected.scores.length); - const firstDetection = result.detection.detections[0]!; - expect(firstDetection.labelId).toBe(reference.realImage.expected.labels[0]); - const parity = { - iou: boxIou(firstDetection.box), - maxScoreDelta: Math.abs(firstDetection.score - reference.realImage.expected.scores[0]!), - meanPolygonPointDistancePixels: meanPolygonPointDistance(firstDetection.polygon) - }; - expect(parity.iou).toBeGreaterThanOrEqual(parityThresholds.iou); - expect(parity.maxScoreDelta).toBeLessThanOrEqual(parityThresholds.maxScoreDelta); - expect(parity.meanPolygonPointDistancePixels).toBeLessThanOrEqual( - parityThresholds.meanPolygonPointDistancePixels + { + acceptedManifest, + backend, + fixtures: fixturesLock.fixtures, + origin, + precision, + targetManifest + } ); + expect(result.runtime).toMatchObject({ backend, fallbacks: [], precision }); + expect(result.model.sha256).toBe(manifestVariant!.sha256); + expect(result.fixtures).toHaveLength(fixturesLock.fixtures.length); + const fixtureEvidence = result.fixtures.map(({ detections, ...fixture }) => { + expect(fixture.detectionCount).toBe(fixture.expectedDetectionCount); + expect(fixture.labelSequenceEqual).toBe(true); + expect(fixture.readingOrderEqual).toBe(true); + expect(fixture.parity).toBe("passed"); + expect(fixture.outputSha256).toMatch(/^[a-f0-9]{64}$/); + if (fixture.filename !== "table.png") return fixture; + + const firstDetection = detections[0]!; + expect(firstDetection.labelId).toBe(reference.realImage.expected.labels[0]); + const parityMetrics = { + iou: boxIou(firstDetection.box), + maxScoreDelta: Math.abs(firstDetection.score - reference.realImage.expected.scores[0]!), + meanPolygonPointDistancePixels: meanPolygonPointDistance(firstDetection.polygon) + }; + expect(parityMetrics.iou).toBeGreaterThanOrEqual(parityThresholds.iou); + expect(parityMetrics.maxScoreDelta).toBeLessThanOrEqual(parityThresholds.maxScoreDelta); + expect(parityMetrics.meanPolygonPointDistancePixels).toBeLessThanOrEqual( + parityThresholds.meanPolygonPointDistancePixels + ); + return { ...fixture, parityMetrics, parityThresholds }; + }); + + const sdkCommit = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: repositoryRoot, + encoding: "utf8" + }).trim(); const report = { schemaVersion: 1, - id: mode, status: "passed", - generatedAt: new Date().toISOString(), - sdkCommit: execFileSync("git", ["rev-parse", "HEAD"], { - cwd: repositoryRoot, - encoding: "utf8" - }).trim(), - environment: { - browser: { name: "Chromium", version: browser.version(), userAgent: result.browser }, - capabilities: result.runtime.capabilities, - cpu: cpus()[0]?.model ?? "unknown", - hardware: - backend === "webgpu" - ? (result.adapter?.description ?? result.adapter?.architecture ?? result.adapter?.vendor) - : (cpus()[0]?.model ?? "unknown"), - os: `${platform()} ${release()}` - }, - ort: { version: "1.27.0" }, - model: { - bytes: result.model.bytes, - precision: result.model.precision, - sha256: result.model.sha256 - }, - coldLoad: result.coldLoad, - warmLoad: result.warmLoad, - detection: { - count: result.detection.detections.length, - parity: "passed", - parityMetrics: parity, - parityThresholds, - timings: result.detection.timings - }, + executionProvider: backend, + precision, + fallbacks: result.runtime.fallbacks, + modelBytes: result.model.bytes, + modelSha256: result.model.sha256, + onnxruntimeWebVersion: "1.27.0", adapter: result.adapter, - peakMemory: { - bytes: null, - reason: "Chromium does not expose reliable per-inference peak memory." - } + adapterFeatures: result.adapterFeatures, + browser: { name: "Chromium", version: browser.version(), userAgent: result.browser }, + operatingSystem: `${platform()} ${release()}`, + fixtures: fixtureEvidence, + timingsMs: result.timings, + sdkCommit, + capabilities: result.runtime.capabilities, + cpu: cpus()[0]?.model ?? "unknown", + generatedAt: new Date().toISOString(), + id: mode }; mkdirSync(outputRoot, { recursive: true }); writeFileSync(join(outputRoot, `${mode}.json`), `${JSON.stringify(report, null, 2)}\n`); diff --git a/tools/model-pipeline/reports/1.0.1/browser-evidence.json b/tools/model-pipeline/reports/1.0.1/browser-evidence.json new file mode 100644 index 0000000..7ce80cd --- /dev/null +++ b/tools/model-pipeline/reports/1.0.1/browser-evidence.json @@ -0,0 +1,497 @@ +{ + "schemaVersion": 1, + "fp16Webgpu": { + "status": "passed", + "adapter": { + "architecture": "blackwell", + "description": null, + "device": null, + "isFallbackAdapter": null, + "subgroupMaxSize": 32, + "subgroupMinSize": 32, + "vendor": "nvidia" + }, + "adapterFeatures": [ + "bgra8unorm-storage", + "clip-distances", + "core-features-and-limits", + "depth-clip-control", + "depth32float-stencil8", + "dual-source-blending", + "float32-blendable", + "float32-filterable", + "indirect-first-instance", + "primitive-index", + "rg11b10ufloat-renderable", + "shader-f16", + "subgroups", + "texture-component-swizzle", + "texture-compression-bc", + "texture-compression-bc-sliced-3d", + "texture-formats-tier1", + "texture-formats-tier2", + "timestamp-query" + ], + "browser": { + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36", + "userAgentData": { + "brands": [ + { + "brand": "Not=A?Brand", + "version": "99" + }, + { + "brand": "Google Chrome", + "version": "151" + }, + { + "brand": "Chromium", + "version": "151" + } + ], + "mobile": false, + "platform": "Windows" + } + }, + "executionProvider": "webgpu", + "input": { + "dimensions": [1, 3, 800, 800], + "name": "pixel_values", + "type": "float32" + }, + "modelBytes": 74279796, + "modelSha256": "463ba56faa555baf84271b4002b33b0c5fcc50776fe4f39344235eccb72073f2", + "onnxruntimeWebVersion": "1.27.0", + "outputs": { + "logits": { + "allFinite": true, + "dimensions": [1, 300, 25], + "sha256": "2d1c470358fd8162ac3e6025f99658c4727a62e5a80eea1b6199989bebf1337f", + "type": "float32" + }, + "order_logits": { + "allFinite": true, + "dimensions": [1, 300, 300], + "sha256": "f039615990d24938fb35c9dfe142be13846caa48002c7c37beed61f37970c4d2", + "type": "float32" + }, + "out_masks": { + "allFinite": true, + "dimensions": [1, 300, 200, 200], + "sha256": "8e27be8e4b2294a27028b15e7c03679364f1bf2f158e41c5dbe7c67a9be47ad8", + "type": "float32" + }, + "pred_boxes": { + "allFinite": true, + "dimensions": [1, 300, 4], + "sha256": "fd8ec5511d6b5f840ec86967a8912aa483873ce43d28e6119ff3692cf79ae9b5", + "type": "float32" + } + }, + "timingsMs": { + "download": 439.9899999946356, + "inference": 681.9950000047684, + "sessionCreate": 1785.0250000059605 + }, + "validatedAt": "2026-08-11T09:25:16.542Z" + }, + "fp32Wasm": { + "schemaVersion": 1, + "status": "passed", + "executionProvider": "wasm", + "precision": "fp32", + "fallbacks": [], + "modelBytes": 142574928, + "modelSha256": "476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269", + "onnxruntimeWebVersion": "1.27.0", + "adapter": null, + "adapterFeatures": [], + "browser": { + "name": "Chromium", + "version": "151.0.7922.34", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.7922.34 Safari/537.36" + }, + "operatingSystem": "win32 10.0.26200", + "fixtures": [ + { + "detectionCount": 12, + "expectedDetectionCount": 12, + "filename": "curved-document.jpg", + "labelSequenceEqual": true, + "outputSha256": "98601e9b45ffca68a280c95604d67f40bca747aefa7f07ec502369969dfd4025", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 10.405000001192093, + "inferenceMs": 7859.410000003874, + "postprocessMs": 39.41499999910593, + "preprocessMs": 127.48499999940395, + "totalMs": 8045.539999999106 + } + }, + { + "detectionCount": 59, + "expectedDetectionCount": 59, + "filename": "doc-formula.png", + "labelSequenceEqual": true, + "outputSha256": "8bba1fb0a794a86ec0b2f4d1b1425d28e000a49beab399e4c55bbb04b4cf8cf4", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 10.439999997615814, + "inferenceMs": 7962.560000002384, + "postprocessMs": 35.59000000357628, + "preprocessMs": 115.08500000089407, + "totalMs": 8132.585000000894 + } + }, + { + "detectionCount": 44, + "expectedDetectionCount": 44, + "filename": "image-layout.jpg", + "labelSequenceEqual": true, + "outputSha256": "da3c2a041cc94c4b5616637b7acd9af116c0882befd434920ebe8aaf96be626d", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 27.730000004172325, + "inferenceMs": 7777.990000002086, + "postprocessMs": 59.399999998509884, + "preprocessMs": 105.34000000357628, + "totalMs": 7979.435000002384 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "layout-demo.jpg", + "labelSequenceEqual": true, + "outputSha256": "c6051575214356859bcb9f87be446f27f32dc405a22bdc99ef9f887866c5ddb7", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 31.45500000566244, + "inferenceMs": 7889.280000001192, + "postprocessMs": 99.87999999523163, + "preprocessMs": 172.27000000327826, + "totalMs": 8202.32500000298 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "screen-photo.jpg", + "labelSequenceEqual": true, + "outputSha256": "b036ef27908bd3b94406d9a3106cb5ef6ecca8030874f2fceacac7a0a6407a90", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 11.390000000596046, + "inferenceMs": 8091.3499999940395, + "postprocessMs": 44.645000003278255, + "preprocessMs": 122.03499999642372, + "totalMs": 8279.155000001192 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "skew-document.jpg", + "labelSequenceEqual": true, + "outputSha256": "07f0c613f0d87b91597984e01e83acdca5c5bf440d3efcc19873153b9250fa82", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 11.479999996721745, + "inferenceMs": 7944.47500000149, + "postprocessMs": 52.269999995827675, + "preprocessMs": 125.89500000327826, + "totalMs": 8142.655000001192 + } + }, + { + "detectionCount": 1, + "expectedDetectionCount": 1, + "filename": "table.png", + "labelSequenceEqual": true, + "outputSha256": "8ff528db91eb3893ec1fbf50d69ac23aac17bc2aefec79e70cf0c11f0602550a", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 3.0499999970197678, + "inferenceMs": 7915.270000003278, + "postprocessMs": 55.57499999552965, + "preprocessMs": 76.70500000566244, + "totalMs": 8060.77499999851 + }, + "parityMetrics": { + "iou": 0.9999961699392192, + "maxScoreDelta": 0.000013727193068024945, + "meanPolygonPointDistancePixels": 0 + }, + "parityThresholds": { + "iou": 0.95, + "maxScoreDelta": 0.02, + "meanPolygonPointDistancePixels": 2 + } + } + ], + "timingsMs": { + "coldLoad": { + "capabilitiesMs": 3.530000001192093, + "integrityMs": 366.4150000065565, + "manifestMs": 1.589999996125698, + "modelCacheMs": 0.8250000029802322, + "modelDownloadMs": 1204.2849999964237, + "modelMs": 1968.9499999955297, + "modelSource": "network", + "sessionMs": 1593.4200000017881, + "totalMs": 3568.1999999955297 + }, + "warmLoad": { + "capabilitiesMs": 1.2349999994039536, + "integrityMs": 361.66499999910593, + "manifestMs": 0.0949999988079071, + "modelCacheMs": 79.13000000268221, + "modelDownloadMs": 0, + "modelMs": 440.8799999952316, + "modelSource": "cache", + "sessionMs": 510.9950000047684, + "totalMs": 953.2649999931455 + } + }, + "sdkCommit": "8c47754068ff4ec7b34451cb7d562e6e9a8b1c8a", + "capabilities": { + "crossOriginIsolated": true, + "diagnostics": [ + "webgpu: unavailable because no adapter was returned", + "webgpu-fp16: shader-f16 is unavailable", + "wasm: supported", + "wasm-simd: supported", + "wasm-threads: supported", + "worker: unavailable in this environment" + ], + "wasm": true, + "wasmSimd": true, + "wasmThreads": true, + "webgpu": false, + "webgpuFp16": false, + "worker": false + }, + "cpu": "Intel(R) Core(TM) i5-10400F CPU @ 2.90GHz", + "generatedAt": "2026-08-14T17:26:16.905Z", + "id": "wasm-fp32" + }, + "fp32Webgpu": { + "schemaVersion": 1, + "status": "passed", + "executionProvider": "webgpu", + "precision": "fp32", + "fallbacks": [], + "modelBytes": 142574928, + "modelSha256": "476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269", + "onnxruntimeWebVersion": "1.27.0", + "adapter": { + "architecture": "blackwell", + "description": null, + "device": null, + "vendor": "nvidia" + }, + "adapterFeatures": [ + "bgra8unorm-storage", + "clip-distances", + "core-features-and-limits", + "depth-clip-control", + "depth32float-stencil8", + "dual-source-blending", + "float32-blendable", + "float32-filterable", + "indirect-first-instance", + "primitive-index", + "rg11b10ufloat-renderable", + "shader-f16", + "subgroups", + "texture-component-swizzle", + "texture-compression-bc", + "texture-compression-bc-sliced-3d", + "texture-formats-tier1", + "texture-formats-tier2", + "timestamp-query" + ], + "browser": { + "name": "Chromium", + "version": "151.0.7922.138", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" + }, + "operatingSystem": "win32 10.0.26200", + "fixtures": [ + { + "detectionCount": 12, + "expectedDetectionCount": 12, + "filename": "curved-document.jpg", + "labelSequenceEqual": true, + "outputSha256": "49d017eb3a0946b85dbf0076cf3fc8af88ad12faa1c46b2e5063b54f0b082393", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 15.869999997317791, + "inferenceMs": 5850.69999999553, + "postprocessMs": 68.39000000059605, + "preprocessMs": 161.90999999642372, + "totalMs": 6113.865000002086 + } + }, + { + "detectionCount": 59, + "expectedDetectionCount": 59, + "filename": "doc-formula.png", + "labelSequenceEqual": true, + "outputSha256": "70e57bf726376f2e3aad8fd35a6d492d679b6b38087df95de2648aee09babc51", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 8.570000000298023, + "inferenceMs": 315.6899999976158, + "postprocessMs": 50.16499999910593, + "preprocessMs": 119.58000000566244, + "totalMs": 504.45499999821186 + } + }, + { + "detectionCount": 44, + "expectedDetectionCount": 44, + "filename": "image-layout.jpg", + "labelSequenceEqual": true, + "outputSha256": "16320eaccc62ae6d9509d498ade7996febe9fa72cc7543f716c91d8577b06b00", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 24.344999998807907, + "inferenceMs": 278.85999999940395, + "postprocessMs": 72.42000000178814, + "preprocessMs": 121.79500000178814, + "totalMs": 508.01500000059605 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "layout-demo.jpg", + "labelSequenceEqual": true, + "outputSha256": "24d11a2ed203d8ee72ca4e14e803cab560ef57855976ead40efa65a07bd0fbb7", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 30.639999993145466, + "inferenceMs": 290.7799999937415, + "postprocessMs": 94.3399999961257, + "preprocessMs": 177.5300000011921, + "totalMs": 606.9549999982119 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "screen-photo.jpg", + "labelSequenceEqual": true, + "outputSha256": "6f0fe242016734de274a9ac9d002ee166694e8360cb52cc3cb1d0b4000d1d160", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 11.925000004470348, + "inferenceMs": 290.51500000059605, + "postprocessMs": 44.42000000178814, + "preprocessMs": 144, + "totalMs": 502.4949999973178 + } + }, + { + "detectionCount": 13, + "expectedDetectionCount": 13, + "filename": "skew-document.jpg", + "labelSequenceEqual": true, + "outputSha256": "840d7c61343deee9fa966b541e17a19671d5987aa27280ecec8ca2943c133c47", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 11.689999997615814, + "inferenceMs": 230.11499999463558, + "postprocessMs": 57.30500000715256, + "preprocessMs": 138.96000000089407, + "totalMs": 451.71000000089407 + } + }, + { + "detectionCount": 1, + "expectedDetectionCount": 1, + "filename": "table.png", + "labelSequenceEqual": true, + "outputSha256": "7978ca0ec7143f98fc24ef2f4613c72d785a82ca4a1b31bcf97fc1952ffb9e8c", + "parity": "passed", + "readingOrderEqual": true, + "timings": { + "decodeMs": 3.0450000017881393, + "inferenceMs": 355.1149999946356, + "postprocessMs": 53.979999996721745, + "preprocessMs": 83.14499999582767, + "totalMs": 504.5949999988079 + }, + "parityMetrics": { + "iou": 0.9999962574460338, + "maxScoreDelta": 0.00001386435552097609, + "meanPolygonPointDistancePixels": 0 + }, + "parityThresholds": { + "iou": 0.95, + "maxScoreDelta": 0.02, + "meanPolygonPointDistancePixels": 2 + } + } + ], + "timingsMs": { + "coldLoad": { + "capabilitiesMs": 168.0949999988079, + "integrityMs": 539.7800000011921, + "manifestMs": 2.3250000029802322, + "modelCacheMs": 1.089999996125698, + "modelDownloadMs": 3077.2300000041723, + "modelMs": 4335.57499999553, + "modelSource": "network", + "sessionMs": 3461.4400000050664, + "totalMs": 7968.234999999404 + }, + "warmLoad": { + "capabilitiesMs": 0.5600000023841858, + "integrityMs": 363.2800000011921, + "manifestMs": 0.11999999731779099, + "modelCacheMs": 102.08500000089407, + "modelDownloadMs": 0, + "modelMs": 465.5, + "modelSource": "cache", + "sessionMs": 1073.929999999702, + "totalMs": 1540.1750000044703 + } + }, + "sdkCommit": "8c47754068ff4ec7b34451cb7d562e6e9a8b1c8a", + "capabilities": { + "crossOriginIsolated": true, + "diagnostics": [ + "webgpu: supported by an acquired adapter", + "webgpu-fp16: shader-f16 is supported", + "wasm: supported", + "wasm-simd: supported", + "wasm-threads: supported", + "worker: supported" + ], + "wasm": true, + "wasmSimd": true, + "wasmThreads": true, + "webgpu": true, + "webgpuFp16": true, + "worker": true + }, + "cpu": "Intel(R) Core(TM) i5-10400F CPU @ 2.90GHz", + "generatedAt": "2026-08-14T17:22:41.632Z", + "id": "webgpu-fp32" + } +} From 0f1c7edca80acc5910da8d292bc6e28d8ac07a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 01:52:25 +0800 Subject: [PATCH 09/13] feat(models): generate validated model 1.0.1 manifest --- models/pp-doclayoutv3/1.0.1/manifest.json | 160 ++++++++ .../ppdoclayout/build_manifest.py | 123 +++++- .../ppdoclayout/variant_validation.py | 41 +- .../reports/1.0.1/variant-validation.json | 354 ++++++++++++++++++ tools/model-pipeline/tests/test_manifest.py | 50 ++- tools/model-pipeline/tests/test_variants.py | 63 +++- 6 files changed, 745 insertions(+), 46 deletions(-) create mode 100644 models/pp-doclayoutv3/1.0.1/manifest.json create mode 100644 tools/model-pipeline/reports/1.0.1/variant-validation.json diff --git a/models/pp-doclayoutv3/1.0.1/manifest.json b/models/pp-doclayoutv3/1.0.1/manifest.json new file mode 100644 index 0000000..348eec0 --- /dev/null +++ b/models/pp-doclayoutv3/1.0.1/manifest.json @@ -0,0 +1,160 @@ +{ + "input": { + "dtype": "float32", + "name": "pixel_values", + "shape": [ + 1, + 3, + 800, + 800 + ] + }, + "labels": [ + "abstract", + "algorithm", + "aside_text", + "chart", + "content", + "formula", + "doc_title", + "figure_title", + "footer", + "footer", + "footnote", + "formula_number", + "header", + "header", + "image", + "formula", + "number", + "paragraph_title", + "reference", + "reference_content", + "seal", + "table", + "text", + "text", + "vision_footnote" + ], + "minSdkVersion": "1.0.0", + "model": { + "architecture": "PPDocLayoutV3ForObjectDetection", + "id": "pp-doclayoutv3", + "modelType": "pp_doclayout_v3", + "parameterCount": 33175165, + "version": "1.0.1" + }, + "outputs": [ + { + "dtype": "float32", + "name": "logits", + "shape": [ + 1, + 300, + 25 + ] + }, + { + "dtype": "float32", + "name": "pred_boxes", + "shape": [ + 1, + 300, + 4 + ] + }, + { + "dtype": "float32", + "name": "order_logits", + "shape": [ + 1, + 300, + 300 + ] + }, + { + "dtype": "float32", + "name": "out_masks", + "shape": [ + 1, + 300, + 200, + 200 + ] + } + ], + "preprocessing": { + "doNormalize": true, + "doRescale": true, + "doResize": true, + "imageMean": [ + 0, + 0, + 0 + ], + "imageStd": [ + 1, + 1, + 1 + ], + "resample": 3, + "rescaleFactor": 0.00392156862745098, + "size": { + "height": 800, + "width": 800 + } + }, + "schemaVersion": 1, + "source": { + "files": { + "config.json": "4aeed1ded82e4d18a3801a2f771ff31648ba302d1e736cafb065d666b1d3d131", + "inference.yml": "aa6afd4938b83e97a14a9d7da39deecfc16ad0f8be103316fa578cc3b2a89f09", + "model.safetensors": "5ea422c6cc5fe759a47e1357c35639b58173508e025a3131cbe4b6ac59e2b85e", + "preprocessor_config.json": "3b9074dd30b642c406309e21f030b496a948a6baedb02dfb72d6d1b4ef03c47b" + }, + "license": "Apache-2.0", + "name": "PaddlePaddle/PP-DocLayoutV3_safetensors", + "url": "https://huggingface.co/PaddlePaddle/PP-DocLayoutV3_safetensors" + }, + "variantPriority": [ + "fp16", + "fp32" + ], + "variants": [ + { + "backendCompatibility": [ + "webgpu" + ], + "bytes": 74279796, + "filename": "model-fp16.onnx", + "id": "fp16", + "opset": 18, + "precision": "fp16", + "sha256": "463ba56faa555baf84271b4002b33b0c5fcc50776fe4f39344235eccb72073f2", + "url": "https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/releases/download/v1.0.1-models/model-fp16.onnx", + "validation": { + "included": true, + "pass": true, + "report": "tools/model-pipeline/reports/1.0.1/variant-validation.json" + } + }, + { + "backendCompatibility": [ + "wasm", + "webgpu" + ], + "bytes": 142574928, + "filename": "model-fp32.onnx", + "id": "fp32", + "opset": 18, + "precision": "fp32", + "sha256": "476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269", + "url": "https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/releases/download/v1.0.1-models/model-fp32.onnx", + "validation": { + "included": true, + "pass": true, + "report": "tools/model-pipeline/reports/1.0.1/fp32-validation.json" + } + } + ] +} diff --git a/tools/model-pipeline/ppdoclayout/build_manifest.py b/tools/model-pipeline/ppdoclayout/build_manifest.py index 67700f4..1683b93 100644 --- a/tools/model-pipeline/ppdoclayout/build_manifest.py +++ b/tools/model-pipeline/ppdoclayout/build_manifest.py @@ -3,6 +3,7 @@ import argparse import hashlib import json +import re from pathlib import Path from typing import Any @@ -10,12 +11,9 @@ MODEL_ID = "pp-doclayoutv3" -MODEL_VERSION = "1.0.0" MIN_SDK_VERSION = "1.0.0" -RELEASE_BASE_URL = ( - "https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/" - "releases/download/v1.0.0-models/" -) +SEMVER = re.compile(r"^\d+\.\d+\.\d+$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") EXPECTED_INPUT_NAME = "pixel_values" EXPECTED_INPUT_SHAPE = [1, 3, 800, 800] EXPECTED_OUTPUT_NAMES = ["logits", "pred_boxes", "order_logits", "out_masks"] @@ -24,6 +22,17 @@ ROOT = Path(__file__).parents[3] +def release_base_url(model_version: str, release_tag: str) -> str: + if not SEMVER.fullmatch(model_version): + raise ValueError(f"Invalid model version: {model_version}") + if release_tag != f"v{model_version}-models": + raise ValueError("Release tag must match model version") + return ( + "https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/" + f"releases/download/{release_tag}/" + ) + + def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: @@ -138,16 +147,78 @@ def _validation_status(report: dict[str, Any], name: str) -> tuple[bool, bool]: return passed, included +def _validate_fp32_browser_evidence( + report: dict[str, Any], fp32: dict[str, Any] +) -> None: + if report.get("schemaVersion") != 1: + raise ValueError("Unsupported browser evidence schema") + + for key, backend in (("fp32Wasm", "wasm"), ("fp32Webgpu", "webgpu")): + evidence = report.get(key) + if not isinstance(evidence, dict): + raise ValueError(f"FP32 {backend} browser evidence is missing") + if evidence.get("status") != "passed": + raise ValueError(f"FP32 {backend} browser validation did not pass") + if evidence.get("executionProvider") != backend: + raise ValueError(f"FP32 {backend} execution provider does not match") + if evidence.get("precision") != "fp32": + raise ValueError(f"FP32 {backend} precision does not match") + if evidence.get("fallbacks") != []: + raise ValueError(f"FP32 {backend} browser evidence contains fallback records") + if evidence.get("modelBytes") != fp32["bytes"]: + raise ValueError(f"FP32 {backend} browser byte size does not match") + if evidence.get("modelSha256") != fp32["sha256"]: + raise ValueError(f"FP32 {backend} browser SHA-256 does not match") + if evidence.get("onnxruntimeWebVersion") != "1.27.0": + raise ValueError(f"FP32 {backend} ONNX Runtime Web version does not match") + + fixtures = evidence.get("fixtures") + if not isinstance(fixtures, list) or len(fixtures) != 7: + raise ValueError(f"FP32 {backend} evidence must contain seven fixtures") + filenames = set() + for fixture in fixtures: + if not isinstance(fixture, dict) or fixture.get("parity") != "passed": + raise ValueError(f"FP32 {backend} fixture parity did not pass") + filename = fixture.get("filename") + if not isinstance(filename, str) or not filename or filename in filenames: + raise ValueError(f"FP32 {backend} fixture identity is invalid") + filenames.add(filename) + digest = fixture.get("outputSha256") + if not isinstance(digest, str) or not SHA256.fullmatch(digest): + raise ValueError(f"FP32 {backend} fixture output hash is invalid") + + if backend == "webgpu": + adapter = evidence.get("adapter") + if not isinstance(adapter, dict) or not any( + isinstance(adapter.get(name), str) and adapter[name] + for name in ("architecture", "description", "device", "vendor") + ): + raise ValueError("FP32 WebGPU adapter identity is missing") + features = evidence.get("adapterFeatures") + if ( + not isinstance(features, list) + or not features + or any(not isinstance(feature, str) or not feature for feature in features) + or features != sorted(set(features)) + ): + raise ValueError("FP32 WebGPU adapter feature list is invalid") + + def build_manifest( *, contract_path: Path, fp32_report_path: Path, variant_report_path: Path, + browser_report_path: Path, model_dir: Path, + model_version: str, + release_tag: str, ) -> dict[str, Any]: + release_url = release_base_url(model_version, release_tag) contract = _load_json(contract_path) fp32_report = _load_json(fp32_report_path) variant_report = _load_json(variant_report_path) + browser_report = _load_json(browser_report_path) fp32_path = model_dir / "model-fp32.onnx" fp16_path = model_dir / "model-fp16.onnx" fp32 = _inspect_onnx(fp32_path) @@ -161,6 +232,7 @@ def build_manifest( fp32_source_hashes = fp32_report.get("sourceHashes", {}) if fp32_source_hashes.get("onnx") != fp32["sha256"]: raise ValueError("FP32 report SHA-256 does not match the ONNX artifact") + _validate_fp32_browser_evidence(browser_report, fp32) source_files = contract.get("source", {}).get("files") if not isinstance(source_files, dict) or "model.safetensors" not in source_files: @@ -190,11 +262,11 @@ def build_manifest( "opset": fp32["opset"], "precision": "fp32", "sha256": fp32["sha256"], - "url": RELEASE_BASE_URL + fp32_path.name, + "url": release_url + fp32_path.name, "validation": { "included": True, "pass": True, - "report": "tools/model-pipeline/reports/fp32-validation.json", + "report": f"tools/model-pipeline/reports/{model_version}/fp32-validation.json", }, } ] @@ -226,11 +298,11 @@ def build_manifest( "opset": fp16["opset"], "precision": "fp16", "sha256": fp16["sha256"], - "url": RELEASE_BASE_URL + fp16_path.name, + "url": release_url + fp16_path.name, "validation": { "included": fp16_included, "pass": fp16_pass, - "report": "tools/model-pipeline/reports/variant-validation.json", + "report": f"tools/model-pipeline/reports/{model_version}/variant-validation.json", }, } ) @@ -265,7 +337,7 @@ def build_manifest( "id": MODEL_ID, "modelType": contract.get("modelType"), "parameterCount": contract.get("parameterCount"), - "version": MODEL_VERSION, + "version": model_version, }, "outputs": outputs, "preprocessing": preprocessing, @@ -286,14 +358,20 @@ def write_manifest( contract_path: Path, fp32_report_path: Path, variant_report_path: Path, + browser_report_path: Path, model_dir: Path, output_path: Path, + model_version: str, + release_tag: str, ) -> None: manifest = build_manifest( contract_path=contract_path, fp32_report_path=fp32_report_path, variant_report_path=variant_report_path, + browser_report_path=browser_report_path, model_dir=model_dir, + model_version=model_version, + release_tag=release_tag, ) output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_bytes(canonical_json(manifest)) @@ -301,8 +379,9 @@ def write_manifest( def parse_args() -> argparse.Namespace: pipeline_dir = ROOT / "tools" / "model-pipeline" - model_dir = ROOT / "models" / MODEL_ID / MODEL_VERSION parser = argparse.ArgumentParser(description="Build the versioned model manifest") + parser.add_argument("--model-version", default="1.0.1") + parser.add_argument("--release-tag", default="v1.0.1-models") parser.add_argument( "--contract", type=Path, @@ -311,16 +390,25 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--fp32-report", type=Path, - default=pipeline_dir / "reports" / "fp32-validation.json", + default=None, ) parser.add_argument( "--variant-report", type=Path, - default=pipeline_dir / "reports" / "variant-validation.json", + default=None, ) - parser.add_argument("--model-dir", type=Path, default=model_dir) - parser.add_argument("--output", type=Path, default=model_dir / "manifest.json") - return parser.parse_args() + parser.add_argument("--browser-report", type=Path, default=None) + parser.add_argument("--model-dir", type=Path, default=None) + parser.add_argument("--output", type=Path, default=None) + args = parser.parse_args() + model_dir = ROOT / "models" / MODEL_ID / args.model_version + report_dir = pipeline_dir / "reports" / args.model_version + args.model_dir = args.model_dir or model_dir + args.fp32_report = args.fp32_report or report_dir / "fp32-validation.json" + args.variant_report = args.variant_report or report_dir / "variant-validation.json" + args.browser_report = args.browser_report or report_dir / "browser-evidence.json" + args.output = args.output or model_dir / "manifest.json" + return args def main() -> None: @@ -329,8 +417,11 @@ def main() -> None: contract_path=args.contract.resolve(), fp32_report_path=args.fp32_report.resolve(), variant_report_path=args.variant_report.resolve(), + browser_report_path=args.browser_report.resolve(), model_dir=args.model_dir.resolve(), output_path=args.output.resolve(), + model_version=args.model_version, + release_tag=args.release_tag, ) diff --git a/tools/model-pipeline/ppdoclayout/variant_validation.py b/tools/model-pipeline/ppdoclayout/variant_validation.py index 862823f..b7c8393 100644 --- a/tools/model-pipeline/ppdoclayout/variant_validation.py +++ b/tools/model-pipeline/ppdoclayout/variant_validation.py @@ -2,6 +2,7 @@ import argparse import json +from copy import deepcopy from pathlib import Path from typing import Any @@ -197,19 +198,35 @@ def build_variant_report( model_path: Path, fp32_path: Path, fp16_path: Path, - int8_path: Path, + accepted_variant_report_path: Path, fixtures_lock: Path, browser_evidence: dict[str, Any], ) -> dict[str, Any]: + accepted_report = json.loads( + accepted_variant_report_path.read_text(encoding="utf-8") + ) + if accepted_report.get("schemaVersion") != 1: + raise ValueError("Unsupported accepted variant report schema") + if ( + accepted_report.get("thresholds", {}).get("int8") + != VARIANT_THRESHOLDS["int8"] + ): + raise ValueError("Accepted INT8 thresholds do not match current thresholds") + accepted_int8 = accepted_report.get("variants", {}).get("int8") + if not isinstance(accepted_int8, dict): + raise ValueError("Accepted INT8 validation evidence is missing") + if ( + accepted_int8.get("pass") is not False + or accepted_int8.get("included") is not False + ): + raise ValueError("Accepted INT8 evidence must remain rejected and excluded") + int8 = deepcopy(accepted_int8) + fp16 = evaluate_variant( model_path, fp16_path, fixtures_lock, VARIANT_THRESHOLDS["fp16"]["iou"] ) - int8 = evaluate_variant( - model_path, int8_path, fixtures_lock, VARIANT_THRESHOLDS["int8"]["iou"] - ) fp32_bytes = fp32_path.stat().st_size fp16.update(_file_metadata(fp16_path, fp32_bytes)) - int8.update(_file_metadata(int8_path, fp32_bytes)) fp16["blockedOps"] = DEFAULT_BLOCKED_OPS import onnx @@ -238,7 +255,6 @@ def build_variant_report( } ] fp16["browser"] = {"webgpu": browser_evidence.get("fp16Webgpu", {"status": "pending"})} - int8["browser"] = {"wasm": browser_evidence.get("int8Wasm", {"status": "pending"})} fp16_browser_errors = _browser_evidence_errors( fp16["browser"]["webgpu"], fp16_path, "fp16" ) @@ -247,15 +263,6 @@ def build_variant_report( fp16["included"] = fp16["pass"] fp16["cpu"]["acceptanceStatus"] = "passed" if _passes(fp16, "fp16") else "failed" - int8_browser_errors = _browser_evidence_errors( - int8["browser"]["wasm"], int8_path, "int8" - ) - int8["browser"]["wasm"]["validationErrors"] = int8_browser_errors - int8["pass"] = _passes(int8, "int8") and not int8_browser_errors - int8["included"] = int8["pass"] - int8["cpu"]["acceptanceStatus"] = "passed" if _passes(int8, "int8") else "failed" - int8["exclusionReasons"] = [] if int8["pass"] else _exclusion_reasons(int8, "int8") - return { "schemaVersion": 1, "threshold": THRESHOLD, @@ -375,7 +382,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--model", required=True, type=Path) parser.add_argument("--fp32", required=True, type=Path) parser.add_argument("--fp16", required=True, type=Path) - parser.add_argument("--int8", required=True, type=Path) + parser.add_argument("--accepted-variant-report", required=True, type=Path) parser.add_argument("--fixtures-lock", required=True, type=Path) parser.add_argument("--browser-evidence", type=Path) parser.add_argument("--output", required=True, type=Path) @@ -393,7 +400,7 @@ def main() -> None: args.model.resolve(), args.fp32.resolve(), args.fp16.resolve(), - args.int8.resolve(), + args.accepted_variant_report.resolve(), args.fixtures_lock.resolve(), evidence, ) diff --git a/tools/model-pipeline/reports/1.0.1/variant-validation.json b/tools/model-pipeline/reports/1.0.1/variant-validation.json new file mode 100644 index 0000000..be038a4 --- /dev/null +++ b/tools/model-pipeline/reports/1.0.1/variant-validation.json @@ -0,0 +1,354 @@ +{ + "schemaVersion": 1, + "source": { + "fixturesLockSha256": "e1884d55c83fe24f707675d044dce0a200de74961159364802147e7b4f42b811", + "fp32Sha256": "476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269" + }, + "threshold": 0.5, + "thresholds": { + "fp16": { + "iou": 0.95, + "matchedDetectionRatio": 0.99, + "maxScoreDelta": 0.02, + "meanPolygonPointDistancePixels": 2.0 + }, + "int8": { + "iou": 0.9, + "matchedDetectionRatio": 0.97, + "maxScoreDelta": 0.05, + "maxSizeRatio": 0.6, + "meanPolygonPointDistancePixels": 4.0, + "minMedianWasmSpeedup": 0.1 + } + }, + "variants": { + "fp16": { + "blockedNodes": [ + "node__to_copy_11", + "node__to_copy_12", + "node__to_copy_13", + "node__to_copy_4", + "node_convert_element_type_default", + "node_convert_element_type_default_2" + ], + "blockedOpEvidence": [ + { + "opType": "Resize", + "strategy": "preserve operator inputs and outputs as FP32", + "withoutBlock": { + "configuration": { + "blockedOps": [] + }, + "error": "Resize scale tensor type mismatch: expected float, actual float16", + "nodes": [ + "node_upsample_nearest2d_4", + "node_upsample_nearest2d_5", + "node_upsample_bilinear2d", + "node_upsample_bilinear2d_2", + "node_upsample_bilinear2d_3", + "node_upsample_bilinear2d_5" + ], + "onnxVersion": "1.22.0", + "stage": "onnx.shape_inference", + "status": "failed" + } + } + ], + "blockedOps": [ + "Resize" + ], + "browser": { + "webgpu": { + "adapter": { + "architecture": "blackwell", + "description": null, + "device": null, + "isFallbackAdapter": null, + "subgroupMaxSize": 32, + "subgroupMinSize": 32, + "vendor": "nvidia" + }, + "adapterFeatures": [ + "bgra8unorm-storage", + "clip-distances", + "core-features-and-limits", + "depth-clip-control", + "depth32float-stencil8", + "dual-source-blending", + "float32-blendable", + "float32-filterable", + "indirect-first-instance", + "primitive-index", + "rg11b10ufloat-renderable", + "shader-f16", + "subgroups", + "texture-component-swizzle", + "texture-compression-bc", + "texture-compression-bc-sliced-3d", + "texture-formats-tier1", + "texture-formats-tier2", + "timestamp-query" + ], + "browser": { + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36", + "userAgentData": { + "brands": [ + { + "brand": "Not=A?Brand", + "version": "99" + }, + { + "brand": "Google Chrome", + "version": "151" + }, + { + "brand": "Chromium", + "version": "151" + } + ], + "mobile": false, + "platform": "Windows" + } + }, + "executionProvider": "webgpu", + "input": { + "dimensions": [ + 1, + 3, + 800, + 800 + ], + "name": "pixel_values", + "type": "float32" + }, + "modelBytes": 74279796, + "modelSha256": "463ba56faa555baf84271b4002b33b0c5fcc50776fe4f39344235eccb72073f2", + "onnxruntimeWebVersion": "1.27.0", + "outputs": { + "logits": { + "allFinite": true, + "dimensions": [ + 1, + 300, + 25 + ], + "sha256": "2d1c470358fd8162ac3e6025f99658c4727a62e5a80eea1b6199989bebf1337f", + "type": "float32" + }, + "order_logits": { + "allFinite": true, + "dimensions": [ + 1, + 300, + 300 + ], + "sha256": "f039615990d24938fb35c9dfe142be13846caa48002c7c37beed61f37970c4d2", + "type": "float32" + }, + "out_masks": { + "allFinite": true, + "dimensions": [ + 1, + 300, + 200, + 200 + ], + "sha256": "8e27be8e4b2294a27028b15e7c03679364f1bf2f158e41c5dbe7c67a9be47ad8", + "type": "float32" + }, + "pred_boxes": { + "allFinite": true, + "dimensions": [ + 1, + 300, + 4 + ], + "sha256": "fd8ec5511d6b5f840ec86967a8912aa483873ce43d28e6119ff3692cf79ae9b5", + "type": "float32" + } + }, + "status": "passed", + "timingsMs": { + "download": 439.9899999946356, + "inference": 681.9950000047684, + "sessionCreate": 1785.0250000059605 + }, + "validatedAt": "2026-08-11T09:25:16.542Z", + "validationErrors": [] + } + }, + "bytes": 74279796, + "candidateDetections": 155, + "cpu": { + "acceptanceStatus": "passed", + "executionStatus": "passed", + "provider": "CPUExecutionProvider" + }, + "filename": "model-fp16.onnx", + "fixtures": [ + { + "candidateDetections": 12, + "filename": "curved-document.jpg", + "matches": 12, + "maxScoreDelta": 0.0022199153900146484, + "meanPolygonPointDistancePixels": 0.11666666666666665, + "officialDetections": 12 + }, + { + "candidateDetections": 59, + "filename": "doc-formula.png", + "matches": 59, + "maxScoreDelta": 0.002554774284362793, + "meanPolygonPointDistancePixels": 0.1554630895275069, + "officialDetections": 59 + }, + { + "candidateDetections": 44, + "filename": "image-layout.jpg", + "matches": 44, + "maxScoreDelta": 0.0022498369216918945, + "meanPolygonPointDistancePixels": 0.2632143695883176, + "officialDetections": 44 + }, + { + "candidateDetections": 13, + "filename": "layout-demo.jpg", + "matches": 13, + "maxScoreDelta": 0.002132594585418701, + "meanPolygonPointDistancePixels": 0.26153846153846155, + "officialDetections": 13 + }, + { + "candidateDetections": 13, + "filename": "screen-photo.jpg", + "matches": 13, + "maxScoreDelta": 0.0010916590690612793, + "meanPolygonPointDistancePixels": 0.38057254408239094, + "officialDetections": 13 + }, + { + "candidateDetections": 13, + "filename": "skew-document.jpg", + "matches": 13, + "maxScoreDelta": 0.002375006675720215, + "meanPolygonPointDistancePixels": 0.3606471755368814, + "officialDetections": 13 + }, + { + "candidateDetections": 1, + "filename": "table.png", + "matches": 1, + "maxScoreDelta": 6.61015510559082e-5, + "meanPolygonPointDistancePixels": 0.0, + "officialDetections": 1 + } + ], + "included": true, + "matchedDetectionPrecision": 1.0, + "matchedDetectionRatio": 1.0, + "matchedDetections": 155, + "maxScoreDelta": 0.002554774284362793, + "meanPolygonPointDistancePixels": 0.22833214658755624, + "officialDetections": 155, + "pass": true, + "sha256": "463ba56faa555baf84271b4002b33b0c5fcc50776fe4f39344235eccb72073f2", + "sizeRatio": 0.5209877854541157, + "unmatchedCandidateDetections": 0 + }, + "int8": { + "browser": { + "wasm": { + "reason": "CPU precision acceptance failed before browser validation", + "status": "skipped", + "validationErrors": [ + "browser wasm validation did not pass" + ] + } + }, + "bytes": 45313310, + "candidateDetections": 0, + "cpu": { + "acceptanceStatus": "failed", + "executionStatus": "passed", + "provider": "CPUExecutionProvider" + }, + "exclusionReasons": [ + "matched detection ratio 0.000000 is below 0.97", + "matched detection precision 0.000000 is below 0.97", + "score delta acceptance failed", + "polygon distance acceptance failed", + "browser wasm validation did not pass" + ], + "filename": "model-int8.onnx", + "fixtures": [ + { + "candidateDetections": 0, + "filename": "curved-document.jpg", + "matches": 0, + "maxScoreDelta": null, + "meanPolygonPointDistancePixels": null, + "officialDetections": 12 + }, + { + "candidateDetections": 0, + "filename": "doc-formula.png", + "matches": 0, + "maxScoreDelta": null, + "meanPolygonPointDistancePixels": null, + "officialDetections": 59 + }, + { + "candidateDetections": 0, + "filename": "image-layout.jpg", + "matches": 0, + "maxScoreDelta": null, + "meanPolygonPointDistancePixels": null, + "officialDetections": 44 + }, + { + "candidateDetections": 0, + "filename": "layout-demo.jpg", + "matches": 0, + "maxScoreDelta": null, + "meanPolygonPointDistancePixels": null, + "officialDetections": 13 + }, + { + "candidateDetections": 0, + "filename": "screen-photo.jpg", + "matches": 0, + "maxScoreDelta": null, + "meanPolygonPointDistancePixels": null, + "officialDetections": 13 + }, + { + "candidateDetections": 0, + "filename": "skew-document.jpg", + "matches": 0, + "maxScoreDelta": null, + "meanPolygonPointDistancePixels": null, + "officialDetections": 13 + }, + { + "candidateDetections": 0, + "filename": "table.png", + "matches": 0, + "maxScoreDelta": null, + "meanPolygonPointDistancePixels": null, + "officialDetections": 1 + } + ], + "included": false, + "matchedDetectionPrecision": 0.0, + "matchedDetectionRatio": 0.0, + "matchedDetections": 0, + "maxScoreDelta": null, + "meanPolygonPointDistancePixels": null, + "officialDetections": 155, + "pass": false, + "sha256": "20fd15891d32fcfcec90ec379a513ae6c606178d8c4f648a0cadf4118a096933", + "sizeRatio": 0.3163981475155894, + "unmatchedCandidateDetections": 0 + } + } +} diff --git a/tools/model-pipeline/tests/test_manifest.py b/tools/model-pipeline/tests/test_manifest.py index 100ba59..579bda6 100644 --- a/tools/model-pipeline/tests/test_manifest.py +++ b/tools/model-pipeline/tests/test_manifest.py @@ -13,11 +13,14 @@ ROOT = Path(__file__).parents[3] PIPELINE_DIR = ROOT / "tools" / "model-pipeline" -MODEL_DIR = ROOT / "models" / "pp-doclayoutv3" / "1.0.0" +MODEL_VERSION = "1.0.1" +RELEASE_TAG = "v1.0.1-models" +MODEL_DIR = ROOT / "models" / "pp-doclayoutv3" / MODEL_VERSION MANIFEST_PATH = MODEL_DIR / "manifest.json" CONTRACT_PATH = PIPELINE_DIR / "artifacts" / "model-contract.json" -FP32_REPORT_PATH = PIPELINE_DIR / "reports" / "fp32-validation.json" -VARIANT_REPORT_PATH = PIPELINE_DIR / "reports" / "variant-validation.json" +FP32_REPORT_PATH = PIPELINE_DIR / "reports" / MODEL_VERSION / "fp32-validation.json" +VARIANT_REPORT_PATH = PIPELINE_DIR / "reports" / MODEL_VERSION / "variant-validation.json" +BROWSER_REPORT_PATH = PIPELINE_DIR / "reports" / MODEL_VERSION / "browser-evidence.json" EXPECTED_OUTPUTS = ["logits", "pred_boxes", "order_logits", "out_masks"] EXPECTED_VARIANTS = { "fp16": { @@ -25,12 +28,19 @@ "sha256": "463ba56faa555baf84271b4002b33b0c5fcc50776fe4f39344235eccb72073f2", }, "fp32": { - "bytes": 143216104, - "sha256": "fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b", + "bytes": 142574928, + "sha256": "476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269", }, } SOURCE_SHA256 = "5ea422c6cc5fe759a47e1357c35639b58173508e025a3131cbe4b6ac59e2b85e" +HISTORICAL_FP32_SHA256 = ( + "fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b" +) RELEASE_BASE = ( + "https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/" + f"releases/download/{RELEASE_TAG}/" +) +HISTORICAL_RELEASE_BASE = ( "https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/" "releases/download/v1.0.0-models/" ) @@ -53,12 +63,16 @@ def build_from_paths( contract_path: Path = CONTRACT_PATH, fp32_report_path: Path = FP32_REPORT_PATH, variant_report_path: Path = VARIANT_REPORT_PATH, + browser_report_path: Path = BROWSER_REPORT_PATH, ) -> dict: return build_manifest( contract_path=contract_path, fp32_report_path=fp32_report_path, variant_report_path=variant_report_path, + browser_report_path=browser_report_path, model_dir=MODEL_DIR, + model_version=MODEL_VERSION, + release_tag=RELEASE_TAG, ) @@ -67,7 +81,7 @@ def test_manifest_has_stable_browser_runtime_contract() -> None: assert manifest["schemaVersion"] == 1 assert manifest["model"]["id"] == "pp-doclayoutv3" - assert manifest["model"]["version"] == "1.0.0" + assert manifest["model"]["version"] == MODEL_VERSION assert manifest["minSdkVersion"] == "1.0.0" assert len(manifest["labels"]) == 25 assert manifest["input"] == { @@ -157,18 +171,38 @@ def test_rejected_int8_is_not_publishable() -> None: assert "int8" not in {variant["id"] for variant in manifest["variants"]} +def test_fp32_requires_strict_wasm_and_webgpu_evidence(tmp_path: Path) -> None: + evidence = json.loads(BROWSER_REPORT_PATH.read_text(encoding="utf-8")) + evidence["fp32Webgpu"]["fallbacks"] = [{"provider": "wasm"}] + path = tmp_path / "browser-evidence.json" + path.write_text(json.dumps(evidence), encoding="utf-8") + + with pytest.raises(ValueError, match="fallback"): + build_from_paths(browser_report_path=path) + + +def test_manifest_advertises_validated_fp32_for_both_backends() -> None: + manifest = build_from_paths() + fp32 = next(item for item in manifest["variants"] if item["id"] == "fp32") + + assert manifest["model"]["version"] == MODEL_VERSION + assert manifest["variantPriority"] == ["fp16", "fp32"] + assert fp32["backendCompatibility"] == ["wasm", "webgpu"] + assert fp32["url"].endswith(f"/{RELEASE_TAG}/model-fp32.onnx") + + def test_model_readme_documents_distribution_and_reproducibility() -> None: readme = (ROOT / "models" / "README.md").read_text(encoding="utf-8") assert readme.startswith("# 模型文件") assert "Model files" in readme assert "143216104" in readme - assert EXPECTED_VARIANTS["fp32"]["sha256"] in readme + assert HISTORICAL_FP32_SHA256 in readme assert "74279796" in readme assert EXPECTED_VARIANTS["fp16"]["sha256"] in readme assert "INT8" in readme and "不发布" in readme assert "python.exe -m ppdoclayout.build_manifest" in readme - assert RELEASE_BASE in readme + assert HISTORICAL_RELEASE_BASE in readme assert "latest" in readme assert "自定义微调模型" in readme assert "Apache-2.0" in readme diff --git a/tools/model-pipeline/tests/test_variants.py b/tools/model-pipeline/tests/test_variants.py index 0b83614..053a454 100644 --- a/tools/model-pipeline/tests/test_variants.py +++ b/tools/model-pipeline/tests/test_variants.py @@ -15,8 +15,14 @@ ROOT = Path(__file__).parents[3] -MODEL_DIR = ROOT / "models" / "pp-doclayoutv3" / "1.0.0" -REPORT_PATH = ROOT / "tools" / "model-pipeline" / "reports" / "variant-validation.json" +OLD_MODEL_DIR = ROOT / "models" / "pp-doclayoutv3" / "1.0.0" +MODEL_DIR = ROOT / "models" / "pp-doclayoutv3" / "1.0.1" +REPORT_PATH = ( + ROOT / "tools" / "model-pipeline" / "reports" / "1.0.1" / "variant-validation.json" +) +ACCEPTED_REPORT_PATH = ( + ROOT / "tools" / "model-pipeline" / "reports" / "variant-validation.json" +) def test_fp32_semantic_casts_are_preserved() -> None: @@ -84,11 +90,58 @@ def test_fp16_conversion_is_byte_reproducible(tmp_path: Path) -> None: regenerated = tmp_path / "model-fp16.onnx" with warnings.catch_warnings(): warnings.filterwarnings("ignore", module="onnxconverter_common.float16") - convert_fp16(MODEL_DIR / "model-fp32.onnx", regenerated) + convert_fp16(OLD_MODEL_DIR / "model-fp32.onnx", regenerated) assert sha256_file(regenerated) == sha256_file(MODEL_DIR / "model-fp16.onnx") +def test_model_1_0_1_reuses_accepted_fp16_bytes() -> None: + assert (MODEL_DIR / "model-fp16.onnx").read_bytes() == ( + OLD_MODEL_DIR / "model-fp16.onnx" + ).read_bytes() + + +def test_rejected_int8_evidence_is_carried_forward_without_binary() -> None: + accepted = json.loads(ACCEPTED_REPORT_PATH.read_text(encoding="utf-8"))["variants"][ + "int8" + ] + candidate = json.loads(REPORT_PATH.read_text(encoding="utf-8"))["variants"]["int8"] + + assert accepted["pass"] is False + assert accepted["included"] is False + assert candidate == accepted + assert not (MODEL_DIR / "model-int8.onnx").exists() + + +def test_validation_cli_uses_accepted_int8_report( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "variant_validation", + "--model", + str(tmp_path / "model"), + "--fp32", + str(tmp_path / "fp32.onnx"), + "--fp16", + str(tmp_path / "fp16.onnx"), + "--accepted-variant-report", + str(tmp_path / "accepted.json"), + "--fixtures-lock", + str(tmp_path / "fixtures.json"), + "--output", + str(tmp_path / "report.json"), + ], + ) + + args = variant_validation.parse_args() + + assert args.accepted_variant_report == tmp_path / "accepted.json" + assert not hasattr(args, "int8") + + def test_report_and_browser_evidence_are_bound_to_fp16_artifact() -> None: report = json.loads(REPORT_PATH.read_text(encoding="utf-8")) evidence = json.loads( @@ -185,7 +238,7 @@ def test_int8_accepts_speed_alternative_when_size_is_over_limit() -> None: def test_validation_cli_fails_without_passing_browser_evidence( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - paths = [tmp_path / name for name in ("model", "fp32", "fp16", "int8", "lock")] + paths = [tmp_path / name for name in ("model", "fp32", "fp16", "accepted", "lock")] for path in paths: path.write_bytes(b"placeholder") monkeypatch.setattr( @@ -195,7 +248,7 @@ def test_validation_cli_fails_without_passing_browser_evidence( model=paths[0], fp32=paths[1], fp16=paths[2], - int8=paths[3], + accepted_variant_report=paths[3], fixtures_lock=paths[4], browser_evidence=None, output=tmp_path / "report.json", From 9e50afdc0a3d47390bd4ffdacece6c9664fb88ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 01:59:18 +0800 Subject: [PATCH 10/13] ci(models): publish versioned immutable assets --- .github/workflows/model-validation.yml | 67 +++++++++++++++++++------- scripts/verify-release.mjs | 42 ++++++++++++---- scripts/verify-release.test.mjs | 24 ++++++++- 3 files changed, 104 insertions(+), 29 deletions(-) diff --git a/.github/workflows/model-validation.yml b/.github/workflows/model-validation.yml index 716be9f..b23d88c 100644 --- a/.github/workflows/model-validation.yml +++ b/.github/workflows/model-validation.yml @@ -3,8 +3,18 @@ name: Model validation and assets on: workflow_dispatch: inputs: + model_version: + description: Model version to validate and publish + required: true + type: string + default: "1.0.1" + release_tag: + description: Immutable GitHub Release tag + required: true + type: string + default: "v1.0.1-models" upload_assets: - description: Upload verified model assets to v1.0.0-models + description: Create the immutable model release required: true type: boolean default: false @@ -27,17 +37,28 @@ jobs: node-version-file: .nvmrc cache: pnpm - run: pnpm install --frozen-lockfile + - name: Validate immutable release identity + shell: bash + run: | + set -euo pipefail + if [[ "${RELEASE_TAG}" != "v${MODEL_VERSION}-models" ]]; then + echo "Release tag must be v${MODEL_VERSION}-models." >&2 + exit 1 + fi + env: + MODEL_VERSION: ${{ inputs.model_version }} + RELEASE_TAG: ${{ inputs.release_tag }} - name: Verify model hashes and validation reports - run: node scripts/verify-release.mjs --models + run: node scripts/verify-release.mjs --models "${{ inputs.model_version }}" - uses: actions/upload-artifact@v7 with: - name: pp-doclayoutv3-model-validation + name: pp-doclayoutv3-${{ inputs.model_version }}-validation if-no-files-found: error path: | - models/pp-doclayoutv3/1.0.0/manifest.json - tools/model-pipeline/reports/browser-evidence.json - tools/model-pipeline/reports/fp32-validation.json - tools/model-pipeline/reports/variant-validation.json + models/pp-doclayoutv3/${{ inputs.model_version }}/manifest.json + tools/model-pipeline/reports/${{ inputs.model_version }}/browser-evidence.json + tools/model-pipeline/reports/${{ inputs.model_version }}/fp32-validation.json + tools/model-pipeline/reports/${{ inputs.model_version }}/variant-validation.json upload-model-assets: if: inputs.upload_assets @@ -57,16 +78,26 @@ jobs: node-version-file: .nvmrc cache: pnpm - run: pnpm install --frozen-lockfile - - run: node scripts/verify-release.mjs --models - - name: Upload assets to the existing model release - run: >- - gh release upload v1.0.0-models - models/pp-doclayoutv3/1.0.0/manifest.json - models/pp-doclayoutv3/1.0.0/model-fp16.onnx - models/pp-doclayoutv3/1.0.0/model-fp32.onnx - tools/model-pipeline/reports/browser-evidence.json - tools/model-pipeline/reports/fp32-validation.json - tools/model-pipeline/reports/variant-validation.json - --clobber + - run: node scripts/verify-release.mjs --models "${{ inputs.model_version }}" + - name: Create immutable model release + shell: bash + run: | + set -euo pipefail + if gh release view "${RELEASE_TAG}" >/dev/null 2>&1; then + echo "Release ${RELEASE_TAG} already exists; immutable assets will not be overwritten." >&2 + exit 1 + fi + gh release create "${RELEASE_TAG}" \ + "models/pp-doclayoutv3/${MODEL_VERSION}/manifest.json" \ + "models/pp-doclayoutv3/${MODEL_VERSION}/model-fp16.onnx" \ + "models/pp-doclayoutv3/${MODEL_VERSION}/model-fp32.onnx" \ + "tools/model-pipeline/reports/${MODEL_VERSION}/browser-evidence.json" \ + "tools/model-pipeline/reports/${MODEL_VERSION}/fp32-validation.json" \ + "tools/model-pipeline/reports/${MODEL_VERSION}/variant-validation.json" \ + --target main \ + --title "PP-DocLayoutV3 model ${MODEL_VERSION}" \ + --notes "Immutable PP-DocLayoutV3 ${MODEL_VERSION} browser model assets." env: GH_TOKEN: ${{ github.token }} + MODEL_VERSION: ${{ inputs.model_version }} + RELEASE_TAG: ${{ inputs.release_tag }} diff --git a/scripts/verify-release.mjs b/scripts/verify-release.mjs index e383288..fc06d7c 100644 --- a/scripts/verify-release.mjs +++ b/scripts/verify-release.mjs @@ -204,12 +204,18 @@ async function sha256(path) { return hash.digest("hex"); } -async function verifyModels(manifest) { +async function verifyModels(modelVersion) { + const modelRoot = `models/pp-doclayoutv3/${modelVersion}`; + const reportRoot = + modelVersion === "1.0.0" + ? "tools/model-pipeline/reports" + : `tools/model-pipeline/reports/${modelVersion}`; + const manifest = JSON.parse(read(`${modelRoot}/manifest.json`)); const manifestVariants = Object.fromEntries( manifest.variants.map((variant) => [variant.id, variant]) ); for (const variant of manifest.variants) { - const path = join(repositoryRoot, "models/pp-doclayoutv3/1.0.0", variant.filename); + const path = join(repositoryRoot, modelRoot, variant.filename); const size = statSync(path).size; if (size < 1024) fail(`${variant.filename} is probably an unresolved Git LFS pointer`); if (size !== variant.bytes) @@ -218,9 +224,9 @@ async function verifyModels(manifest) { if (digest !== variant.sha256) fail(`${variant.filename} SHA-256 does not match the manifest`); } - const fp32 = JSON.parse(read("tools/model-pipeline/reports/fp32-validation.json")); - const variants = JSON.parse(read("tools/model-pipeline/reports/variant-validation.json")); - const browser = JSON.parse(read("tools/model-pipeline/reports/browser-evidence.json")); + const fp32 = JSON.parse(read(`${reportRoot}/fp32-validation.json`)); + const variants = JSON.parse(read(`${reportRoot}/variant-validation.json`)); + const browser = JSON.parse(read(`${reportRoot}/browser-evidence.json`)); if (fp32.overallPass !== true) fail("FP32 validation report did not pass"); if (variants.variants?.fp16?.pass !== true) fail("FP16 validation report did not pass"); if (browser.fp16Webgpu?.status !== "passed") fail("FP16 hardware WebGPU evidence is missing"); @@ -232,6 +238,17 @@ async function verifyModels(manifest) { fail("variant validation report does not match the FP16 manifest SHA-256"); if (browser.fp16Webgpu?.modelSha256 !== manifestVariants.fp16?.sha256) fail("browser evidence does not match the FP16 manifest SHA-256"); + + if (modelVersion === "1.0.1") { + if (browser.fp32Wasm?.status !== "passed") fail("strict FP32 WASM evidence is missing"); + if (browser.fp32Webgpu?.status !== "passed") fail("strict FP32 WebGPU evidence is missing"); + if (browser.fp32Wasm?.fallbacks?.length !== 0) fail("FP32 WASM evidence contains fallback"); + if (browser.fp32Webgpu?.fallbacks?.length !== 0) fail("FP32 WebGPU evidence contains fallback"); + if (manifestVariants.fp32?.backendCompatibility.join(",") !== "wasm,webgpu") { + fail("FP32 manifest compatibility must be wasm,webgpu"); + } + } + return manifest; } function runPnpm(args) { @@ -255,13 +272,18 @@ function verifyTag(tag) { const [mode, value, ...extraArguments] = process.argv.slice(2); if (extraArguments.length > 0 || ![undefined, "--static", "--models", "--release"].includes(mode)) { - fail("usage: verify-release.mjs [--static | --models | --release vX.Y.Z]"); + fail("usage: verify-release.mjs [--static | --models X.Y.Z | --release vX.Y.Z]"); } if (mode === "--release" && value === undefined) fail("--release requires a tag"); -if (mode !== "--release" && value !== undefined) fail(`${mode} does not accept a value`); +if (mode === "--models" && value === undefined) fail("--models requires a version"); +if (mode === "--models" && !/^\d+\.\d+\.\d+$/.test(value)) + fail(`model version ${value} is not semver`); +if (!["--models", "--release"].includes(mode) && value !== undefined) + fail(`${mode} does not accept a value`); -const manifest = verifyStaticContract(); -if (mode !== "--static") await verifyModels(manifest); +const staticManifest = verifyStaticContract(); +const modelVersion = mode === "--models" ? value : "1.0.0"; +const manifest = mode === "--static" ? staticManifest : await verifyModels(modelVersion); if (mode === "--release") verifyTag(value); if (mode === undefined || mode === "--release") { runPnpm(["run", "verify"]); @@ -269,5 +291,5 @@ if (mode === undefined || mode === "--release") { } console.log( - `Release contract verified: ${requiredWorkflows.length} workflows, ${manifest.variants.length} model variants.` + `Release contract verified: ${requiredWorkflows.length} workflows, ${manifest.variants.length} model variants, model ${modelVersion}.` ); diff --git a/scripts/verify-release.test.mjs b/scripts/verify-release.test.mjs index b238713..bad716e 100644 --- a/scripts/verify-release.test.mjs +++ b/scripts/verify-release.test.mjs @@ -63,6 +63,28 @@ describe("release workflow contract", () => { assert.match(output, /4 workflows, 2 model variants/); }); + test("verifies model 1.0.1 without changing the SDK 1.0.4 default", () => { + const output = execFileSync( + process.execPath, + [resolve(repositoryRoot, "scripts/verify-release.mjs"), "--models", "1.0.1"], + { cwd: repositoryRoot, encoding: "utf8" } + ); + + assert.match(output, /model 1\.0\.1/); + }); + + test("creates the immutable model release without clobber", () => { + const workflow = readFileSync( + resolve(repositoryRoot, ".github/workflows/model-validation.yml"), + "utf8" + ); + + assert.match(workflow, /model_version:[\s\S]*default:\s*["']?1\.0\.1/); + assert.match(workflow, /release_tag:[\s\S]*default:\s*["']?v1\.0\.1-models/); + assert.match(workflow, /gh release create/); + assert.doesNotMatch(workflow, /--clobber/); + }); + test("requires package repository metadata to match GitHub provenance", () => { const packageMetadata = JSON.parse( readFileSync(resolve(repositoryRoot, "packages/sdk/package.json"), "utf8") @@ -98,7 +120,7 @@ describe("release workflow contract", () => { assert.match(pages, /^\s*- uses: actions\/upload-pages-artifact@v5\r?$/m); const setupNodeWorkflows = [ - ["benchmark.yml", 3], + ["benchmark.yml", 4], ["ci.yml", 3], ["model-validation.yml", 2], ["pages.yml", 1], From cb7a71c3516823f3f0952a4415d79e968d610a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 02:12:42 +0800 Subject: [PATCH 11/13] docs(models): record FP32 sanitation evidence --- docs/en/conversion.md | 16 +++++++++++++++- docs/zh-CN/conversion.md | 16 +++++++++++++++- models/README.md | 29 +++++++++++++++++++++++++++++ scripts/check-doc-parity.test.mjs | 20 ++++++++++++++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) diff --git a/docs/en/conversion.md b/docs/en/conversion.md index f3bbb68..15d6273 100644 --- a/docs/en/conversion.md +++ b/docs/en/conversion.md @@ -16,4 +16,18 @@ python -m ppdoclayout.build_manifest Use each module's `--help` for exact local model paths and arguments. Validation reports bind source and ONNX SHA-256 values, opset, tensor names/shapes, detection matching, and browser runtime evidence. Only accepted variants may enter the manifest. -FP32 acceptance covers seven licensed fixtures: at threshold 0.5, detection counts, label sequences, and reading order all match; browser WASM execution also passed. FP16 passed the same fixture set and real WebGPU execution. No physical WebGPU FP32 validation has been recorded, so the default manifest limits FP32 to WASM. The INT8 candidate failed detection matching and is not distributed. +## Model 1.0.1 WebGPU FP32 provenance + +The upstream `torch_dtype` is float32; this is not FP64 inference, and FP64 inference is not supported. The sanitizer converts only the four positional DOUBLE initializers `sin`, `cos`, `sin_1`, and `cos_1`, each with shape `[625, 64]`, to FLOAT before the existing FLOAT Cast. Learned initializers and the graph input/output contract remain unchanged. The source FP32 SHA-256 is `fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b`; the sanitized artifact SHA-256 is `476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269`. + +Model `1.0.1` is prepared for publication under the immutable `v1.0.1-models` release. Its FP32 artifact passed CPU parity on seven licensed fixtures, strict browser WASM, and a physical WebGPU adapter with no fallback. The FP16 artifact is byte-identical to the accepted `1.0.0` FP16 artifact. Historical `v1.0.0-models` assets remain immutable. + +Reproduce the sanitized graph from the repository root: + +```powershell +.\.venv-model\Scripts\python.exe tools/model-pipeline/ppdoclayout/sanitize_fp32.py ` + --source models/pp-doclayoutv3/1.0.0/model-fp32.onnx ` + --output models/pp-doclayoutv3/1.0.1/model-fp32.onnx +``` + +Then run the versioned FP32 parity, browser evidence, variant validation, and manifest commands with `tools/model-pipeline/reports/1.0.1/`; the manifest generator rejects any hash, fixture, provider, precision, or fallback mismatch. diff --git a/docs/zh-CN/conversion.md b/docs/zh-CN/conversion.md index c232f9d..7fd906b 100644 --- a/docs/zh-CN/conversion.md +++ b/docs/zh-CN/conversion.md @@ -16,4 +16,18 @@ python -m ppdoclayout.build_manifest 具体参数与本地模型路径以各模块的 `--help` 为准。验证报告必须绑定源文件 SHA-256、ONNX SHA-256、opset、输入输出名称/形状、检测匹配和浏览器运行证据。只有通过验收的变体才能写入清单。 -FP32 验收覆盖 7 张授权测试图，在阈值 0.5 下检测数量、标签序列和阅读顺序全部一致，并通过浏览器 WASM 运行。FP16 验收覆盖同一集合并通过真实 WebGPU 运行。FP32 尚未记录物理 WebGPU 验证，因此默认清单仅把 FP32 用于 WASM。INT8 候选未达到检测匹配阈值，因此没有发布。 +## 模型 1.0.1 的 WebGPU FP32 来源与复现 + +上游 `torch_dtype` 是 float32，不是 FP64 推理；FP64 推理不支持。sanitizer 只把位置编码路径中的四个 DOUBLE initializer `sin`、`cos`、`sin_1`、`cos_1` 转为 FLOAT，每个形状都是 `[625, 64]`，并保留原有 FLOAT Cast。学习参数以及图输入输出契约不变。源 FP32 SHA-256 为 `fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b`，sanitized 产物 SHA-256 为 `476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269`。 + +模型 `1.0.1` 准备发布到不可覆盖的 `v1.0.1-models` release。FP32 在 7 张授权图片上通过 CPU parity、严格浏览器 WASM 和物理 WebGPU，且没有 fallback。FP16 与已验收的 `1.0.0` FP16 文件字节完全一致。历史 `v1.0.0-models` 资产保持不变。 + +在仓库根目录复现 sanitized 图： + +```powershell +.\.venv-model\Scripts\python.exe tools/model-pipeline/ppdoclayout/sanitize_fp32.py ` + --source models/pp-doclayoutv3/1.0.0/model-fp32.onnx ` + --output models/pp-doclayoutv3/1.0.1/model-fp32.onnx +``` + +随后使用 `tools/model-pipeline/reports/1.0.1/` 下的 FP32 parity、浏览器证据、变体验证和 manifest 命令；manifest 生成器会拒绝任何哈希、样本、provider、精度或 fallback 不一致。 diff --git a/models/README.md b/models/README.md index bda6e14..05f7241 100644 --- a/models/README.md +++ b/models/README.md @@ -19,6 +19,29 @@ INT8 候选没有通过精度验收，浏览器 WASM 验证因此未执行；它 默认清单和模型 URL 绝不使用可变的 `latest` 地址。文件下载后应先按清单中的字节数和 SHA-256 校验，再创建 ONNX Runtime session。 +### 1.0.1 WebGPU FP32 变体 + +`1.0.1` 是独立的、不可覆盖的模型资产版本，发布前缀为 +`https://github.com/chenmohan123/web-sdk-PP-DocLayoutV3/releases/download/v1.0.1-models/`。 +它不会改写 `v1.0.0-models` 或 SDK `1.0.4`。 + +上游 `torch_dtype` 是 float32，这不是 FP64 推理；FP64 inference 不支持。为使 FP32 图在 WebGPU 上可执行，只转换位置编码路径中的 `sin`、`cos`、`sin_1`、`cos_1` 四个 DOUBLE initializer，每个形状都是 `[625, 64]`，在原有 FLOAT Cast 之前转换为 FLOAT。学习参数、图输入输出名称/形状和 opset 18 保持不变。源 FP32 SHA-256 为 `fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b`，sanitized FP32 SHA-256 为 `476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269`。 + +| 文件 | 精度 | 兼容后端 | 字节数 | SHA-256 | 验证 | +| ----------------- | ---- | ------------ | --------: | ------------------------------------------------------------------ | -------------------------------------------- | +| `model-fp16.onnx` | FP16 | WebGPU | 74279796 | `463ba56faa555baf84271b4002b33b0c5fcc50776fe4f39344235eccb72073f2` | 与已验收的 1.0.0 FP16 字节完全一致 | +| `model-fp32.onnx` | FP32 | WASM、WebGPU | 142574928 | `476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269` | 7 张授权图片、严格 WASM 和物理 WebGPU 均通过 | + +复现 sanitizer（仓库根目录）： + +```powershell +.\.venv-model\Scripts\python.exe tools/model-pipeline/ppdoclayout/sanitize_fp32.py ` + --source models/pp-doclayoutv3/1.0.0/model-fp32.onnx ` + --output models/pp-doclayoutv3/1.0.1/model-fp32.onnx +``` + +生成器会拒绝额外 DOUBLE、错误形状或错误拓扑；只有通过七样本 CPU parity、严格浏览器 WASM 和物理 WebGPU 证据的版本才能进入 `1.0.1/manifest.json`。 + ### 重现清单 清单由模型契约、FP32 验证报告、变体验证报告和实际 ONNX 文件生成，不能手工编辑。在仓库根目录执行： @@ -40,4 +63,10 @@ This directory contains versioned PP-DocLayoutV3 ONNX artifacts and a generated The ONNX files are available through Git LFS and are intended for publication at the immutable `v1.0.0-models` GitHub Release URLs above. Regenerate `manifest.json` with the command above; the generator binds report claims to the actual ONNX size, SHA-256, graph contract, and opset. Planned SDK support for custom fine-tuned manifests will require the same explicit runtime contract. +### Model 1.0.1 WebGPU FP32 provenance + +Model `1.0.1` is prepared for publication under the immutable `v1.0.1-models` release and does not alter historical `v1.0.0-models` assets or the SDK `1.0.4` default. The upstream `torch_dtype` is float32; this is not FP64 inference, and FP64 inference is not supported. The sanitizer converts only `sin`, `cos`, `sin_1`, and `cos_1`, each DOUBLE with shape `[625, 64]`, to FLOAT before the existing FLOAT Cast. Learned initializers and the graph input/output contract remain unchanged. The source FP32 SHA-256 is `fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b`; the sanitized artifact SHA-256 is `476da6d3892bc6211ec90f53df1f68722626b3cf67af77d1c75bd0bd2ee8d269`. + +The FP16 artifact is byte-identical to the accepted `1.0.0` FP16 artifact. The new FP32 artifact passed seven licensed fixtures in strict browser WASM and on a physical WebGPU adapter. Reproduce the sanitizer with the command shown above; the generated manifest is gated by the CPU parity and browser reports. + The upstream PaddlePaddle model is identified as Apache-2.0 by its official metadata. See `THIRD_PARTY_NOTICES.md` for attribution and citation details. diff --git a/scripts/check-doc-parity.test.mjs b/scripts/check-doc-parity.test.mjs index 395936b..0322fb6 100644 --- a/scripts/check-doc-parity.test.mjs +++ b/scripts/check-doc-parity.test.mjs @@ -42,4 +42,24 @@ describe("documentation contract", () => { assert.match(englishModels, /FP32\s+\| 143,216,104 bytes \| WASM/); assert.match(chineseModels, /FP32 \| 143,216,104 字节 \| WASM/); }); + + it("records model 1.0.1 sanitation provenance in both languages", () => { + const modelReadme = readFileSync(new URL("models/README.md", repositoryRoot), "utf8"); + const englishConversion = readFileSync( + new URL("docs/en/conversion.md", repositoryRoot), + "utf8" + ); + const chineseConversion = readFileSync( + new URL("docs/zh-CN/conversion.md", repositoryRoot), + "utf8" + ); + + for (const document of [modelReadme, englishConversion, chineseConversion]) { + assert.match(document, /1\.0\.1/); + assert.match(document, /v1\.0\.1-models/); + assert.match(document, /sin.*cos.*sin_1.*cos_1/s); + assert.match(document, /625.*64/s); + assert.match(document, /FP64.*不支持|FP64.*not supported/is); + } + }); }); From 000707fe10d33a5e6797cd144e18ff68dc73f50d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 02:43:03 +0800 Subject: [PATCH 12/13] style(models): format validation reports --- .../reports/1.0.1/fp32-validation.json | 175 +++--------------- .../reports/1.0.1/variant-validation.json | 40 +--- 2 files changed, 35 insertions(+), 180 deletions(-) diff --git a/tools/model-pipeline/reports/1.0.1/fp32-validation.json b/tools/model-pipeline/reports/1.0.1/fp32-validation.json index e40fc1e..c06288b 100644 --- a/tools/model-pipeline/reports/1.0.1/fp32-validation.json +++ b/tools/model-pipeline/reports/1.0.1/fp32-validation.json @@ -32,11 +32,7 @@ "candidateSha256": "e4133d98a74db6ccfe23940501bd0244be713af040a1465124529e0f8281142b", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 25 - ] + "shape": [1, 300, 25] }, "order_logits": { "acceptedSha256": "ee1887bf86d95b06a7b2c55fa4c94e02776d7c16130cf6ce4270786cd33c12e5", @@ -44,11 +40,7 @@ "candidateSha256": "ee1887bf86d95b06a7b2c55fa4c94e02776d7c16130cf6ce4270786cd33c12e5", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 300 - ] + "shape": [1, 300, 300] }, "out_masks": { "acceptedSha256": "494abcc172eccfc3427b7dfca80d906e1a4d019a7af399becc080f8503ed80ef", @@ -56,12 +48,7 @@ "candidateSha256": "494abcc172eccfc3427b7dfca80d906e1a4d019a7af399becc080f8503ed80ef", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 200, - 200 - ] + "shape": [1, 300, 200, 200] }, "pred_boxes": { "acceptedSha256": "4f72bfd2294edaf7f1b4141d9f6078b098b92cdf7be72eac947e356051d70d7b", @@ -69,11 +56,7 @@ "candidateSha256": "4f72bfd2294edaf7f1b4141d9f6078b098b92cdf7be72eac947e356051d70d7b", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 4 - ] + "shape": [1, 300, 4] } } }, @@ -106,11 +89,7 @@ "candidateSha256": "5fce7480a5f6ecbe926bbed9b93aade165fedf9de47d39caabe41e2e623e5e55", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 25 - ] + "shape": [1, 300, 25] }, "order_logits": { "acceptedSha256": "fe70871e6e847638b70942ddd123ba8a219655e84fa4fbc722f97a963b27ef33", @@ -118,11 +97,7 @@ "candidateSha256": "fe70871e6e847638b70942ddd123ba8a219655e84fa4fbc722f97a963b27ef33", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 300 - ] + "shape": [1, 300, 300] }, "out_masks": { "acceptedSha256": "c25be644690feb8ae9bb1f218edeb6b569b341134f9667763e5305e3513d9563", @@ -130,12 +105,7 @@ "candidateSha256": "c25be644690feb8ae9bb1f218edeb6b569b341134f9667763e5305e3513d9563", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 200, - 200 - ] + "shape": [1, 300, 200, 200] }, "pred_boxes": { "acceptedSha256": "66a9d7e1de1591d1666ff4b55394f4529d99681308bb9583bdb5dc4f32c27e4e", @@ -143,11 +113,7 @@ "candidateSha256": "66a9d7e1de1591d1666ff4b55394f4529d99681308bb9583bdb5dc4f32c27e4e", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 4 - ] + "shape": [1, 300, 4] } } }, @@ -180,11 +146,7 @@ "candidateSha256": "d69d92e4d0066136c71a99a7daa9968629c1668b21fbb357dd6133cffd212104", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 25 - ] + "shape": [1, 300, 25] }, "order_logits": { "acceptedSha256": "b05f2433ebf57aab16b1e58769fd13887a37b26b31e3309b34366da151bc8c95", @@ -192,11 +154,7 @@ "candidateSha256": "b05f2433ebf57aab16b1e58769fd13887a37b26b31e3309b34366da151bc8c95", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 300 - ] + "shape": [1, 300, 300] }, "out_masks": { "acceptedSha256": "8f9a13d537b60255e4f654f0415ef3642874f3deb0ad9d03f448e77a7be2f7eb", @@ -204,12 +162,7 @@ "candidateSha256": "8f9a13d537b60255e4f654f0415ef3642874f3deb0ad9d03f448e77a7be2f7eb", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 200, - 200 - ] + "shape": [1, 300, 200, 200] }, "pred_boxes": { "acceptedSha256": "9795d8be3223d62eac914b4588b418177065102e4888c4cbd4e16b2b88b0fca3", @@ -217,11 +170,7 @@ "candidateSha256": "9795d8be3223d62eac914b4588b418177065102e4888c4cbd4e16b2b88b0fca3", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 4 - ] + "shape": [1, 300, 4] } } }, @@ -254,11 +203,7 @@ "candidateSha256": "ff761ba437202de2aabb28bd557bf30f07365e079d8e83e7157a72f890e2779e", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 25 - ] + "shape": [1, 300, 25] }, "order_logits": { "acceptedSha256": "9fa299dfa5f12f0425286cfaad7147887c0c184661c3971eb5731fae5c0e2274", @@ -266,11 +211,7 @@ "candidateSha256": "9fa299dfa5f12f0425286cfaad7147887c0c184661c3971eb5731fae5c0e2274", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 300 - ] + "shape": [1, 300, 300] }, "out_masks": { "acceptedSha256": "18c48f9938beb0c3e527768630148d793f667e195bef16fe6d5f9f16018b5ad6", @@ -278,12 +219,7 @@ "candidateSha256": "18c48f9938beb0c3e527768630148d793f667e195bef16fe6d5f9f16018b5ad6", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 200, - 200 - ] + "shape": [1, 300, 200, 200] }, "pred_boxes": { "acceptedSha256": "3592393014e402b2bcaf7f89e07a9020e6d69913705c0095401c62c72a276a2e", @@ -291,11 +227,7 @@ "candidateSha256": "3592393014e402b2bcaf7f89e07a9020e6d69913705c0095401c62c72a276a2e", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 4 - ] + "shape": [1, 300, 4] } } }, @@ -328,11 +260,7 @@ "candidateSha256": "dae5933ceecea367ac8b4c5778e471bd8f372bff91593ec054b8c5e5daba557b", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 25 - ] + "shape": [1, 300, 25] }, "order_logits": { "acceptedSha256": "0c600a04387f1f0ac88f279ecdde51d5315fdd105409ff7247b99d3e429dd624", @@ -340,11 +268,7 @@ "candidateSha256": "0c600a04387f1f0ac88f279ecdde51d5315fdd105409ff7247b99d3e429dd624", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 300 - ] + "shape": [1, 300, 300] }, "out_masks": { "acceptedSha256": "4f166c99f61dfc69e53a0637543b4a5d7faf75f971203c041af059559fa9a71c", @@ -352,12 +276,7 @@ "candidateSha256": "4f166c99f61dfc69e53a0637543b4a5d7faf75f971203c041af059559fa9a71c", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 200, - 200 - ] + "shape": [1, 300, 200, 200] }, "pred_boxes": { "acceptedSha256": "32c2ee356e4f4780542ee2ec60373d38429a01f60090f52d6f35cbd5841456f2", @@ -365,11 +284,7 @@ "candidateSha256": "32c2ee356e4f4780542ee2ec60373d38429a01f60090f52d6f35cbd5841456f2", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 4 - ] + "shape": [1, 300, 4] } } }, @@ -402,11 +317,7 @@ "candidateSha256": "a22dc8e94aa4e809480c29aedbeef057e2e7d0a3b694cc63231235a2ae0f17cb", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 25 - ] + "shape": [1, 300, 25] }, "order_logits": { "acceptedSha256": "4e4e6af629a74b8d84c3caa465fd5d84590c74c7e1033b2d3317a6c7d9952933", @@ -414,11 +325,7 @@ "candidateSha256": "4e4e6af629a74b8d84c3caa465fd5d84590c74c7e1033b2d3317a6c7d9952933", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 300 - ] + "shape": [1, 300, 300] }, "out_masks": { "acceptedSha256": "73e2bd39e4ab3363d48106cb7aab38869c20f00dc51e582e24f493e9ee651aae", @@ -426,12 +333,7 @@ "candidateSha256": "73e2bd39e4ab3363d48106cb7aab38869c20f00dc51e582e24f493e9ee651aae", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 200, - 200 - ] + "shape": [1, 300, 200, 200] }, "pred_boxes": { "acceptedSha256": "e2a4d30dee1acc953e7637201c11501bb8dea51ed857bc88d93c627a502c9513", @@ -439,11 +341,7 @@ "candidateSha256": "e2a4d30dee1acc953e7637201c11501bb8dea51ed857bc88d93c627a502c9513", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 4 - ] + "shape": [1, 300, 4] } } }, @@ -476,11 +374,7 @@ "candidateSha256": "9292025c692bf595631a317965035be42e574caf7c27f9f5a47b9b140b21ceca", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 25 - ] + "shape": [1, 300, 25] }, "order_logits": { "acceptedSha256": "83256b094d74df5e84153923390f1ee9c0b4a53dec38d961ce4578db537b1a6d", @@ -488,11 +382,7 @@ "candidateSha256": "83256b094d74df5e84153923390f1ee9c0b4a53dec38d961ce4578db537b1a6d", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 300 - ] + "shape": [1, 300, 300] }, "out_masks": { "acceptedSha256": "5bfcfee296f00fb650ae37161d26518a1fbf2c0f1d6bed024b02a663a35e79c8", @@ -500,12 +390,7 @@ "candidateSha256": "5bfcfee296f00fb650ae37161d26518a1fbf2c0f1d6bed024b02a663a35e79c8", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 200, - 200 - ] + "shape": [1, 300, 200, 200] }, "pred_boxes": { "acceptedSha256": "8dffcef7a09ae707ec68239245bbcc8708532bac7e5b3bcc5fd269b11be907e5", @@ -513,11 +398,7 @@ "candidateSha256": "8dffcef7a09ae707ec68239245bbcc8708532bac7e5b3bcc5fd269b11be907e5", "dtype": "float32", "maxAbsoluteDelta": 0.0, - "shape": [ - 1, - 300, - 4 - ] + "shape": [1, 300, 4] } } }, diff --git a/tools/model-pipeline/reports/1.0.1/variant-validation.json b/tools/model-pipeline/reports/1.0.1/variant-validation.json index be038a4..1a19675 100644 --- a/tools/model-pipeline/reports/1.0.1/variant-validation.json +++ b/tools/model-pipeline/reports/1.0.1/variant-validation.json @@ -54,9 +54,7 @@ } } ], - "blockedOps": [ - "Resize" - ], + "blockedOps": ["Resize"], "browser": { "webgpu": { "adapter": { @@ -112,12 +110,7 @@ }, "executionProvider": "webgpu", "input": { - "dimensions": [ - 1, - 3, - 800, - 800 - ], + "dimensions": [1, 3, 800, 800], "name": "pixel_values", "type": "float32" }, @@ -127,42 +120,25 @@ "outputs": { "logits": { "allFinite": true, - "dimensions": [ - 1, - 300, - 25 - ], + "dimensions": [1, 300, 25], "sha256": "2d1c470358fd8162ac3e6025f99658c4727a62e5a80eea1b6199989bebf1337f", "type": "float32" }, "order_logits": { "allFinite": true, - "dimensions": [ - 1, - 300, - 300 - ], + "dimensions": [1, 300, 300], "sha256": "f039615990d24938fb35c9dfe142be13846caa48002c7c37beed61f37970c4d2", "type": "float32" }, "out_masks": { "allFinite": true, - "dimensions": [ - 1, - 300, - 200, - 200 - ], + "dimensions": [1, 300, 200, 200], "sha256": "8e27be8e4b2294a27028b15e7c03679364f1bf2f158e41c5dbe7c67a9be47ad8", "type": "float32" }, "pred_boxes": { "allFinite": true, - "dimensions": [ - 1, - 300, - 4 - ], + "dimensions": [1, 300, 4], "sha256": "fd8ec5511d6b5f840ec86967a8912aa483873ce43d28e6119ff3692cf79ae9b5", "type": "float32" } @@ -260,9 +236,7 @@ "wasm": { "reason": "CPU precision acceptance failed before browser validation", "status": "skipped", - "validationErrors": [ - "browser wasm validation did not pass" - ] + "validationErrors": ["browser wasm validation did not pass"] } }, "bytes": 45313310, From c85b6be0ce9557bd73ebdada414c8be1c5302c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=BB=98=E6=B6=B5?= <21739308@qq.com> Date: Sat, 15 Aug 2026 03:21:12 +0800 Subject: [PATCH 13/13] fix(models): enforce fp32 evidence release gates --- .github/workflows/model-validation.yml | 36 +- benchmarks/1.0.1/wasm-fp32.json | 193 ++++++--- benchmarks/1.0.1/webgpu-fp32.json | 193 ++++++--- scripts/benchmark-contract.test.mjs | 28 +- scripts/verify-release.mjs | 152 ++++++- scripts/verify-release.test.mjs | 88 +++- tests/browser/benchmark.spec.ts | 98 ++++- .../reports/1.0.1/browser-evidence.json | 390 +++++++++++++----- 8 files changed, 944 insertions(+), 234 deletions(-) diff --git a/.github/workflows/model-validation.yml b/.github/workflows/model-validation.yml index b23d88c..2e49921 100644 --- a/.github/workflows/model-validation.yml +++ b/.github/workflows/model-validation.yml @@ -29,6 +29,7 @@ jobs: - uses: actions/checkout@v7 with: lfs: true + ref: ${{ github.sha }} - uses: pnpm/action-setup@v6 with: version: 11.16.0 @@ -45,9 +46,18 @@ jobs: echo "Release tag must be v${MODEL_VERSION}-models." >&2 exit 1 fi + if [[ "${UPLOAD_ASSETS}" == "true" && "${GITHUB_REF}" != "refs/heads/main" ]]; then + echo "Model assets may only be published from refs/heads/main." >&2 + exit 1 + fi + if [[ "$(git rev-parse HEAD)" != "${GITHUB_SHA}" ]]; then + echo "Checked out commit does not match GITHUB_SHA." >&2 + exit 1 + fi env: MODEL_VERSION: ${{ inputs.model_version }} RELEASE_TAG: ${{ inputs.release_tag }} + UPLOAD_ASSETS: ${{ inputs.upload_assets }} - name: Verify model hashes and validation reports run: node scripts/verify-release.mjs --models "${{ inputs.model_version }}" - uses: actions/upload-artifact@v7 @@ -70,6 +80,7 @@ jobs: - uses: actions/checkout@v7 with: lfs: true + ref: ${{ github.sha }} - uses: pnpm/action-setup@v6 with: version: 11.16.0 @@ -83,6 +94,29 @@ jobs: shell: bash run: | set -euo pipefail + if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then + echo "Model assets may only be published from refs/heads/main." >&2 + exit 1 + fi + if [[ "$(git rev-parse HEAD)" != "${GITHUB_SHA}" ]]; then + echo "Checked out commit does not match GITHUB_SHA." >&2 + exit 1 + fi + git fetch --no-tags origin main + if [[ "$(git rev-parse origin/main)" != "${GITHUB_SHA}" ]]; then + echo "GITHUB_SHA is not the current origin/main commit." >&2 + exit 1 + fi + if git ls-remote --exit-code --tags origin "refs/tags/${RELEASE_TAG}" >/dev/null 2>&1; then + echo "Git tag ${RELEASE_TAG} already exists; it will not be reused." >&2 + exit 1 + else + tag_status=$? + if [[ "${tag_status}" -ne 2 ]]; then + echo "Unable to verify whether Git tag ${RELEASE_TAG} exists." >&2 + exit "${tag_status}" + fi + fi if gh release view "${RELEASE_TAG}" >/dev/null 2>&1; then echo "Release ${RELEASE_TAG} already exists; immutable assets will not be overwritten." >&2 exit 1 @@ -94,7 +128,7 @@ jobs: "tools/model-pipeline/reports/${MODEL_VERSION}/browser-evidence.json" \ "tools/model-pipeline/reports/${MODEL_VERSION}/fp32-validation.json" \ "tools/model-pipeline/reports/${MODEL_VERSION}/variant-validation.json" \ - --target main \ + --target "${GITHUB_SHA}" \ --title "PP-DocLayoutV3 model ${MODEL_VERSION}" \ --notes "Immutable PP-DocLayoutV3 ${MODEL_VERSION} browser model assets." env: diff --git a/benchmarks/1.0.1/wasm-fp32.json b/benchmarks/1.0.1/wasm-fp32.json index f0760c8..18f604e 100644 --- a/benchmarks/1.0.1/wasm-fp32.json +++ b/benchmarks/1.0.1/wasm-fp32.json @@ -1,6 +1,7 @@ { "schemaVersion": 1, "status": "passed", + "acceptedModelSha256": "fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b", "executionProvider": "wasm", "precision": "fp32", "fallbacks": [], @@ -17,122 +18,206 @@ "operatingSystem": "win32 10.0.26200", "fixtures": [ { + "acceptedOutputSha256": "98601e9b45ffca68a280c95604d67f40bca747aefa7f07ec502369969dfd4025", "detectionCount": 12, "expectedDetectionCount": 12, "filename": "curved-document.jpg", + "fixtureSha256": "fce39d864ff7b0612f7073415c2a7c656f7790a26f96d539831f1bb1a994a069", "labelSequenceEqual": true, "outputSha256": "98601e9b45ffca68a280c95604d67f40bca747aefa7f07ec502369969dfd4025", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 10.405000001192093, - "inferenceMs": 7859.410000003874, - "postprocessMs": 39.41499999910593, - "preprocessMs": 127.48499999940395, - "totalMs": 8045.539999999106 + "decodeMs": 10.409999996423721, + "inferenceMs": 7663.39999999851, + "postprocessMs": 39.59000000357628, + "preprocessMs": 130.2800000011921, + "totalMs": 7852.52499999851 } }, { + "acceptedOutputSha256": "8bba1fb0a794a86ec0b2f4d1b1425d28e000a49beab399e4c55bbb04b4cf8cf4", "detectionCount": 59, "expectedDetectionCount": 59, "filename": "doc-formula.png", + "fixtureSha256": "6b07d28527dc9e930804fa73df562f1a81599c6b8a1a8bbc2a80742fa9f26e80", "labelSequenceEqual": true, "outputSha256": "8bba1fb0a794a86ec0b2f4d1b1425d28e000a49beab399e4c55bbb04b4cf8cf4", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 10.439999997615814, - "inferenceMs": 7962.560000002384, - "postprocessMs": 35.59000000357628, - "preprocessMs": 115.08500000089407, - "totalMs": 8132.585000000894 + "decodeMs": 9.110000006854534, + "inferenceMs": 7679.560000002384, + "postprocessMs": 33.74000000208616, + "preprocessMs": 108.30499999970198, + "totalMs": 7839.179999999702 } }, { + "acceptedOutputSha256": "da3c2a041cc94c4b5616637b7acd9af116c0882befd434920ebe8aaf96be626d", "detectionCount": 44, "expectedDetectionCount": 44, "filename": "image-layout.jpg", + "fixtureSha256": "cfebd4e0716da8ef01ad29c6f5bf7ed0dcc7d3a07bd38e32219c3b10645798de", "labelSequenceEqual": true, "outputSha256": "da3c2a041cc94c4b5616637b7acd9af116c0882befd434920ebe8aaf96be626d", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 27.730000004172325, - "inferenceMs": 7777.990000002086, - "postprocessMs": 59.399999998509884, - "preprocessMs": 105.34000000357628, - "totalMs": 7979.435000002384 + "decodeMs": 25.179999999701977, + "inferenceMs": 7650.9000000059605, + "postprocessMs": 58.375, + "preprocessMs": 108.12999999523163, + "totalMs": 7851.259999997914 } }, { + "acceptedOutputSha256": "c6051575214356859bcb9f87be446f27f32dc405a22bdc99ef9f887866c5ddb7", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "layout-demo.jpg", + "fixtureSha256": "785b7d19f158dcb636342dd3378ed3a4cddb7333d2d71688f0baa5c25a88ad51", "labelSequenceEqual": true, "outputSha256": "c6051575214356859bcb9f87be446f27f32dc405a22bdc99ef9f887866c5ddb7", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 31.45500000566244, - "inferenceMs": 7889.280000001192, - "postprocessMs": 99.87999999523163, - "preprocessMs": 172.27000000327826, - "totalMs": 8202.32500000298 + "decodeMs": 30.03499999642372, + "inferenceMs": 7656.109999999404, + "postprocessMs": 96.92999999970198, + "preprocessMs": 170.16999999433756, + "totalMs": 7961.7099999934435 } }, { + "acceptedOutputSha256": "b036ef27908bd3b94406d9a3106cb5ef6ecca8030874f2fceacac7a0a6407a90", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "screen-photo.jpg", + "fixtureSha256": "f27a8ad40192f2bff4bcc3605beaddf246bc35b07355e688defde1a2de333aa1", "labelSequenceEqual": true, "outputSha256": "b036ef27908bd3b94406d9a3106cb5ef6ecca8030874f2fceacac7a0a6407a90", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 11.390000000596046, - "inferenceMs": 8091.3499999940395, - "postprocessMs": 44.645000003278255, - "preprocessMs": 122.03499999642372, - "totalMs": 8279.155000001192 + "decodeMs": 11.574999995529652, + "inferenceMs": 7703.579999998212, + "postprocessMs": 40.46999999880791, + "preprocessMs": 123.03999999910593, + "totalMs": 7887.454999998212 } }, { + "acceptedOutputSha256": "07f0c613f0d87b91597984e01e83acdca5c5bf440d3efcc19873153b9250fa82", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "skew-document.jpg", + "fixtureSha256": "4ae0d5bebbe152a9cca8add806e376b3eb3314c3a55d6b7ccba70d9c4de97a1e", "labelSequenceEqual": true, "outputSha256": "07f0c613f0d87b91597984e01e83acdca5c5bf440d3efcc19873153b9250fa82", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 11.479999996721745, - "inferenceMs": 7944.47500000149, - "postprocessMs": 52.269999995827675, - "preprocessMs": 125.89500000327826, - "totalMs": 8142.655000001192 + "decodeMs": 11.339999996125698, + "inferenceMs": 7624.465000003576, + "postprocessMs": 52.63500000536442, + "preprocessMs": 124.5899999961257, + "totalMs": 7821.585000000894 } }, { + "acceptedOutputSha256": "8ff528db91eb3893ec1fbf50d69ac23aac17bc2aefec79e70cf0c11f0602550a", "detectionCount": 1, "expectedDetectionCount": 1, "filename": "table.png", + "fixtureSha256": "6d50148ceccb2d5cecc50b084b5105e3167f2d55a8899b29e04c3ebe46e88fa8", "labelSequenceEqual": true, "outputSha256": "8ff528db91eb3893ec1fbf50d69ac23aac17bc2aefec79e70cf0c11f0602550a", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 3.0499999970197678, - "inferenceMs": 7915.270000003278, - "postprocessMs": 55.57499999552965, - "preprocessMs": 76.70500000566244, - "totalMs": 8060.77499999851 + "decodeMs": 3.030000001192093, + "inferenceMs": 7697.104999996722, + "postprocessMs": 55.03499999642372, + "preprocessMs": 74.86999999731779, + "totalMs": 7838.77499999851 }, - "parityMetrics": { + "referenceMetrics": { "iou": 0.9999961699392192, "maxScoreDelta": 0.000013727193068024945, "meanPolygonPointDistancePixels": 0 }, - "parityThresholds": { + "referenceThresholds": { "iou": 0.95, "maxScoreDelta": 0.02, "meanPolygonPointDistancePixels": 2 @@ -141,29 +226,29 @@ ], "timingsMs": { "coldLoad": { - "capabilitiesMs": 3.530000001192093, - "integrityMs": 366.4150000065565, - "manifestMs": 1.589999996125698, - "modelCacheMs": 0.8250000029802322, - "modelDownloadMs": 1204.2849999964237, - "modelMs": 1968.9499999955297, + "capabilitiesMs": 3.7400000020861626, + "integrityMs": 374.54500000178814, + "manifestMs": 1.2549999952316284, + "modelCacheMs": 0.6650000065565109, + "modelDownloadMs": 1105.3500000014901, + "modelMs": 1880.4549999982119, "modelSource": "network", - "sessionMs": 1593.4200000017881, - "totalMs": 3568.1999999955297 + "sessionMs": 1531.2849999964237, + "totalMs": 3417.314999997616 }, "warmLoad": { - "capabilitiesMs": 1.2349999994039536, - "integrityMs": 361.66499999910593, - "manifestMs": 0.0949999988079071, - "modelCacheMs": 79.13000000268221, + "capabilitiesMs": 0.5850000008940697, + "integrityMs": 352.85499999672174, + "manifestMs": 0.10999999940395355, + "modelCacheMs": 74.875, "modelDownloadMs": 0, - "modelMs": 440.8799999952316, + "modelMs": 427.80500000715256, "modelSource": "cache", - "sessionMs": 510.9950000047684, - "totalMs": 953.2649999931455 + "sessionMs": 488.1549999937415, + "totalMs": 916.7250000014901 } }, - "sdkCommit": "8c47754068ff4ec7b34451cb7d562e6e9a8b1c8a", + "sdkCommit": "000707fe10d33a5e6797cd144e18ff68dc73f50d", "capabilities": { "crossOriginIsolated": true, "diagnostics": [ @@ -182,6 +267,6 @@ "worker": false }, "cpu": "Intel(R) Core(TM) i5-10400F CPU @ 2.90GHz", - "generatedAt": "2026-08-14T17:26:16.905Z", + "generatedAt": "2026-08-14T19:09:41.586Z", "id": "wasm-fp32" } diff --git a/benchmarks/1.0.1/webgpu-fp32.json b/benchmarks/1.0.1/webgpu-fp32.json index 13e16a5..6377d0e 100644 --- a/benchmarks/1.0.1/webgpu-fp32.json +++ b/benchmarks/1.0.1/webgpu-fp32.json @@ -1,6 +1,7 @@ { "schemaVersion": 1, "status": "passed", + "acceptedModelSha256": "fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b", "executionProvider": "webgpu", "precision": "fp32", "fallbacks": [], @@ -42,122 +43,206 @@ "operatingSystem": "win32 10.0.26200", "fixtures": [ { + "acceptedOutputSha256": "98601e9b45ffca68a280c95604d67f40bca747aefa7f07ec502369969dfd4025", "detectionCount": 12, "expectedDetectionCount": 12, "filename": "curved-document.jpg", + "fixtureSha256": "fce39d864ff7b0612f7073415c2a7c656f7790a26f96d539831f1bb1a994a069", "labelSequenceEqual": true, "outputSha256": "49d017eb3a0946b85dbf0076cf3fc8af88ad12faa1c46b2e5063b54f0b082393", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.00030517578125, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0.0000016508409746984753 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 15.869999997317791, - "inferenceMs": 5850.69999999553, - "postprocessMs": 68.39000000059605, - "preprocessMs": 161.90999999642372, - "totalMs": 6113.865000002086 + "decodeMs": 10.104999996721745, + "inferenceMs": 5305.939999997616, + "postprocessMs": 70.9050000011921, + "preprocessMs": 134.60000000149012, + "totalMs": 5532.594999998808 } }, { + "acceptedOutputSha256": "8bba1fb0a794a86ec0b2f4d1b1425d28e000a49beab399e4c55bbb04b4cf8cf4", "detectionCount": 59, "expectedDetectionCount": 59, "filename": "doc-formula.png", + "fixtureSha256": "6b07d28527dc9e930804fa73df562f1a81599c6b8a1a8bbc2a80742fa9f26e80", "labelSequenceEqual": true, "outputSha256": "70e57bf726376f2e3aad8fd35a6d492d679b6b38087df95de2648aee09babc51", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.0003662109375, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0.000001028899655031168 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 8.570000000298023, - "inferenceMs": 315.6899999976158, - "postprocessMs": 50.16499999910593, - "preprocessMs": 119.58000000566244, - "totalMs": 504.45499999821186 + "decodeMs": 7.435000002384186, + "inferenceMs": 279.95000000298023, + "postprocessMs": 59.94500000029802, + "preprocessMs": 115.23999999463558, + "totalMs": 472.6200000047684 } }, { + "acceptedOutputSha256": "da3c2a041cc94c4b5616637b7acd9af116c0882befd434920ebe8aaf96be626d", "detectionCount": 44, "expectedDetectionCount": 44, "filename": "image-layout.jpg", + "fixtureSha256": "cfebd4e0716da8ef01ad29c6f5bf7ed0dcc7d3a07bd38e32219c3b10645798de", "labelSequenceEqual": true, "outputSha256": "16320eaccc62ae6d9509d498ade7996febe9fa72cc7543f716c91d8577b06b00", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.00048828125, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0.0000013091050572455742 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 24.344999998807907, - "inferenceMs": 278.85999999940395, - "postprocessMs": 72.42000000178814, - "preprocessMs": 121.79500000178814, - "totalMs": 508.01500000059605 + "decodeMs": 23.509999997913837, + "inferenceMs": 272.554999999702, + "postprocessMs": 87.42999999970198, + "preprocessMs": 105.24500000476837, + "totalMs": 498.9349999949336 } }, { + "acceptedOutputSha256": "c6051575214356859bcb9f87be446f27f32dc405a22bdc99ef9f887866c5ddb7", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "layout-demo.jpg", + "fixtureSha256": "785b7d19f158dcb636342dd3378ed3a4cddb7333d2d71688f0baa5c25a88ad51", "labelSequenceEqual": true, "outputSha256": "24d11a2ed203d8ee72ca4e14e803cab560ef57855976ead40efa65a07bd0fbb7", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.000640869140625, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 9.07715355213945e-7 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 30.639999993145466, - "inferenceMs": 290.7799999937415, - "postprocessMs": 94.3399999961257, - "preprocessMs": 177.5300000011921, - "totalMs": 606.9549999982119 + "decodeMs": 28.810000002384186, + "inferenceMs": 275.41499999910593, + "postprocessMs": 120.25999999791384, + "preprocessMs": 213.1600000038743, + "totalMs": 649.9800000041723 } }, { + "acceptedOutputSha256": "b036ef27908bd3b94406d9a3106cb5ef6ecca8030874f2fceacac7a0a6407a90", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "screen-photo.jpg", + "fixtureSha256": "f27a8ad40192f2bff4bcc3605beaddf246bc35b07355e688defde1a2de333aa1", "labelSequenceEqual": true, "outputSha256": "6f0fe242016734de274a9ac9d002ee166694e8360cb52cc3cb1d0b4000d1d160", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.0003662109375, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 6.820948376118352e-7 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 11.925000004470348, - "inferenceMs": 290.51500000059605, - "postprocessMs": 44.42000000178814, - "preprocessMs": 144, - "totalMs": 502.4949999973178 + "decodeMs": 11.094999998807907, + "inferenceMs": 263.5649999976158, + "postprocessMs": 48.474999994039536, + "preprocessMs": 125.89499999582767, + "totalMs": 459.28999999910593 } }, { + "acceptedOutputSha256": "07f0c613f0d87b91597984e01e83acdca5c5bf440d3efcc19873153b9250fa82", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "skew-document.jpg", + "fixtureSha256": "4ae0d5bebbe152a9cca8add806e376b3eb3314c3a55d6b7ccba70d9c4de97a1e", "labelSequenceEqual": true, "outputSha256": "840d7c61343deee9fa966b541e17a19671d5987aa27280ecec8ca2943c133c47", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.00042724609375, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0.000001394110166863527 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 11.689999997615814, - "inferenceMs": 230.11499999463558, - "postprocessMs": 57.30500000715256, - "preprocessMs": 138.96000000089407, - "totalMs": 451.71000000089407 + "decodeMs": 11.559999994933605, + "inferenceMs": 296.1850000023842, + "postprocessMs": 42.184999994933605, + "preprocessMs": 130.05499999970198, + "totalMs": 491.1299999952316 } }, { + "acceptedOutputSha256": "8ff528db91eb3893ec1fbf50d69ac23aac17bc2aefec79e70cf0c11f0602550a", "detectionCount": 1, "expectedDetectionCount": 1, "filename": "table.png", + "fixtureSha256": "6d50148ceccb2d5cecc50b084b5105e3167f2d55a8899b29e04c3ebe46e88fa8", "labelSequenceEqual": true, "outputSha256": "7978ca0ec7143f98fc24ef2f4613c72d785a82ca4a1b31bcf97fc1952ffb9e8c", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.0000616908073425293, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 1.3716245295114504e-7 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 3.0450000017881393, - "inferenceMs": 355.1149999946356, - "postprocessMs": 53.979999996721745, - "preprocessMs": 83.14499999582767, - "totalMs": 504.5949999988079 + "decodeMs": 2.894999995827675, + "inferenceMs": 278.054999999702, + "postprocessMs": 39.74499999731779, + "preprocessMs": 84.19500000029802, + "totalMs": 413.9699999988079 }, - "parityMetrics": { + "referenceMetrics": { "iou": 0.9999962574460338, "maxScoreDelta": 0.00001386435552097609, "meanPolygonPointDistancePixels": 0 }, - "parityThresholds": { + "referenceThresholds": { "iou": 0.95, "maxScoreDelta": 0.02, "meanPolygonPointDistancePixels": 2 @@ -166,29 +251,29 @@ ], "timingsMs": { "coldLoad": { - "capabilitiesMs": 168.0949999988079, - "integrityMs": 539.7800000011921, - "manifestMs": 2.3250000029802322, - "modelCacheMs": 1.089999996125698, - "modelDownloadMs": 3077.2300000041723, - "modelMs": 4335.57499999553, + "capabilitiesMs": 83.23000000417233, + "integrityMs": 364.6900000050664, + "manifestMs": 1.1050000041723251, + "modelCacheMs": 0.6649999991059303, + "modelDownloadMs": 1039.9349999949336, + "modelMs": 1792.8499999940395, "modelSource": "network", - "sessionMs": 3461.4400000050664, - "totalMs": 7968.234999999404 + "sessionMs": 1653.2999999970198, + "totalMs": 3531.1350000053644 }, "warmLoad": { - "capabilitiesMs": 0.5600000023841858, - "integrityMs": 363.2800000011921, - "manifestMs": 0.11999999731779099, - "modelCacheMs": 102.08500000089407, + "capabilitiesMs": 0.5700000002980232, + "integrityMs": 356.2550000026822, + "manifestMs": 0.08999999612569809, + "modelCacheMs": 109.79999999701977, "modelDownloadMs": 0, - "modelMs": 465.5, + "modelMs": 466.1850000023842, "modelSource": "cache", - "sessionMs": 1073.929999999702, - "totalMs": 1540.1750000044703 + "sessionMs": 1042.445000000298, + "totalMs": 1509.3650000020862 } }, - "sdkCommit": "8c47754068ff4ec7b34451cb7d562e6e9a8b1c8a", + "sdkCommit": "000707fe10d33a5e6797cd144e18ff68dc73f50d", "capabilities": { "crossOriginIsolated": true, "diagnostics": [ @@ -207,6 +292,6 @@ "worker": true }, "cpu": "Intel(R) Core(TM) i5-10400F CPU @ 2.90GHz", - "generatedAt": "2026-08-14T17:22:41.632Z", + "generatedAt": "2026-08-14T19:06:49.320Z", "id": "webgpu-fp32" } diff --git a/scripts/benchmark-contract.test.mjs b/scripts/benchmark-contract.test.mjs index 50f5706..9f259f9 100644 --- a/scripts/benchmark-contract.test.mjs +++ b/scripts/benchmark-contract.test.mjs @@ -104,12 +104,38 @@ describe("1.0.0 benchmark release contract", () => { }); test("publishes seven-fixture evidence for model 1.0.1 FP32 runtimes", () => { + const fixtureLock = JSON.parse( + readFileSync(join(repositoryRoot, "tools/model-pipeline/fixtures/fixtures.lock.json"), "utf8") + ); + const fixtureHashes = new Map( + fixtureLock.fixtures.map((fixture) => [fixture.filename, fixture.sha256]) + ); + const thresholds = { + maxBoxCoordinateDeltaPixels: 1, + maxPolygonCoordinateDeltaPixels: 1.5, + maxScoreDelta: 0.001 + }; for (const name of ["wasm-fp32.json", "webgpu-fp32.json"]) { const report = readJson(name, "1.0.1"); assert.equal(report.status, "passed"); assert.equal(report.fallbacks.length, 0); assert.equal(report.fixtures.length, 7); - assert.ok(report.fixtures.every((fixture) => fixture.parity === "passed")); + for (const fixture of report.fixtures) { + assert.equal(fixture.parity, "passed"); + assert.equal(fixture.fixtureSha256, fixtureHashes.get(fixture.filename)); + assert.match(fixture.acceptedOutputSha256, /^[a-f0-9]{64}$/); + assert.match(fixture.outputSha256, /^[a-f0-9]{64}$/); + assert.deepEqual(fixture.parityThresholds, thresholds); + assert.ok( + fixture.parityMetrics.maxBoxCoordinateDeltaPixels <= + thresholds.maxBoxCoordinateDeltaPixels + ); + assert.ok( + fixture.parityMetrics.maxPolygonCoordinateDeltaPixels <= + thresholds.maxPolygonCoordinateDeltaPixels + ); + assert.ok(fixture.parityMetrics.maxScoreDelta <= thresholds.maxScoreDelta); + } } }); diff --git a/scripts/verify-release.mjs b/scripts/verify-release.mjs index fc06d7c..8d880ee 100644 --- a/scripts/verify-release.mjs +++ b/scripts/verify-release.mjs @@ -18,6 +18,30 @@ function requireMatch(source, pattern, message) { if (!pattern.test(source)) fail(message); } +function canonicalJson(value) { + if (Array.isArray(value)) return value.map(canonicalJson); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, canonicalJson(entry)]) + ); + } + return value; +} + +function withoutVolatileEvidenceFields(value) { + if (Array.isArray(value)) return value.map(withoutVolatileEvidenceFields); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .filter(([key]) => key !== "generatedAt") + .map(([key, entry]) => [key, withoutVolatileEvidenceFields(entry)]) + ); + } + return value; +} + function read(relativePath) { try { return readFileSync(join(repositoryRoot, relativePath), "utf8"); @@ -150,6 +174,22 @@ function verifyStaticContract() { "model uploads must require an explicit boolean input" ); requireMatch(model, /contents:\s+write/, "model upload job needs contents: write"); + requireMatch( + model, + /UPLOAD_ASSETS[\s\S]*GITHUB_REF[\s\S]*refs\/heads\/main/, + "model uploads must be restricted to main" + ); + requireMatch( + model, + /git ls-remote --exit-code --tags origin "refs\/tags\/\$\{RELEASE_TAG\}"/, + "model uploads must reject an existing Git tag" + ); + requireMatch( + model, + /--target "\$\{GITHUB_SHA\}"/, + "model releases must target the validated commit SHA" + ); + if (/--target main/.test(model)) fail("model releases must not target a moving branch name"); const release = workflows["release.yml"]; requireMatch(release, /tags:\s*\n\s+-\s+["']v\*["']/, "npm release must use v* tags"); @@ -204,6 +244,95 @@ async function sha256(path) { return hash.digest("hex"); } +const fp32BrowserParityThresholds = { + maxBoxCoordinateDeltaPixels: 1, + maxPolygonCoordinateDeltaPixels: 1.5, + maxScoreDelta: 0.001 +}; + +function verifyFp32BrowserEvidence({ + acceptedFp32Sha256, + benchmark, + evidence, + fixtures, + manifestVariant, + provider +}) { + const displayName = provider === "wasm" ? "FP32 WASM" : "FP32 WebGPU"; + if (evidence?.schemaVersion !== 1) fail(`${displayName} browser evidence schema is invalid`); + if (evidence.status !== "passed") fail(`${displayName} browser evidence did not pass`); + if (evidence.executionProvider !== provider) { + fail(`${displayName} browser evidence execution provider is invalid`); + } + if (evidence.precision !== "fp32") fail(`${displayName} browser evidence precision is invalid`); + if (!Array.isArray(evidence.fallbacks) || evidence.fallbacks.length !== 0) { + fail(`${displayName} browser evidence contains fallback`); + } + if (evidence.modelBytes !== manifestVariant?.bytes) { + fail(`${displayName} browser evidence byte size does not match the manifest`); + } + if (evidence.modelSha256 !== manifestVariant?.sha256) { + fail(`${displayName} browser evidence does not match the manifest`); + } + + const expectedFilenames = fixtures.map(({ filename }) => filename); + if (!Array.isArray(evidence.fixtures)) { + fail(`${displayName} browser evidence fixture set is incomplete`); + } + const actualFilenames = evidence.fixtures.map(({ filename }) => filename); + if (JSON.stringify(actualFilenames) !== JSON.stringify(expectedFilenames)) { + fail(`${displayName} browser evidence fixture set is incomplete`); + } + if (evidence.fixtures.some((fixture) => fixture.parity !== "passed")) { + fail(`${displayName} browser evidence fixture parity failed`); + } + if ( + JSON.stringify(canonicalJson(withoutVolatileEvidenceFields(evidence))) !== + JSON.stringify(canonicalJson(withoutVolatileEvidenceFields(benchmark))) + ) { + fail(`${displayName} browser evidence differs from the benchmark artifact`); + } + if (evidence.acceptedModelSha256 !== acceptedFp32Sha256) { + fail(`${displayName} browser evidence accepted model SHA-256 is invalid`); + } + + for (const [index, fixture] of evidence.fixtures.entries()) { + const lockedFixture = fixtures[index]; + if (fixture.fixtureSha256 !== lockedFixture.sha256) { + fail(`${displayName} browser evidence fixture SHA-256 is invalid`); + } + if (!/^[a-f0-9]{64}$/.test(fixture.acceptedOutputSha256 ?? "")) { + fail(`${displayName} browser evidence accepted output SHA-256 is invalid`); + } + if (!/^[a-f0-9]{64}$/.test(fixture.outputSha256 ?? "")) { + fail(`${displayName} browser evidence output SHA-256 is invalid`); + } + if ( + fixture.detectionCount !== fixture.expectedDetectionCount || + fixture.labelSequenceEqual !== true || + fixture.readingOrderEqual !== true + ) { + fail(`${displayName} browser evidence structural parity failed`); + } + if (JSON.stringify(fixture.parityThresholds) !== JSON.stringify(fp32BrowserParityThresholds)) { + fail(`${displayName} browser evidence parity thresholds are invalid`); + } + const metrics = fixture.parityMetrics; + if ( + !Number.isFinite(metrics?.maxBoxCoordinateDeltaPixels) || + !Number.isFinite(metrics?.maxPolygonCoordinateDeltaPixels) || + !Number.isFinite(metrics?.maxScoreDelta) || + metrics.maxBoxCoordinateDeltaPixels > + fp32BrowserParityThresholds.maxBoxCoordinateDeltaPixels || + metrics.maxPolygonCoordinateDeltaPixels > + fp32BrowserParityThresholds.maxPolygonCoordinateDeltaPixels || + metrics.maxScoreDelta > fp32BrowserParityThresholds.maxScoreDelta + ) { + fail(`${displayName} browser evidence numeric parity failed`); + } + } +} + async function verifyModels(modelVersion) { const modelRoot = `models/pp-doclayoutv3/${modelVersion}`; const reportRoot = @@ -240,10 +369,25 @@ async function verifyModels(modelVersion) { fail("browser evidence does not match the FP16 manifest SHA-256"); if (modelVersion === "1.0.1") { - if (browser.fp32Wasm?.status !== "passed") fail("strict FP32 WASM evidence is missing"); - if (browser.fp32Webgpu?.status !== "passed") fail("strict FP32 WebGPU evidence is missing"); - if (browser.fp32Wasm?.fallbacks?.length !== 0) fail("FP32 WASM evidence contains fallback"); - if (browser.fp32Webgpu?.fallbacks?.length !== 0) fail("FP32 WebGPU evidence contains fallback"); + const fixtureLock = JSON.parse(read("tools/model-pipeline/fixtures/fixtures.lock.json")); + const acceptedManifest = JSON.parse(read("models/pp-doclayoutv3/1.0.0/manifest.json")); + const acceptedFp32 = acceptedManifest.variants.find(({ id }) => id === "fp32"); + verifyFp32BrowserEvidence({ + acceptedFp32Sha256: acceptedFp32?.sha256, + benchmark: JSON.parse(read("benchmarks/1.0.1/wasm-fp32.json")), + evidence: browser.fp32Wasm, + fixtures: fixtureLock.fixtures, + manifestVariant: manifestVariants.fp32, + provider: "wasm" + }); + verifyFp32BrowserEvidence({ + acceptedFp32Sha256: acceptedFp32?.sha256, + benchmark: JSON.parse(read("benchmarks/1.0.1/webgpu-fp32.json")), + evidence: browser.fp32Webgpu, + fixtures: fixtureLock.fixtures, + manifestVariant: manifestVariants.fp32, + provider: "webgpu" + }); if (manifestVariants.fp32?.backendCompatibility.join(",") !== "wasm,webgpu") { fail("FP32 manifest compatibility must be wasm,webgpu"); } diff --git a/scripts/verify-release.test.mjs b/scripts/verify-release.test.mjs index bad716e..719ea9b 100644 --- a/scripts/verify-release.test.mjs +++ b/scripts/verify-release.test.mjs @@ -1,8 +1,7 @@ import assert from "node:assert/strict"; import { execFileSync, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { readFileSync } from "node:fs"; -import { mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { describe, test } from "node:test"; import { dirname, resolve } from "node:path"; @@ -14,6 +13,23 @@ function sha256(bytes) { return createHash("sha256").update(bytes).digest("hex"); } +function verifyWithJsonMutation(relativePath, mutate) { + const path = resolve(repositoryRoot, relativePath); + const original = readFileSync(path, "utf8"); + const value = JSON.parse(original); + mutate(value); + try { + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); + return spawnSync( + process.execPath, + [resolve(repositoryRoot, "scripts/verify-release.mjs"), "--models", "1.0.1"], + { cwd: repositoryRoot, encoding: "utf8" } + ); + } finally { + writeFileSync(path, original); + } +} + function modelStagingFixture({ corruptFp32 = false } = {}) { const fp16 = Buffer.from("fp16-model"); const fp32 = Buffer.from("fp32-model"); @@ -83,6 +99,74 @@ describe("release workflow contract", () => { assert.match(workflow, /release_tag:[\s\S]*default:\s*["']?v1\.0\.1-models/); assert.match(workflow, /gh release create/); assert.doesNotMatch(workflow, /--clobber/); + assert.match(workflow, /GITHUB_REF[\s\S]*refs\/heads\/main/); + assert.match( + workflow, + /git ls-remote --exit-code --tags origin "refs\/tags\/\$\{RELEASE_TAG\}"/ + ); + assert.match(workflow, /--target "\$\{GITHUB_SHA\}"/); + assert.doesNotMatch(workflow, /--target main/); + }); + + test("rejects FP32 browser evidence for a different model", () => { + const result = verifyWithJsonMutation( + "tools/model-pipeline/reports/1.0.1/browser-evidence.json", + (evidence) => { + evidence.fp32Wasm.modelSha256 = "0".repeat(64); + } + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /FP32 WASM browser evidence does not match the manifest/); + }); + + test("rejects incomplete or inconsistent FP32 browser evidence", () => { + const evidencePath = "tools/model-pipeline/reports/1.0.1/browser-evidence.json"; + const cases = [ + [ + (evidence) => { + evidence.fp32Wasm.modelBytes += 1; + }, + /FP32 WASM browser evidence byte size does not match the manifest/ + ], + [ + (evidence) => { + evidence.fp32Wasm.executionProvider = "webgpu"; + }, + /FP32 WASM browser evidence execution provider is invalid/ + ], + [ + (evidence) => { + evidence.fp32Wasm.precision = "fp16"; + }, + /FP32 WASM browser evidence precision is invalid/ + ], + [ + (evidence) => { + evidence.fp32Wasm.fixtures.pop(); + }, + /FP32 WASM browser evidence fixture set is incomplete/ + ], + [ + (evidence) => { + evidence.fp32Wasm.fixtures[0].parity = "failed"; + }, + /FP32 WASM browser evidence fixture parity failed/ + ] + ]; + + for (const [mutate, expectedError] of cases) { + const result = verifyWithJsonMutation(evidencePath, mutate); + assert.notEqual(result.status, 0); + assert.match(result.stderr, expectedError); + } + }); + + test("rejects browser evidence that differs from the committed benchmark", () => { + const result = verifyWithJsonMutation("benchmarks/1.0.1/wasm-fp32.json", (report) => { + report.modelBytes += 1; + }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /FP32 WASM browser evidence differs from the benchmark artifact/); }); test("requires package repository metadata to match GitHub provenance", () => { diff --git a/tests/browser/benchmark.spec.ts b/tests/browser/benchmark.spec.ts index 03a454f..c9e802b 100644 --- a/tests/browser/benchmark.spec.ts +++ b/tests/browser/benchmark.spec.ts @@ -57,12 +57,18 @@ let server: Server; test.use(mode?.startsWith("webgpu-") ? { channel: "chrome" } : {}); -const parityThresholds = { +const referenceThresholds = { iou: 0.95, maxScoreDelta: 0.02, meanPolygonPointDistancePixels: 2 } as const; +const acceptedParityThresholds = { + maxBoxCoordinateDeltaPixels: 1, + maxPolygonCoordinateDeltaPixels: 1.5, + maxScoreDelta: 0.001 +} as const; + function sha256File(path: string): string { return createHash("sha256").update(readFileSync(path)).digest("hex"); } @@ -222,6 +228,7 @@ test("records strict seven-fixture browser evidence", async ({ browser, page }) const result = await page.evaluate( async ({ acceptedManifest, + acceptedParityThresholds: browserParityThresholds, backend, fixtures, origin: browserOrigin, @@ -235,6 +242,55 @@ test("records strict seven-fixture browser evidence", async ({ browser, page }) .join(""); } + type DetectionForParity = { + box: { xMax: number; xMin: number; yMax: number; yMin: number }; + polygon: Array<{ x: number; y: number }>; + score: number; + }; + + function compareDetections( + acceptedDetections: DetectionForParity[], + candidateDetections: DetectionForParity[] + ) { + let maxBoxCoordinateDeltaPixels = 0; + let maxPolygonCoordinateDeltaPixels = 0; + let maxScoreDelta = 0; + if (acceptedDetections.length !== candidateDetections.length) { + return { + maxBoxCoordinateDeltaPixels: Number.POSITIVE_INFINITY, + maxPolygonCoordinateDeltaPixels: Number.POSITIVE_INFINITY, + maxScoreDelta: Number.POSITIVE_INFINITY + }; + } + for (const [index, candidate] of candidateDetections.entries()) { + const accepted = acceptedDetections[index]!; + for (const coordinate of ["xMin", "xMax", "yMin", "yMax"] as const) { + maxBoxCoordinateDeltaPixels = Math.max( + maxBoxCoordinateDeltaPixels, + Math.abs(candidate.box[coordinate] - accepted.box[coordinate]) + ); + } + maxScoreDelta = Math.max(maxScoreDelta, Math.abs(candidate.score - accepted.score)); + if (candidate.polygon.length !== accepted.polygon.length) { + maxPolygonCoordinateDeltaPixels = Number.POSITIVE_INFINITY; + continue; + } + for (const [pointIndex, point] of candidate.polygon.entries()) { + const acceptedPoint = accepted.polygon[pointIndex]!; + maxPolygonCoordinateDeltaPixels = Math.max( + maxPolygonCoordinateDeltaPixels, + Math.abs(point.x - acceptedPoint.x), + Math.abs(point.y - acceptedPoint.y) + ); + } + } + return { + maxBoxCoordinateDeltaPixels, + maxPolygonCoordinateDeltaPixels, + maxScoreDelta + }; + } + const targetOptions = { allowFallback: false, backend, @@ -287,19 +343,33 @@ test("records strict seven-fixture browser evidence", async ({ browser, page }) const readingOrder = detection.detections.map(({ readingOrder }) => readingOrder); const labelSequenceEqual = JSON.stringify(labels) === JSON.stringify(acceptedLabels); const readingOrderEqual = JSON.stringify(readingOrder) === JSON.stringify(acceptedOrder); + const parityMetrics = compareDetections(acceptedDetection.detections, detection.detections); + const numericParity = + parityMetrics.maxBoxCoordinateDeltaPixels <= + browserParityThresholds.maxBoxCoordinateDeltaPixels && + parityMetrics.maxPolygonCoordinateDeltaPixels <= + browserParityThresholds.maxPolygonCoordinateDeltaPixels && + parityMetrics.maxScoreDelta <= browserParityThresholds.maxScoreDelta; + const acceptedDetectionJson = JSON.stringify(acceptedDetection.detections); const detectionJson = JSON.stringify(detection.detections); + const acceptedOutputSha256 = await sha256(new TextEncoder().encode(acceptedDetectionJson)); const outputSha256 = await sha256(new TextEncoder().encode(detectionJson)); fixtureResults.push({ + acceptedOutputSha256, detectionCount: detection.detections.length, detections: detection.detections, expectedDetectionCount: acceptedDetection.detections.length, filename: fixture.filename, + fixtureSha256: fixture.sha256, labelSequenceEqual, outputSha256, + parityMetrics, + parityThresholds: browserParityThresholds, parity: detection.detections.length === acceptedDetection.detections.length && labelSequenceEqual && - readingOrderEqual + readingOrderEqual && + numericParity ? "passed" : "failed", readingOrderEqual, @@ -341,6 +411,7 @@ test("records strict seven-fixture browser evidence", async ({ browser, page }) }, { acceptedManifest, + acceptedParityThresholds, backend, fixtures: fixturesLock.fixtures, origin, @@ -357,22 +428,32 @@ test("records strict seven-fixture browser evidence", async ({ browser, page }) expect(fixture.labelSequenceEqual).toBe(true); expect(fixture.readingOrderEqual).toBe(true); expect(fixture.parity).toBe("passed"); + expect(fixture.acceptedOutputSha256).toMatch(/^[a-f0-9]{64}$/); expect(fixture.outputSha256).toMatch(/^[a-f0-9]{64}$/); + expect(fixture.parityMetrics.maxBoxCoordinateDeltaPixels).toBeLessThanOrEqual( + acceptedParityThresholds.maxBoxCoordinateDeltaPixels + ); + expect(fixture.parityMetrics.maxPolygonCoordinateDeltaPixels).toBeLessThanOrEqual( + acceptedParityThresholds.maxPolygonCoordinateDeltaPixels + ); + expect(fixture.parityMetrics.maxScoreDelta).toBeLessThanOrEqual( + acceptedParityThresholds.maxScoreDelta + ); if (fixture.filename !== "table.png") return fixture; const firstDetection = detections[0]!; expect(firstDetection.labelId).toBe(reference.realImage.expected.labels[0]); - const parityMetrics = { + const referenceMetrics = { iou: boxIou(firstDetection.box), maxScoreDelta: Math.abs(firstDetection.score - reference.realImage.expected.scores[0]!), meanPolygonPointDistancePixels: meanPolygonPointDistance(firstDetection.polygon) }; - expect(parityMetrics.iou).toBeGreaterThanOrEqual(parityThresholds.iou); - expect(parityMetrics.maxScoreDelta).toBeLessThanOrEqual(parityThresholds.maxScoreDelta); - expect(parityMetrics.meanPolygonPointDistancePixels).toBeLessThanOrEqual( - parityThresholds.meanPolygonPointDistancePixels + expect(referenceMetrics.iou).toBeGreaterThanOrEqual(referenceThresholds.iou); + expect(referenceMetrics.maxScoreDelta).toBeLessThanOrEqual(referenceThresholds.maxScoreDelta); + expect(referenceMetrics.meanPolygonPointDistancePixels).toBeLessThanOrEqual( + referenceThresholds.meanPolygonPointDistancePixels ); - return { ...fixture, parityMetrics, parityThresholds }; + return { ...fixture, referenceMetrics, referenceThresholds }; }); const sdkCommit = execFileSync("git", ["rev-parse", "HEAD"], { @@ -382,6 +463,7 @@ test("records strict seven-fixture browser evidence", async ({ browser, page }) const report = { schemaVersion: 1, status: "passed", + acceptedModelSha256: acceptedManifest.variants.find(({ id }) => id === "fp32")!.sha256, executionProvider: backend, precision, fallbacks: result.runtime.fallbacks, diff --git a/tools/model-pipeline/reports/1.0.1/browser-evidence.json b/tools/model-pipeline/reports/1.0.1/browser-evidence.json index 7ce80cd..3acb1c4 100644 --- a/tools/model-pipeline/reports/1.0.1/browser-evidence.json +++ b/tools/model-pipeline/reports/1.0.1/browser-evidence.json @@ -98,6 +98,7 @@ "fp32Wasm": { "schemaVersion": 1, "status": "passed", + "acceptedModelSha256": "fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b", "executionProvider": "wasm", "precision": "fp32", "fallbacks": [], @@ -114,122 +115,206 @@ "operatingSystem": "win32 10.0.26200", "fixtures": [ { + "acceptedOutputSha256": "98601e9b45ffca68a280c95604d67f40bca747aefa7f07ec502369969dfd4025", "detectionCount": 12, "expectedDetectionCount": 12, "filename": "curved-document.jpg", + "fixtureSha256": "fce39d864ff7b0612f7073415c2a7c656f7790a26f96d539831f1bb1a994a069", "labelSequenceEqual": true, "outputSha256": "98601e9b45ffca68a280c95604d67f40bca747aefa7f07ec502369969dfd4025", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 10.405000001192093, - "inferenceMs": 7859.410000003874, - "postprocessMs": 39.41499999910593, - "preprocessMs": 127.48499999940395, - "totalMs": 8045.539999999106 + "decodeMs": 10.409999996423721, + "inferenceMs": 7663.39999999851, + "postprocessMs": 39.59000000357628, + "preprocessMs": 130.2800000011921, + "totalMs": 7852.52499999851 } }, { + "acceptedOutputSha256": "8bba1fb0a794a86ec0b2f4d1b1425d28e000a49beab399e4c55bbb04b4cf8cf4", "detectionCount": 59, "expectedDetectionCount": 59, "filename": "doc-formula.png", + "fixtureSha256": "6b07d28527dc9e930804fa73df562f1a81599c6b8a1a8bbc2a80742fa9f26e80", "labelSequenceEqual": true, "outputSha256": "8bba1fb0a794a86ec0b2f4d1b1425d28e000a49beab399e4c55bbb04b4cf8cf4", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 10.439999997615814, - "inferenceMs": 7962.560000002384, - "postprocessMs": 35.59000000357628, - "preprocessMs": 115.08500000089407, - "totalMs": 8132.585000000894 + "decodeMs": 9.110000006854534, + "inferenceMs": 7679.560000002384, + "postprocessMs": 33.74000000208616, + "preprocessMs": 108.30499999970198, + "totalMs": 7839.179999999702 } }, { + "acceptedOutputSha256": "da3c2a041cc94c4b5616637b7acd9af116c0882befd434920ebe8aaf96be626d", "detectionCount": 44, "expectedDetectionCount": 44, "filename": "image-layout.jpg", + "fixtureSha256": "cfebd4e0716da8ef01ad29c6f5bf7ed0dcc7d3a07bd38e32219c3b10645798de", "labelSequenceEqual": true, "outputSha256": "da3c2a041cc94c4b5616637b7acd9af116c0882befd434920ebe8aaf96be626d", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 27.730000004172325, - "inferenceMs": 7777.990000002086, - "postprocessMs": 59.399999998509884, - "preprocessMs": 105.34000000357628, - "totalMs": 7979.435000002384 + "decodeMs": 25.179999999701977, + "inferenceMs": 7650.9000000059605, + "postprocessMs": 58.375, + "preprocessMs": 108.12999999523163, + "totalMs": 7851.259999997914 } }, { + "acceptedOutputSha256": "c6051575214356859bcb9f87be446f27f32dc405a22bdc99ef9f887866c5ddb7", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "layout-demo.jpg", + "fixtureSha256": "785b7d19f158dcb636342dd3378ed3a4cddb7333d2d71688f0baa5c25a88ad51", "labelSequenceEqual": true, "outputSha256": "c6051575214356859bcb9f87be446f27f32dc405a22bdc99ef9f887866c5ddb7", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 31.45500000566244, - "inferenceMs": 7889.280000001192, - "postprocessMs": 99.87999999523163, - "preprocessMs": 172.27000000327826, - "totalMs": 8202.32500000298 + "decodeMs": 30.03499999642372, + "inferenceMs": 7656.109999999404, + "postprocessMs": 96.92999999970198, + "preprocessMs": 170.16999999433756, + "totalMs": 7961.7099999934435 } }, { + "acceptedOutputSha256": "b036ef27908bd3b94406d9a3106cb5ef6ecca8030874f2fceacac7a0a6407a90", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "screen-photo.jpg", + "fixtureSha256": "f27a8ad40192f2bff4bcc3605beaddf246bc35b07355e688defde1a2de333aa1", "labelSequenceEqual": true, "outputSha256": "b036ef27908bd3b94406d9a3106cb5ef6ecca8030874f2fceacac7a0a6407a90", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 11.390000000596046, - "inferenceMs": 8091.3499999940395, - "postprocessMs": 44.645000003278255, - "preprocessMs": 122.03499999642372, - "totalMs": 8279.155000001192 + "decodeMs": 11.574999995529652, + "inferenceMs": 7703.579999998212, + "postprocessMs": 40.46999999880791, + "preprocessMs": 123.03999999910593, + "totalMs": 7887.454999998212 } }, { + "acceptedOutputSha256": "07f0c613f0d87b91597984e01e83acdca5c5bf440d3efcc19873153b9250fa82", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "skew-document.jpg", + "fixtureSha256": "4ae0d5bebbe152a9cca8add806e376b3eb3314c3a55d6b7ccba70d9c4de97a1e", "labelSequenceEqual": true, "outputSha256": "07f0c613f0d87b91597984e01e83acdca5c5bf440d3efcc19873153b9250fa82", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 11.479999996721745, - "inferenceMs": 7944.47500000149, - "postprocessMs": 52.269999995827675, - "preprocessMs": 125.89500000327826, - "totalMs": 8142.655000001192 + "decodeMs": 11.339999996125698, + "inferenceMs": 7624.465000003576, + "postprocessMs": 52.63500000536442, + "preprocessMs": 124.5899999961257, + "totalMs": 7821.585000000894 } }, { + "acceptedOutputSha256": "8ff528db91eb3893ec1fbf50d69ac23aac17bc2aefec79e70cf0c11f0602550a", "detectionCount": 1, "expectedDetectionCount": 1, "filename": "table.png", + "fixtureSha256": "6d50148ceccb2d5cecc50b084b5105e3167f2d55a8899b29e04c3ebe46e88fa8", "labelSequenceEqual": true, "outputSha256": "8ff528db91eb3893ec1fbf50d69ac23aac17bc2aefec79e70cf0c11f0602550a", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 0 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 3.0499999970197678, - "inferenceMs": 7915.270000003278, - "postprocessMs": 55.57499999552965, - "preprocessMs": 76.70500000566244, - "totalMs": 8060.77499999851 + "decodeMs": 3.030000001192093, + "inferenceMs": 7697.104999996722, + "postprocessMs": 55.03499999642372, + "preprocessMs": 74.86999999731779, + "totalMs": 7838.77499999851 }, - "parityMetrics": { + "referenceMetrics": { "iou": 0.9999961699392192, - "maxScoreDelta": 0.000013727193068024945, + "maxScoreDelta": 1.3727193068024945e-5, "meanPolygonPointDistancePixels": 0 }, - "parityThresholds": { + "referenceThresholds": { "iou": 0.95, "maxScoreDelta": 0.02, "meanPolygonPointDistancePixels": 2 @@ -238,29 +323,29 @@ ], "timingsMs": { "coldLoad": { - "capabilitiesMs": 3.530000001192093, - "integrityMs": 366.4150000065565, - "manifestMs": 1.589999996125698, - "modelCacheMs": 0.8250000029802322, - "modelDownloadMs": 1204.2849999964237, - "modelMs": 1968.9499999955297, + "capabilitiesMs": 3.7400000020861626, + "integrityMs": 374.54500000178814, + "manifestMs": 1.2549999952316284, + "modelCacheMs": 0.6650000065565109, + "modelDownloadMs": 1105.3500000014901, + "modelMs": 1880.4549999982119, "modelSource": "network", - "sessionMs": 1593.4200000017881, - "totalMs": 3568.1999999955297 + "sessionMs": 1531.2849999964237, + "totalMs": 3417.314999997616 }, "warmLoad": { - "capabilitiesMs": 1.2349999994039536, - "integrityMs": 361.66499999910593, - "manifestMs": 0.0949999988079071, - "modelCacheMs": 79.13000000268221, + "capabilitiesMs": 0.5850000008940697, + "integrityMs": 352.85499999672174, + "manifestMs": 0.10999999940395355, + "modelCacheMs": 74.875, "modelDownloadMs": 0, - "modelMs": 440.8799999952316, + "modelMs": 427.80500000715256, "modelSource": "cache", - "sessionMs": 510.9950000047684, - "totalMs": 953.2649999931455 + "sessionMs": 488.1549999937415, + "totalMs": 916.7250000014901 } }, - "sdkCommit": "8c47754068ff4ec7b34451cb7d562e6e9a8b1c8a", + "sdkCommit": "000707fe10d33a5e6797cd144e18ff68dc73f50d", "capabilities": { "crossOriginIsolated": true, "diagnostics": [ @@ -279,12 +364,13 @@ "worker": false }, "cpu": "Intel(R) Core(TM) i5-10400F CPU @ 2.90GHz", - "generatedAt": "2026-08-14T17:26:16.905Z", + "generatedAt": "2026-08-14T19:09:41.586Z", "id": "wasm-fp32" }, "fp32Webgpu": { "schemaVersion": 1, "status": "passed", + "acceptedModelSha256": "fc2eebdc2153ad4e6993766f914f78f47a737fed123a78731bc9c57f7a6c806b", "executionProvider": "webgpu", "precision": "fp32", "fallbacks": [], @@ -326,122 +412,206 @@ "operatingSystem": "win32 10.0.26200", "fixtures": [ { + "acceptedOutputSha256": "98601e9b45ffca68a280c95604d67f40bca747aefa7f07ec502369969dfd4025", "detectionCount": 12, "expectedDetectionCount": 12, "filename": "curved-document.jpg", + "fixtureSha256": "fce39d864ff7b0612f7073415c2a7c656f7790a26f96d539831f1bb1a994a069", "labelSequenceEqual": true, "outputSha256": "49d017eb3a0946b85dbf0076cf3fc8af88ad12faa1c46b2e5063b54f0b082393", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.00030517578125, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 1.6508409746984753e-6 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 15.869999997317791, - "inferenceMs": 5850.69999999553, - "postprocessMs": 68.39000000059605, - "preprocessMs": 161.90999999642372, - "totalMs": 6113.865000002086 + "decodeMs": 10.104999996721745, + "inferenceMs": 5305.939999997616, + "postprocessMs": 70.9050000011921, + "preprocessMs": 134.60000000149012, + "totalMs": 5532.594999998808 } }, { + "acceptedOutputSha256": "8bba1fb0a794a86ec0b2f4d1b1425d28e000a49beab399e4c55bbb04b4cf8cf4", "detectionCount": 59, "expectedDetectionCount": 59, "filename": "doc-formula.png", + "fixtureSha256": "6b07d28527dc9e930804fa73df562f1a81599c6b8a1a8bbc2a80742fa9f26e80", "labelSequenceEqual": true, "outputSha256": "70e57bf726376f2e3aad8fd35a6d492d679b6b38087df95de2648aee09babc51", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.0003662109375, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 1.028899655031168e-6 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 8.570000000298023, - "inferenceMs": 315.6899999976158, - "postprocessMs": 50.16499999910593, - "preprocessMs": 119.58000000566244, - "totalMs": 504.45499999821186 + "decodeMs": 7.435000002384186, + "inferenceMs": 279.95000000298023, + "postprocessMs": 59.94500000029802, + "preprocessMs": 115.23999999463558, + "totalMs": 472.6200000047684 } }, { + "acceptedOutputSha256": "da3c2a041cc94c4b5616637b7acd9af116c0882befd434920ebe8aaf96be626d", "detectionCount": 44, "expectedDetectionCount": 44, "filename": "image-layout.jpg", + "fixtureSha256": "cfebd4e0716da8ef01ad29c6f5bf7ed0dcc7d3a07bd38e32219c3b10645798de", "labelSequenceEqual": true, "outputSha256": "16320eaccc62ae6d9509d498ade7996febe9fa72cc7543f716c91d8577b06b00", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.00048828125, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 1.3091050572455742e-6 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 24.344999998807907, - "inferenceMs": 278.85999999940395, - "postprocessMs": 72.42000000178814, - "preprocessMs": 121.79500000178814, - "totalMs": 508.01500000059605 + "decodeMs": 23.509999997913837, + "inferenceMs": 272.554999999702, + "postprocessMs": 87.42999999970198, + "preprocessMs": 105.24500000476837, + "totalMs": 498.9349999949336 } }, { + "acceptedOutputSha256": "c6051575214356859bcb9f87be446f27f32dc405a22bdc99ef9f887866c5ddb7", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "layout-demo.jpg", + "fixtureSha256": "785b7d19f158dcb636342dd3378ed3a4cddb7333d2d71688f0baa5c25a88ad51", "labelSequenceEqual": true, "outputSha256": "24d11a2ed203d8ee72ca4e14e803cab560ef57855976ead40efa65a07bd0fbb7", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.000640869140625, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 9.07715355213945e-7 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 30.639999993145466, - "inferenceMs": 290.7799999937415, - "postprocessMs": 94.3399999961257, - "preprocessMs": 177.5300000011921, - "totalMs": 606.9549999982119 + "decodeMs": 28.810000002384186, + "inferenceMs": 275.41499999910593, + "postprocessMs": 120.25999999791384, + "preprocessMs": 213.1600000038743, + "totalMs": 649.9800000041723 } }, { + "acceptedOutputSha256": "b036ef27908bd3b94406d9a3106cb5ef6ecca8030874f2fceacac7a0a6407a90", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "screen-photo.jpg", + "fixtureSha256": "f27a8ad40192f2bff4bcc3605beaddf246bc35b07355e688defde1a2de333aa1", "labelSequenceEqual": true, "outputSha256": "6f0fe242016734de274a9ac9d002ee166694e8360cb52cc3cb1d0b4000d1d160", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.0003662109375, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 6.820948376118352e-7 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 11.925000004470348, - "inferenceMs": 290.51500000059605, - "postprocessMs": 44.42000000178814, - "preprocessMs": 144, - "totalMs": 502.4949999973178 + "decodeMs": 11.094999998807907, + "inferenceMs": 263.5649999976158, + "postprocessMs": 48.474999994039536, + "preprocessMs": 125.89499999582767, + "totalMs": 459.28999999910593 } }, { + "acceptedOutputSha256": "07f0c613f0d87b91597984e01e83acdca5c5bf440d3efcc19873153b9250fa82", "detectionCount": 13, "expectedDetectionCount": 13, "filename": "skew-document.jpg", + "fixtureSha256": "4ae0d5bebbe152a9cca8add806e376b3eb3314c3a55d6b7ccba70d9c4de97a1e", "labelSequenceEqual": true, "outputSha256": "840d7c61343deee9fa966b541e17a19671d5987aa27280ecec8ca2943c133c47", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 0.00042724609375, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 1.394110166863527e-6 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 11.689999997615814, - "inferenceMs": 230.11499999463558, - "postprocessMs": 57.30500000715256, - "preprocessMs": 138.96000000089407, - "totalMs": 451.71000000089407 + "decodeMs": 11.559999994933605, + "inferenceMs": 296.1850000023842, + "postprocessMs": 42.184999994933605, + "preprocessMs": 130.05499999970198, + "totalMs": 491.1299999952316 } }, { + "acceptedOutputSha256": "8ff528db91eb3893ec1fbf50d69ac23aac17bc2aefec79e70cf0c11f0602550a", "detectionCount": 1, "expectedDetectionCount": 1, "filename": "table.png", + "fixtureSha256": "6d50148ceccb2d5cecc50b084b5105e3167f2d55a8899b29e04c3ebe46e88fa8", "labelSequenceEqual": true, "outputSha256": "7978ca0ec7143f98fc24ef2f4613c72d785a82ca4a1b31bcf97fc1952ffb9e8c", + "parityMetrics": { + "maxBoxCoordinateDeltaPixels": 6.16908073425293e-5, + "maxPolygonCoordinateDeltaPixels": 0, + "maxScoreDelta": 1.3716245295114504e-7 + }, + "parityThresholds": { + "maxBoxCoordinateDeltaPixels": 1, + "maxPolygonCoordinateDeltaPixels": 1.5, + "maxScoreDelta": 0.001 + }, "parity": "passed", "readingOrderEqual": true, "timings": { - "decodeMs": 3.0450000017881393, - "inferenceMs": 355.1149999946356, - "postprocessMs": 53.979999996721745, - "preprocessMs": 83.14499999582767, - "totalMs": 504.5949999988079 + "decodeMs": 2.894999995827675, + "inferenceMs": 278.054999999702, + "postprocessMs": 39.74499999731779, + "preprocessMs": 84.19500000029802, + "totalMs": 413.9699999988079 }, - "parityMetrics": { + "referenceMetrics": { "iou": 0.9999962574460338, - "maxScoreDelta": 0.00001386435552097609, + "maxScoreDelta": 1.386435552097609e-5, "meanPolygonPointDistancePixels": 0 }, - "parityThresholds": { + "referenceThresholds": { "iou": 0.95, "maxScoreDelta": 0.02, "meanPolygonPointDistancePixels": 2 @@ -450,29 +620,29 @@ ], "timingsMs": { "coldLoad": { - "capabilitiesMs": 168.0949999988079, - "integrityMs": 539.7800000011921, - "manifestMs": 2.3250000029802322, - "modelCacheMs": 1.089999996125698, - "modelDownloadMs": 3077.2300000041723, - "modelMs": 4335.57499999553, + "capabilitiesMs": 83.23000000417233, + "integrityMs": 364.6900000050664, + "manifestMs": 1.1050000041723251, + "modelCacheMs": 0.6649999991059303, + "modelDownloadMs": 1039.9349999949336, + "modelMs": 1792.8499999940395, "modelSource": "network", - "sessionMs": 3461.4400000050664, - "totalMs": 7968.234999999404 + "sessionMs": 1653.2999999970198, + "totalMs": 3531.1350000053644 }, "warmLoad": { - "capabilitiesMs": 0.5600000023841858, - "integrityMs": 363.2800000011921, - "manifestMs": 0.11999999731779099, - "modelCacheMs": 102.08500000089407, + "capabilitiesMs": 0.5700000002980232, + "integrityMs": 356.2550000026822, + "manifestMs": 0.08999999612569809, + "modelCacheMs": 109.79999999701977, "modelDownloadMs": 0, - "modelMs": 465.5, + "modelMs": 466.1850000023842, "modelSource": "cache", - "sessionMs": 1073.929999999702, - "totalMs": 1540.1750000044703 + "sessionMs": 1042.445000000298, + "totalMs": 1509.3650000020862 } }, - "sdkCommit": "8c47754068ff4ec7b34451cb7d562e6e9a8b1c8a", + "sdkCommit": "000707fe10d33a5e6797cd144e18ff68dc73f50d", "capabilities": { "crossOriginIsolated": true, "diagnostics": [ @@ -491,7 +661,7 @@ "worker": true }, "cpu": "Intel(R) Core(TM) i5-10400F CPU @ 2.90GHz", - "generatedAt": "2026-08-14T17:22:41.632Z", + "generatedAt": "2026-08-14T19:06:49.32Z", "id": "webgpu-fp32" } }