From a1178bdf89b278ccee37d9d33ab626b26618dd1f Mon Sep 17 00:00:00 2001 From: Piotr Wolnowski Date: Mon, 27 Jul 2026 15:10:45 +0200 Subject: [PATCH 1/5] feat(inference): add ov_smoke integration test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three-stage pytest contract that validates 'OpenVINO works with physicalai' against the three published HuggingFace exports: OpenVINO/act-fp16-ov — no tokenizer OpenVINO/pi05-libero-fp16-ov — tokenizer-bearing (VLA) OpenVINO/smolvla-libero-fp16-ov — tokenizer-bearing (VLA) Stages: 1. load — OpenVINOAdapter.load (read_model + compile_model) 2. tokenizer — OVTokenizer (openvino_tokenizers extension loads and matches core OpenVINO version) 3. predict — predict_action_chunk returns finite outputs The tokenizer stage catches the exact failure mode that caused the openvino pin: under a core/tokenizer mismatch ACT passes while pi05 fails at stage 2, pinpointing the broken version pair. Also adds: - docs/design/openvino-validation.md — design proposal for the full OV validation pipeline (smoke, golden-action, perf, gym E2E) - ov_smoke pytest marker in pyproject.toml - tests/integration/ package skeleton Invoke with: pytest -m ov_smoke -s OV_SMOKE_CACHE_DIR=~/.cache/hf pytest -m ov_smoke --- docs/design/openvino-validation.md | 211 +++++++++++++ pyproject.toml | 1 + tests/integration/__init__.py | 2 + tests/integration/inference/__init__.py | 2 + tests/integration/inference/test_ov_smoke.py | 310 +++++++++++++++++++ 5 files changed, 526 insertions(+) create mode 100644 docs/design/openvino-validation.md create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/inference/__init__.py create mode 100644 tests/integration/inference/test_ov_smoke.py diff --git a/docs/design/openvino-validation.md b/docs/design/openvino-validation.md new file mode 100644 index 00000000..458b0aad --- /dev/null +++ b/docs/design/openvino-validation.md @@ -0,0 +1,211 @@ +# OpenVINO Validation — Proposal + +**Status:** Draft proposal · **Owner:** physicalai team · **Date:** 2026-06-29 + +Proposal for integrating physicalai tests into OpenVINO's automated CI/CD. It +defines the validation contract, who owns what, the test tiers, and the target +platforms, frequency, and reporting for the OV pipeline. Piotr Wolnowski owns +the pipeline implementation and contributes test cases via the physicalai repo. + +## Coverage + +Three areas, mapped to the tiers below: + +| Area | Question | Tier | +| ------------------------------- | ----------------------------------------------- | ------------- | +| **Correctness + compatibility** | does physicalai load + run on this OV? | smoke | +| **Accuracy** | same numbers on target HW (per device/version)? | golden-action | +| **Functional** | does it still do the task end-to-end? | gym E2E | +| **Performance** | latency / throughput on target HW? | benchmark | + +## Why + +An OpenVINO upgrade broke physicalai inference — hence the current +`openvino==2026.1` pin in [`pyproject.toml`](../../pyproject.toml). We need an +automated signal that an OpenVINO change still works with physicalai _before_ we +adopt it. + +## The Contract + +"OpenVINO works with physicalai" means a real exported policy loads and runs +through the physicalai OpenVINO code paths. Three stages must pass: + +| Stage | Code path | What it proves | +| ------------- | ------------------------- | ---------------------------------------------------------------- | +| **load** | `OpenVINOAdapter.load` | `read_model` + `compile_model` succeed | +| **tokenizer** | `OVTokenizer` | `openvino_tokenizers` extension loads + matches core OpenVINO | +| **predict** | `OpenVINOAdapter.predict` | inference runs, outputs have expected shape/dtype and are finite | + +The tokenizer stage matters because `openvino` and `openvino_tokenizers` are +versioned **independently** and must stay a matched pair — a mismatch is what +broke us, and it only affects models that carry a tokenizer. + +## Who Owns What + +| Concern | Owner | +| ------------------------------------------------------------- | ------------------- | +| Test **logic + code location** (contract, stages, assertions) | **physicalai** repo | +| Pass/fail criteria | **physicalai** | +| PR merge gate | **physicalai** CI | +| Early-warning on OV bump PRs (Renovate-triggered) | **physicalai** CI | +| Running the same test in pre-release matrix | **OpenVINO** | +| Weekly + per-RC runs on target hardware | **OpenVINO** CI | +| Orchestration (workflow YAMLs, runners) + reporting | **OpenVINO** repo | +| Pre-release wheels + breakage notification | **OpenVINO** | + +**Decision:** split by layer. + +- **Test logic lives in physicalai** — this is the only way to guarantee its + quality and relevancy. Only physicalai knows its real OpenVINO usage, and the + test sits next to the code it protects, so it changes in the same PR as that + code. The OV team does not author test logic in-house. +- **Orchestration and reporting live in OpenVINO** — the workflow YAMLs, runners, + matrix, and dashboards are OV-side tooling. OV consumes the physicalai test by + pinned tag (`git clone physicalai@` → `pytest -m ov_smoke`). + +Test-case contributions from the OV team land as PRs into the +physicalai repo and are reviewed by physicalai, so logic stays single-sourced +and quality-controlled while OV helps build it. + +The test is self-contained (pytest + `ov_smoke` marker + model discovery via env +var), so OV can invoke it without pulling in physicalai internals. + +Same test, multiple runners: physicalai's PR gate and Renovate early-warning +catch _our_ changes and released OV bumps; OpenVINO's pre-release matrix catches +_their_ changes before release. + +**What OV runs:** smoke runs by pinned tag, so OV adds it to their pre-release +matrix. Golden-action runs on each device, comparing the OpenVINO output to a +stored PyTorch reference. Gym E2E uses the in-house LIBERO gyms on OV's GPU +runners. PAI maintains all tiers; OV provides the hardware. + +## Smoke Test + +A pytest integration test (marker `ov_smoke`) that runs the three contract +stages against discovered model exports and records the `openvino` / +`openvino_tokenizers` versions in each result. + +Coverage needs **two models** to cover the contract: + +| Model | Covers | +| ----------------------------------------- | -------------------------- | +| **ACT** (no tokenizer) | load + predict | +| **a tokenizer-bearing model** (e.g. pi05) | load + tokenizer + predict | + +Verified 2026-06-26: under a core/tokenizer mismatch, ACT passes and the +tokenizer model fails at the tokenizer stage — so this pair detects the exact +break that caused the pin. + +**Devices:** run the smoke across the device spectrum — CPU on hosted runners, +GPU on OV runners. CPU covers the version-pair contract; GPU catches +device-plugin regressions. + +## Early Warning + +physicalai CI runs `ov_smoke` when a Renovate PR bumps `openvino*`, against the +PR's resolved versions — event-driven, alert-only, fires exactly when a new +version appears. OV's weekly-vs-nightly and per-RC matrix (below) covers +pre-release wheels, so physicalai does not duplicate a weekly run. + +## Golden-Action + +A regression test that pins the OpenVINO output to a stored reference. It answers +a different question than smoke: smoke proves "it ran", golden proves it produced +the **right numbers**. + +**Approach:** + +1. **Export deterministically.** Build the test model with the PyTorch + deterministic-denoising toggle on, so the policy uses fixed (non-random) noise. + This removes the in-graph sampling that otherwise makes every inference + different (see below). +2. **Record one reference.** Run the source **PyTorch** model (e.g. on CPU) on a + fixed observation and store the action chunk as the ground-truth reference — + recorded once, not per device. +3. **Check every run.** Run the OpenVINO export on the target device for the same + observation and compare against the reference with an **L2 tolerance**. + +Exact equality only holds for a device compared against itself; across devices, +kernel and precision differences make the numbers drift, so the L2 tolerance +absorbs that drift while still catching a real regression. One PyTorch reference +covers the whole device matrix. + +**Why a deterministic export is needed (verified 2026-06-30).** Without it the +output is non-deterministic, and the cause is inside the exported graph — not in +physicalai's Python code, and not in any model input. The pi05 IR draws its +denoising noise from a single `RandomUniform` op exported with +`global_seed=0, op_seed=0`, which the OpenVINO spec defines as a non-deterministic +sequence; the stock model varies ~0.3 abs across identical calls. We confirmed the +noise can also be pinned at the IR level — seeding that op makes the OpenVINO +output reproducible bit-for-bit — which proves determinism is achievable, but +exporting deterministically from PyTorch is the cleaner source-level fix and also +makes the PyTorch reference itself stable. + +## Performance + +Latency / throughput on target hardware, built on the existing +`InferenceLatencyBenchmark`. Reports median/p99 per-iteration time and FPS per +model per device, tracked as a trend. Threshold-based and alert-only — hosted +variance makes absolute pass/fail unreliable, so flag large regressions, never +gate. Runs on the OV GPU runners across the platform matrix. + +## End-to-End (gym) + +Beyond smoke (loads + runs) and golden-action (same numbers), a gym rollout +checks the policy still **does the task** end-to-end: closed-loop +`reset → predict_action_chunk → step`, exercising the full pre/post pipeline and +the runtime action queue. + +- **Gyms exist in-house:** LIBERO for SmolVLA and pi05 (a newer gym is in + progress for later). +- **Reduced mode:** a small, fixed set of seeded episodes to bound runtime — a + pass/fail signal, not a full acceptance benchmark. +- **Where:** OV GPU runners, using the in-house LIBERO gyms. Reduced/seeded, + per-device reference, threshold-based, alert-only. Never on the PR gate — a + distinct tier from smoke. + +## Target Platforms + +All targets are GPU — the pipeline runs on OV-provided Intel hardware: + +| Platform | Type | +| -------- | --------------------------- | +| PTL iGPU | Panther Lake integrated GPU | +| B580 | Arc Battlemage discrete | +| B60 | Arc Battlemage discrete | +| B70 | Arc Battlemage discrete | + +CPU smoke stays on hosted runners as the portable baseline; GPU/accuracy/perf +tiers run across this matrix. + +## Frequency + +| Trigger | Scope | Gates? | +| ---------------------------- | ------------------------------------ | ------------ | +| OV bump PR / PAI PR | smoke (CPU) | yes | +| **Weekly** vs OV **nightly** | smoke + golden + perf, all platforms | alert | +| **Each RC** (pre-release) | full suite, all platforms | release gate | + +## Reporting + +Per-run record of OV / `openvino_tokenizers` versions, model, platform, tier, +and pass/fail + metrics. Failures notify the owner; perf/accuracy tracked as a +trend across versions. Dashboards and notification wiring are OV-side. + +## Next + +- Build the `ov_smoke` test (three contract stages, model discovery, version logging). +- Lock `openvino` + `openvino_tokenizers` as a matched pair in + [`pyproject.toml`](../../pyproject.toml) — they are versioned independently + and must be binary-compatible; constraining the tokenizer to the matching + minor stops a resolver from installing the mismatch that broke us. +- Add a small tokenizer-bearing model the gate can run — the tokenizer break + only shows on a model with an OV tokenizer; pi05 is 6.25 GB (too large for + hosted CI) and ACT has none, so the gate needs a small tokenizer model. +- Wire `ov_smoke` into [`library.yml`](../../.github/workflows/library.yml) as a + PR gate, plus the Renovate-triggered early-warning run. +- Add the golden-action tier — export the test model with deterministic denoising, + store a PyTorch reference, and L2-compare the OpenVINO output per device. +- Provision OV GPU runners (PTL iGPU, B580, B60, B70) for GPU smoke, perf, and gym E2E. +- Add the performance tier across the platform matrix. +- Review with Piotr Wolnowski; agree pipeline ownership + RC release gate. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d2f5dde5..0ad4a737 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -208,6 +208,7 @@ markers = [ "integration: marks tests as integration tests", "v4l2: marks tests requiring V4L2 mocking (Linux only)", "transport: marks tests requiring iceoryx2", + "ov_smoke: OpenVINO smoke tests — three-stage contract (load, tokenizer, predict) against published HF exports", ] [tool.coverage.report] diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..91da480a --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/integration/inference/__init__.py b/tests/integration/inference/__init__.py new file mode 100644 index 00000000..91da480a --- /dev/null +++ b/tests/integration/inference/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/integration/inference/test_ov_smoke.py b/tests/integration/inference/test_ov_smoke.py new file mode 100644 index 00000000..032635b2 --- /dev/null +++ b/tests/integration/inference/test_ov_smoke.py @@ -0,0 +1,310 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""OpenVINO smoke tests — ov_smoke marker. + +Validates the three-stage contract "OpenVINO works with physicalai" against the +three published model exports in the OpenVINO/physical-ai HuggingFace collection: + + Stage 1 — load: OpenVINOAdapter.load() — read_model + compile_model succeed + Stage 2 — tokenizer: OVTokenizer — openvino_tokenizers extension loads and is + compatible with the installed core OpenVINO version + Stage 3 — predict: InferenceModel.predict_action_chunk() — outputs are finite + +Models tested: + OpenVINO/act-fp16-ov → stages 1, 3 (no tokenizer) + OpenVINO/pi05-libero-fp16-ov → stages 1, 2, 3 (tokenizer-bearing, VLA) + OpenVINO/smolvla-libero-fp16-ov → stages 1, 2, 3 (tokenizer-bearing, VLA) + +The tokenizer stage is critical: openvino and openvino_tokenizers are versioned +independently. A mismatch silently passes for ACT (no tokenizer) but breaks at the +tokenizer stage for pi05 / smolvla — exactly the failure mode that caused the +openvino pin in pyproject.toml. + +Run: + pytest -m ov_smoke -s # all models, all stages + pytest -m ov_smoke -k pi05 # pi05 only + pytest -m ov_smoke -k "load or tokenizer" # stages 1 and 2 only + +Environment: + OV_SMOKE_CACHE_DIR optional path for the HuggingFace model cache + (avoids re-downloading on repeated runs) +""" + +from __future__ import annotations + +import importlib.metadata +import logging +import os +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest + +from physicalai.inference.constants import TASK, TOKENIZED_PROMPT, TOKENIZED_PROMPT_MASK +from physicalai.inference.manifest import Manifest +from physicalai.inference.model import InferenceModel +from physicalai.inference.preprocessors import OVTokenizer +from physicalai.inference.utils._hub import download_from_hub + +if TYPE_CHECKING: + pass + +log = logging.getLogger(__name__) + +# All tests in this module carry both markers. +pytestmark = [pytest.mark.ov_smoke, pytest.mark.integration] + +# Skip the entire module when OpenVINO is not installed — these tests require +# the real runtime, not a mock. +pytest.importorskip("openvino", reason="openvino not installed — skipping ov_smoke suite") + +# --------------------------------------------------------------------------- +# Model specifications +# --------------------------------------------------------------------------- + +_RNG = np.random.default_rng(42) + +_LIBERO_IMAGES = { + "images.image": _RNG.random((1, 3, 256, 256)).astype(np.float32), + "images.image2": _RNG.random((1, 3, 256, 256)).astype(np.float32), +} +_LIBERO_STATE = {"state": np.zeros((1, 8), dtype=np.float32)} +_LIBERO_TASK = {TASK: ["pick up the red block and place it in the bowl"]} + + +@dataclass(frozen=True) +class _ModelSpec: + """Specification for one smoke-tested model export.""" + + repo_id: str + has_tokenizer: bool + observation: dict[str, Any] + + @property + def short_id(self) -> str: + """Return the repo slug (last path component) for test IDs and logging.""" + return self.repo_id.split("/")[-1] + + +_ALL_MODELS: list[_ModelSpec] = [ + _ModelSpec( + repo_id="OpenVINO/act-fp16-ov", + has_tokenizer=False, + observation={**_LIBERO_IMAGES, **_LIBERO_STATE}, + ), + _ModelSpec( + repo_id="OpenVINO/pi05-libero-fp16-ov", + has_tokenizer=True, + observation={**_LIBERO_IMAGES, **_LIBERO_STATE, **_LIBERO_TASK}, + ), + _ModelSpec( + repo_id="OpenVINO/smolvla-libero-fp16-ov", + has_tokenizer=True, + observation={**_LIBERO_IMAGES, **_LIBERO_STATE, **_LIBERO_TASK}, + ), +] + +_TOKENIZER_MODELS: list[_ModelSpec] = [m for m in _ALL_MODELS if m.has_tokenizer] + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _pkg_version(pkg: str) -> str: + """Return the installed version of *pkg*, or ``'not installed'``.""" + try: + return importlib.metadata.version(pkg) + except importlib.metadata.PackageNotFoundError: + return "not installed" + + +def _find_tokenizer_artifact(export_dir: Path) -> Path | None: + """Return the absolute path to the OVTokenizer artifact in *export_dir*. + + Parses ``manifest.json`` and looks for a preprocessor with + ``type == "ov_tokenizer"`` or a class_path containing ``OVTokenizer``. + Returns ``None`` when no tokenizer preprocessor is declared. + """ + manifest = Manifest.load(export_dir) + for spec in manifest.model.preprocessors: + is_ov_tokenizer = spec.type == "ov_tokenizer" or "OVTokenizer" in spec.class_path + if not is_ov_tokenizer: + continue + artifact = spec.flat_params.get("artifact") or spec.init_args.get("artifact") + if artifact: + return (export_dir / artifact).resolve() + return None + + +# --------------------------------------------------------------------------- +# Session fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def smoke_cache_dir() -> Path | None: + """Optional HuggingFace cache directory from the environment.""" + env = os.environ.get("OV_SMOKE_CACHE_DIR") + return Path(env) if env else None + + +@pytest.fixture(scope="session", autouse=True) +def _report_ov_versions() -> None: + """Log the installed openvino / openvino-tokenizers version pair once per session.""" + ov_ver = _pkg_version("openvino") + tok_ver = _pkg_version("openvino-tokenizers") + log.info("ov_smoke | openvino==%s | openvino-tokenizers==%s", ov_ver, tok_ver) + + +@pytest.fixture( + scope="session", + params=_ALL_MODELS, + ids=lambda m: m.short_id, +) +def all_exports( + request: pytest.FixtureRequest, + smoke_cache_dir: Path | None, +) -> tuple[Path, _ModelSpec]: + """Download (or reuse cached) export dir for each model in *_ALL_MODELS*.""" + spec: _ModelSpec = request.param + export_dir = download_from_hub(spec.repo_id, cache_dir=smoke_cache_dir) + return export_dir, spec + + +@pytest.fixture( + scope="session", + params=_TOKENIZER_MODELS, + ids=lambda m: m.short_id, +) +def tokenizer_exports( + request: pytest.FixtureRequest, + smoke_cache_dir: Path | None, +) -> tuple[Path, _ModelSpec]: + """Download (or reuse cached) export dir for tokenizer-bearing models.""" + spec: _ModelSpec = request.param + export_dir = download_from_hub(spec.repo_id, cache_dir=smoke_cache_dir) + return export_dir, spec + + +# --------------------------------------------------------------------------- +# Stage 1 — load +# --------------------------------------------------------------------------- + + +def test_load(all_exports: tuple[Path, _ModelSpec]) -> None: + """Stage 1: OpenVINOAdapter.load — read_model + compile_model succeed. + + Loads the policy XML via the OpenVINO runtime. Preprocessors and + postprocessors are suppressed so this stage only exercises the adapter. + A failure here means the core OpenVINO runtime cannot read or compile the + exported model IR. + """ + export_dir, spec = all_exports + log.info("ov_smoke | stage=load | model=%s | openvino==%s", spec.short_id, _pkg_version("openvino")) + + # Bypass preprocessors/postprocessors — isolates adapter.load() from the tokenizer stage. + model = InferenceModel(export_dir=export_dir, device="CPU", preprocessors=[], postprocessors=[]) + + assert model.adapter.compiled_model is not None, ( + f"[{spec.short_id}] compiled_model is None after load — " + "read_model or compile_model failed silently" + ) + assert model.adapter.input_names, f"[{spec.short_id}] adapter reports no input names after load" + assert model.adapter.output_names, f"[{spec.short_id}] adapter reports no output names after load" + + +# --------------------------------------------------------------------------- +# Stage 2 — tokenizer +# --------------------------------------------------------------------------- + + +def test_tokenizer(tokenizer_exports: tuple[Path, _ModelSpec]) -> None: + """Stage 2: OVTokenizer — openvino_tokenizers extension loads and matches core OpenVINO. + + Instantiates OVTokenizer with the artifact declared in the manifest and + runs a single tokenisation pass. An incompatible openvino / openvino_tokenizers + version pair raises an exception during OVTokenizer.__init__ (the import + registers the runtime extension); this stage catches exactly the failure mode + that caused the openvino pin. + """ + export_dir, spec = tokenizer_exports + ov_ver = _pkg_version("openvino") + tok_ver = _pkg_version("openvino-tokenizers") + log.info( + "ov_smoke | stage=tokenizer | model=%s | openvino==%s | openvino-tokenizers==%s", + spec.short_id, + ov_ver, + tok_ver, + ) + + tokenizer_path = _find_tokenizer_artifact(export_dir) + assert tokenizer_path is not None, ( + f"[{spec.short_id}] No ov_tokenizer artifact found in manifest — " + "model is declared as tokenizer-bearing but manifest.json has no " + "'ov_tokenizer' preprocessor entry" + ) + assert tokenizer_path.exists(), ( + f"[{spec.short_id}] Tokenizer artifact declared in manifest but not on disk: {tokenizer_path}" + ) + + # Constructing OVTokenizer imports openvino_tokenizers, registering the OV runtime + # extension. A core/tokenizer version mismatch raises here. + tokenizer = OVTokenizer(tokenizer_path) + + # Run a single tokenisation to confirm the loaded model produces valid outputs. + outputs = tokenizer({TASK: ["pick up the red block and place it in the bowl"]}) + + assert TOKENIZED_PROMPT in outputs, ( + f"[{spec.short_id}] OVTokenizer output is missing the '{TOKENIZED_PROMPT}' key" + ) + assert TOKENIZED_PROMPT_MASK in outputs, ( + f"[{spec.short_id}] OVTokenizer output is missing the '{TOKENIZED_PROMPT_MASK}' key" + ) + + ids = outputs[TOKENIZED_PROMPT] + assert ids.ndim == 2, ( # noqa: PLR2004 + f"[{spec.short_id}] Expected 2-D token ids (batch, seq_len), got shape {ids.shape}" + ) + assert ids.shape[0] == 1, ( + f"[{spec.short_id}] Expected batch size 1, got {ids.shape[0]}" + ) + assert ids.shape[1] > 0, f"[{spec.short_id}] Token sequence length must be > 0" + + +# --------------------------------------------------------------------------- +# Stage 3 — predict +# --------------------------------------------------------------------------- + + +def test_predict(all_exports: tuple[Path, _ModelSpec]) -> None: + """Stage 3: OpenVINOAdapter.predict — inference runs and outputs are finite. + + Loads the full pipeline (including tokenizer for VLA models) and calls + predict_action_chunk. Validates that the returned action chunk is a finite + numpy array with a non-zero action dimension. A failure here means inference + either crashes or produces NaN / Inf — both indicate a broken OV version. + """ + export_dir, spec = all_exports + log.info( + "ov_smoke | stage=predict | model=%s | openvino==%s", + spec.short_id, + _pkg_version("openvino"), + ) + + model = InferenceModel(export_dir=export_dir, device="CPU") + model.reset() + chunk = model.predict_action_chunk(spec.observation) + + assert isinstance(chunk, np.ndarray), ( + f"[{spec.short_id}] predict_action_chunk returned {type(chunk).__name__}, expected np.ndarray" + ) + assert chunk.ndim >= 1, f"[{spec.short_id}] Action chunk must have at least one dimension" + assert chunk.shape[-1] > 0, f"[{spec.short_id}] Action dimension must be > 0" + assert np.isfinite(chunk).all(), ( + f"[{spec.short_id}] predict_action_chunk returned non-finite values (NaN or Inf) — " + f"non-finite count: {(~np.isfinite(chunk)).sum()}" + ) From d483c4589ce51d5c48e126be5b8d4445f2ff5f72 Mon Sep 17 00:00:00 2001 From: Piotr Wolnowski Date: Mon, 27 Jul 2026 15:24:18 +0200 Subject: [PATCH 2/5] fix(ov_smoke): preserve HF Hub symlinks in _find_tokenizer_artifact Use os.path.normpath instead of Path.resolve() when resolving the tokenizer artifact path from the manifest, mirroring the same pattern used by component_factory._resolve_artifact_path. Path.resolve() followed the snapshot symlink into the blobs/ store, giving a raw blob path with no .bin sibling. OpenVINO's read_model then raised 'Empty weights data in bin file or bin file cannot be found'. normpath keeps the path in the snapshot dir where the .xml and .bin symlinks coexist. Verified: all 8 ov_smoke tests pass on openvino==2026.2.0 with this fix. --- tests/integration/inference/test_ov_smoke.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/integration/inference/test_ov_smoke.py b/tests/integration/inference/test_ov_smoke.py index 032635b2..837758ca 100644 --- a/tests/integration/inference/test_ov_smoke.py +++ b/tests/integration/inference/test_ov_smoke.py @@ -123,11 +123,15 @@ def _pkg_version(pkg: str) -> str: def _find_tokenizer_artifact(export_dir: Path) -> Path | None: - """Return the absolute path to the OVTokenizer artifact in *export_dir*. - - Parses ``manifest.json`` and looks for a preprocessor with - ``type == "ov_tokenizer"`` or a class_path containing ``OVTokenizer``. - Returns ``None`` when no tokenizer preprocessor is declared. + """Return the path to the OVTokenizer artifact in *export_dir*. + + Uses os.path.normpath (not Path.resolve) deliberately — same reasoning as + component_factory._resolve_artifact_path: HuggingFace Hub snapshot dirs + contain symlinks that point into a sibling blobs/ store. Resolving the + symlink would give the raw blob path which has no `.bin` sibling, causing + OpenVINO's read_model to fail with "Empty weights data". normpath keeps + the path inside the snapshot dir where the `.xml` and `.bin` symlinks + coexist. """ manifest = Manifest.load(export_dir) for spec in manifest.model.preprocessors: @@ -136,7 +140,7 @@ def _find_tokenizer_artifact(export_dir: Path) -> Path | None: continue artifact = spec.flat_params.get("artifact") or spec.init_args.get("artifact") if artifact: - return (export_dir / artifact).resolve() + return Path(os.path.normpath(export_dir / artifact)) return None From 9c3afbdf32ba493078ee1b59f4a39b32b32158e5 Mon Sep 17 00:00:00 2001 From: Piotr Wolnowski Date: Mon, 27 Jul 2026 15:35:09 +0200 Subject: [PATCH 3/5] chore: remove design doc from PR (tracked separately) --- docs/design/openvino-validation.md | 211 ----------------------------- 1 file changed, 211 deletions(-) delete mode 100644 docs/design/openvino-validation.md diff --git a/docs/design/openvino-validation.md b/docs/design/openvino-validation.md deleted file mode 100644 index 458b0aad..00000000 --- a/docs/design/openvino-validation.md +++ /dev/null @@ -1,211 +0,0 @@ -# OpenVINO Validation — Proposal - -**Status:** Draft proposal · **Owner:** physicalai team · **Date:** 2026-06-29 - -Proposal for integrating physicalai tests into OpenVINO's automated CI/CD. It -defines the validation contract, who owns what, the test tiers, and the target -platforms, frequency, and reporting for the OV pipeline. Piotr Wolnowski owns -the pipeline implementation and contributes test cases via the physicalai repo. - -## Coverage - -Three areas, mapped to the tiers below: - -| Area | Question | Tier | -| ------------------------------- | ----------------------------------------------- | ------------- | -| **Correctness + compatibility** | does physicalai load + run on this OV? | smoke | -| **Accuracy** | same numbers on target HW (per device/version)? | golden-action | -| **Functional** | does it still do the task end-to-end? | gym E2E | -| **Performance** | latency / throughput on target HW? | benchmark | - -## Why - -An OpenVINO upgrade broke physicalai inference — hence the current -`openvino==2026.1` pin in [`pyproject.toml`](../../pyproject.toml). We need an -automated signal that an OpenVINO change still works with physicalai _before_ we -adopt it. - -## The Contract - -"OpenVINO works with physicalai" means a real exported policy loads and runs -through the physicalai OpenVINO code paths. Three stages must pass: - -| Stage | Code path | What it proves | -| ------------- | ------------------------- | ---------------------------------------------------------------- | -| **load** | `OpenVINOAdapter.load` | `read_model` + `compile_model` succeed | -| **tokenizer** | `OVTokenizer` | `openvino_tokenizers` extension loads + matches core OpenVINO | -| **predict** | `OpenVINOAdapter.predict` | inference runs, outputs have expected shape/dtype and are finite | - -The tokenizer stage matters because `openvino` and `openvino_tokenizers` are -versioned **independently** and must stay a matched pair — a mismatch is what -broke us, and it only affects models that carry a tokenizer. - -## Who Owns What - -| Concern | Owner | -| ------------------------------------------------------------- | ------------------- | -| Test **logic + code location** (contract, stages, assertions) | **physicalai** repo | -| Pass/fail criteria | **physicalai** | -| PR merge gate | **physicalai** CI | -| Early-warning on OV bump PRs (Renovate-triggered) | **physicalai** CI | -| Running the same test in pre-release matrix | **OpenVINO** | -| Weekly + per-RC runs on target hardware | **OpenVINO** CI | -| Orchestration (workflow YAMLs, runners) + reporting | **OpenVINO** repo | -| Pre-release wheels + breakage notification | **OpenVINO** | - -**Decision:** split by layer. - -- **Test logic lives in physicalai** — this is the only way to guarantee its - quality and relevancy. Only physicalai knows its real OpenVINO usage, and the - test sits next to the code it protects, so it changes in the same PR as that - code. The OV team does not author test logic in-house. -- **Orchestration and reporting live in OpenVINO** — the workflow YAMLs, runners, - matrix, and dashboards are OV-side tooling. OV consumes the physicalai test by - pinned tag (`git clone physicalai@` → `pytest -m ov_smoke`). - -Test-case contributions from the OV team land as PRs into the -physicalai repo and are reviewed by physicalai, so logic stays single-sourced -and quality-controlled while OV helps build it. - -The test is self-contained (pytest + `ov_smoke` marker + model discovery via env -var), so OV can invoke it without pulling in physicalai internals. - -Same test, multiple runners: physicalai's PR gate and Renovate early-warning -catch _our_ changes and released OV bumps; OpenVINO's pre-release matrix catches -_their_ changes before release. - -**What OV runs:** smoke runs by pinned tag, so OV adds it to their pre-release -matrix. Golden-action runs on each device, comparing the OpenVINO output to a -stored PyTorch reference. Gym E2E uses the in-house LIBERO gyms on OV's GPU -runners. PAI maintains all tiers; OV provides the hardware. - -## Smoke Test - -A pytest integration test (marker `ov_smoke`) that runs the three contract -stages against discovered model exports and records the `openvino` / -`openvino_tokenizers` versions in each result. - -Coverage needs **two models** to cover the contract: - -| Model | Covers | -| ----------------------------------------- | -------------------------- | -| **ACT** (no tokenizer) | load + predict | -| **a tokenizer-bearing model** (e.g. pi05) | load + tokenizer + predict | - -Verified 2026-06-26: under a core/tokenizer mismatch, ACT passes and the -tokenizer model fails at the tokenizer stage — so this pair detects the exact -break that caused the pin. - -**Devices:** run the smoke across the device spectrum — CPU on hosted runners, -GPU on OV runners. CPU covers the version-pair contract; GPU catches -device-plugin regressions. - -## Early Warning - -physicalai CI runs `ov_smoke` when a Renovate PR bumps `openvino*`, against the -PR's resolved versions — event-driven, alert-only, fires exactly when a new -version appears. OV's weekly-vs-nightly and per-RC matrix (below) covers -pre-release wheels, so physicalai does not duplicate a weekly run. - -## Golden-Action - -A regression test that pins the OpenVINO output to a stored reference. It answers -a different question than smoke: smoke proves "it ran", golden proves it produced -the **right numbers**. - -**Approach:** - -1. **Export deterministically.** Build the test model with the PyTorch - deterministic-denoising toggle on, so the policy uses fixed (non-random) noise. - This removes the in-graph sampling that otherwise makes every inference - different (see below). -2. **Record one reference.** Run the source **PyTorch** model (e.g. on CPU) on a - fixed observation and store the action chunk as the ground-truth reference — - recorded once, not per device. -3. **Check every run.** Run the OpenVINO export on the target device for the same - observation and compare against the reference with an **L2 tolerance**. - -Exact equality only holds for a device compared against itself; across devices, -kernel and precision differences make the numbers drift, so the L2 tolerance -absorbs that drift while still catching a real regression. One PyTorch reference -covers the whole device matrix. - -**Why a deterministic export is needed (verified 2026-06-30).** Without it the -output is non-deterministic, and the cause is inside the exported graph — not in -physicalai's Python code, and not in any model input. The pi05 IR draws its -denoising noise from a single `RandomUniform` op exported with -`global_seed=0, op_seed=0`, which the OpenVINO spec defines as a non-deterministic -sequence; the stock model varies ~0.3 abs across identical calls. We confirmed the -noise can also be pinned at the IR level — seeding that op makes the OpenVINO -output reproducible bit-for-bit — which proves determinism is achievable, but -exporting deterministically from PyTorch is the cleaner source-level fix and also -makes the PyTorch reference itself stable. - -## Performance - -Latency / throughput on target hardware, built on the existing -`InferenceLatencyBenchmark`. Reports median/p99 per-iteration time and FPS per -model per device, tracked as a trend. Threshold-based and alert-only — hosted -variance makes absolute pass/fail unreliable, so flag large regressions, never -gate. Runs on the OV GPU runners across the platform matrix. - -## End-to-End (gym) - -Beyond smoke (loads + runs) and golden-action (same numbers), a gym rollout -checks the policy still **does the task** end-to-end: closed-loop -`reset → predict_action_chunk → step`, exercising the full pre/post pipeline and -the runtime action queue. - -- **Gyms exist in-house:** LIBERO for SmolVLA and pi05 (a newer gym is in - progress for later). -- **Reduced mode:** a small, fixed set of seeded episodes to bound runtime — a - pass/fail signal, not a full acceptance benchmark. -- **Where:** OV GPU runners, using the in-house LIBERO gyms. Reduced/seeded, - per-device reference, threshold-based, alert-only. Never on the PR gate — a - distinct tier from smoke. - -## Target Platforms - -All targets are GPU — the pipeline runs on OV-provided Intel hardware: - -| Platform | Type | -| -------- | --------------------------- | -| PTL iGPU | Panther Lake integrated GPU | -| B580 | Arc Battlemage discrete | -| B60 | Arc Battlemage discrete | -| B70 | Arc Battlemage discrete | - -CPU smoke stays on hosted runners as the portable baseline; GPU/accuracy/perf -tiers run across this matrix. - -## Frequency - -| Trigger | Scope | Gates? | -| ---------------------------- | ------------------------------------ | ------------ | -| OV bump PR / PAI PR | smoke (CPU) | yes | -| **Weekly** vs OV **nightly** | smoke + golden + perf, all platforms | alert | -| **Each RC** (pre-release) | full suite, all platforms | release gate | - -## Reporting - -Per-run record of OV / `openvino_tokenizers` versions, model, platform, tier, -and pass/fail + metrics. Failures notify the owner; perf/accuracy tracked as a -trend across versions. Dashboards and notification wiring are OV-side. - -## Next - -- Build the `ov_smoke` test (three contract stages, model discovery, version logging). -- Lock `openvino` + `openvino_tokenizers` as a matched pair in - [`pyproject.toml`](../../pyproject.toml) — they are versioned independently - and must be binary-compatible; constraining the tokenizer to the matching - minor stops a resolver from installing the mismatch that broke us. -- Add a small tokenizer-bearing model the gate can run — the tokenizer break - only shows on a model with an OV tokenizer; pi05 is 6.25 GB (too large for - hosted CI) and ACT has none, so the gate needs a small tokenizer model. -- Wire `ov_smoke` into [`library.yml`](../../.github/workflows/library.yml) as a - PR gate, plus the Renovate-triggered early-warning run. -- Add the golden-action tier — export the test model with deterministic denoising, - store a PyTorch reference, and L2-compare the OpenVINO output per device. -- Provision OV GPU runners (PTL iGPU, B580, B60, B70) for GPU smoke, perf, and gym E2E. -- Add the performance tier across the platform matrix. -- Review with Piotr Wolnowski; agree pipeline ownership + RC release gate. \ No newline at end of file From 12b99eee06fd76dc505ce6e8537d5a157b927654 Mon Sep 17 00:00:00 2001 From: Piotr Wolnowski Date: Mon, 27 Jul 2026 15:52:19 +0200 Subject: [PATCH 4/5] fix(ov_smoke): resolve CI type errors and np.bool deprecation - ov_tokenizer.py: np.bool -> np.bool_ (removed in NumPy 1.24) - test_ov_smoke.py: add isinstance(model.adapter, OpenVINOAdapter) so pyrefly can narrow RuntimeAdapter before accessing compiled_model (fixes missing-attribute error) - test_ov_smoke.py: suppress bad-argument-type for OVTokenizer call with list[str] TASK value (base Preprocessor signature is dict[str, ndarray] which does not reflect that OVTokenizer accepts list[str] for TASK) --- .../inference/preprocessors/ov_tokenizer.py | 2 +- tests/integration/inference/test_ov_smoke.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/physicalai/inference/preprocessors/ov_tokenizer.py b/src/physicalai/inference/preprocessors/ov_tokenizer.py index 0f91b0d2..790539ba 100644 --- a/src/physicalai/inference/preprocessors/ov_tokenizer.py +++ b/src/physicalai/inference/preprocessors/ov_tokenizer.py @@ -67,7 +67,7 @@ def __call__(self, inputs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: adapter_output = self._adapter.predict({self._input_name: batch_tasks}) outputs[TOKENIZED_PROMPT] = adapter_output["input_ids"] - outputs[TOKENIZED_PROMPT_MASK] = adapter_output["attention_mask"].astype(np.bool) + outputs[TOKENIZED_PROMPT_MASK] = adapter_output["attention_mask"].astype(np.bool_) return outputs diff --git a/tests/integration/inference/test_ov_smoke.py b/tests/integration/inference/test_ov_smoke.py index 837758ca..bb257ebb 100644 --- a/tests/integration/inference/test_ov_smoke.py +++ b/tests/integration/inference/test_ov_smoke.py @@ -38,20 +38,18 @@ import os from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any import numpy as np import pytest +from physicalai.inference.adapters import OpenVINOAdapter from physicalai.inference.constants import TASK, TOKENIZED_PROMPT, TOKENIZED_PROMPT_MASK from physicalai.inference.manifest import Manifest from physicalai.inference.model import InferenceModel from physicalai.inference.preprocessors import OVTokenizer from physicalai.inference.utils._hub import download_from_hub -if TYPE_CHECKING: - pass - log = logging.getLogger(__name__) # All tests in this module carry both markers. @@ -213,6 +211,11 @@ def test_load(all_exports: tuple[Path, _ModelSpec]) -> None: # Bypass preprocessors/postprocessors — isolates adapter.load() from the tokenizer stage. model = InferenceModel(export_dir=export_dir, device="CPU", preprocessors=[], postprocessors=[]) + # Cast to OpenVINOAdapter so static type checkers (pyrefly) can see compiled_model. + # The backend is always openvino for these exports so this assertion always holds. + assert isinstance(model.adapter, OpenVINOAdapter), ( + f"[{spec.short_id}] expected OpenVINOAdapter, got {type(model.adapter).__name__}" + ) assert model.adapter.compiled_model is not None, ( f"[{spec.short_id}] compiled_model is None after load — " "read_model or compile_model failed silently" @@ -260,6 +263,7 @@ def test_tokenizer(tokenizer_exports: tuple[Path, _ModelSpec]) -> None: tokenizer = OVTokenizer(tokenizer_path) # Run a single tokenisation to confirm the loaded model produces valid outputs. + # pyrefly: ignore [bad-argument-type] outputs = tokenizer({TASK: ["pick up the red block and place it in the bowl"]}) assert TOKENIZED_PROMPT in outputs, ( From 1b6bc8fa280e7cbf19f1fae7eac9ec3f3fe14df8 Mon Sep 17 00:00:00 2001 From: Piotr Wolnowski Date: Mon, 27 Jul 2026 16:00:51 +0200 Subject: [PATCH 5/5] fix(ov_smoke): add requires_download marker Tests download multi-GB artifacts via download_from_hub(); adding the marker lets developers and CI deselect them with -m 'not requires_download'. --- tests/integration/inference/test_ov_smoke.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/inference/test_ov_smoke.py b/tests/integration/inference/test_ov_smoke.py index bb257ebb..c470adfb 100644 --- a/tests/integration/inference/test_ov_smoke.py +++ b/tests/integration/inference/test_ov_smoke.py @@ -53,7 +53,7 @@ log = logging.getLogger(__name__) # All tests in this module carry both markers. -pytestmark = [pytest.mark.ov_smoke, pytest.mark.integration] +pytestmark = [pytest.mark.ov_smoke, pytest.mark.integration, pytest.mark.requires_download] # Skip the entire module when OpenVINO is not installed — these tests require # the real runtime, not a mock.