diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index beace27..3171f8f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,12 @@ jobs: assert schematron_registry.count_rules() > 0 PY + - name: Smoke test packaged oracle dispatcher + run: | + for engine in word excel pptx odf gsuite preflight eurooffice; do + .release-venv/bin/openxml-audit-oracle "$engine" --help >/dev/null + done + - name: Upload dist artifacts uses: actions/upload-artifact@v6 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index b6be299..8ca7dc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.8] - 2026-08-09 + +Euro-Office conversion evidence and a packaging repair for the public oracle +dispatcher. PyPI installs can now run every dispatcher engine outside a source +checkout, including the new authenticated Euro-Office conversion oracle. + +### Added +- **Euro-Office conversion oracle (Spec 038)** — a dependency-free + `openxml_audit.eurooffice` client and `openxml-audit-oracle eurooffice` + engine for the current Document Server `/converter` contract. The format + model is pinned to Nextcloud connector 11.0.1 and document-formats 3.2.0: + native OOXML editing, lossy ODT/ODS/ODP-to-OOXML editing, and ODG + view/conversion only. JWT secrets are environment-only, returned OOXML is + validated, and same-format conversions receive canonical per-part diffs. + Reports state their limit explicitly: this proves conversion, not browser + editing, save callbacks, or coauthoring. The first six-file live baseline + records two validator-clean native conversions, three returned packages with + validator findings (including unchanged source findings for PPTX), and one + ODP conversion failure; it also exposes the live 9.3.1.37 versus upstream + 9.3.3 deployment-version gap. + +### Fixed +- **Packaged oracle dispatcher** — wheels now include the `tools.oracle` + implementation package used by `openxml-audit-oracle`. PyPI installs no + longer fail with `ModuleNotFoundError: No module named 'tools'` when an + engine is selected outside a repository checkout. + ## [0.7.7] - 2026-07-20 A reported bug that turned out to be the visible corner of a larger diff --git a/README.md b/README.md index 60650e2..ed10438 100644 --- a/README.md +++ b/README.md @@ -365,6 +365,58 @@ The report classifies each roundtrip across a `LossClass` taxonomy: Drive uploads are deleted in `finally` after each roundtrip — the oracle never leaves files in your account. +## Euro-Office Conversion Oracle + +The `eurooffice` engine exercises Euro-Office Document Server's +synchronous `/converter` endpoint, downloads the result, validates the +returned OOXML package, and records a canonical per-part diff when the +source and target formats match. + +The capability model is pinned to Nextcloud connector **11.0.1** and +its `ONLYOFFICE/document-formats` **3.2.0** dependency: + +| Connector behavior | Formats | Conversion target | +|---|---|---| +| Native edit | DOCX/DOCM/DOTX/DOTM, XLSX/XLSM/XLTX/XLTM, PPTX/PPTM/POTX/POTM/PPSX/PPSM | Same format | +| Lossy edit | ODT/OTT, ODS/OTS, ODP/OTP | DOCX, XLSX, PPTX | +| View/convert only | ODG | PPTX | + +This distinction matters: an ODT shown as editable by the connector is +converted to DOCX before editing, while ODG is not registered as an +editable format. See +[`specs/038-eurooffice-conversion-oracle.md`](specs/038-eurooffice-conversion-oracle.md) +for the pinned sources and evidence limits. + +Configure the endpoint and JWT through environment variables. The +secret deliberately has no CLI flag, keeping it out of shell history +and process listings: + +```bash +export EUROOFFICE_ORACLE_URL=https://office.example.test/ +export EUROOFFICE_ORACLE_JWT_SECRET='read-from-your-secret-store' + +# URL inputs can be fetched directly by Document Server. +openxml-audit-oracle eurooffice \ + https://files.example.test/sample.odt \ + --output eurooffice-report.json + +# Local inputs require a public base URL that serves their basenames. +export EUROOFFICE_ORACLE_SOURCE_BASE_URL=https://files.example.test/corpus/ +openxml-audit-oracle eurooffice ./corpus/ --output eurooffice-report.json +``` + +`euro-office` and `euro` are aliases. A successful observation proves +the Document Server fetch/conversion path and output validity. It does +not exercise the browser editor, Nextcloud save callback, or +coauthoring, so it is not an end-to-end editing claim. + +The first live six-file snapshot is committed at +[`tools/oracle/baselines/eurooffice/2026-08-09.json`](tools/oracle/baselines/eurooffice/2026-08-09.json). +It records the live server's reported 9.3.1.37 build separately from the +current 9.3.3 upstream release, plus validator-clean DOCX/XLSX conversion, +pre-existing PPTX findings, ODF-target findings, and the observed ODP +conversion failure. + ## Open XML SDK (Standalone) Run the .NET SDK validator separately (requires .NET SDK 8.x or Docker): diff --git a/pyproject.toml b/pyproject.toml index 7b2eb43..f4d0b6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "openxml-audit" -version = "0.7.7" +version = "0.7.8" description = "Validate OOXML and ODF files in pure Python — no .NET required" readme = "README.md" license = {file = "LICENSE"} @@ -76,6 +76,10 @@ packages = ["src/openxml_audit"] [tool.hatch.build.targets.wheel.force-include] "data/openxml" = "openxml_audit/data/openxml" "data/pptx_oracle" = "openxml_audit/data/pptx_oracle" +# The packaged dispatcher imports ``tools.oracle.*``. Keep that implementation +# package in the wheel as well as the thin entry point under ``openxml_audit``; +# otherwise ``openxml-audit-oracle `` only works from a repository checkout. +"tools/oracle" = "tools/oracle" [tool.ruff] target-version = "py310" diff --git a/specs/038-eurooffice-conversion-oracle.md b/specs/038-eurooffice-conversion-oracle.md new file mode 100644 index 0000000..f4d14a6 --- /dev/null +++ b/specs/038-eurooffice-conversion-oracle.md @@ -0,0 +1,138 @@ +# Spec 038: Euro-Office Conversion Oracle + +## Status + +Implemented (August 9, 2026): pinned format model, authenticated client, +packaged dispatcher engine, unit contract tests, and live conversion +baseline. The live server reported 9.3.1.37; upstream 9.3.3 is modeled and +recorded separately rather than mislabeling the snapshot. + +## Problem + +`openxml-audit` has local application oracles for Microsoft Office and +LibreOffice, plus a Google Workspace import/export oracle. The Opus95 office +path now uses Euro-Office Document Server with the Nextcloud connector, but the +repository had no executable description of that connector's current format +semantics and no way to capture conversion-path evidence. + +Calling ODF simply "editable" is misleading. The connector converts ODT, ODS, +and ODP to OOXML before editing; ODG is viewable and convertible but is not an +editable format. The oracle must preserve those distinctions and must not turn +a conversion API check into a claim about the browser editor. + +## Pinned Upstream Contract + +This implementation is calibrated to: + +- Euro-Office/Document Server 9.3.3 (released August 4, 2026) +- ONLYOFFICE Nextcloud connector 11.0.1 (released June 30, 2026) +- `ONLYOFFICE/document-formats` 3.2.0, connector-pinned commit + `7d7576a3fe2337c30f4c9b40fae70a69dc68ba08` + +Primary upstream references: + +- +- +- + +The format matrix relevant to this validator is: + +| Mode | Word | Cell | Slide | +|---|---|---|---| +| Native edit | DOCX, DOCM, DOTX, DOTM | XLSX, XLSM, XLTX, XLTM | PPTX, PPTM, POTX, POTM, PPSX, PPSM | +| Lossy edit | ODT, OTT -> DOCX | ODS, OTS -> XLSX | ODP, OTP -> PPTX | +| View/convert only | - | - | ODG -> PPTX | + +`src/openxml_audit/eurooffice/formats.py` is the machine-readable version of +this table. Formats outside it are reported as `unsupported`; the oracle does +not infer capability from a file merely being a ZIP/XML container. + +## Client Contract + +`EuroOfficeClient` uses only the Python standard library and implements four +operations: + +1. `GET /healthcheck` +2. `POST /coauthoring/CommandService.ashx` with `{"c":"version"}` +3. `POST /converter?shardKey=` with the connector-compatible synchronous + conversion body +4. download of the returned `fileUrl` + +When `EUROOFFICE_ORACLE_JWT_SECRET` is set, requests receive both forms used by +connector 11.0.1: the configured header token (Authorization/Bearer by default) +and the signed `token` field in the JSON body. HS256 is implemented with +stdlib `hmac`; no JWT dependency is added. + +Security rules: + +- secrets have no command-line option; +- error messages omit response bodies and URL query strings; +- reports never contain JWTs, response URLs, or the source URL; +- conversion keys are deterministic SHA-256 prefixes, restricted to the + Document Server key character/length contract. + +Unsigned requests remain possible when the target server has JWT disabled. + +## Oracle Semantics + +For each supported input, `tools/oracle/eurooffice_conversion_oracle.py`: + +1. resolves a URL that Document Server can fetch; +2. calls synchronous conversion using the matrix target; +3. downloads the result; +4. validates the returned OOXML package with `OpenXmlValidator`; +5. for same-format conversion, runs `package_diff.compare_packages` over all + XML and relationship parts; +6. emits a JSON observation and aggregate outcome counts. + +Outcomes: + +- `preserved`: same-format output has no canonical XML/relationship diff; +- `rewritten`: same-format output is valid but canonical parts changed; +- `converted`: a cross-format target is valid; +- `unsupported`: the pinned connector matrix has no matching capability; +- `request_failed`, `download_failed`, `invalid_output`, + `source_unavailable`: operational failures, all producing exit status 1. + +Unsupported capability is data, not a run failure. Native OOXML, lossy ODF, +and ODG view-only modes remain separate report fields even when conversion +succeeds. + +## Source Reachability + +Document Server fetches the input itself. HTTP(S) inputs can therefore be +passed directly. A local input needs `--source-base-url` or +`EUROOFFICE_ORACLE_SOURCE_BASE_URL`; the server must be able to retrieve the +local file's URL-escaped basename from that base URL. The oracle copies the +local bytes only for validation and diffing—it does not start an HTTP server or +upload source files. + +## Evidence Boundary + +This is a **conversion-path oracle**, not a roundtrip editor oracle. It verifies +Document Server health/version, remote fetch, conversion, artifact download, +and static output validity. It does not exercise: + +- Nextcloud discovery or connector configuration; +- browser editor boot and WebSocket/coauthoring paths; +- user interaction, comments, macros, or collaborative edits; +- the Nextcloud callback and persistence of an edited file; +- visual or semantic fidelity after conversion. + +A full user-path claim still requires a real browser/editor open-edit-save +check against Nextcloud. The report embeds this limitation in +`evidence_scope` so downstream summaries cannot silently erase it. + +## Acceptance + +- Exact positive and negative format-matrix tests. +- JWT envelope, health/version, conversion-error, and secret non-leakage tests. +- Network-free oracle tests for native preservation, lossy ODF conversion, ODG + view-only conversion, unsupported formats, and report scope. +- `eurooffice`, `euro-office`, and `euro` dispatcher routes work from an + installed wheel. +- A built sdist-to-wheel artifact contains `tools.oracle` and all dispatcher + engines import outside the checkout. +- Live health, version, and representative OOXML/ODF conversions are recorded + in `tools/oracle/baselines/eurooffice/2026-08-09.json` without committing + credentials; failures and version drift remain visible evidence. diff --git a/src/openxml_audit/__init__.py b/src/openxml_audit/__init__.py index 9c76811..7c2560c 100644 --- a/src/openxml_audit/__init__.py +++ b/src/openxml_audit/__init__.py @@ -60,7 +60,7 @@ from openxml_audit.validator import OpenXmlValidator, is_valid_pptx, validate_pptx from openxml_audit.verdict import AppPrediction, AppVerdict, predict -__version__ = "0.7.7" +__version__ = "0.7.8" __all__ = [ # Main API diff --git a/src/openxml_audit/eurooffice/__init__.py b/src/openxml_audit/eurooffice/__init__.py new file mode 100644 index 0000000..66a9121 --- /dev/null +++ b/src/openxml_audit/eurooffice/__init__.py @@ -0,0 +1,35 @@ +"""Euro-Office format capabilities and Document Server client.""" + +from openxml_audit.eurooffice.client import ( + ConversionResult, + EuroOfficeClient, + EuroOfficeConfigError, + EuroOfficeConversionError, + EuroOfficeError, + EuroOfficeRequestError, +) +from openxml_audit.eurooffice.formats import ( + CONNECTOR_VERSION, + DOCUMENT_FORMATS_COMMIT, + DOCUMENT_FORMATS_VERSION, + DOCUMENT_SERVER_RELEASE, + EuroOfficeFormatMode, + EuroOfficeFormatSupport, + format_support, +) + +__all__ = [ + "CONNECTOR_VERSION", + "DOCUMENT_SERVER_RELEASE", + "DOCUMENT_FORMATS_COMMIT", + "DOCUMENT_FORMATS_VERSION", + "ConversionResult", + "EuroOfficeClient", + "EuroOfficeConfigError", + "EuroOfficeConversionError", + "EuroOfficeError", + "EuroOfficeFormatMode", + "EuroOfficeFormatSupport", + "EuroOfficeRequestError", + "format_support", +] diff --git a/src/openxml_audit/eurooffice/client.py b/src/openxml_audit/eurooffice/client.py new file mode 100644 index 0000000..9c00953 --- /dev/null +++ b/src/openxml_audit/eurooffice/client.py @@ -0,0 +1,284 @@ +"""Small, dependency-free client for the Euro-Office Document Server API. + +The request shape mirrors Nextcloud connector 11.0.1. JWT credentials are +read from the environment by :meth:`EuroOfficeClient.from_env`; the oracle +does not accept secrets on its command line, where process listings and shell +history could expose them. +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any, cast + + +class EuroOfficeError(Exception): + """Base class for Euro-Office client failures.""" + + +class EuroOfficeConfigError(EuroOfficeError): + """Missing or invalid client configuration.""" + + +class EuroOfficeRequestError(EuroOfficeError): + """Transport or malformed-response failure.""" + + +class EuroOfficeConversionError(EuroOfficeError): + """Document Server rejected or failed a conversion.""" + + +@dataclass(frozen=True) +class ConversionResult: + """Successful synchronous conversion response.""" + + file_url: str + percent: int | None = None + end_convert: bool | None = None + + +_CONVERSION_ERRORS = { + -1: "unknown conversion error", + -2: "conversion timed out", + -3: "conversion failed", + -4: "source document could not be downloaded", + -5: "incorrect document password", + -6: "conversion database error", + -7: "invalid conversion input", + -8: "invalid JWT token", +} + +_USER_AGENT = "openxml-audit-eurooffice-oracle/1" + + +def _base64url(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _encode_jwt(payload: dict[str, Any], secret: str) -> str: + """Encode an HS256 JWT without adding a third-party dependency.""" + + header = {"alg": "HS256", "typ": "JWT"} + header_part = _base64url(json.dumps(header, separators=(",", ":")).encode()) + payload_part = _base64url(json.dumps(payload, separators=(",", ":")).encode()) + signing_input = f"{header_part}.{payload_part}".encode("ascii") + signature = hmac.new(secret.encode(), signing_input, hashlib.sha256).digest() + return f"{header_part}.{payload_part}.{_base64url(signature)}" + + +def _endpoint_label(url: str) -> str: + parsed = urllib.parse.urlsplit(url) + return parsed.path or "/" + + +class EuroOfficeClient: + """Client for health, version, conversion, and artifact download calls.""" + + def __init__( + self, + server_url: str, + *, + jwt_secret: str | None = None, + jwt_header: str = "Authorization", + timeout: float = 120.0, + ) -> None: + if not server_url.strip(): + raise EuroOfficeConfigError("Euro-Office server URL is required") + parsed = urllib.parse.urlsplit(server_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise EuroOfficeConfigError("Euro-Office server URL must use HTTP(S)") + if not jwt_header.strip(): + raise EuroOfficeConfigError("JWT header name cannot be empty") + if timeout <= 0: + raise EuroOfficeConfigError("timeout must be greater than zero") + + self._server_url = server_url.rstrip("/") + "/" + self._jwt_secret = jwt_secret + self._jwt_header = jwt_header + self._timeout = timeout + + @property + def server_url(self) -> str: + """Normalized, non-secret server URL.""" + + return self._server_url + + @classmethod + def from_env( + cls, + server_url: str | None = None, + *, + timeout: float = 120.0, + ) -> EuroOfficeClient: + """Build a client from explicit URL plus secret-safe environment vars.""" + + resolved_url = server_url or os.environ.get("EUROOFFICE_ORACLE_URL") + if not resolved_url: + raise EuroOfficeConfigError("set EUROOFFICE_ORACLE_URL or pass --server-url") + return cls( + resolved_url, + jwt_secret=os.environ.get("EUROOFFICE_ORACLE_JWT_SECRET"), + jwt_header=os.environ.get("EUROOFFICE_ORACLE_JWT_HEADER", "Authorization"), + timeout=timeout, + ) + + def _url(self, relative: str) -> str: + return urllib.parse.urljoin(self._server_url, relative) + + def _request( + self, + url: str, + *, + body: bytes | None = None, + headers: dict[str, str] | None = None, + decode_json: bool = False, + ) -> Any: + method = "POST" if body is not None else "GET" + request_headers = {"User-Agent": _USER_AGENT} + request_headers.update(headers or {}) + request = urllib.request.Request(url, data=body, method=method, headers=request_headers) + try: + with urllib.request.urlopen(request, timeout=self._timeout) as response: + payload = response.read() + except urllib.error.HTTPError as exc: + # Do not echo response bodies or signed query strings: either can contain + # credentials supplied by the server or upstream source URL. + raise EuroOfficeRequestError( + f"Euro-Office {_endpoint_label(url)} returned HTTP {exc.code}" + ) from exc + except urllib.error.URLError as exc: + raise EuroOfficeRequestError( + f"Euro-Office {_endpoint_label(url)} request failed: {exc.reason}" + ) from exc + + if not decode_json: + return payload + try: + decoded = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise EuroOfficeRequestError( + f"Euro-Office {_endpoint_label(url)} returned invalid JSON" + ) from exc + if not isinstance(decoded, dict): + raise EuroOfficeRequestError( + f"Euro-Office {_endpoint_label(url)} returned a non-object response" + ) + return decoded + + def _authenticated_payload( + self, + data: dict[str, Any], + ) -> tuple[dict[str, Any], dict[str, str]]: + headers = {"Content-Type": "application/json"} + if not self._jwt_secret: + return data, headers + + now = int(time.time()) + expires = now + 300 + body_data = {**data, "iat": now, "exp": expires} + body_data["token"] = _encode_jwt(body_data, self._jwt_secret) + outer_token = _encode_jwt( + {"payload": body_data, "iat": now, "exp": expires}, + self._jwt_secret, + ) + prefix = "Bearer " if self._jwt_header.lower() == "authorization" else "" + headers[self._jwt_header] = f"{prefix}{outer_token}" + return body_data, headers + + def healthcheck(self) -> bool: + """Return whether ``/healthcheck`` reports a healthy server.""" + + payload = cast(bytes, self._request(self._url("healthcheck"))) + return payload.decode("utf-8", errors="replace").strip().lower() == "true" + + def version(self) -> str: + """Return the Document Server version from CommandService.""" + + data, headers = self._authenticated_payload({"c": "version"}) + payload = self._request( + self._url("coauthoring/CommandService.ashx"), + body=json.dumps(data, separators=(",", ":")).encode(), + headers=headers, + decode_json=True, + ) + error = payload.get("error") + if isinstance(error, int) and error != 0: + raise EuroOfficeRequestError(f"Euro-Office version request returned error {error}") + version = payload.get("version") + if not isinstance(version, str) or not version: + raise EuroOfficeRequestError("Euro-Office version response omitted version") + build = payload.get("buildNumber") + return f"{version}.{build}" if isinstance(build, int | str) and str(build) else version + + def convert( + self, + *, + source_url: str, + source_format: str, + target_format: str, + key: str, + title: str | None = None, + ) -> ConversionResult: + """Synchronously convert a URL-addressable document.""" + + if not 1 <= len(key) <= 20 or not all(char.isalnum() or char in "-._=" for char in key): + raise EuroOfficeConfigError( + "conversion key must be 1-20 ASCII letters, digits, or -._=" + ) + source_format = source_format.lower().lstrip(".") + target_format = target_format.lower().lstrip(".") + data: dict[str, Any] = { + "async": False, + "url": source_url, + "outputtype": target_format, + "filetype": source_format, + "title": title or f"document.{source_format}", + "key": key, + } + request_data, headers = self._authenticated_payload(data) + converter_url = self._url("converter?" + urllib.parse.urlencode({"shardKey": key})) + payload = self._request( + converter_url, + body=json.dumps(request_data, separators=(",", ":")).encode(), + headers=headers, + decode_json=True, + ) + + error = payload.get("error", 0) + if isinstance(error, int) and error != 0: + message = _CONVERSION_ERRORS.get(error, "unrecognized conversion error") + raise EuroOfficeConversionError(f"Euro-Office conversion error {error}: {message}") + file_url = payload.get("fileUrl") + if not isinstance(file_url, str) or not file_url: + raise EuroOfficeRequestError("Euro-Office conversion response omitted fileUrl") + percent = payload.get("percent") + end_convert = payload.get("endConvert") + return ConversionResult( + file_url=file_url, + percent=percent if isinstance(percent, int) else None, + end_convert=end_convert if isinstance(end_convert, bool) else None, + ) + + def download(self, file_url: str) -> bytes: + """Download a conversion artifact returned by Document Server.""" + + return cast(bytes, self._request(file_url)) + + +__all__ = [ + "ConversionResult", + "EuroOfficeClient", + "EuroOfficeConfigError", + "EuroOfficeConversionError", + "EuroOfficeError", + "EuroOfficeRequestError", +] diff --git a/src/openxml_audit/eurooffice/formats.py b/src/openxml_audit/eurooffice/formats.py new file mode 100644 index 0000000..32b4424 --- /dev/null +++ b/src/openxml_audit/eurooffice/formats.py @@ -0,0 +1,137 @@ +"""Euro-Office/ONLYOFFICE connector format capability matrix. + +The matrix is pinned to Nextcloud connector 11.0.1 and its +ONLYOFFICE/document-formats 3.2.0 dependency. It deliberately distinguishes +native OOXML editing from ODF editing through lossy conversion: an ``.odt`` +that appears editable in Nextcloud is converted to ``.docx`` before editing. +``.odg`` is available for viewing/conversion, but is not an editable format. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from urllib.parse import urlparse + +CONNECTOR_VERSION = "11.0.1" +DOCUMENT_SERVER_RELEASE = "9.3.3" +DOCUMENT_FORMATS_VERSION = "3.2.0" +DOCUMENT_FORMATS_COMMIT = "7d7576a3fe2337c30f4c9b40fae70a69dc68ba08" + + +class EuroOfficeFormatMode(str, Enum): + """How the current connector exposes a format.""" + + NATIVE_EDIT = "native-edit" + LOSSY_EDIT = "lossy-edit" + VIEW_ONLY = "view-only" + UNSUPPORTED = "unsupported" + + +@dataclass(frozen=True) +class EuroOfficeFormatSupport: + """Connector capability for one filename extension.""" + + extension: str + document_type: str + mode: EuroOfficeFormatMode + conversion_target: str | None + actions: tuple[str, ...] + + +_NATIVE_EDIT: dict[str, str] = { + "docx": "word", + "docm": "word", + "dotx": "word", + "dotm": "word", + "xlsx": "cell", + "xlsm": "cell", + "xltx": "cell", + "xltm": "cell", + "pptx": "slide", + "pptm": "slide", + "potx": "slide", + "potm": "slide", + "ppsx": "slide", + "ppsm": "slide", +} + +_LOSSY_EDIT: dict[str, tuple[str, str]] = { + "odt": ("word", "docx"), + "ott": ("word", "docx"), + "ods": ("cell", "xlsx"), + "ots": ("cell", "xlsx"), + "odp": ("slide", "pptx"), + "otp": ("slide", "pptx"), +} + +_VIEW_ONLY: dict[str, tuple[str, str]] = { + "odg": ("slide", "pptx"), +} + + +def _extension(value: str | Path) -> str: + raw = str(value).strip() + if not raw: + return "" + if raw.startswith(".") and "/" not in raw and "\\" not in raw: + return raw[1:].lower() + + parsed_path = urlparse(raw).path if "://" in raw else raw + suffix = Path(parsed_path).suffix + if suffix: + return suffix[1:].lower() + if "/" not in raw and "\\" not in raw: + return raw.lower() + return "" + + +def format_support(value: str | Path) -> EuroOfficeFormatSupport: + """Return the pinned connector capability for a path or extension.""" + + extension = _extension(value) + if extension in _NATIVE_EDIT: + return EuroOfficeFormatSupport( + extension=extension, + document_type=_NATIVE_EDIT[extension], + mode=EuroOfficeFormatMode.NATIVE_EDIT, + conversion_target=extension, + actions=("view", "edit"), + ) + if extension in _LOSSY_EDIT: + document_type, target = _LOSSY_EDIT[extension] + return EuroOfficeFormatSupport( + extension=extension, + document_type=document_type, + mode=EuroOfficeFormatMode.LOSSY_EDIT, + conversion_target=target, + actions=("view", "edit", "auto-convert"), + ) + if extension in _VIEW_ONLY: + document_type, target = _VIEW_ONLY[extension] + return EuroOfficeFormatSupport( + extension=extension, + document_type=document_type, + mode=EuroOfficeFormatMode.VIEW_ONLY, + conversion_target=target, + actions=("view", "auto-convert"), + ) + return EuroOfficeFormatSupport( + extension=extension, + document_type="unknown", + mode=EuroOfficeFormatMode.UNSUPPORTED, + conversion_target=None, + actions=(), + ) + + +__all__ = [ + "CONNECTOR_VERSION", + "DOCUMENT_SERVER_RELEASE", + "DOCUMENT_FORMATS_COMMIT", + "DOCUMENT_FORMATS_VERSION", + "EuroOfficeFormatMode", + "EuroOfficeFormatSupport", + "format_support", +] diff --git a/src/openxml_audit/oracle/__main__.py b/src/openxml_audit/oracle/__main__.py index 53705be..b72c20e 100644 --- a/src/openxml_audit/oracle/__main__.py +++ b/src/openxml_audit/oracle/__main__.py @@ -7,6 +7,7 @@ python -m openxml_audit.oracle pptx FILES... [--output X.json] python -m openxml_audit.oracle odf FILES... [--output X.json] python -m openxml_audit.oracle gsuite FILES... [--output X.json] + python -m openxml_audit.oracle eurooffice FILES... [--output X.json] Each subcommand defers to the format's existing CLI in `tools/oracle/`. This module is a thin dispatcher introduced in 0.6.8 @@ -17,6 +18,11 @@ domain-wide delegation configured — see `specs/031-gsuite-roundtrip-oracle.md`. +The `eurooffice` engine (aliases `euro-office` and `euro`) exercises +Euro-Office Document Server's conversion endpoint. It reads JWT +credentials from environment variables only; see +`specs/038-eurooffice-conversion-oracle.md`. + Use `python -m openxml_audit.oracle preflight` to run the macOS permission / install check across the desktop-app engines before a corpus walk. @@ -24,11 +30,9 @@ from __future__ import annotations -import argparse import sys from pathlib import Path - _REPO_ROOT = Path(__file__).resolve().parents[3] _TOOLS_DIR = _REPO_ROOT / "tools" @@ -80,6 +84,14 @@ def _run_gsuite(args: list[str]) -> int: return gsuite_main() +def _run_eurooffice(args: list[str]) -> int: + _ensure_tools_on_path() + from tools.oracle.eurooffice_conversion_oracle import main as eurooffice_main + + sys.argv = ["eurooffice_conversion_oracle.py", *args] + return eurooffice_main() + + def _run_preflight(args: list[str]) -> int: _ensure_tools_on_path() from tools.oracle.preflight import main as preflight_main @@ -97,6 +109,9 @@ def _run_preflight(args: list[str]) -> int: "odf": _run_odf, "gsuite": _run_gsuite, "google": _run_gsuite, # alias + "eurooffice": _run_eurooffice, + "euro-office": _run_eurooffice, # alias + "euro": _run_eurooffice, # alias "preflight": _run_preflight, } @@ -104,8 +119,10 @@ def _run_preflight(args: list[str]) -> int: def main() -> int: if len(sys.argv) < 2 or sys.argv[1] in {"-h", "--help"}: print(__doc__, file=sys.stderr) - print("\nAvailable engines: " + ", ".join(sorted(_DISPATCH.keys())), - file=sys.stderr) + print( + "\nAvailable engines: " + ", ".join(sorted(_DISPATCH.keys())), + file=sys.stderr, + ) return 0 if len(sys.argv) >= 2 else 2 engine = sys.argv[1].lower() diff --git a/tests/test_eurooffice.py b/tests/test_eurooffice.py new file mode 100644 index 0000000..00b63bd --- /dev/null +++ b/tests/test_eurooffice.py @@ -0,0 +1,194 @@ +"""Euro-Office format matrix and Document Server client contract tests.""" + +from __future__ import annotations + +import base64 +import json +import urllib.error +import urllib.request +from typing import Any + +import pytest + +from openxml_audit.eurooffice import ( + EuroOfficeClient, + EuroOfficeConversionError, + EuroOfficeFormatMode, + EuroOfficeRequestError, + format_support, +) +from openxml_audit.eurooffice.client import _encode_jwt + + +@pytest.mark.parametrize( + ("extension", "document_type"), + [ + *((extension, "word") for extension in ("docx", "docm", "dotx", "dotm")), + *((extension, "cell") for extension in ("xlsx", "xlsm", "xltx", "xltm")), + *((extension, "slide") for extension in ("pptx", "pptm", "potx", "potm", "ppsx", "ppsm")), + ], +) +def test_native_edit_matrix(extension: str, document_type: str) -> None: + support = format_support(extension) + assert support.document_type == document_type + assert support.mode is EuroOfficeFormatMode.NATIVE_EDIT + assert support.conversion_target == extension + assert support.actions == ("view", "edit") + + +@pytest.mark.parametrize( + ("extension", "document_type", "target"), + [ + ("odt", "word", "docx"), + ("ott", "word", "docx"), + ("ods", "cell", "xlsx"), + ("ots", "cell", "xlsx"), + ("odp", "slide", "pptx"), + ("otp", "slide", "pptx"), + ], +) +def test_lossy_edit_matrix(extension: str, document_type: str, target: str) -> None: + support = format_support(f"EXAMPLE.{extension.upper()}") + assert support.document_type == document_type + assert support.mode is EuroOfficeFormatMode.LOSSY_EDIT + assert support.conversion_target == target + assert support.actions == ("view", "edit", "auto-convert") + + +def test_odg_is_view_convert_only() -> None: + support = format_support("https://files.example.test/drawing.ODG?download=1") + assert support.mode is EuroOfficeFormatMode.VIEW_ONLY + assert support.conversion_target == "pptx" + assert support.actions == ("view", "auto-convert") + + +@pytest.mark.parametrize("extension", ["odm", "xlam", "thmx", "ppam", "txt", ""]) +def test_unsupported_formats_are_explicit(extension: str) -> None: + support = format_support(extension) + assert support.mode is EuroOfficeFormatMode.UNSUPPORTED + assert support.conversion_target is None + assert support.actions == () + + +def _decode_jwt_part(token: str, part: int) -> dict[str, Any]: + encoded = token.split(".")[part] + padding = "=" * (-len(encoded) % 4) + return json.loads(base64.urlsafe_b64decode(encoded + padding)) + + +class _Response: + def __init__(self, payload: bytes) -> None: + self._payload = payload + + def __enter__(self) -> _Response: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self) -> bytes: + return self._payload + + +def test_convert_matches_connector_jwt_request_shape(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, Any] = {} + + def fake_urlopen(request: urllib.request.Request, *, timeout: float) -> _Response: + captured["request"] = request + captured["timeout"] = timeout + return _Response( + json.dumps({"error": 0, "fileUrl": "https://result.test/out.docx"}).encode() + ) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + monkeypatch.setattr("openxml_audit.eurooffice.client.time.time", lambda: 1_700_000_000) + client = EuroOfficeClient( + "https://office.example.test/", + jwt_secret="test-secret", + timeout=42, + ) + result = client.convert( + source_url="https://files.example.test/input.odt", + source_format="odt", + target_format="docx", + key="0123456789abcdef0123", + title="input.odt", + ) + + request = captured["request"] + assert isinstance(request, urllib.request.Request) + assert request.full_url == ( + "https://office.example.test/converter?shardKey=0123456789abcdef0123" + ) + assert captured["timeout"] == 42 + assert request.get_method() == "POST" + assert request.headers["User-agent"] == "openxml-audit-eurooffice-oracle/1" + assert request.headers["Content-type"] == "application/json" + assert request.headers["Authorization"].startswith("Bearer ") + + body = json.loads(request.data) + assert body["async"] is False + assert body["filetype"] == "odt" + assert body["outputtype"] == "docx" + assert body["iat"] == 1_700_000_000 + assert body["exp"] == 1_700_000_300 + body_claims = {key: value for key, value in body.items() if key != "token"} + assert body["token"] == _encode_jwt(body_claims, "test-secret") + + outer = request.headers["Authorization"].removeprefix("Bearer ") + outer_claims = _decode_jwt_part(outer, 1) + assert outer_claims == { + "payload": body, + "iat": 1_700_000_000, + "exp": 1_700_000_300, + } + assert result.file_url == "https://result.test/out.docx" + + +def test_healthcheck_and_version(monkeypatch: pytest.MonkeyPatch) -> None: + responses = iter( + [ + _Response(b"true"), + _Response(json.dumps({"error": 0, "version": "9.3.3", "buildNumber": 8}).encode()), + ] + ) + monkeypatch.setattr(urllib.request, "urlopen", lambda *_args, **_kwargs: next(responses)) + client = EuroOfficeClient("https://office.example.test") + assert client.healthcheck() + assert client.version() == "9.3.3.8" + + +def test_conversion_error_is_classified(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + urllib.request, + "urlopen", + lambda *_args, **_kwargs: _Response(b'{"error":-4}'), + ) + client = EuroOfficeClient("https://office.example.test") + with pytest.raises(EuroOfficeConversionError, match="could not be downloaded"): + client.convert( + source_url="https://files.example.test/missing.odt", + source_format="odt", + target_format="docx", + key="valid-key", + ) + + +def test_http_error_does_not_leak_query_or_body(monkeypatch: pytest.MonkeyPatch) -> None: + secret = "do-not-leak-this" + + def fail(*_args: object, **_kwargs: object) -> _Response: + raise urllib.error.HTTPError( + f"https://result.test/file?token={secret}", + 403, + "Forbidden", + {}, + None, + ) + + monkeypatch.setattr(urllib.request, "urlopen", fail) + client = EuroOfficeClient("https://office.example.test", jwt_secret=secret) + with pytest.raises(EuroOfficeRequestError) as caught: + client.download(f"https://result.test/file?token={secret}") + assert secret not in str(caught.value) + assert "?" not in str(caught.value) diff --git a/tests/test_eurooffice_oracle.py b/tests/test_eurooffice_oracle.py new file mode 100644 index 0000000..81b60c0 --- /dev/null +++ b/tests/test_eurooffice_oracle.py @@ -0,0 +1,146 @@ +"""Tests for the Euro-Office conversion oracle (Spec 038).""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from openxml_audit.eurooffice import ConversionResult + +REPO_ROOT = Path(__file__).resolve().parents[1] +TOOLS = REPO_ROOT / "tools" +if str(TOOLS) not in sys.path: + sys.path.insert(0, str(TOOLS)) + +from oracle.eurooffice_conversion_oracle import ( # noqa: E402 + build_report, + observe, +) + + +class _FakeClient: + def __init__(self, output: bytes) -> None: + self.output = output + self.calls: list[dict[str, str]] = [] + + def healthcheck(self) -> bool: + return True + + def version(self) -> str: + return "9.3.3" + + def convert(self, **kwargs: str) -> ConversionResult: + self.calls.append(kwargs) + return ConversionResult("https://result.example.test/converted") + + def download(self, file_url: str) -> bytes: + return self.output + + +def test_native_same_format_conversion_is_diffed( + minimal_pptx: Path, + tmp_path: Path, +) -> None: + client = _FakeClient(minimal_pptx.read_bytes()) + observation = observe( + minimal_pptx, + client=client, + server_version="9.3.3", + source_base_url="https://files.example.test/corpus/", + work_root=tmp_path / "work", + ) + + assert observation.outcome == "preserved" + assert observation.format_mode == "native-edit" + assert observation.source_valid is True + assert observation.source_error_count == 0 + assert observation.target_valid is True + assert observation.changed_parts == [] + assert observation.sha256_in == observation.sha256_out + assert client.calls[0]["source_url"].endswith("/minimal.pptx") + assert client.calls[0]["target_format"] == "pptx" + + +def test_odf_edit_path_is_reported_as_lossy_conversion( + minimal_odt: Path, + tmp_path: Path, +) -> None: + valid_docx = REPO_ROOT / "data" / "corpus" / "tokenmoulds_v0.7.2" / "word" / "acme-us.docx" + client = _FakeClient(valid_docx.read_bytes()) + observation = observe( + minimal_odt, + client=client, + server_version="9.3.3", + source_base_url="https://files.example.test/corpus/", + work_root=tmp_path / "work", + ) + + assert observation.outcome == "converted" + assert observation.format_mode == "lossy-edit" + assert observation.source_format == "odt" + assert observation.target_format == "docx" + assert observation.target_valid is True + assert any("converts odt to docx" in note for note in observation.notes) + + +def test_odg_is_conversion_only(minimal_pptx: Path, tmp_path: Path) -> None: + source = tmp_path / "drawing.odg" + source.write_bytes(b"synthetic ODG source") + observation = observe( + source, + client=_FakeClient(minimal_pptx.read_bytes()), + source_base_url="https://files.example.test/", + work_root=tmp_path / "work", + ) + assert observation.outcome == "converted" + assert observation.format_mode == "view-only" + assert observation.target_format == "pptx" + assert any("not an editable" in note for note in observation.notes) + + +def test_invalid_same_format_output_still_records_diff( + minimal_pptx: Path, + invalid_pptx_missing_presentation: Path, + tmp_path: Path, +) -> None: + observation = observe( + minimal_pptx, + client=_FakeClient(invalid_pptx_missing_presentation.read_bytes()), + source_base_url="https://files.example.test/", + work_root=tmp_path / "work", + ) + assert observation.outcome == "invalid_output" + assert observation.source_valid is True + assert observation.target_valid is False + assert "ppt/presentation.xml" in observation.removed_parts + + +def test_unsupported_format_does_not_call_server(tmp_path: Path) -> None: + source = tmp_path / "database.odb" + source.write_bytes(b"not sent") + client = _FakeClient(b"") + observation = observe(source, client=client) + assert observation.outcome == "unsupported" + assert observation.format_mode == "unsupported" + assert client.calls == [] + + +def test_report_keeps_conversion_evidence_scope(minimal_pptx: Path, tmp_path: Path) -> None: + observation = observe( + minimal_pptx, + client=_FakeClient(minimal_pptx.read_bytes()), + source_base_url="https://files.example.test/", + work_root=tmp_path / "work", + ) + report = build_report([observation], server_version="9.3.3") + assert report["engine"] == "eurooffice" + assert report["evidence_scope"] == "Document Server conversion endpoint; not browser editing" + assert report["upstream"]["document_server_release"] == "9.3.3" + assert report["summary"] == {"total": 1, "outcomes": {"preserved": 1}} + + +def test_dispatcher_has_eurooffice_aliases() -> None: + from openxml_audit.oracle.__main__ import _DISPATCH + + assert _DISPATCH["eurooffice"] is _DISPATCH["euro-office"] + assert _DISPATCH["eurooffice"] is _DISPATCH["euro"] diff --git a/tests/test_packaging_contract.py b/tests/test_packaging_contract.py new file mode 100644 index 0000000..b39364e --- /dev/null +++ b/tests/test_packaging_contract.py @@ -0,0 +1,19 @@ +"""Regression checks for files required by installed console scripts.""" + +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_wheel_includes_oracle_dispatcher_implementations() -> None: + """The public oracle entry point imports a package outside ``src``. + + Keep the Hatch force-include mapping explicit: source-checkout tests can + import ``tools.oracle`` from the repository root and would otherwise miss + the broken-wheel regression. + """ + pyproject = (REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8") + + assert '"tools/oracle" = "tools/oracle"' in pyproject diff --git a/tools/oracle/README.md b/tools/oracle/README.md index 366ed5e..95674fe 100644 --- a/tools/oracle/README.md +++ b/tools/oracle/README.md @@ -1,4 +1,21 @@ -# Word Roundtrip Oracle +# Oracle Tools + +This directory contains the app and service-backed oracle engines used by +`openxml-audit-oracle`. It ships in the wheel because the packaged dispatcher +imports these modules; individual engines still have their own platform, +application, network, and credential requirements. + +The Euro-Office conversion engine is service-backed and cross-platform: + +```bash +openxml-audit-oracle eurooffice https://files.example.test/sample.odt +``` + +It reads `EUROOFFICE_ORACLE_URL` and (when enabled by the server) +`EUROOFFICE_ORACLE_JWT_SECRET`. See +[`specs/038-eurooffice-conversion-oracle.md`](../../specs/038-eurooffice-conversion-oracle.md). + +## Word Roundtrip Oracle Spec: [`specs/011-word-roundtrip-oracle.md`](../../specs/011-word-roundtrip-oracle.md) @@ -7,8 +24,9 @@ Mac via osascript, opens a DOCX, saves it through Word, and returns the post-Word file. The diff between input and post-Word is the empirical oracle for any "would Word repair this?" question. -The package is intentionally outside `src/openxml_audit/` — it does not -ship in the wheel and cannot run in CI. +The Word engine is developer-machine infrastructure: although its module ships +for dispatcher availability, it needs Microsoft Word for Mac and cannot run in +ordinary Linux CI. ## Why It Exists diff --git a/tools/oracle/baselines/README.md b/tools/oracle/baselines/README.md index 89f6b34..82577de 100644 --- a/tools/oracle/baselines/README.md +++ b/tools/oracle/baselines/README.md @@ -1,9 +1,9 @@ # Roundtrip Oracle Baselines -Per-format observation snapshots from running the four roundtrip oracles +Per-format observation snapshots from running the desktop roundtrip oracles (`tools/oracle/{word,odf,pptx,xlsx}_repair_oracle.py` / `word_repair_corpus.py`) -against curated corpora. Each subdirectory holds dated JSON reports -matching the schema in `RoundtripObservation`. +and service-backed oracles against curated corpora. Each subdirectory holds +dated JSON reports matching its engine's observation schema. ## Layout @@ -14,6 +14,7 @@ tools/oracle/baselines/ odf/.json pptx/.json xlsx/.json + eurooffice/.json word__pairwise.json ← scenario-matrix oracle output (e.g. word_trpr_pairwise.json, word_tblpr_pairwise.json, ...) ``` @@ -35,11 +36,44 @@ python tools/oracle/odf_repair_oracle.py /path/to/odf/corpus --output ... python tools/oracle/word_repair_corpus.py /path/to/docx/corpus --output ... python tools/oracle/pptx_repair_oracle.py /path/to/pptx/corpus --output ... python tools/oracle/xlsx_repair_oracle.py /path/to/xlsx/corpus --output ... +openxml-audit-oracle eurooffice https://files.example.test/corpus/... --output ... ``` Permissions setup is required for the three Microsoft Office oracles — see `docs/oracle_permissions.md`. The ODF oracle uses headless soffice -and needs no special grants. +and needs no special grants. Euro-Office needs a reachable Document Server, +an environment-provided JWT secret when enabled, and source URLs that tolerate +the server's configured outbox Authorization header. + +## 2026-08-09 Euro-Office conversion baseline + +[`eurooffice/2026-08-09.json`](eurooffice/2026-08-09.json) records six +TokenMoulds v0.7.2 `acme-us` fixtures against the live Opus95 Euro-Office +conversion endpoint. The server reported **9.3.1.37**; the current upstream +release pinned by the oracle is **9.3.3**, so this snapshot also records a +deployment-version gap rather than pretending it ran the newest server build. + +| Source | Mode | Target | Outcome | Source errors | Target errors | +|---|---|---|---|---:|---:| +| DOCX | native edit | DOCX | preserved | 0 | 0 | +| XLSX | native edit | XLSX | preserved | 0 | 0 | +| PPTX | native edit | PPTX | invalid output | 2 | 2 | +| ODT | lossy edit | DOCX | invalid output | n/a | 139 | +| ODS | lossy edit | XLSX | invalid output | n/a | 6 | +| ODP | lossy edit | PPTX | request failed (`-3`) | n/a | n/a | + +`preserved` means canonical XML and relationship parts were unchanged; the ZIP +bytes and sizes did change. The PPTX's two findings were already present in the +source, were identical in the target, and had a zero canonical part diff—the +converter introduced no new reported finding. ODT and ODS produced downloadable +OOXML but those targets did not pass `openxml-audit`; ODP conversion failed in +the converter before an artifact was returned. + +These are conversion-endpoint observations, not claims about the Nextcloud +browser editor or save callback. The source fixtures used a static CDN because +the live server attaches its outbox bearer token to downloads: raw GitHub treats +that unrelated token as invalid GitHub authentication and deliberately returns +404, while the CDN ignores it. ## 2026-04-29 baseline run — first across all four formats diff --git a/tools/oracle/baselines/eurooffice/2026-08-09.json b/tools/oracle/baselines/eurooffice/2026-08-09.json new file mode 100644 index 0000000..8c27611 --- /dev/null +++ b/tools/oracle/baselines/eurooffice/2026-08-09.json @@ -0,0 +1,169 @@ +{ + "engine": "eurooffice", + "evidence_scope": "Document Server conversion endpoint; not browser editing", + "observations": [ + { + "added_parts": [], + "artifact_path": null, + "changed_parts": [], + "diff_dir": null, + "duration_seconds": 0.7658650000230409, + "format_mode": "native-edit", + "notes": [], + "outcome": "preserved", + "removed_parts": [], + "server_version": "9.3.1.37", + "sha256_in": "900bcb1349b385871a2fb7cfac6a03003ffb998675bfc63acbff2ef4d3c7042a", + "sha256_out": "b76f285e656327ca77d7e6670199729272dcf9fef9d5f513a75292fed81a536b", + "size_in": 14233, + "size_out": 15812, + "source_error_count": 0, + "source_format": "docx", + "source_relpath": "acme-us.docx", + "source_valid": true, + "target_error_count": 0, + "target_format": "docx", + "target_valid": true + }, + { + "added_parts": [], + "artifact_path": null, + "changed_parts": [], + "diff_dir": null, + "duration_seconds": 0.5409165840246715, + "format_mode": "native-edit", + "notes": [], + "outcome": "preserved", + "removed_parts": [], + "server_version": "9.3.1.37", + "sha256_in": "fe2ee89d3b3355befcf68a427424b7ad4f01a39a204e2a2accaeab90231e04e5", + "sha256_out": "76772ab7a5e5553b9a7fc769da9f8da52ec1519ab5c1dda4079970f6e3e86001", + "size_in": 9480, + "size_out": 10897, + "source_error_count": 0, + "source_format": "xlsx", + "source_relpath": "acme-us.xlsx", + "source_valid": true, + "target_error_count": 0, + "target_format": "xlsx", + "target_valid": true + }, + { + "added_parts": [], + "artifact_path": null, + "changed_parts": [], + "diff_dir": null, + "duration_seconds": 0.8998033750103787, + "format_mode": "native-edit", + "notes": [ + "converted package failed openxml-audit validation", + "target validator findings match the source; no new finding was introduced" + ], + "outcome": "invalid_output", + "removed_parts": [], + "server_version": "9.3.1.37", + "sha256_in": "a79d8643368e3a84d9fecea95058ace9300071b27a15351067dddd78fd1b9cb6", + "sha256_out": "b521f97f91e99d78e7a9b11308f3b879b5b6ebb6cb544860deb980563d90ef0c", + "size_in": 67694, + "size_out": 74733, + "source_error_count": 2, + "source_format": "pptx", + "source_relpath": "acme-us.pptx", + "source_valid": false, + "target_error_count": 2, + "target_format": "pptx", + "target_valid": false + }, + { + "added_parts": [], + "artifact_path": null, + "changed_parts": [], + "diff_dir": null, + "duration_seconds": 0.5312673750449903, + "format_mode": "lossy-edit", + "notes": [ + "converted package failed openxml-audit validation" + ], + "outcome": "invalid_output", + "removed_parts": [], + "server_version": "9.3.1.37", + "sha256_in": "1ad9d06e1345eb1245f41af5c3c32381d46aabd0e7c9a769d64e3f64c2c285a0", + "sha256_out": "56242b5d0f2923e11ff93607f4dc9648c0e8ef16d93f5cfdb8288a2d1ed781a6", + "size_in": 6048, + "size_out": 9438, + "source_error_count": null, + "source_format": "odt", + "source_relpath": "acme-us.odt", + "source_valid": null, + "target_error_count": 139, + "target_format": "docx", + "target_valid": false + }, + { + "added_parts": [], + "artifact_path": null, + "changed_parts": [], + "diff_dir": null, + "duration_seconds": 0.3968550830031745, + "format_mode": "lossy-edit", + "notes": [ + "converted package failed openxml-audit validation" + ], + "outcome": "invalid_output", + "removed_parts": [], + "server_version": "9.3.1.37", + "sha256_in": "eda3ee5d48cf1053ee61ed6af0a1971e12b2ccb17e89fd72776694f0de078c52", + "sha256_out": "c4d6fd415c7fdc20261d85eccefec408442f1ca5e97269fb7fe57ef32715d65c", + "size_in": 3252, + "size_out": 5326, + "source_error_count": null, + "source_format": "ods", + "source_relpath": "acme-us.ods", + "source_valid": null, + "target_error_count": 6, + "target_format": "xlsx", + "target_valid": false + }, + { + "added_parts": [], + "artifact_path": null, + "changed_parts": [], + "diff_dir": null, + "duration_seconds": 0.27520333300344646, + "format_mode": "lossy-edit", + "notes": [ + "Euro-Office conversion error -3: conversion failed" + ], + "outcome": "request_failed", + "removed_parts": [], + "server_version": "9.3.1.37", + "sha256_in": "8c32e6d925b7d8d74246b1a04d6846618b27b0c69bdfe44b99fc719d741b6094", + "sha256_out": null, + "size_in": 3473, + "size_out": null, + "source_error_count": null, + "source_format": "odp", + "source_relpath": "acme-us.odp", + "source_valid": null, + "target_error_count": null, + "target_format": "pptx", + "target_valid": null + } + ], + "schema_version": 1, + "summary": { + "outcomes": { + "invalid_output": 3, + "preserved": 2, + "request_failed": 1 + }, + "total": 6 + }, + "upstream": { + "document_formats_commit": "7d7576a3fe2337c30f4c9b40fae70a69dc68ba08", + "document_formats_version": "3.2.0", + "document_server_release": "9.3.3", + "document_server_version": "9.3.1.37", + "nextcloud_connector_version": "11.0.1" + } +} diff --git a/tools/oracle/eurooffice_conversion_oracle.py b/tools/oracle/eurooffice_conversion_oracle.py new file mode 100644 index 0000000..0c60811 --- /dev/null +++ b/tools/oracle/eurooffice_conversion_oracle.py @@ -0,0 +1,482 @@ +"""Euro-Office Document Server conversion oracle. + +This engine exercises the released ``/converter`` contract. It establishes +that Document Server can fetch and convert a file, then validates the returned +OOXML package and (for same-format conversions) records a canonical package +diff. It does *not* drive the browser editor, a save callback, or coauthoring, +so its observations must not be presented as end-to-end editing evidence. + +Secrets are intentionally environment-only:: + + export EUROOFFICE_ORACLE_URL=https://office.example.test/ + export EUROOFFICE_ORACLE_JWT_SECRET=... + openxml-audit-oracle eurooffice FILES... --source-base-url https://files.example.test/ + +HTTP(S) input URLs need no ``--source-base-url``. Local inputs do: Document +Server must be able to fetch them from a URL reachable from its own container. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import sys +import tempfile +import time +import urllib.parse +import zipfile +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Literal, Protocol, cast + +from openxml_audit.eurooffice import ( + CONNECTOR_VERSION, + DOCUMENT_FORMATS_COMMIT, + DOCUMENT_FORMATS_VERSION, + DOCUMENT_SERVER_RELEASE, + ConversionResult, + EuroOfficeClient, + EuroOfficeError, + EuroOfficeFormatMode, + format_support, +) +from openxml_audit.package_diff import compare_packages +from openxml_audit.validator import OpenXmlValidator + +Outcome = Literal[ + "preserved", + "rewritten", + "converted", + "unsupported", + "request_failed", + "download_failed", + "invalid_output", + "source_unavailable", +] + +_OFFICE_EXTENSIONS = { + "docx", + "docm", + "dotx", + "dotm", + "xlsx", + "xlsm", + "xltx", + "xltm", + "pptx", + "pptm", + "potx", + "potm", + "ppsx", + "ppsm", + "odt", + "ott", + "ods", + "ots", + "odp", + "otp", + "odg", + # Validator-recognized families that the pinned connector does not edit. + "odm", + "oth", + "odc", + "odi", + "odf", + "odb", + "otm", + "otg", + "xlam", + "thmx", + "ppam", +} + + +class ConversionClient(Protocol): + """Surface needed by the orchestrator; enables network-free tests.""" + + def healthcheck(self) -> bool: ... + + def version(self) -> str: ... + + def convert( + self, + *, + source_url: str, + source_format: str, + target_format: str, + key: str, + title: str | None = None, + ) -> ConversionResult: ... + + def download(self, file_url: str) -> bytes: ... + + +@dataclass +class EuroOfficeConversionObservation: + """One conversion-path observation.""" + + source_relpath: str + source_format: str + target_format: str | None + format_mode: str + outcome: Outcome + duration_seconds: float + server_version: str | None = None + source_valid: bool | None = None + source_error_count: int | None = None + target_valid: bool | None = None + target_error_count: int | None = None + changed_parts: list[str] = field(default_factory=list) + added_parts: list[str] = field(default_factory=list) + removed_parts: list[str] = field(default_factory=list) + size_in: int | None = None + size_out: int | None = None + sha256_in: str | None = None + sha256_out: str | None = None + artifact_path: str | None = None + diff_dir: str | None = None + notes: list[str] = field(default_factory=list) + + +def _is_url(value: str) -> bool: + return urllib.parse.urlsplit(value).scheme in {"http", "https"} + + +def _source_name(value: str) -> str: + if _is_url(value): + return urllib.parse.unquote(Path(urllib.parse.urlsplit(value).path).name) + return Path(value).name + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _conversion_key(source_url: str, target_format: str) -> str: + return hashlib.sha256(f"{source_url}\0{target_format}".encode()).hexdigest()[:20] + + +def _local_source_url(path: Path, source_base_url: str | None) -> str: + if not source_base_url: + raise ValueError( + "local inputs require --source-base-url or EUROOFFICE_ORACLE_SOURCE_BASE_URL" + ) + parsed = urllib.parse.urlsplit(source_base_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("source base URL must use HTTP(S)") + return urllib.parse.urljoin( + source_base_url.rstrip("/") + "/", + urllib.parse.quote(path.name), + ) + + +def _prepare_source( + source: str, + *, + client: ConversionClient, + work_dir: Path, + source_base_url: str | None, +) -> tuple[str, Path | None, bytes | None, list[str]]: + notes: list[str] = [] + if not _is_url(source): + path = Path(source).expanduser().resolve() + data = path.read_bytes() + staged = work_dir / f"source{path.suffix.lower()}" + shutil.copy2(path, staged) + return _local_source_url(path, source_base_url), staged, data, notes + + suffix = Path(urllib.parse.urlsplit(source).path).suffix.lower() + staged = work_dir / f"source{suffix}" + try: + data = client.download(source) + except EuroOfficeError: + notes.append("source was not downloadable locally; package diff is unavailable") + return source, None, None, notes + staged.write_bytes(data) + return source, staged, data, notes + + +def _validate_ooxml(path: Path) -> tuple[bool, int, frozenset[tuple[str, str, str, str]]]: + result = OpenXmlValidator().validate(path) + signatures = frozenset( + ( + error.severity.value, + error.error_type.value, + error.part_uri, + error.description, + ) + for error in result.errors + ) + return result.is_valid, result.error_count, signatures + + +def observe( + source: str | Path, + *, + client: ConversionClient, + server_version: str | None = None, + source_base_url: str | None = None, + work_root: Path | None = None, + keep_artifacts: bool = False, +) -> EuroOfficeConversionObservation: + """Exercise one file through the synchronous conversion endpoint.""" + + started = time.perf_counter() + source_text = str(source) + name = _source_name(source_text) + capability = format_support(name) + observation = EuroOfficeConversionObservation( + source_relpath=name or source_text, + source_format=capability.extension, + target_format=capability.conversion_target, + format_mode=capability.mode.value, + outcome="unsupported", + duration_seconds=0.0, + server_version=server_version, + ) + if capability.mode is EuroOfficeFormatMode.UNSUPPORTED: + observation.notes.append( + f"not supported by connector {CONNECTOR_VERSION}'s pinned format matrix" + ) + observation.duration_seconds = time.perf_counter() - started + return observation + + root = work_root.expanduser().resolve() if work_root else None + if root: + root.mkdir(parents=True, exist_ok=True) + work_dir = Path(tempfile.mkdtemp(prefix="eurooffice-oracle-", dir=root)) + try: + try: + source_url, source_path, source_bytes, source_notes = _prepare_source( + source_text, + client=client, + work_dir=work_dir, + source_base_url=source_base_url, + ) + except (OSError, ValueError) as exc: + observation.outcome = "source_unavailable" + observation.notes.append(str(exc)) + return observation + + observation.notes.extend(source_notes) + if source_bytes is not None: + observation.size_in = len(source_bytes) + observation.sha256_in = _sha256(source_bytes) + + target_format = capability.conversion_target + if target_format is None: # guarded above; keeps the invariant explicit + observation.outcome = "unsupported" + return observation + source_findings: frozenset[tuple[str, str, str, str]] | None = None + if source_path is not None and capability.extension == target_format: + source_valid, source_error_count, source_findings = _validate_ooxml(source_path) + observation.source_valid = source_valid + observation.source_error_count = source_error_count + key = _conversion_key(source_url, target_format) + try: + result = client.convert( + source_url=source_url, + source_format=capability.extension, + target_format=target_format, + key=key, + title=name, + ) + except EuroOfficeError as exc: + observation.outcome = "request_failed" + observation.notes.append(str(exc)) + return observation + + try: + output_bytes = client.download(result.file_url) + except EuroOfficeError as exc: + observation.outcome = "download_failed" + observation.notes.append(str(exc)) + return observation + + output_path = work_dir / f"converted.{target_format}" + output_path.write_bytes(output_bytes) + observation.size_out = len(output_bytes) + observation.sha256_out = _sha256(output_bytes) + + same_format_diff: dict[str, object] | None = None + if source_path is not None and capability.extension == target_format: + diff_dir = work_dir / "package-diff" + try: + same_format_diff = compare_packages( + base_path=source_path, + head_path=output_path, + output_dir=diff_dir, + ) + except (OSError, ValueError, zipfile.BadZipFile): + observation.notes.append("converted package could not be diffed") + else: + observation.changed_parts = list(same_format_diff["changed_files"]) + observation.added_parts = list(same_format_diff["added_files"]) + observation.removed_parts = list(same_format_diff["removed_files"]) + if keep_artifacts: + observation.diff_dir = str(diff_dir) + + valid, error_count, target_findings = _validate_ooxml(output_path) + observation.target_valid = valid + observation.target_error_count = error_count + if not valid: + observation.outcome = "invalid_output" + observation.notes.append("converted package failed openxml-audit validation") + if source_findings is not None and source_findings == target_findings: + observation.notes.append( + "target validator findings match the source; no new finding was introduced" + ) + return observation + + if same_format_diff is not None: + observation.outcome = ( + "rewritten" + if observation.changed_parts or observation.added_parts or observation.removed_parts + else "preserved" + ) + elif source_path is None and capability.extension == target_format: + observation.outcome = "source_unavailable" + else: + observation.outcome = "converted" + + if capability.mode is EuroOfficeFormatMode.LOSSY_EDIT: + observation.notes.append( + f"connector editing path converts {capability.extension} to {target_format}" + ) + elif capability.mode is EuroOfficeFormatMode.VIEW_ONLY: + observation.notes.append( + f"{capability.extension} is view/convert-only, not an editable connector format" + ) + return observation + finally: + observation.duration_seconds = time.perf_counter() - started + if keep_artifacts: + converted = work_dir / f"converted.{capability.conversion_target}" + observation.artifact_path = str(converted) if converted.exists() else None + else: + shutil.rmtree(work_dir, ignore_errors=True) + + +def build_report( + observations: list[EuroOfficeConversionObservation], + *, + server_version: str | None, +) -> dict[str, object]: + """Build the stable JSON report envelope.""" + + counts: dict[str, int] = {} + for observation in observations: + counts[observation.outcome] = counts.get(observation.outcome, 0) + 1 + return { + "schema_version": 1, + "engine": "eurooffice", + "evidence_scope": "Document Server conversion endpoint; not browser editing", + "upstream": { + "document_server_version": server_version, + "document_server_release": DOCUMENT_SERVER_RELEASE, + "nextcloud_connector_version": CONNECTOR_VERSION, + "document_formats_version": DOCUMENT_FORMATS_VERSION, + "document_formats_commit": DOCUMENT_FORMATS_COMMIT, + }, + "observations": [asdict(observation) for observation in observations], + "summary": {"total": len(observations), "outcomes": counts}, + } + + +def _expand_inputs(values: list[str]) -> list[str]: + inputs: list[str] = [] + for value in values: + if _is_url(value): + inputs.append(value) + continue + path = Path(value).expanduser() + if path.is_dir(): + inputs.extend( + str(candidate) + for candidate in sorted(path.rglob("*")) + if candidate.is_file() + and candidate.suffix.lower().lstrip(".") in _OFFICE_EXTENSIONS + ) + elif path.is_file(): + inputs.append(str(path)) + return inputs + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", nargs="+", help="office files, directories, or HTTP(S) URLs") + parser.add_argument( + "--server-url", + default=None, + help="Document Server base URL; defaults to EUROOFFICE_ORACLE_URL", + ) + parser.add_argument( + "--source-base-url", + default=None, + help="public base URL serving local inputs; defaults to EUROOFFICE_ORACLE_SOURCE_BASE_URL", + ) + parser.add_argument("--output", type=Path, default=None, help="write JSON report here") + parser.add_argument("--work-root", type=Path, default=None, help="parent for temporary runs") + parser.add_argument("--timeout", type=float, default=120.0, help="HTTP timeout in seconds") + parser.add_argument( + "--keep-artifacts", action="store_true", help="retain converted files and XML diffs" + ) + args = parser.parse_args() + + inputs = _expand_inputs(args.input) + if not inputs: + print("no office inputs found", file=sys.stderr) + return 2 + + try: + client = EuroOfficeClient.from_env(args.server_url, timeout=args.timeout) + if not client.healthcheck(): + print("Euro-Office healthcheck did not return true", file=sys.stderr) + return 1 + server_version = client.version() + except EuroOfficeError as exc: + print(f"Euro-Office preflight failed: {exc}", file=sys.stderr) + return 1 + + source_base_url = args.source_base_url + if source_base_url is None: + source_base_url = os.environ.get("EUROOFFICE_ORACLE_SOURCE_BASE_URL") + + observations = [ + observe( + item, + client=client, + server_version=server_version, + source_base_url=source_base_url, + work_root=args.work_root, + keep_artifacts=args.keep_artifacts, + ) + for item in inputs + ] + report = build_report(observations, server_version=server_version) + rendered = json.dumps(report, indent=2, sort_keys=True) + if args.output: + args.output.write_text(rendered + "\n", encoding="utf-8") + print(f"wrote report to {args.output}", file=sys.stderr) + else: + print(rendered) + + summary = cast(dict[str, object], report["summary"]) + outcomes = cast(dict[str, int], summary["outcomes"]) + print( + "eurooffice-oracle: " + + " ".join(f"{name}={count}" for name, count in sorted(outcomes.items())), + file=sys.stderr, + ) + hard_failures = sum( + outcomes.get(name, 0) + for name in ("request_failed", "download_failed", "invalid_output", "source_unavailable") + ) + return 1 if hard_failures else 0 + + +if __name__ == "__main__": + sys.exit(main())