From 5020432f980d0fb461cbd64a6c79fa22deb50eb2 Mon Sep 17 00:00:00 2001 From: Liam Crumm Date: Tue, 11 Aug 2026 00:42:47 +0000 Subject: [PATCH 01/10] feat: make streaming reachable from every supported language Section 18.1 stream mediation shipped in 0.4.0-alpha.2 as Rust only. The `streaming` cargo feature is off by default and none of the three binding crates enabled it, so a Python, Node or .NET consumer installing a prebuilt binary had no way to turn it on and no API to call. Enable it in each binding and expose the session. Add thirteen stream session entry points to the C ABI, following the handle shape `AcsActivatedPolicy` already uses. Scalar queries return i64 so an absent value needs no allocation, distinguishing absent from failed, because a settled session legitimately has no safe offset. Structured queries return JSON, so the wire contract is owned at the boundary rather than derived from Rust layout. Bind that surface in .NET, and bind the engine directly through pyo3 and napi for Python and Node. All four spell an absent release point in their own vocabulary rather than as 0 or -1, both of which read as permission. Ship the engine binary in the NuGet package. The managed assembly reaches the engine through agent_control_spec_ffi, and the package carried no native library, so every call threw DllNotFoundException on a clean install. Nothing in the .NET SDK worked as published, streaming or otherwise. Build one library per runtime identifier in the release matrix and pack them under runtimes/, and refuse at pack time to build a managed-only package, which installs cleanly and fails in someone else's process. Verify the artifact rather than the checkout. The existing suite runs with the engine on the loader path, which a consumer does not have, so it passed throughout. scripts/verify-dotnet-package.sh builds a console app outside the repository, restores the packed artifact, and calls both a plain entry point and the session. Assert the four languages agree. They reach the engine through four mechanisms and each converts enums, offsets and the absent release point at its own boundary, so agreement is not structural. cross_language_parity.py runs one scenario in all four and diffs, using an astral-plane character whose rune count differs from its UTF-16 length. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Liam Crumm --- .github/workflows/ci.yml | 46 ++ .github/workflows/release.yml | 59 +- .gitignore | 3 + README.md | 4 + docs/STREAMING.md | 170 ++++ scripts/verify-dotnet-package.sh | 81 ++ .../AgentControlSpec/AgentControlSpec.csproj | 30 + sdk/dotnet/src/AgentControlSpec/Native.cs | 175 ++++- .../src/AgentControlSpec/StreamSession.cs | 298 +++++++ .../StreamSessionTests.cs | 183 +++++ sdk/ffi/Cargo.toml | 2 +- sdk/ffi/src/lib.rs | 730 +++++++++++++++++- sdk/node/binding.d.ts | 91 +++ sdk/node/binding.js | 12 + sdk/node/native/Cargo.toml | 2 +- sdk/node/native/src/lib.rs | 344 ++++++++- sdk/node/src/index.ts | 405 ++++++++++ sdk/node/test/stream-session.test.mjs | 295 +++++++ sdk/python/Cargo.toml | 2 +- sdk/python/agent_control_spec/__init__.py | 212 ++++- sdk/python/src/lib.rs | 326 +++++++- sdk/python/tests/test_stream_session.py | 341 ++++++++ .../streaming/cross_language_parity.py | 283 +++++++ 23 files changed, 4074 insertions(+), 20 deletions(-) create mode 100644 docs/STREAMING.md create mode 100755 scripts/verify-dotnet-package.sh create mode 100644 sdk/dotnet/src/AgentControlSpec/StreamSession.cs create mode 100644 sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs create mode 100644 sdk/node/test/stream-session.test.mjs create mode 100644 sdk/python/tests/test_stream_session.py create mode 100644 tests/conformance/streaming/cross_language_parity.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f50519..bf37429 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,3 +151,49 @@ jobs: working-directory: sdk/dotnet env: LD_LIBRARY_PATH: ${{ github.workspace }}/target/release + # The suite above runs with the engine on the loader path, which a + # consumer installing from NuGet does not have. Pack and run the + # artifact itself so a package that omits the native library fails + # here rather than in someone else's process. + - name: Verify the packed package works without a local engine build + run: | + set -euo pipefail + mkdir -p sdk/dotnet/native/runtimes/linux-x64/native + cp target/release/libagent_control_spec_ffi.so \ + sdk/dotnet/native/runtimes/linux-x64/native/ + dotnet pack sdk/dotnet/src/AgentControlSpec -c Release -o "$PWD/ci-feed" \ + --nologo -p:AcsNativeAssetsRequired=true -p:Version=0.0.0-ci + bash scripts/verify-dotnet-package.sh "$PWD/ci-feed" 0.0.0-ci + + # Streaming reaches each language through a different binding + # mechanism, so agreement between them is not structural. Run one + # scenario in all four and diff. + streaming-parity: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: "8.0.x" + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - run: cargo build --release --locked -p agent-control-spec-ffi + - name: Build the node binding + run: | + npm ci + npm run build + working-directory: sdk/node + - name: Build and install the python wheel + run: | + pip install maturin==1.8.7 + maturin build --release -m sdk/python/Cargo.toml -o dist + pip install dist/*.whl + - name: Compare every language against the same stream + run: python tests/conformance/streaming/cross_language_parity.py + env: + LD_LIBRARY_PATH: ${{ github.workspace }}/target/release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 969f87a..1bb1e78 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -226,7 +226,43 @@ jobs: fi working-directory: sdk/node + # The managed assembly calls the engine through agent_control_spec_ffi, + # so a package without the native library installs cleanly and then + # throws DllNotFoundException on the first call. Build one library per + # supported RID and let the pack step assemble them. + dotnet-native: + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, rid: linux-x64, lib: libagent_control_spec_ffi.so } + - { os: ubuntu-24.04-arm, target: aarch64-unknown-linux-gnu, rid: linux-arm64, lib: libagent_control_spec_ffi.so } + - { os: macos-latest, target: x86_64-apple-darwin, rid: osx-x64, lib: libagent_control_spec_ffi.dylib } + - { os: macos-latest, target: aarch64-apple-darwin, rid: osx-arm64, lib: libagent_control_spec_ffi.dylib } + - { os: windows-latest, target: x86_64-pc-windows-msvc, rid: win-x64, lib: agent_control_spec_ffi.dll } + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master; stable toolchain + with: + toolchain: stable + targets: ${{ matrix.target }} + - name: Build the engine binding for ${{ matrix.rid }} + shell: bash + run: cargo build --release --locked -p agent-control-spec-ffi --target ${{ matrix.target }} + - name: Stage under its runtime identifier + shell: bash + run: | + mkdir -p "sdk/dotnet/native/runtimes/${{ matrix.rid }}/native" + cp "target/${{ matrix.target }}/release/${{ matrix.lib }}" \ + "sdk/dotnet/native/runtimes/${{ matrix.rid }}/native/${{ matrix.lib }}" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dotnet-native-${{ matrix.rid }} + path: sdk/dotnet/native/runtimes/${{ matrix.rid }}/native/${{ matrix.lib }} + dotnet: + needs: dotnet-native runs-on: ubuntu-latest environment: release permissions: @@ -237,11 +273,32 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: { dotnet-version: "8.0.x" } + - name: Collect the native libraries + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + pattern: dotnet-native-* + path: sdk/dotnet/native/staging + - name: Lay them out by runtime identifier + shell: bash + run: | + set -euo pipefail + for dir in sdk/dotnet/native/staging/dotnet-native-*; do + rid="${dir##*/dotnet-native-}" + mkdir -p "sdk/dotnet/native/runtimes/$rid/native" + cp "$dir"/* "sdk/dotnet/native/runtimes/$rid/native/" + done + find sdk/dotnet/native/runtimes -type f | sort - name: Pack run: | dotnet restore --nologo - dotnet pack src/AgentControlSpec -c Release -o dist --nologo + dotnet pack src/AgentControlSpec -c Release -o dist --nologo -p:AcsNativeAssetsRequired=true working-directory: sdk/dotnet + - name: Prove the packed artifact runs without a local engine build + shell: bash + run: | + set -euo pipefail + version=$(ls sdk/dotnet/dist/*.nupkg | sed -E 's/.*ResponsibleAI\.AgentControlSpec\.(.*)\.nupkg/\1/') + bash scripts/verify-dotnet-package.sh "$PWD/sdk/dotnet/dist" "$version" - name: SBOM uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0 with: diff --git a/.gitignore b/.gitignore index 32e9518..6c18f25 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ __pycache__/ dist/ bin/ obj/ + +# Per-RID engine binaries staged for the .NET package (built, never committed). +/sdk/dotnet/native/ diff --git a/README.md b/README.md index ddd6cef..f08cd6f 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,14 @@ those are host obligations defined by agent-hooks. | Path | Contents | | --- | --- | | `engine/` | Rust evaluation core (`agent-control-spec` crate): manifest, dispatchers, annotators, policy-output normalization, the `AcsInterceptor` | +| `sdk/ffi/` | C ABI over the engine (`agent-control-spec-ffi`), which the .NET binding calls | | `sdk/python/` | Python binding: `agent_control_spec` package wrapping the engine as an `agent_hooks` interceptor | +| `sdk/node/` | Node binding: `@responsibleai/agent-control-spec` | +| `sdk/dotnet/` | .NET binding: `ResponsibleAI.AgentControlSpec` | | `spec/` | The ACS specification (policy plane) and schemas | | `policy/` | Cedar and Rego policy libraries | | `fixtures/` | Evaluation fixtures | +| `docs/STREAMING.md` | The incremental stream profile (section 18.1) in every language | | `docs/EXTRACTION.md` | Provenance map from the previous tree | ## Status diff --git a/docs/STREAMING.md b/docs/STREAMING.md new file mode 100644 index 0000000..f05bf6c --- /dev/null +++ b/docs/STREAMING.md @@ -0,0 +1,170 @@ +# Streaming + +Specification section 18.1 defines the incremental stream profile: a host +that emits a response while it is still arriving needs to know how much +of it is safe to release so far. `StreamSession` answers that question. + +The session is release accounting, not evaluation. Each segment is one +ordinary stateless evaluation through the normal runtime; the session +records what those evaluations cleared and reports the offset that every +mediating task has agreed to. It holds no text, so emitting released +runes and applying a rewrite stay host obligations. + +## Availability + +| Language | Package | Streaming | +| --- | --- | --- | +| Rust | `agent-control-spec` | opt in with `features = ["streaming"]` | +| Python | `agent-control-spec` | always present | +| Node | `@responsibleai/agent-control-spec` | always present | +| .NET | `ResponsibleAI.AgentControlSpec` | always present | + +The feature is off by default in the Rust crate because a `StreamSession` +carries per-stream state, which the default surface deliberately does +not, and because a host that assembles a whole payload before evaluating +it never needs the module. The bindings enable it unconditionally: a +Python, Node or .NET consumer installs a prebuilt binary and cannot flip +a cargo feature. + +## The shape of a session + +Every language exposes the same eight operations. + +| Operation | What it does | +| --- | --- | +| `observe` / `observe_text` | report that runes arrived on a track | +| `record_outcome` | report what one task decided about one span | +| `record_verdict` | the same, taking a verdict as the runtime returns it | +| `advance` | recompute a track's watermark against recorded outcomes | +| `safe_offset` | the offset safe to release, as of the last `advance` | +| `watermark` | how far a track got, and which tasks must clear it | +| `end_of_payloads` | no further payload will arrive | +| `finish` | settle, returning why the stream ended | + +Two details decide whether a host is correct. + +**Recording an outcome does not release anything.** `advance` moves the +watermark; `safe_offset` reads it. A host that records outcomes and polls +`safe_offset` without ever calling `advance` will release nothing and +appear to hang. + +**An absent safe offset is not zero.** Once a session ends, whether +cleanly, by refusal, or by rewrite, there is no offset anyone may emit +through, and `safe_offset` says so in the language's own vocabulary: +`None` in Rust and Python, `null` in Node, `null` in .NET. That is a +value, not an error. A host that polls until it stops advancing stops on +its own. Reading it as `0` or as `-1` would release text no task +evaluated. + +The offset the track reached stays readable through `watermark` after +settlement, so an audit record can still say how far the stream got. + +## Rust + +```toml +agent-control-spec = { version = "0.4", features = ["streaming"] } +``` + +```rust +use agent_control_spec::stream_session::*; + +let mut session = StreamSession::new(StreamSessionConfig { + safety_level: SafetyLevel::Blocking, + request_start_rune_offset: 0, + response_start_rune_offset: 0, + request_tasks: vec![], + response_tasks: vec!["pii".to_string()], +})?; + +let received = session.observe_text(StreamSourceType::ModelGenerated, "hello")?; +let span = StreamSpan::new(StreamSourceType::ModelGenerated, 0, received)?; +session.record_outcome("pii", &span, SegmentOutcome::Cleared)?; + +session.advance(StreamTrack::Response); +assert_eq!(session.safe_offset(StreamTrack::Response), Some(5)); +``` + +## Python + +```python +from agent_control_spec import StreamSession + +session = StreamSession(safety_level="blocking", response_tasks=["pii"]) + +received = session.observe_text("model_generated", "hello") +session.record_outcome("pii", "model_generated", 0, received, "cleared") + +session.advance("response") +assert session.safe_offset("response") == 5 + +session.finish() +assert session.safe_offset("response") is None +``` + +## Node + +```ts +import { StreamSession } from "@responsibleai/agent-control-spec"; + +const session = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], +}); + +const received = session.observeText("model_generated", "hello"); +session.recordOutcome("pii", "model_generated", 0, received, "cleared"); + +session.advance("response"); +console.log(session.safeOffset("response")); // 5 + +session.finish(); +console.log(session.safeOffset("response")); // null +``` + +## .NET + +```csharp +using AgentControlSpec; + +using var session = new StreamSession( + SafetyLevel.Blocking, responseTasks: ["pii"]); + +var received = session.ObserveText(StreamSourceType.ModelGenerated, "hello"); +session.RecordOutcome( + "pii", StreamSourceType.ModelGenerated, 0, received, SegmentOutcome.Cleared); + +session.Advance(StreamTrack.Response); +Console.WriteLine(session.SafeOffset(StreamTrack.Response)); // 5 + +session.Finish(); +Console.WriteLine(session.SafeOffset(StreamTrack.Response)); // null +``` + +## Counting runes + +Offsets are counted in runes, meaning Unicode scalar values, not code +units and not bytes. The distinction is load bearing: an emoji is one +rune, two UTF-16 code units and four UTF-8 bytes, so a .NET host using +`string.Length` or a Node host using `String.prototype.length` would +release twice what a task evaluated. + +Prefer `observe_text`, which hands the text to the engine and lets it +count. Reserve `observe` for a host that already has a rune count from +the same source of truth. + +## Verdicts feed straight back + +`record_verdict` takes a verdict exactly as the runtime returns one, so a +host that evaluates a segment through `ActivatedPolicy` passes the result +in without translating it. `allow` clears, `deny` refuses, and +`transform` records a rewrite, which is terminal. + +## Cross-language agreement + +The four bindings reach the engine through four different mechanisms: a +direct crate dependency, pyo3, napi, and a C ABI with P/Invoke over it. +Each converts enums, offsets and the absent release point at its own +boundary, so agreement is not structural. + +`tests/conformance/streaming/cross_language_parity.py` runs one scenario +in all four and fails on any divergence. It is part of CI. diff --git a/scripts/verify-dotnet-package.sh b/scripts/verify-dotnet-package.sh new file mode 100755 index 0000000..c566d41 --- /dev/null +++ b/scripts/verify-dotnet-package.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Prove a packed ResponsibleAI.AgentControlSpec really works. +# +# The managed assembly reaches the engine through agent_control_spec_ffi. +# A package that omits the native library still restores, still compiles +# against, and still passes any test run from a repository checkout that +# happens to have the library on its loader path. It fails only in a +# consumer's process, on the first call. 0.4.0-alpha.2 shipped that way. +# +# So this builds a throwaway console app outside the repository, restores +# the packed artifact from a local feed, and calls both a plain entry +# point and the streaming session. No LD_LIBRARY_PATH, no engine build, +# nothing from the checkout on the loader path. +# +# Usage: verify-dotnet-package.sh + +set -euo pipefail + +FEED="${1:?usage: verify-dotnet-package.sh }" +VERSION="${2:?usage: verify-dotnet-package.sh }" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +cat >"$WORK/NuGet.config" < + + + + + + + +XML + +dotnet new console -o "$WORK/app" >/dev/null +cat >"$WORK/app/Program.cs" <<'CS' +using AgentControlSpec; + +// A plain entry point: proves the native library resolved at all. +var versions = AcsManifest.SupportedVersions(); +if (versions.Count == 0) +{ + throw new InvalidOperationException("no supported manifest versions"); +} + +// The streaming session: proves the engine was built with the feature +// its consumers cannot enable for themselves. +using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); +if (session.ObserveText(StreamSourceType.ModelGenerated, "hello") != 5) +{ + throw new InvalidOperationException("observe_text did not count five runes"); +} + +session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); +if (session.Advance(StreamTrack.Response) != 5 || session.SafeOffset(StreamTrack.Response) != 5) +{ + throw new InvalidOperationException("a cleared span did not release"); +} + +if (!session.Finish().IsClean) +{ + throw new InvalidOperationException("a clean stream did not settle clean"); +} + +if (session.SafeOffset(StreamTrack.Response) is not null) +{ + throw new InvalidOperationException("a settled session still offered a safe offset"); +} + +Console.WriteLine("package verified: non-streaming and streaming both reachable"); +CS + +( + cd "$WORK/app" + dotnet add package ResponsibleAI.AgentControlSpec --version "$VERSION" >/dev/null + dotnet run --nologo +) diff --git a/sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj b/sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj index c002b01..febdfea 100644 --- a/sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj +++ b/sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj @@ -10,8 +10,38 @@ Agent Control Specification: a policy decision runtime plugging into agent-hooks as an interceptor. .NET wrapper over the Rust engine (native library agent_control_spec_ffi resolved from the loader path). MIT https://github.com/responsibleai/agent-control-spec + + $(MSBuildThisFileDirectory)..\..\native\ + + + <_AcsNativeAsset Include="$(AcsNativeDir)runtimes\**\*" /> + + + + + + + + diff --git a/sdk/dotnet/src/AgentControlSpec/Native.cs b/sdk/dotnet/src/AgentControlSpec/Native.cs index 235a49f..4372bf1 100644 --- a/sdk/dotnet/src/AgentControlSpec/Native.cs +++ b/sdk/dotnet/src/AgentControlSpec/Native.cs @@ -312,16 +312,175 @@ internal static string PolicyInterventionPoints(ActivatedPolicyHandle handle) } internal static void PolicyFree(IntPtr handle) => acs_policy_free(handle); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_stream_session_new(string configJson, out IntPtr errOut); + + [LibraryImport(Lib)] + private static partial void acs_stream_session_free(IntPtr handle); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial long acs_stream_session_observe( + IntPtr handle, string sourceType, uint runes, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial long acs_stream_session_observe_text( + IntPtr handle, string sourceType, string text, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial int acs_stream_session_record_outcome( + IntPtr handle, string task, string sourceType, uint start, uint end, string outcome, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial int acs_stream_session_record_verdict( + IntPtr handle, string task, string sourceType, uint start, uint end, string verdictJson, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial long acs_stream_session_advance(IntPtr handle, string track, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial long acs_stream_session_safe_offset(IntPtr handle, string track, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial long acs_stream_session_pending(IntPtr handle, string track, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_stream_session_watermark(IntPtr handle, string track, out IntPtr errOut); + + [LibraryImport(Lib)] + private static partial IntPtr acs_stream_session_state(IntPtr handle, out IntPtr errOut); + + [LibraryImport(Lib)] + private static partial int acs_stream_session_end_of_payloads(IntPtr handle, out IntPtr errOut); + + [LibraryImport(Lib)] + private static partial IntPtr acs_stream_session_finish(IntPtr handle, out IntPtr errOut); + + internal static StreamSessionHandle StreamSessionNew(string configJson) + { + var handle = acs_stream_session_new(configJson, out var err); + if (handle == IntPtr.Zero) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("stream session creation returned no handle"); + } + + return new StreamSessionHandle(handle); + } + + internal static void StreamSessionFree(IntPtr handle) => acs_stream_session_free(handle); + + // A scalar query answers with the value, -1 for absent, or -2 for a + // boundary failure. Absent is a real answer (a settled session has no + // safe offset), so it becomes null rather than an exception. + private static long? Scalar(long value, IntPtr err) + { + if (value == -2) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("stream session call failed without a message"); + } + + return value == -1 ? null : value; + } + + internal static int StreamObserve(StreamSessionHandle handle, string sourceType, uint runes) + { + var received = acs_stream_session_observe(handle.DangerousGetHandle(), sourceType, runes, out var err); + return checked((int)Scalar(received, err)!.Value); + } + + internal static int StreamObserveText(StreamSessionHandle handle, string sourceType, string text) + { + var received = acs_stream_session_observe_text(handle.DangerousGetHandle(), sourceType, text, out var err); + return checked((int)Scalar(received, err)!.Value); + } + + internal static void StreamRecordOutcome( + StreamSessionHandle handle, string task, string sourceType, uint start, uint end, string outcome) + { + if (acs_stream_session_record_outcome( + handle.DangerousGetHandle(), task, sourceType, start, end, outcome, out var err) != 0) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("recording the outcome failed without a message"); + } + } + + internal static void StreamRecordVerdict( + StreamSessionHandle handle, string task, string sourceType, uint start, uint end, string verdictJson) + { + if (acs_stream_session_record_verdict( + handle.DangerousGetHandle(), task, sourceType, start, end, verdictJson, out var err) != 0) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("recording the verdict failed without a message"); + } + } + + internal static int? StreamAdvance(StreamSessionHandle handle, string track) + { + var offset = acs_stream_session_advance(handle.DangerousGetHandle(), track, out var err); + return (int?)Scalar(offset, err); + } + + internal static int? StreamSafeOffset(StreamSessionHandle handle, string track) + { + var offset = acs_stream_session_safe_offset(handle.DangerousGetHandle(), track, out var err); + return (int?)Scalar(offset, err); + } + + internal static int StreamPending(StreamSessionHandle handle, string track) + { + var pending = acs_stream_session_pending(handle.DangerousGetHandle(), track, out var err); + return checked((int)Scalar(pending, err)!.Value); + } + + internal static string StreamWatermark(StreamSessionHandle handle, string track) + { + var json = acs_stream_session_watermark(handle.DangerousGetHandle(), track, out var err); + ThrowIfError(err); + return TakeString(json); + } + + internal static string StreamState(StreamSessionHandle handle) + { + var json = acs_stream_session_state(handle.DangerousGetHandle(), out var err); + ThrowIfError(err); + return TakeString(json); + } + + internal static void StreamEndOfPayloads(StreamSessionHandle handle) + { + if (acs_stream_session_end_of_payloads(handle.DangerousGetHandle(), out var err) != 0) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("closing the payload stream failed without a message"); + } + } + + internal static string StreamFinish(StreamSessionHandle handle) + { + var json = acs_stream_session_finish(handle.DangerousGetHandle(), out var err); + ThrowIfError(err); + return TakeString(json); + } +} + +internal sealed class StreamSessionHandle : SafeHandle +{ + internal StreamSessionHandle(IntPtr handle) + : base(IntPtr.Zero, ownsHandle: true) => SetHandle(handle); + + public override bool IsInvalid => handle == IntPtr.Zero; + + protected override bool ReleaseHandle() + { + Native.StreamSessionFree(handle); + return true; + } } -/// Owns one activated policy version's native allocation. -/// -/// A rather than the bare pointer the -/// interceptor keeps, because an activated policy is shared across -/// threads by design: the ref count holds the pointer alive for the -/// duration of every in-flight evaluation, so disposing while another -/// thread evaluates frees after that call rather than under it. -/// internal sealed class ActivatedPolicyHandle : SafeHandle { internal ActivatedPolicyHandle(IntPtr handle) diff --git a/sdk/dotnet/src/AgentControlSpec/StreamSession.cs b/sdk/dotnet/src/AgentControlSpec/StreamSession.cs new file mode 100644 index 0000000..fa433ec --- /dev/null +++ b/sdk/dotnet/src/AgentControlSpec/StreamSession.cs @@ -0,0 +1,298 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Incremental stream mediation (specification section 18.1). +// +// This is a binding, not an implementation. Every decision about what +// may be released is made by the Rust engine through the C ABI, so a +// host on this SDK and a host on any other answer identically by +// construction rather than by agreement. + +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AgentControlSpec; + +/// How much a host withholds while a stream is mediated. +public enum SafetyLevel +{ + /// Withhold every rune until a task clears it. + Blocking, + + /// Withhold until the whole payload has been evaluated. + Complete, + + /// Emit on arrival and evaluate behind the stream. + Deferred, +} + +/// Which side of the exchange a payload belongs to. +public enum StreamSourceType +{ + /// Text authored by the caller. + UserRequest, + + /// Text generated by the model. + ModelGenerated, +} + +/// The two independently accounted halves of a stream. +public enum StreamTrack +{ + /// Runes flowing toward the model. + Request, + + /// Runes flowing back to the caller. + Response, +} + +/// What a task decided about one evaluated span. +public enum SegmentOutcome +{ + /// The span may be released unchanged. + Cleared, + + /// The host substitutes its own text. This ends the stream. + Transformed, + + /// The span is refused. This ends the stream. + Denied, +} + +/// How far one track got and what still owes a decision. +/// The track described. +/// Runes every task has cleared. +/// Runes observed. +/// Runes observed but not yet released. +/// The tasks that must clear a span before it releases. +public sealed record StreamWatermark( + [property: JsonPropertyName("track")] string Track, + [property: JsonPropertyName("confirmed")] int Confirmed, + [property: JsonPropertyName("received")] int Received, + [property: JsonPropertyName("pending")] int Pending, + [property: JsonPropertyName("tasks")] IReadOnlyList Tasks); + +/// Why a session ended. +/// One of complete, denied, rewritten or failed. +/// The track carrying the span, when a span ended it. +/// The task that decided, when a task ended it. +/// Start rune of the deciding span. +/// End rune of the deciding span. +/// The engine's reason code, when the session failed. +/// The engine's message, when the session failed. +public sealed record StreamEndReason( + [property: JsonPropertyName("kind")] string Kind, + [property: JsonPropertyName("track")] string? Track = null, + [property: JsonPropertyName("task")] string? Task = null, + [property: JsonPropertyName("start")] int? Start = null, + [property: JsonPropertyName("end")] int? End = null, + [property: JsonPropertyName("reason")] string? Reason = null, + [property: JsonPropertyName("message")] string? Message = null); + +/// Terminal settlement of a session. +/// Why the session ended. +/// Whether the emitted text is not verbatim model output. +/// Whether the stream ran to completion without a refusal. +public sealed record StreamCompletion( + [property: JsonPropertyName("reason")] StreamEndReason Reason, + [property: JsonPropertyName("transformed")] bool Transformed, + [property: JsonPropertyName("is_clean")] bool IsClean); + +/// The offsets and task sets a session was opened with. +/// The withholding level. +/// Where request accounting begins. +/// Where response accounting begins. +/// Tasks mediating the request track. +/// Tasks mediating the response track. +public sealed record StreamSessionConfig( + [property: JsonPropertyName("safety_level")] string SafetyLevel, + [property: JsonPropertyName("request_start_rune_offset")] int RequestStartRuneOffset, + [property: JsonPropertyName("response_start_rune_offset")] int ResponseStartRuneOffset, + [property: JsonPropertyName("request_tasks")] IReadOnlyList RequestTasks, + [property: JsonPropertyName("response_tasks")] IReadOnlyList ResponseTasks); + +/// Live state of a session. +/// Whether the session has settled. +/// Whether a rewrite ended it. +/// Why it ended, or null while it is live. +/// The configuration in force. +public sealed record StreamSessionState( + [property: JsonPropertyName("is_ended")] bool IsEnded, + [property: JsonPropertyName("transformed")] bool Transformed, + [property: JsonPropertyName("end_reason")] StreamEndReason? EndReason, + [property: JsonPropertyName("config")] StreamSessionConfig Config); + +/// +/// One mediated stream. +/// +/// A host reports arriving payloads, reports what each task decided +/// about each evaluated span, and reads back how far the stream is safe +/// to release. The session holds no text, so applying a rewrite and +/// emitting released runes stay the host's job. +/// +/// +/// +/// using var session = new StreamSession( +/// SafetyLevel.Blocking, responseTasks: ["pii"]); +/// session.ObserveText(StreamSourceType.ModelGenerated, "hello"); +/// session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); +/// int? release = session.SafeOffset(StreamTrack.Response); +/// +/// +public sealed class StreamSession : IDisposable +{ + private static readonly JsonSerializerOptions Json = new() + { + PropertyNameCaseInsensitive = true, + }; + + private readonly StreamSessionHandle _handle; + + /// Open a session. + /// How much to withhold while mediating. + /// Tasks mediating the request track. Empty leaves it unmediated. + /// Tasks mediating the response track. Empty leaves it unmediated. + /// Where request accounting begins, for a resumed stream. + /// Where response accounting begins, for a resumed stream. + /// + /// The engine refused the configuration, which includes leaving both + /// tracks unmediated, because a session that evaluates nothing + /// releases everything. + /// + public StreamSession( + SafetyLevel safetyLevel = SafetyLevel.Blocking, + IEnumerable? requestTasks = null, + IEnumerable? responseTasks = null, + int requestStartRuneOffset = 0, + int responseStartRuneOffset = 0) + { + var config = JsonSerializer.Serialize(new Dictionary + { + ["safety_level"] = Wire(safetyLevel), + ["request_start_rune_offset"] = requestStartRuneOffset, + ["response_start_rune_offset"] = responseStartRuneOffset, + ["request_tasks"] = requestTasks?.ToArray() ?? [], + ["response_tasks"] = responseTasks?.ToArray() ?? [], + }); + _handle = Native.StreamSessionNew(config); + } + + internal static string Wire(SafetyLevel level) => level switch + { + AgentControlSpec.SafetyLevel.Blocking => "blocking", + AgentControlSpec.SafetyLevel.Complete => "complete", + AgentControlSpec.SafetyLevel.Deferred => "deferred", + _ => throw new ArgumentOutOfRangeException(nameof(level), level, "unknown safety level"), + }; + + internal static string Wire(StreamSourceType source) => source switch + { + StreamSourceType.UserRequest => "user_request", + StreamSourceType.ModelGenerated => "model_generated", + _ => throw new ArgumentOutOfRangeException(nameof(source), source, "unknown stream source type"), + }; + + internal static string Wire(StreamTrack track) => track switch + { + StreamTrack.Request => "request", + StreamTrack.Response => "response", + _ => throw new ArgumentOutOfRangeException(nameof(track), track, "unknown stream track"), + }; + + internal static string Wire(SegmentOutcome outcome) => outcome switch + { + SegmentOutcome.Cleared => "cleared", + SegmentOutcome.Transformed => "transformed", + SegmentOutcome.Denied => "denied", + _ => throw new ArgumentOutOfRangeException(nameof(outcome), outcome, "unknown segment outcome"), + }; + + /// Record that more runes arrived. + /// The track's received offset. + public int Observe(StreamSourceType sourceType, int runes) => + Native.StreamObserve(_handle, Wire(sourceType), checked((uint)runes)); + + /// + /// Record an arriving payload by its text, counting runes the way the + /// engine does so a host never counts them itself. A rune is a Unicode + /// scalar, so an astral-plane character counts once even though it + /// occupies two UTF-16 code units in a .NET string. + /// + /// The track's received offset. + public int ObserveText(StreamSourceType sourceType, string text) => + Native.StreamObserveText(_handle, Wire(sourceType), text); + + /// Record what decided about a span. + public void RecordOutcome( + string task, StreamSourceType sourceType, int start, int end, SegmentOutcome outcome) => + Native.StreamRecordOutcome( + _handle, task, Wire(sourceType), checked((uint)start), checked((uint)end), Wire(outcome)); + + /// + /// Record a verdict against a span, mapping its decision onto an + /// outcome. Takes a verdict exactly as + /// returns one, so a decision feeds straight back with nothing to + /// translate. + /// + public void RecordVerdict( + string task, StreamSourceType sourceType, int start, int end, string verdictJson) => + Native.StreamRecordVerdict( + _handle, task, Wire(sourceType), checked((uint)start), checked((uint)end), verdictJson); + + /// + /// Recompute a track's watermark against the outcomes recorded so far. + /// This is the call that moves the releasable offset; recording an + /// outcome on its own does not. + /// + /// The offset it advanced to, or null when it did not advance. + public int? Advance(StreamTrack track) => Native.StreamAdvance(_handle, Wire(track)); + + /// + /// The offset safe to release, as of the last . + /// + /// + /// The releasable offset, or null once the session has ended. Null + /// means release nothing further; it is not an error, and a settled + /// session reports it rather than a stale number that would release + /// text no task cleared. + /// + public int? SafeOffset(StreamTrack track) => Native.StreamSafeOffset(_handle, Wire(track)); + + /// Runes observed on a track but not yet released. + public int Pending(StreamTrack track) => Native.StreamPending(_handle, Wire(track)); + + /// + /// A track's watermark. The confirmed offset stays readable after + /// settlement, so an audit record can still say how far the stream got. + /// + public StreamWatermark Watermark(StreamTrack track) => + JsonSerializer.Deserialize(Native.StreamWatermark(_handle, Wire(track)), Json) + ?? throw new AgentControlSpecNativeException("watermark did not deserialize"); + + /// Live session state. + public StreamSessionState State => + JsonSerializer.Deserialize(Native.StreamState(_handle), Json) + ?? throw new AgentControlSpecNativeException("state did not deserialize"); + + /// Whether the session has settled. + public bool IsEnded => State.IsEnded; + + /// Whether a rewrite ended the session. + public bool Transformed => State.Transformed; + + /// Why the session ended, or null while it is live. + public StreamEndReason? EndReason => State.EndReason; + + /// The configuration in force. + public StreamSessionConfig Config => State.Config; + + /// Declare that no further payload will arrive. + public void EndOfPayloads() => Native.StreamEndOfPayloads(_handle); + + /// Settle the session. Settling twice returns the same completion. + public StreamCompletion Finish() => + JsonSerializer.Deserialize(Native.StreamFinish(_handle), Json) + ?? throw new AgentControlSpecNativeException("completion did not deserialize"); + + /// Release the native session. + public void Dispose() => _handle.Dispose(); +} diff --git a/sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs b/sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs new file mode 100644 index 0000000..66390cf --- /dev/null +++ b/sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Streaming reaches .NET through the C ABI, so these run the real engine. +// They are the .NET half of a suite that asserts the same scenarios in +// every supported language. + +using AgentControlSpec; +using Xunit; + +namespace AgentControlSpec.Tests; + +public sealed class StreamSessionTests +{ + [Fact] + public void AClearedSpanReleasesUpToItsEnd() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + + Assert.Equal(5, session.ObserveText(StreamSourceType.ModelGenerated, "hello")); + Assert.Equal(0, session.SafeOffset(StreamTrack.Response)); + + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); + + Assert.Equal(5, session.Advance(StreamTrack.Response)); + Assert.Equal(5, session.SafeOffset(StreamTrack.Response)); + Assert.Equal(0, session.Pending(StreamTrack.Response)); + + var completion = session.Finish(); + Assert.Equal("complete", completion.Reason.Kind); + Assert.True(completion.IsClean); + Assert.False(completion.Transformed); + } + + [Fact] + public void ARefusalEndsTheSessionAndStillReportsHowFarItGot() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hello world"); + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); + session.Advance(StreamTrack.Response); + Assert.Equal(5, session.SafeOffset(StreamTrack.Response)); + + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 5, 11, SegmentOutcome.Denied); + + Assert.True(session.IsEnded); + Assert.Null(session.SafeOffset(StreamTrack.Response)); + + // The audit path: the offset the stream reached survives settlement. + Assert.Equal(5, session.Watermark(StreamTrack.Response).Confirmed); + + var reason = session.EndReason; + Assert.NotNull(reason); + Assert.Equal("denied", reason!.Kind); + Assert.Equal("pii", reason.Task); + Assert.Equal("response", reason.Track); + Assert.False(session.Finish().IsClean); + } + + [Fact] + public void EveryTaskMustClearASpanBeforeItReleases() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii", "harm"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hello"); + + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); + Assert.Null(session.Advance(StreamTrack.Response)); + Assert.Equal(0, session.SafeOffset(StreamTrack.Response)); + + session.RecordOutcome("harm", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); + Assert.Equal(5, session.Advance(StreamTrack.Response)); + Assert.Equal(5, session.SafeOffset(StreamTrack.Response)); + } + + [Fact] + public void ObserveTextCountsRunesNotUtf16CodeUnits() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + + // One astral-plane scalar. .NET stores it as two UTF-16 code units, + // so a host counting string.Length would release twice what was + // evaluated. + const string Emoji = "\U0001F600"; + Assert.Equal(2, Emoji.Length); + Assert.Equal(1, session.ObserveText(StreamSourceType.ModelGenerated, Emoji)); + } + + [Fact] + public void APayloadOnAnUnmediatedTrackIsRefused() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + + // No task mediates the request track, so nothing would ever clear + // text sent there. The engine refuses the payload rather than + // releasing it unevaluated. + var error = Assert.Throws(() => + session.ObserveText(StreamSourceType.UserRequest, "hi")); + Assert.Contains("unmediated", error.Message); + Assert.Empty(session.Watermark(StreamTrack.Request).Tasks); + } + + [Fact] + public void TheTwoTracksAccountIndependently() + { + using var session = new StreamSession( + SafetyLevel.Blocking, requestTasks: ["pii"], responseTasks: ["pii"]); + + session.ObserveText(StreamSourceType.UserRequest, "abc"); + session.ObserveText(StreamSourceType.ModelGenerated, "defghi"); + + session.RecordOutcome("pii", StreamSourceType.UserRequest, 0, 3, SegmentOutcome.Cleared); + session.Advance(StreamTrack.Request); + + Assert.Equal(3, session.SafeOffset(StreamTrack.Request)); + Assert.Equal(0, session.SafeOffset(StreamTrack.Response)); + Assert.Equal(6, session.Watermark(StreamTrack.Response).Received); + } + + [Fact] + public void AResumedStreamStartsFromItsRecordedOffsets() + { + using var session = new StreamSession( + SafetyLevel.Blocking, responseTasks: ["pii"], responseStartRuneOffset: 10); + + Assert.Equal(10, session.Config.ResponseStartRuneOffset); + session.ObserveText(StreamSourceType.ModelGenerated, "abc"); + Assert.Equal(13, session.Watermark(StreamTrack.Response).Received); + } + + [Fact] + public void ARewriteIsTerminalAndReportsItself() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hello"); + + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Transformed); + + Assert.True(session.IsEnded); + Assert.True(session.Transformed); + Assert.Equal("rewritten", session.EndReason!.Kind); + Assert.Null(session.SafeOffset(StreamTrack.Response)); + } + + [Fact] + public void AVerdictFeedsBackWithoutTranslation() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hello"); + + // Shaped as ActivatedPolicy.Evaluate returns one. + session.RecordVerdict( + "pii", StreamSourceType.ModelGenerated, 0, 5, + """{"decision":"allow","reasons":[]}"""); + + Assert.Equal(5, session.Advance(StreamTrack.Response)); + Assert.Equal(5, session.SafeOffset(StreamTrack.Response)); + } + + [Fact] + public void AnUnknownTaskIsRefusedRatherThanIgnored() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hello"); + + Assert.Throws(() => + session.RecordOutcome("nope", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared)); + } + + [Fact] + public void ASessionThatEvaluatesNothingIsRefused() + { + Assert.Throws(() => new StreamSession(SafetyLevel.Blocking)); + } + + [Fact] + public void AnOutcomeReachingUnobservedTextIsRefused() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hi"); + + Assert.Throws(() => + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 99, SegmentOutcome.Cleared)); + } +} diff --git a/sdk/ffi/Cargo.toml b/sdk/ffi/Cargo.toml index 4e6f2b3..4a7d24d 100644 --- a/sdk/ffi/Cargo.toml +++ b/sdk/ffi/Cargo.toml @@ -11,5 +11,5 @@ name = "agent_control_spec_ffi" crate-type = ["cdylib", "rlib"] [dependencies] -agent-control-spec = { path = "../../engine", features = ["default-dispatchers"] } +agent-control-spec = { path = "../../engine", features = ["default-dispatchers", "streaming"] } serde_json = "1" diff --git a/sdk/ffi/src/lib.rs b/sdk/ffi/src/lib.rs index 38c0291..8bb7930 100644 --- a/sdk/ffi/src/lib.rs +++ b/sdk/ffi/src/lib.rs @@ -18,9 +18,13 @@ // problems only (bad UTF-8, non-object context, poisoned handle). use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; +use agent_control_spec::stream_session::{ + SafetyLevel, SegmentOutcome, StreamEndReason, StreamSession, StreamSessionConfig, + StreamSourceType, StreamSpan, StreamTrack, +}; use agent_control_spec::{ ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, Runtime, RuntimeError, - SUPPORTED_VERSIONS, + Verdict, SUPPORTED_VERSIONS, }; use serde_json::Value; use std::collections::BTreeMap; @@ -806,6 +810,730 @@ pub unsafe extern "C" fn acs_policy_free(handle: *mut AcsActivatedPolicy) { drop(Box::from_raw(handle)); } +// --------------------------------------------------------------------- +// Incremental stream mediation (specification section 18.1). +// +// A `StreamSession` is stateful, so it follows the handle shape used by +// `AcsActivatedPolicy`: create once, drive it as payloads arrive, free +// exactly once. The runtime underneath stays stateless; the session only +// records what each ordinary evaluation cleared. +// +// Scalar queries return `i64` so an absent value needs no allocation: +// `>= 0` is the value, `-1` is absent (a released offset the caller must +// treat as "release nothing"), and `-2` means the call failed and +// `*err_out` carries why. Absent and failed are distinct because a +// settled session legitimately has no safe offset, which is not an error. +// +// Structured queries return JSON, freed with `acs_free_string`, so the +// wire contract is owned here rather than derived from Rust layout. +// --------------------------------------------------------------------- + +/// Opaque handle to one mediated stream. +pub struct AcsStreamSession { + session: StreamSession, +} + +fn parse_track(value: &str, err_out: *mut *mut c_char) -> Option { + match value { + "request" => Some(StreamTrack::Request), + "response" => Some(StreamTrack::Response), + other => { + set_err(err_out, format!("unknown stream track '{other}'")); + None + } + } +} + +fn parse_outcome(value: &str, err_out: *mut *mut c_char) -> Option { + match value { + "cleared" => Some(SegmentOutcome::Cleared), + "transformed" => Some(SegmentOutcome::Transformed), + "denied" => Some(SegmentOutcome::Denied), + other => { + set_err(err_out, format!("unknown segment outcome '{other}'")); + None + } + } +} + +fn end_reason_json(reason: &StreamEndReason) -> Value { + match reason { + StreamEndReason::Complete => serde_json::json!({ "kind": "complete" }), + StreamEndReason::Denied { track, task, range } => serde_json::json!({ + "kind": "denied", + "track": track.as_str(), + "task": task, + "start": range.start, + "end": range.end, + }), + StreamEndReason::Rewritten { track, task, range } => serde_json::json!({ + "kind": "rewritten", + "track": track.as_str(), + "task": task, + "start": range.start, + "end": range.end, + }), + StreamEndReason::Failed(error) => serde_json::json!({ + "kind": "failed", + "reason": error.reason(), + "message": error.to_string(), + }), + } +} + +/// Open a session from `config_json`. +/// +/// The object takes `safety_level` (`blocking`, `complete` or +/// `deferred`), the per track start offsets `request_start_rune_offset` +/// and `response_start_rune_offset`, and the task name arrays +/// `request_tasks` and `response_tasks`. An absent field takes its +/// default; an empty task array means that track is unmediated. +/// +/// Returns NULL and sets `*err_out` on failure. Free with +/// `acs_stream_session_free`. +/// +/// # Safety +/// `config_json` must be a valid NUL-terminated string. `err_out` must +/// be null or point to a writable pointer. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_new( + config_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut AcsStreamSession { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(raw) = read_utf8(config_json, "config_json", err_out) else { + return std::ptr::null_mut(); + }; + let parsed: Value = match serde_json::from_str(raw) { + Ok(v @ Value::Object(_)) => v, + Ok(_) => { + set_err(err_out, "config_json must be a JSON object".to_string()); + return std::ptr::null_mut(); + } + Err(e) => { + set_err(err_out, format!("config_json does not parse: {e}")); + return std::ptr::null_mut(); + } + }; + let level_raw = parsed + .get("safety_level") + .and_then(Value::as_str) + .unwrap_or("blocking"); + let safety_level = match SafetyLevel::parse(level_raw) { + Ok(l) => l, + Err(e) => { + set_err(err_out, format!("{e}")); + return std::ptr::null_mut(); + } + }; + let offset = |key: &str| -> Result { + match parsed.get(key) { + None | Some(Value::Null) => Ok(0), + Some(v) => v + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| format!("{key} must be a rune offset within u32")), + } + }; + let request_start_rune_offset = match offset("request_start_rune_offset") { + Ok(v) => v, + Err(e) => { + set_err(err_out, e); + return std::ptr::null_mut(); + } + }; + let response_start_rune_offset = match offset("response_start_rune_offset") { + Ok(v) => v, + Err(e) => { + set_err(err_out, e); + return std::ptr::null_mut(); + } + }; + let tasks = |key: &str| -> Result, String> { + match parsed.get(key) { + None | Some(Value::Null) => Ok(Vec::new()), + Some(Value::Array(items)) => items + .iter() + .map(|i| { + i.as_str() + .map(str::to_string) + .ok_or_else(|| format!("{key} must contain only task name strings")) + }) + .collect(), + Some(_) => Err(format!("{key} must be an array of task names")), + } + }; + let request_tasks = match tasks("request_tasks") { + Ok(v) => v, + Err(e) => { + set_err(err_out, e); + return std::ptr::null_mut(); + } + }; + let response_tasks = match tasks("response_tasks") { + Ok(v) => v, + Err(e) => { + set_err(err_out, e); + return std::ptr::null_mut(); + } + }; + let config = StreamSessionConfig { + safety_level, + request_start_rune_offset, + response_start_rune_offset, + request_tasks, + response_tasks, + }; + match StreamSession::new(config) { + Ok(session) => Box::into_raw(Box::new(AcsStreamSession { session })), + Err(e) => { + set_err(err_out, format!("{e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_stream_session_new".to_string(), + ); + std::ptr::null_mut() + } + } +} + +/// Free a session handle. Freeing NULL is a no-op. +/// +/// # Safety +/// `handle` must come from `acs_stream_session_new` and be freed once. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_free(handle: *mut AcsStreamSession) { + if handle.is_null() { + return; + } + let _ = catch_unwind(AssertUnwindSafe(|| drop(Box::from_raw(handle)))); +} + +/// Record that `runes` more runes of `source_type` arrived. Returns the +/// track's received offset, or -2 on failure. +/// +/// # Safety +/// `handle` must be live; `source_type` a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_observe( + handle: *mut AcsStreamSession, + source_type: *const c_char, + runes: u32, + err_out: *mut *mut c_char, +) -> i64 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return -2; + } + let Some(raw) = read_utf8(source_type, "source_type", err_out) else { + return -2; + }; + let source = match StreamSourceType::parse(raw) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -2; + } + }; + match (*handle).session.observe(source, runes) { + Ok(received) => i64::from(received), + Err(e) => { + set_err(err_out, format!("{e}")); + -2 + } + } + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_observe".to_string(), + ); + -2 + }) +} + +/// Record an arriving payload by its text, counting runes the way the +/// engine does so a host never has to count them itself. Returns the +/// track's received offset, or -2 on failure. +/// +/// # Safety +/// `handle` must be live; `source_type` and `text` valid NUL-terminated +/// strings. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_observe_text( + handle: *mut AcsStreamSession, + source_type: *const c_char, + text: *const c_char, + err_out: *mut *mut c_char, +) -> i64 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return -2; + } + let Some(raw) = read_utf8(source_type, "source_type", err_out) else { + return -2; + }; + let source = match StreamSourceType::parse(raw) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -2; + } + }; + let Some(body) = read_utf8(text, "text", err_out) else { + return -2; + }; + match (*handle).session.observe_text(source, body) { + Ok(received) => i64::from(received), + Err(e) => { + set_err(err_out, format!("{e}")); + -2 + } + } + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_observe_text".to_string(), + ); + -2 + }) +} + +/// Record what `task` decided about the span `[start, end)` of +/// `source_type`. `outcome` is `cleared`, `transformed` or `denied`. +/// Returns 0, or -1 on failure. +/// +/// # Safety +/// `handle` must be live; `task`, `source_type` and `outcome` valid +/// NUL-terminated strings. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_record_outcome( + handle: *mut AcsStreamSession, + task: *const c_char, + source_type: *const c_char, + start: u32, + end: u32, + outcome: *const c_char, + err_out: *mut *mut c_char, +) -> i32 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return -1; + } + let Some(task_name) = read_utf8(task, "task", err_out) else { + return -1; + }; + let Some(source_raw) = read_utf8(source_type, "source_type", err_out) else { + return -1; + }; + let source = match StreamSourceType::parse(source_raw) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -1; + } + }; + let Some(outcome_raw) = read_utf8(outcome, "outcome", err_out) else { + return -1; + }; + let Some(outcome) = parse_outcome(outcome_raw, err_out) else { + return -1; + }; + let span = match StreamSpan::new(source, start, end) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -1; + } + }; + match (*handle).session.record_outcome(task_name, &span, outcome) { + Ok(()) => 0, + Err(e) => { + set_err(err_out, format!("{e}")); + -1 + } + } + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_record_outcome".to_string(), + ); + -1 + }) +} + +/// Record an Agent Control Specification verdict against the span +/// `[start, end)` of `source_type`, mapping its decision onto an +/// outcome. `verdict_json` is a verdict as `acs_policy_evaluate` +/// returns one, so a host feeds a decision straight back without +/// translating it. Returns 0, or -1 on failure. +/// +/// # Safety +/// `handle` must be live; `task`, `source_type` and `verdict_json` +/// valid NUL-terminated strings. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_record_verdict( + handle: *mut AcsStreamSession, + task: *const c_char, + source_type: *const c_char, + start: u32, + end: u32, + verdict_json: *const c_char, + err_out: *mut *mut c_char, +) -> i32 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return -1; + } + let Some(task_name) = read_utf8(task, "task", err_out) else { + return -1; + }; + let Some(source_raw) = read_utf8(source_type, "source_type", err_out) else { + return -1; + }; + let source = match StreamSourceType::parse(source_raw) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -1; + } + }; + let Some(raw) = read_utf8(verdict_json, "verdict_json", err_out) else { + return -1; + }; + let verdict: Verdict = match serde_json::from_str(raw) { + Ok(v) => v, + Err(e) => { + set_err(err_out, format!("verdict_json does not parse: {e}")); + return -1; + } + }; + let span = match StreamSpan::new(source, start, end) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -1; + } + }; + match (*handle).session.record_verdict(task_name, &span, &verdict) { + Ok(()) => 0, + Err(e) => { + set_err(err_out, format!("{e}")); + -1 + } + } + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_record_verdict".to_string(), + ); + -1 + }) +} + +/// Recompute `track`'s watermark and return the offset it advanced to, +/// -1 when it did not advance or the session has ended, -2 on failure. +/// +/// # Safety +/// `handle` must be live; `track` a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_advance( + handle: *mut AcsStreamSession, + track: *const c_char, + err_out: *mut *mut c_char, +) -> i64 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return -2; + } + let Some(raw) = read_utf8(track, "track", err_out) else { + return -2; + }; + let Some(track) = parse_track(raw, err_out) else { + return -2; + }; + match (*handle).session.advance(track) { + Some(offset) => i64::from(offset), + None => -1, + } + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_advance".to_string(), + ); + -2 + }) +} + +/// The offset of `track` safe to release, -1 once the session has ended, +/// -2 on failure. A settled session has no safe offset, which is not an +/// error: it means release nothing further. +/// +/// # Safety +/// `handle` must be live; `track` a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_safe_offset( + handle: *const AcsStreamSession, + track: *const c_char, + err_out: *mut *mut c_char, +) -> i64 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return -2; + } + let Some(raw) = read_utf8(track, "track", err_out) else { + return -2; + }; + let Some(track) = parse_track(raw, err_out) else { + return -2; + }; + match (*handle).session.safe_offset(track) { + Some(offset) => i64::from(offset), + None => -1, + } + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_safe_offset".to_string(), + ); + -2 + }) +} + +/// The rune count of `track` observed but not yet released, or -2 on +/// failure. +/// +/// # Safety +/// `handle` must be live; `track` a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_pending( + handle: *const AcsStreamSession, + track: *const c_char, + err_out: *mut *mut c_char, +) -> i64 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return -2; + } + let Some(raw) = read_utf8(track, "track", err_out) else { + return -2; + }; + let Some(track) = parse_track(raw, err_out) else { + return -2; + }; + i64::from((*handle).session.pending(track)) + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_pending".to_string(), + ); + -2 + }) +} + +/// `track`'s watermark as JSON, carrying `track`, `confirmed`, +/// `received`, `pending` and the `tasks` that must clear it. The +/// confirmed offset stays readable after settlement, so an audit record +/// can still say how far the stream got. Freed with `acs_free_string`. +/// +/// # Safety +/// `handle` must be live; `track` a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_watermark( + handle: *const AcsStreamSession, + track: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return std::ptr::null_mut(); + } + let Some(raw) = read_utf8(track, "track", err_out) else { + return std::ptr::null_mut(); + }; + let Some(track) = parse_track(raw, err_out) else { + return std::ptr::null_mut(); + }; + let watermark = (*handle).session.watermark(track); + let payload = serde_json::json!({ + "track": track.as_str(), + "confirmed": watermark.confirmed(), + "received": watermark.received(), + "pending": watermark.pending(), + "tasks": watermark.tasks().collect::>(), + }); + match serde_json::to_string(&payload) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("watermark serialization failed: {e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_stream_session_watermark".to_string(), + ); + std::ptr::null_mut() + } + } +} + +/// Session state as JSON: `is_ended`, `transformed`, `end_reason` and +/// the effective `config`. `end_reason` is null while the session is +/// live. Freed with `acs_free_string`. +/// +/// # Safety +/// `handle` must be a live pointer from `acs_stream_session_new`. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_state( + handle: *const AcsStreamSession, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return std::ptr::null_mut(); + } + let session = &(*handle).session; + let config = session.config(); + let payload = serde_json::json!({ + "is_ended": session.is_ended(), + "transformed": session.transformed(), + "end_reason": session.end_reason().map(end_reason_json), + "config": { + "safety_level": config.safety_level.as_str(), + "request_start_rune_offset": config.request_start_rune_offset, + "response_start_rune_offset": config.response_start_rune_offset, + "request_tasks": config.request_tasks, + "response_tasks": config.response_tasks, + }, + }); + match serde_json::to_string(&payload) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("state serialization failed: {e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_stream_session_state".to_string(), + ); + std::ptr::null_mut() + } + } +} + +/// Declare that no further payload will arrive. Returns 0, or -1 on +/// failure. +/// +/// # Safety +/// `handle` must be a live pointer from `acs_stream_session_new`. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_end_of_payloads( + handle: *mut AcsStreamSession, + err_out: *mut *mut c_char, +) -> i32 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return -1; + } + (*handle).session.end_of_payloads(); + 0 + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_end_of_payloads".to_string(), + ); + -1 + }) +} + +/// Settle the session and return the completion as JSON, carrying +/// `reason`, `transformed` and `is_clean`. Freed with +/// `acs_free_string`. Settling twice returns the same completion. +/// +/// # Safety +/// `handle` must be a live pointer from `acs_stream_session_new`. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_finish( + handle: *mut AcsStreamSession, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return std::ptr::null_mut(); + } + let completion = (*handle).session.finish(); + let payload = serde_json::json!({ + "reason": end_reason_json(&completion.reason), + "transformed": completion.transformed, + "is_clean": completion.reason.is_clean(), + }); + match serde_json::to_string(&payload) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("completion serialization failed: {e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_stream_session_finish".to_string(), + ); + std::ptr::null_mut() + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/sdk/node/binding.d.ts b/sdk/node/binding.d.ts index 6684d84..8d5858d 100644 --- a/sdk/node/binding.d.ts +++ b/sdk/node/binding.d.ts @@ -76,6 +76,97 @@ export declare function policyEvaluate(handle: ExternalObject, poi */ export declare function policyInterventionPoints(handle: ExternalObject): Array +/** + * Recompute `track`'s watermark. Returns the new offset when the + * watermark advanced, `null` when it did not or the session has ended + * (matching the Rust `Option`). + */ +export declare function streamSessionAdvance(handle: ExternalObject, track: string): number | null + +/** Declare that no further payload will arrive. Idempotent. */ +export declare function streamSessionEndOfPayloads(handle: ExternalObject): void + +/** + * Settle the session and return the completion as JSON, carrying + * `reason`, `transformed` and `is_clean`. Settling twice returns the + * same completion. + */ +export declare function streamSessionFinish(handle: ExternalObject): string + +/** + * Open a session from a config JSON object. + * + * Matches `acs_stream_session_new`: takes `safety_level` (`blocking`, + * `complete` or `deferred`), the per-track start offsets + * `request_start_rune_offset` and `response_start_rune_offset`, and + * the task name arrays `request_tasks` and `response_tasks`. An empty + * task array means that track is unmediated; payload on it fails + * closed. A configuration mediating neither track is refused. + * + * The wrapper takes a JSON string rather than a napi object so the + * config surface is identical to the other language SDKs and offset + * coercion happens in one place. + */ +export declare function streamSessionNew(configJson: string): ExternalObject + +/** + * Report that `runes` more runes of `source_type` arrived and return + * the track's new end offset. Boundary failures throw; a streaming + * accounting failure throws with the engine's message and puts the + * session into its terminal state. + */ +export declare function streamSessionObserve(handle: ExternalObject, sourceType: string, runes: number): number + +/** + * Report arriving `text` on `source_type`, counting Unicode scalars so + * a host does not have to. Returns the track's new end offset. + * + * The engine counts runes, not UTF-16 code units. `Utf16String` yields + * UTF-16, so the binding decodes to a `String` before delegating to + * the engine and rune counting stays consistent with every other SDK. + * An astral-plane character is one rune here even though it is two + * UTF-16 code units. + */ +export declare function streamSessionObserveText(handle: ExternalObject, sourceType: string, text: string): number + +/** Runes on `track` observed but not yet released. */ +export declare function streamSessionPending(handle: ExternalObject, track: string): number + +/** + * Record what `task` decided about the span `[start, end)` of + * `source_type`. `outcome` is `cleared`, `transformed` or `denied`. + */ +export declare function streamSessionRecordOutcome(handle: ExternalObject, task: string, sourceType: string, start: number, end: number, outcome: string): void + +/** + * Record an ACS verdict against the span `[start, end)` of + * `source_type`, mapping its decision onto an outcome. A host feeds + * the JSON returned by `policyEvaluate` straight back without + * translating it. + */ +export declare function streamSessionRecordVerdict(handle: ExternalObject, task: string, sourceType: string, start: number, end: number, verdictJson: string): void + +/** + * Offset of `track` the host may release through, or `null` once the + * session has ended. A settled session has no safe offset, which is + * not an error: it means release nothing further. + */ +export declare function streamSessionSafeOffset(handle: ExternalObject, track: string): number | null + +/** + * Session state as JSON: `is_ended`, `transformed`, `end_reason` + * (null while live) and the effective `config`. + */ +export declare function streamSessionState(handle: ExternalObject): string + +/** + * `track`'s watermark as JSON, carrying `track`, `confirmed`, + * `received`, `pending` and the `tasks` that must clear it. The + * confirmed offset stays readable after settlement, so an audit + * record can still say how far the stream got. + */ +export declare function streamSessionWatermark(handle: ExternalObject, track: string): string + /** The manifest grammar versions this engine accepts. */ export declare function supportedManifestVersions(): Array diff --git a/sdk/node/binding.js b/sdk/node/binding.js index 5d14e01..40a6f77 100644 --- a/sdk/node/binding.js +++ b/sdk/node/binding.js @@ -706,6 +706,18 @@ module.exports.policyActivate = nativeBinding.policyActivate module.exports.policyActivateFromMemory = nativeBinding.policyActivateFromMemory module.exports.policyEvaluate = nativeBinding.policyEvaluate module.exports.policyInterventionPoints = nativeBinding.policyInterventionPoints +module.exports.streamSessionAdvance = nativeBinding.streamSessionAdvance +module.exports.streamSessionEndOfPayloads = nativeBinding.streamSessionEndOfPayloads +module.exports.streamSessionFinish = nativeBinding.streamSessionFinish +module.exports.streamSessionNew = nativeBinding.streamSessionNew +module.exports.streamSessionObserve = nativeBinding.streamSessionObserve +module.exports.streamSessionObserveText = nativeBinding.streamSessionObserveText +module.exports.streamSessionPending = nativeBinding.streamSessionPending +module.exports.streamSessionRecordOutcome = nativeBinding.streamSessionRecordOutcome +module.exports.streamSessionRecordVerdict = nativeBinding.streamSessionRecordVerdict +module.exports.streamSessionSafeOffset = nativeBinding.streamSessionSafeOffset +module.exports.streamSessionState = nativeBinding.streamSessionState +module.exports.streamSessionWatermark = nativeBinding.streamSessionWatermark module.exports.supportedManifestVersions = nativeBinding.supportedManifestVersions module.exports.validateManifest = nativeBinding.validateManifest module.exports.validateManifestFile = nativeBinding.validateManifestFile diff --git a/sdk/node/native/Cargo.toml b/sdk/node/native/Cargo.toml index c7f4d91..f23fb40 100644 --- a/sdk/node/native/Cargo.toml +++ b/sdk/node/native/Cargo.toml @@ -11,7 +11,7 @@ name = "agent_control_spec_node" crate-type = ["cdylib"] [dependencies] -agent-control-spec = { path = "../../../engine", features = ["default-dispatchers"] } +agent-control-spec = { path = "../../../engine", features = ["default-dispatchers", "streaming"] } napi = { version = "3", default-features = false, features = ["napi8"] } napi-derive = "3" serde_json = "1" diff --git a/sdk/node/native/src/lib.rs b/sdk/node/native/src/lib.rs index 974322e..efec654 100644 --- a/sdk/node/native/src/lib.rs +++ b/sdk/node/native/src/lib.rs @@ -9,15 +9,17 @@ // context JSON). use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; +use agent_control_spec::Verdict; use agent_control_spec::{ ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, Runtime, RuntimeError, - SUPPORTED_VERSIONS, + SafetyLevel, SegmentOutcome, StreamEndReason, StreamError, StreamSession, StreamSessionConfig, + StreamSourceType, StreamSpan, StreamTrack, SUPPORTED_VERSIONS, }; use napi::bindgen_prelude::{External, Utf16String}; use napi_derive::napi; use serde_json::Value; use std::collections::BTreeMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; pub struct Handle { runtime: Runtime, @@ -257,3 +259,341 @@ pub fn supported_manifest_versions() -> Vec { .map(|v| (*v).to_string()) .collect() } + +// --------------------------------------------------------------------- +// Streaming: incremental release accounting for stream-shaped tracks +// (spec §18.1). A session holds no policy and no text; the host drives +// it: report arriving text, declare the spans its segmenter produced, +// evaluate those spans with the ordinary runtime, feed the outcomes +// back, and ask which prefix is safe to release. Mirrors +// `acs_stream_session_*` in the C ABI and the Python `StreamSession`. +// --------------------------------------------------------------------- + +/// One live streaming session. +/// +/// The session is `&mut` on every meaningful call, so the handle wraps +/// a `Mutex`. Napi may invoke bindings from any worker thread, and a +/// session is cheap to lock: no policy runs behind it. +pub struct StreamHandle { + session: Mutex, +} + +fn parse_track_wire(raw: &str) -> napi::Result { + match raw { + "request" => Ok(StreamTrack::Request), + "response" => Ok(StreamTrack::Response), + other => Err(err(format!("unknown stream track '{other}'"))), + } +} + +fn parse_outcome_wire(raw: &str) -> napi::Result { + match raw { + "cleared" => Ok(SegmentOutcome::Cleared), + "transformed" => Ok(SegmentOutcome::Transformed), + "denied" => Ok(SegmentOutcome::Denied), + other => Err(err(format!("unknown segment outcome '{other}'"))), + } +} + +fn parse_source_wire(raw: &str) -> napi::Result { + StreamSourceType::parse(raw).map_err(|e| err(format!("{e}"))) +} + +fn end_reason_json(reason: &StreamEndReason) -> Value { + match reason { + StreamEndReason::Complete => serde_json::json!({ "kind": "complete" }), + StreamEndReason::Denied { track, task, range } => serde_json::json!({ + "kind": "denied", + "track": track.as_str(), + "task": task, + "start": range.start, + "end": range.end, + }), + StreamEndReason::Rewritten { track, task, range } => serde_json::json!({ + "kind": "rewritten", + "track": track.as_str(), + "task": task, + "start": range.start, + "end": range.end, + }), + StreamEndReason::Failed(error) => serde_json::json!({ + "kind": "failed", + "reason": error.reason(), + "message": error.to_string(), + }), + } +} + +fn stream_err(e: StreamError) -> napi::Error { + err(format!("{e}")) +} + +fn read_offset(parsed: &Value, key: &str) -> napi::Result { + match parsed.get(key) { + None | Some(Value::Null) => Ok(0), + Some(v) => v + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| err(format!("{key} must be a rune offset within u32"))), + } +} + +fn read_tasks(parsed: &Value, key: &str) -> napi::Result> { + match parsed.get(key) { + None | Some(Value::Null) => Ok(Vec::new()), + Some(Value::Array(items)) => items + .iter() + .map(|i| { + i.as_str() + .map(str::to_string) + .ok_or_else(|| err(format!("{key} must contain only task name strings"))) + }) + .collect(), + Some(_) => Err(err(format!("{key} must be an array of task names"))), + } +} + +/// Open a session from a config JSON object. +/// +/// Matches `acs_stream_session_new`: takes `safety_level` (`blocking`, +/// `complete` or `deferred`), the per-track start offsets +/// `request_start_rune_offset` and `response_start_rune_offset`, and +/// the task name arrays `request_tasks` and `response_tasks`. An empty +/// task array means that track is unmediated; payload on it fails +/// closed. A configuration mediating neither track is refused. +/// +/// The wrapper takes a JSON string rather than a napi object so the +/// config surface is identical to the other language SDKs and offset +/// coercion happens in one place. +#[napi] +pub fn stream_session_new(config_json: Utf16String) -> napi::Result> { + let config_json = decode("config_json", &config_json)?; + let parsed: Value = serde_json::from_str(&config_json) + .map_err(|e| err(format!("config_json does not parse: {e}")))?; + if !parsed.is_object() { + return Err(err("config_json must be a JSON object".to_string())); + } + let level_raw = parsed + .get("safety_level") + .and_then(Value::as_str) + .unwrap_or("blocking"); + let safety_level = SafetyLevel::parse(level_raw).map_err(|e| err(format!("{e}")))?; + let request_start_rune_offset = read_offset(&parsed, "request_start_rune_offset")?; + let response_start_rune_offset = read_offset(&parsed, "response_start_rune_offset")?; + let request_tasks = read_tasks(&parsed, "request_tasks")?; + let response_tasks = read_tasks(&parsed, "response_tasks")?; + let config = StreamSessionConfig { + safety_level, + request_start_rune_offset, + response_start_rune_offset, + request_tasks, + response_tasks, + }; + let session = StreamSession::new(config).map_err(stream_err)?; + Ok(External::new(StreamHandle { + session: Mutex::new(session), + })) +} + +fn lock<'a>( + handle: &'a External, +) -> napi::Result> { + handle + .session + .lock() + .map_err(|_| err("stream session mutex was poisoned".to_string())) +} + +/// Report that `runes` more runes of `source_type` arrived and return +/// the track's new end offset. Boundary failures throw; a streaming +/// accounting failure throws with the engine's message and puts the +/// session into its terminal state. +#[napi] +pub fn stream_session_observe( + handle: &External, + source_type: Utf16String, + runes: u32, +) -> napi::Result { + let raw = decode("source_type", &source_type)?; + let source = parse_source_wire(&raw)?; + let mut session = lock(handle)?; + session.observe(source, runes).map_err(stream_err) +} + +/// Report arriving `text` on `source_type`, counting Unicode scalars so +/// a host does not have to. Returns the track's new end offset. +/// +/// The engine counts runes, not UTF-16 code units. `Utf16String` yields +/// UTF-16, so the binding decodes to a `String` before delegating to +/// the engine and rune counting stays consistent with every other SDK. +/// An astral-plane character is one rune here even though it is two +/// UTF-16 code units. +#[napi] +pub fn stream_session_observe_text( + handle: &External, + source_type: Utf16String, + text: Utf16String, +) -> napi::Result { + let source_raw = decode("source_type", &source_type)?; + let source = parse_source_wire(&source_raw)?; + let body = decode("text", &text)?; + let mut session = lock(handle)?; + session.observe_text(source, &body).map_err(stream_err) +} + +/// Record what `task` decided about the span `[start, end)` of +/// `source_type`. `outcome` is `cleared`, `transformed` or `denied`. +#[napi] +pub fn stream_session_record_outcome( + handle: &External, + task: Utf16String, + source_type: Utf16String, + start: u32, + end: u32, + outcome: Utf16String, +) -> napi::Result<()> { + let task = decode("task", &task)?; + let source_raw = decode("source_type", &source_type)?; + let source = parse_source_wire(&source_raw)?; + let outcome_raw = decode("outcome", &outcome)?; + let outcome = parse_outcome_wire(&outcome_raw)?; + let span = StreamSpan::new(source, start, end).map_err(stream_err)?; + let mut session = lock(handle)?; + session + .record_outcome(&task, &span, outcome) + .map_err(stream_err) +} + +/// Record an ACS verdict against the span `[start, end)` of +/// `source_type`, mapping its decision onto an outcome. A host feeds +/// the JSON returned by `policyEvaluate` straight back without +/// translating it. +#[napi] +pub fn stream_session_record_verdict( + handle: &External, + task: Utf16String, + source_type: Utf16String, + start: u32, + end: u32, + verdict_json: Utf16String, +) -> napi::Result<()> { + let task = decode("task", &task)?; + let source_raw = decode("source_type", &source_type)?; + let source = parse_source_wire(&source_raw)?; + let raw = decode("verdict_json", &verdict_json)?; + let verdict: Verdict = + serde_json::from_str(&raw).map_err(|e| err(format!("verdict_json does not parse: {e}")))?; + let span = StreamSpan::new(source, start, end).map_err(stream_err)?; + let mut session = lock(handle)?; + session + .record_verdict(&task, &span, &verdict) + .map_err(stream_err) +} + +/// Recompute `track`'s watermark. Returns the new offset when the +/// watermark advanced, `null` when it did not or the session has ended +/// (matching the Rust `Option`). +#[napi] +pub fn stream_session_advance( + handle: &External, + track: Utf16String, +) -> napi::Result> { + let raw = decode("track", &track)?; + let track = parse_track_wire(&raw)?; + let mut session = lock(handle)?; + Ok(session.advance(track)) +} + +/// Offset of `track` the host may release through, or `null` once the +/// session has ended. A settled session has no safe offset, which is +/// not an error: it means release nothing further. +#[napi] +pub fn stream_session_safe_offset( + handle: &External, + track: Utf16String, +) -> napi::Result> { + let raw = decode("track", &track)?; + let track = parse_track_wire(&raw)?; + let session = lock(handle)?; + Ok(session.safe_offset(track)) +} + +/// Runes on `track` observed but not yet released. +#[napi] +pub fn stream_session_pending( + handle: &External, + track: Utf16String, +) -> napi::Result { + let raw = decode("track", &track)?; + let track = parse_track_wire(&raw)?; + let session = lock(handle)?; + Ok(session.pending(track)) +} + +/// `track`'s watermark as JSON, carrying `track`, `confirmed`, +/// `received`, `pending` and the `tasks` that must clear it. The +/// confirmed offset stays readable after settlement, so an audit +/// record can still say how far the stream got. +#[napi] +pub fn stream_session_watermark( + handle: &External, + track: Utf16String, +) -> napi::Result { + let raw = decode("track", &track)?; + let track = parse_track_wire(&raw)?; + let session = lock(handle)?; + let watermark = session.watermark(track); + let payload = serde_json::json!({ + "track": track.as_str(), + "confirmed": watermark.confirmed(), + "received": watermark.received(), + "pending": watermark.pending(), + "tasks": watermark.tasks().collect::>(), + }); + serde_json::to_string(&payload).map_err(|e| err(format!("watermark serialization failed: {e}"))) +} + +/// Session state as JSON: `is_ended`, `transformed`, `end_reason` +/// (null while live) and the effective `config`. +#[napi] +pub fn stream_session_state(handle: &External) -> napi::Result { + let session = lock(handle)?; + let config = session.config(); + let payload = serde_json::json!({ + "is_ended": session.is_ended(), + "transformed": session.transformed(), + "end_reason": session.end_reason().map(end_reason_json), + "config": { + "safety_level": config.safety_level.as_str(), + "request_start_rune_offset": config.request_start_rune_offset, + "response_start_rune_offset": config.response_start_rune_offset, + "request_tasks": config.request_tasks, + "response_tasks": config.response_tasks, + }, + }); + serde_json::to_string(&payload).map_err(|e| err(format!("state serialization failed: {e}"))) +} + +/// Declare that no further payload will arrive. Idempotent. +#[napi] +pub fn stream_session_end_of_payloads(handle: &External) -> napi::Result<()> { + let mut session = lock(handle)?; + session.end_of_payloads(); + Ok(()) +} + +/// Settle the session and return the completion as JSON, carrying +/// `reason`, `transformed` and `is_clean`. Settling twice returns the +/// same completion. +#[napi] +pub fn stream_session_finish(handle: &External) -> napi::Result { + let mut session = lock(handle)?; + let completion = session.finish(); + let payload = serde_json::json!({ + "reason": end_reason_json(&completion.reason), + "transformed": completion.transformed, + "is_clean": completion.reason.is_clean(), + }); + serde_json::to_string(&payload) + .map_err(|e| err(format!("completion serialization failed: {e}"))) +} diff --git a/sdk/node/src/index.ts b/sdk/node/src/index.ts index c4bedb1..374157f 100644 --- a/sdk/node/src/index.ts +++ b/sdk/node/src/index.ts @@ -34,6 +34,32 @@ const native = require("../binding.js") as { validateManifestFile(path: string): string | null; validateManifest(source: string): string | null; supportedManifestVersions(): string[]; + streamSessionNew(configJson: string): unknown; + streamSessionObserve(handle: unknown, sourceType: string, runes: number): number; + streamSessionObserveText(handle: unknown, sourceType: string, text: string): number; + streamSessionRecordOutcome( + handle: unknown, + task: string, + sourceType: string, + start: number, + end: number, + outcome: string, + ): void; + streamSessionRecordVerdict( + handle: unknown, + task: string, + sourceType: string, + start: number, + end: number, + verdictJson: string, + ): void; + streamSessionAdvance(handle: unknown, track: string): number | null; + streamSessionSafeOffset(handle: unknown, track: string): number | null; + streamSessionPending(handle: unknown, track: string): number; + streamSessionWatermark(handle: unknown, track: string): string; + streamSessionState(handle: unknown): string; + streamSessionEndOfPayloads(handle: unknown): void; + streamSessionFinish(handle: unknown): string; }; export type { @@ -280,3 +306,382 @@ export function validateManifestFile(path: string): void { export function supportedManifestVersions(): readonly string[] { return Object.freeze(native.supportedManifestVersions()); } + +// --------------------------------------------------------------------- +// Streaming: incremental release mediation for stream-shaped tracks +// (spec §18.1). The engine is stateless everywhere else; this session +// is the exception, because it accumulates offsets and holds terminal +// state a host must read across many calls. +// --------------------------------------------------------------------- + +/** How much a host may release ahead of the watermark. */ +export type StreamSafetyLevel = "blocking" | "complete" | "deferred"; + +/** + * Role that produced a span of text. + * + * Only genuinely rune-addressable roles appear here: tool calls and + * tool results are structured values evaluated once per invocation and + * flow through the ordinary snapshot path. + */ +export type StreamSourceType = "user_request" | "model_generated"; + +/** Independent offset space within a session. */ +export type StreamTrack = "request" | "response"; + +/** What the host decided for one span after evaluating it. */ +export type SegmentOutcome = "cleared" | "transformed" | "denied"; + +/** Parameters a host supplies once, before any payload. */ +export interface StreamSessionConfig { + /** How much the host may release ahead of the watermark. */ + readonly safetyLevel: StreamSafetyLevel; + /** + * Offset the first rune of the request track occupies. A retry that + * resumes a partially delivered stream sets this so offsets stay + * comparable with the earlier attempt. + */ + readonly requestStartRuneOffset?: number; + /** Same as {@link requestStartRuneOffset}, for the response track. */ + readonly responseStartRuneOffset?: number; + /** + * Tasks that gate the request track (matching what the host bound at + * `input`). Empty means the request track is not mediated; payload + * on it fails closed. + */ + readonly requestTasks?: readonly string[]; + /** + * Tasks that gate the response track (matching what the host bound + * at `post_model_call`). Empty means the response track is not + * mediated. + */ + readonly responseTasks?: readonly string[]; +} + +/** Watermark snapshot for one track. */ +export interface StreamWatermarkSnapshot { + readonly track: StreamTrack; + /** Highest offset released so far. */ + readonly confirmed: number; + /** End offset of the text the session has been told about. */ + readonly received: number; + /** + * Runes observed but not yet cleared by every task, as of the last + * advance. + */ + readonly pending: number; + /** Task labels this watermark tracks, in deterministic order. */ + readonly tasks: readonly string[]; +} + +/** Terminal reason a session reached its final state. */ +export type StreamEndReason = + | { readonly kind: "complete" } + | { + readonly kind: "denied"; + readonly track: StreamTrack; + readonly task: string; + readonly start: number; + readonly end: number; + } + | { + readonly kind: "rewritten"; + readonly track: StreamTrack; + readonly task: string; + readonly start: number; + readonly end: number; + } + | { + readonly kind: "failed"; + /** The `host_error:*` reason a host records for this failure. */ + readonly reason: string; + readonly message: string; + }; + +/** Terminal settlement of a session. */ +export interface StreamCompletion { + readonly reason: StreamEndReason; + /** + * Whether the host emitted a substitute rather than verbatim model + * output. This is exactly `reason.kind === "rewritten"`. + */ + readonly transformed: boolean; + /** Whether the stream finished without an enforcement action. */ + readonly isClean: boolean; +} + +/** Read-only view of a session's effective configuration. */ +export interface StreamSessionConfigSnapshot { + readonly safetyLevel: StreamSafetyLevel; + readonly requestStartRuneOffset: number; + readonly responseStartRuneOffset: number; + readonly requestTasks: readonly string[]; + readonly responseTasks: readonly string[]; +} + +/** Snapshot of a session's state, matching the C ABI `session_state`. */ +export interface StreamSessionState { + readonly isEnded: boolean; + readonly transformed: boolean; + /** `null` while the session is still live. */ + readonly endReason: StreamEndReason | null; + readonly config: StreamSessionConfigSnapshot; +} + +/** + * Incremental streaming session (spec §18.1). + * + * ACS is stateless on every other path, so a rune-addressable track a + * host emits incrementally cannot ride the ordinary interceptor + * pipeline: a `deny` in the middle of a stream needs to catch a + * specific range, and a `cleared` prefix needs to release without + * waiting for the whole payload. This class is the accounting layer + * that makes both possible. + * + * The session holds no policy and no text. The host drives it: + * + * 1. Observe arriving text ({@link observe}, {@link observeText}) so + * the session knows how many runes exist on each track. + * 2. Segment the text and evaluate each span with the ordinary + * runtime, then record the outcome ({@link recordOutcome}) or + * replay the verdict ({@link recordVerdict}). + * 3. Ask which prefix is safe to release ({@link safeOffset}), and + * read {@link watermark} for the per-task frontier when auditing. + * 4. Call {@link endOfPayloads} at EOF and settle with {@link finish}. + * + * A settled session reports `null` from {@link safeOffset} and + * {@link advance}: the type says "release nothing further" without any + * value a caller could mistake for a permitted offset. The watermark + * stays readable so an audit record can still say how far the stream + * got. + * + * Every non-boundary streaming failure (unknown safety level, payload + * on an unmediated track, offset past the observed end, transform + * after release, uncleared residue at settlement, ...) throws with the + * engine's message and puts the session in its terminal `failed` + * state. The next call sees the session as ended. + */ +export class StreamSession { + private readonly handle: unknown; + + /** + * Open a session. + * + * `config.safetyLevel` selects the release rule. `requestTasks` and + * `responseTasks` are the task labels a host will pass to + * {@link recordOutcome}: matching a task the manifest binds at + * `input` / `post_model_call` respectively. An empty task list + * leaves that track unmediated; payload on it fails closed. + * + * Throws when both task lists are empty (the session would gate + * nothing) or a start offset overflows. + */ + constructor(config: StreamSessionConfig) { + if (config === null || typeof config !== "object") { + throw new TypeError("StreamSession config must be an object"); + } + // Only translate camelCase → wire snake_case here; enum VALUES stay + // lowercase snake as they arrive. + const payload = { + safety_level: config.safetyLevel, + request_start_rune_offset: config.requestStartRuneOffset ?? 0, + response_start_rune_offset: config.responseStartRuneOffset ?? 0, + request_tasks: config.requestTasks ? Array.from(config.requestTasks) : [], + response_tasks: config.responseTasks ? Array.from(config.responseTasks) : [], + }; + this.handle = native.streamSessionNew(JSON.stringify(payload)); + } + + /** + * Report that `runes` more runes of `sourceType` arrived and return + * the track's new end offset. + * + * This only extends the received bound outcomes are checked against. + * It does not release anything and does not decide what the host + * evaluates. Prefer {@link observeText} when the text is at hand: + * counting runes correctly across surrogate pairs is easy to get + * wrong. + */ + observe(sourceType: StreamSourceType, runes: number): number { + return native.streamSessionObserve(this.handle, sourceType, runes); + } + + /** + * Report arriving `text` on `sourceType`, counting Unicode scalars + * the way the engine does, and return the track's new end offset. + * + * The engine counts runes (Unicode scalars), not UTF-16 code units. + * An emoji outside the BMP is one rune here even though it is two + * UTF-16 code units in a JS string. + */ + observeText(sourceType: StreamSourceType, text: string): number { + return native.streamSessionObserveText(this.handle, sourceType, text); + } + + /** + * Record what `task` decided about the span `[start, end)` of + * `sourceType`. `outcome` is `cleared`, `transformed`, or `denied`. + * + * A `denied` outcome ends the session with `endReason.kind === + * "denied"` on the span it refused, and every later `safeOffset` + * returns `null`. A `transformed` outcome is honored only under a + * withholding safety level (`blocking` / `complete`) and only while + * nothing on the track has been released, and it ends the session + * with `endReason.kind === "rewritten"`. + */ + recordOutcome( + task: string, + sourceType: StreamSourceType, + start: number, + end: number, + outcome: SegmentOutcome, + ): void { + native.streamSessionRecordOutcome(this.handle, task, sourceType, start, end, outcome); + } + + /** + * Record an ACS verdict against the span `[start, end)` of + * `sourceType`, mapping its decision onto an outcome. A host feeds + * the verdict returned by {@link ActivatedPolicy.evaluate} straight + * back without translating it. + * + * A verdict whose shape section 5 does not admit (a `transform` + * carrying no transform body, a reserved reason from a policy, ...) + * fails the stream closed rather than clearing the span. + */ + recordVerdict( + task: string, + sourceType: StreamSourceType, + start: number, + end: number, + verdict: Verdict, + ): void { + native.streamSessionRecordVerdict( + this.handle, + task, + sourceType, + start, + end, + JSON.stringify(verdict), + ); + } + + /** + * Recompute `track`'s watermark. Returns the new offset when the + * watermark advanced, `null` when it did not or the session has + * ended. + */ + advance(track: StreamTrack): number | null { + return native.streamSessionAdvance(this.handle, track); + } + + /** + * Offset of `track` the host may release through, or `null` once + * the session has ended. + * + * A settled session has no safe offset: release nothing further. + * The offset the track reached is unaffected and stays available + * for an audit record through {@link watermark}. + */ + safeOffset(track: StreamTrack): number | null { + return native.streamSessionSafeOffset(this.handle, track); + } + + /** Runes on `track` observed but not yet released. */ + pending(track: StreamTrack): number { + return native.streamSessionPending(this.handle, track); + } + + /** + * `track`'s watermark, carrying `confirmed`, `received`, `pending` + * and the `tasks` that must clear it. The confirmed offset stays + * readable after settlement. + */ + watermark(track: StreamTrack): StreamWatermarkSnapshot { + return JSON.parse(native.streamSessionWatermark(this.handle, track)) as StreamWatermarkSnapshot; + } + + /** + * Snapshot of session state: `isEnded`, `transformed`, `endReason` + * (null while live) and the effective `config`. + */ + state(): StreamSessionState { + const raw = JSON.parse(native.streamSessionState(this.handle)) as { + is_ended: boolean; + transformed: boolean; + end_reason: StreamEndReason | null; + config: { + safety_level: StreamSafetyLevel; + request_start_rune_offset: number; + response_start_rune_offset: number; + request_tasks: string[]; + response_tasks: string[]; + }; + }; + return { + isEnded: raw.is_ended, + transformed: raw.transformed, + endReason: raw.end_reason, + config: { + safetyLevel: raw.config.safety_level, + requestStartRuneOffset: raw.config.request_start_rune_offset, + responseStartRuneOffset: raw.config.response_start_rune_offset, + requestTasks: Object.freeze(raw.config.request_tasks), + responseTasks: Object.freeze(raw.config.response_tasks), + }, + }; + } + + /** Whether the session has reached its terminal state. */ + isEnded(): boolean { + return this.state().isEnded; + } + + /** + * Terminal reason, when the session has ended, or `null` while it + * is still live. + */ + endReason(): StreamEndReason | null { + return this.state().endReason; + } + + /** + * Whether a `transformed` outcome ended this session, meaning the + * host emits a substitute rather than verbatim model output. + */ + isTransformed(): boolean { + return this.state().transformed; + } + + /** + * Declare that no further payload will arrive. Idempotent. + * + * A `deferred` host calls this at payload EOF so a classifier + * running behind the stream can still record a denial before + * {@link finish}. + */ + endOfPayloads(): void { + native.streamSessionEndOfPayloads(this.handle); + } + + /** + * Settle the session and return the completion. + * + * Recomputes both watermarks first, so a host that recorded every + * outcome is not failed closed for having skipped an explicit + * {@link advance}. Any rune no task cleared fails the settlement + * closed. Settling twice returns the same completion. + */ + finish(): StreamCompletion { + const raw = JSON.parse(native.streamSessionFinish(this.handle)) as { + reason: StreamEndReason; + transformed: boolean; + is_clean: boolean; + }; + return { + reason: raw.reason, + transformed: raw.transformed, + isClean: raw.is_clean, + }; + } +} diff --git a/sdk/node/test/stream-session.test.mjs b/sdk/node/test/stream-session.test.mjs new file mode 100644 index 0000000..0e43d2b --- /dev/null +++ b/sdk/node/test/stream-session.test.mjs @@ -0,0 +1,295 @@ +// Streaming mediation surface (spec §18.1). +// +// The engine is stateless everywhere else, so a rune-addressable +// track a host emits incrementally cannot ride the ordinary +// interceptor pipeline. `StreamSession` is the accounting layer that +// makes both a mid-stream deny and a cleared-prefix release possible. +// These tests pin the wire contract other language SDKs also +// implement, and the correctness traps a Node-only implementation is +// most likely to fall into: UTF-16 rune-counting drift and a settled +// session leaking a released offset. +import assert from "node:assert/strict"; +import { test } from "node:test"; + +const require = (await import("node:module")).createRequire(import.meta.url); +const { StreamSession } = require("../dist/index.js"); + +test("happy path clears, advances, and finishes clean", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + + assert.equal(s.observeText("model_generated", "hello"), 5); + // Nothing has cleared yet, so the safe offset stays at the start. + assert.equal(s.safeOffset("response"), 0); + assert.equal(s.pending("response"), 5); + + s.recordOutcome("pii", "model_generated", 0, 5, "cleared"); + assert.equal(s.advance("response"), 5); + assert.equal(s.safeOffset("response"), 5); + + const mark = s.watermark("response"); + assert.equal(mark.track, "response"); + assert.equal(mark.confirmed, 5); + assert.equal(mark.received, 5); + assert.equal(mark.pending, 0); + assert.deepEqual(mark.tasks, ["pii"]); + + const completion = s.finish(); + assert.equal(completion.reason.kind, "complete"); + assert.equal(completion.isClean, true); + assert.equal(completion.transformed, false); + // A settled session releases nothing further. `null` says that in + // the type: the caller cannot read it as an offset by accident. + assert.equal(s.safeOffset("response"), null); + assert.equal(s.advance("response"), null); +}); + +test("a deny ends the session and safeOffset becomes null, but watermark still shows how far it got", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + + s.observeText("model_generated", "cleared prefix"); + s.recordOutcome("safety", "model_generated", 0, 7, "cleared"); + s.advance("response"); + assert.equal(s.safeOffset("response"), 7); + + s.observeText("model_generated", "!!!DANGER!!!"); + s.recordOutcome("safety", "model_generated", 14, 26, "denied"); + + // Every rune the host has not already emitted must be withheld, + // including runes a task had cleared. The type says that. + assert.equal(s.safeOffset("response"), null); + assert.equal(s.advance("response"), null); + + // The audit path stays open: the watermark still says how far the + // track got before the deny. + const mark = s.watermark("response"); + assert.equal(mark.confirmed, 7); + assert.equal(mark.received, 26); + + const reason = s.endReason(); + assert.equal(reason.kind, "denied"); + assert.equal(reason.track, "response"); + assert.equal(reason.task, "safety"); + assert.equal(reason.start, 14); + assert.equal(reason.end, 26); + + const completion = s.finish(); + assert.equal(completion.reason.kind, "denied"); + assert.equal(completion.isClean, false); + assert.equal(completion.transformed, false); +}); + +test("a span needs every task to clear before the watermark advances", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii", "safety"], + }); + + s.observeText("model_generated", "hello world"); + s.recordOutcome("pii", "model_generated", 0, 11, "cleared"); + // One task cleared, the other has not, so the confirmed offset must + // not move: releasing the prefix would skip `safety`. + assert.equal(s.advance("response"), null); + assert.equal(s.safeOffset("response"), 0); + + s.recordOutcome("safety", "model_generated", 0, 11, "cleared"); + assert.equal(s.advance("response"), 11); + assert.equal(s.safeOffset("response"), 11); + + const mark = s.watermark("response"); + assert.deepEqual(mark.tasks, ["pii", "safety"]); + assert.equal(mark.confirmed, 11); +}); + +test("observeText counts runes, not UTF-16 code units", () => { + // 🙂 is one Unicode scalar (U+1F642) but two UTF-16 code units in a + // JS string. If observeText leaked the JS length instead of the + // engine's rune count, the received offset would come back as 2 and + // every downstream offset would slide by one for every emoji. + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + + const text = "🙂"; + assert.equal(text.length, 2, "JS string length reports UTF-16 code units"); + assert.equal(s.observeText("model_generated", text), 1); + assert.equal(s.watermark("response").received, 1); + + // Recording an outcome over the UTF-16 length (2) would be past the + // observed end of the track. It must be past the end here. + assert.throws( + () => s.recordOutcome("pii", "model_generated", 0, 2, "cleared"), + /past end|OffsetPastEnd|offset/i, + ); + + // But the actual rune span clears fine. + const s2 = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + s2.observeText("model_generated", "🙂"); + s2.recordOutcome("pii", "model_generated", 0, 1, "cleared"); + assert.equal(s2.advance("response"), 1); + assert.equal(s2.finish().isClean, true); +}); + +test("an unmediated track releases without evaluation", () => { + // Empty response task set: the response track is not mediated at + // all, so text on it fails closed while the request track releases + // as usual. This is the ordinary shape for a host guarding only the + // user prompt. + const s = new StreamSession({ + safetyLevel: "blocking", + requestTasks: ["moderation"], + responseTasks: [], + }); + + s.observeText("user_request", "hello"); + s.recordOutcome("moderation", "user_request", 0, 5, "cleared"); + assert.equal(s.advance("request"), 5); + assert.equal(s.safeOffset("request"), 5); + + // Payload on the unmediated response track fails closed, because + // nothing would gate it. + assert.throws(() => s.observeText("model_generated", "reply"), /NoTasks|not mediated|response/i); + + // The failed observe put the session into its terminal state. + const reason = s.endReason(); + assert.equal(reason.kind, "failed"); + assert.match(reason.reason, /^host_error:/); +}); + +test("unknown safety level and unknown track throw with the engine's message", () => { + assert.throws( + () => new StreamSession({ safetyLevel: "permissive", responseTasks: ["t"] }), + /permissive|unknown/i, + ); + + const s = new StreamSession({ safetyLevel: "blocking", responseTasks: ["t"] }); + assert.throws(() => s.safeOffset("nope"), /nope|unknown/i); + assert.throws(() => s.advance("nope"), /nope|unknown/i); + assert.throws(() => s.watermark("nope"), /nope|unknown/i); + assert.throws(() => s.pending("nope"), /nope|unknown/i); + + assert.throws( + () => new StreamSession({ safetyLevel: "blocking" }), + /NoTracksMediated|neither|no tasks/i, + ); +}); + +test("request and response tracks carry independent offsets", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + requestTasks: ["moderation"], + responseTasks: ["safety"], + }); + + s.observeText("user_request", "user prompt"); + s.observeText("model_generated", "model reply"); + + // Each track has its own confirmed and received frontier. + assert.equal(s.watermark("request").received, 11); + assert.equal(s.watermark("response").received, 11); + assert.equal(s.watermark("request").confirmed, 0); + assert.equal(s.watermark("response").confirmed, 0); + + // Clearing the request must not release the response. + s.recordOutcome("moderation", "user_request", 0, 11, "cleared"); + s.advance("request"); + assert.equal(s.safeOffset("request"), 11); + assert.equal(s.safeOffset("response"), 0); + assert.equal(s.pending("response"), 11); + + // Clearing the response then releases only that track. + s.recordOutcome("safety", "model_generated", 0, 11, "cleared"); + s.advance("response"); + assert.equal(s.safeOffset("response"), 11); + assert.equal(s.finish().isClean, true); +}); + +test("resume offsets keep offsets comparable across a retry", () => { + // A retry that re-sends the prompt and resumes the response reports + // its resume point through `responseStartRuneOffset`. The received + // frontier starts at that offset, so an outcome over the resumed + // range clears without releasing the gap. + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + responseStartRuneOffset: 12, + }); + + const mark = s.watermark("response"); + assert.equal(mark.confirmed, 12); + assert.equal(mark.received, 12); + + s.observeText("model_generated", "continued"); + s.recordOutcome("safety", "model_generated", 12, 21, "cleared"); + s.advance("response"); + assert.equal(s.safeOffset("response"), 21); + assert.equal(s.finish().isClean, true); +}); + +test("recordVerdict routes an allow verdict as a clear", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + s.observeText("model_generated", "hello"); + s.recordVerdict("safety", "model_generated", 0, 5, { + decision: "allow", + warnings: [], + result_labels: [], + }); + assert.equal(s.advance("response"), 5); + assert.equal(s.finish().isClean, true); +}); + +test("recordVerdict deny ends the session with the engine's terminal reason", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + s.observeText("model_generated", "hello"); + s.recordVerdict("safety", "model_generated", 0, 5, { + decision: "deny", + reason: "policy_blocked", + warnings: [], + result_labels: [], + }); + const completion = s.finish(); + assert.equal(completion.reason.kind, "denied"); + assert.equal(completion.reason.task, "safety"); + assert.equal(completion.isClean, false); +}); + +test("transform before any release ends the session rewritten under a withholding level", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + s.observeText("model_generated", "raw"); + s.recordOutcome("safety", "model_generated", 0, 3, "transformed"); + assert.equal(s.isTransformed(), true); + const completion = s.finish(); + assert.equal(completion.reason.kind, "rewritten"); + assert.equal(completion.transformed, true); + assert.equal(completion.isClean, false); +}); + +test("finish twice returns the same completion", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + s.observeText("model_generated", "ok"); + s.recordOutcome("safety", "model_generated", 0, 2, "cleared"); + const first = s.finish(); + const second = s.finish(); + assert.deepEqual(first, second); +}); diff --git a/sdk/python/Cargo.toml b/sdk/python/Cargo.toml index a8a25e9..fac6484 100644 --- a/sdk/python/Cargo.toml +++ b/sdk/python/Cargo.toml @@ -11,7 +11,7 @@ name = "_native" crate-type = ["cdylib"] [dependencies] -agent-control-spec = { version = "0.4.0-alpha.2", path = "../../engine", features = ["default-dispatchers"] } +agent-control-spec = { version = "0.4.0-alpha.2", path = "../../engine", features = ["default-dispatchers", "streaming"] } pyo3 = { version = "0.29", features = ["extension-module", "abi3-py311"] } serde_json = "1" diff --git a/sdk/python/agent_control_spec/__init__.py b/sdk/python/agent_control_spec/__init__.py index 383843a..b0cfd26 100644 --- a/sdk/python/agent_control_spec/__init__.py +++ b/sdk/python/agent_control_spec/__init__.py @@ -14,7 +14,7 @@ from __future__ import annotations import json -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from typing import Any from agent_hooks import Verdict @@ -26,6 +26,7 @@ "ActivatedPolicy", "ManifestInvalidError", "RegoBundle", + "StreamSession", "__version__", "supported_manifest_versions", "validate_manifest", @@ -203,3 +204,212 @@ def supported_manifest_versions() -> tuple[str, ...]: Read it rather than hardcoding the set; it moves with the engine. """ return tuple(_native.supported_manifest_versions()) + + +class StreamSession: + """Host side accounting for one streamed policy target. + + A session holds no policy, performs no evaluation, and stores no + stream text. The host drives it: it reports how much text arrived, + declares the spans its segmenter produced, evaluates those spans + through the ordinary interceptor path, records each outcome, and + reads :meth:`safe_offset` to see how far it may release the track. + This is the incremental profile in specification section 18.1. + + A track with no tasks is unmediated. Payload on such a track fails + closed, which matches the behavior of a host guarding only the model + stream and receiving text on the wrong track. A configuration that + mediates neither track is rejected at construction: it would gate + nothing. + + ``safety_level`` is one of ``"blocking"``, ``"complete"``, or + ``"deferred"``. ``"blocking"`` and ``"complete"`` hold each span + until the watermark covers it. ``"deferred"`` emits payload as it + arrives and evaluates behind the stream, and cannot recall what has + already been emitted. + + The session settles in two steps. :meth:`end_of_payloads` says no + more text is coming while outcomes are still in flight, which is + what a ``"deferred"`` host needs so a late denial can still land. + :meth:`finish` returns the terminal :class:`dict` and marks the + session ended. After :meth:`finish`, :meth:`safe_offset` is + ``None``: a terminated session has no offset a host may emit + through, whatever the reason. The confirmed offset stays available + through :meth:`watermark` for the audit record. + """ + + __slots__ = ("_handle",) + + def __init__( + self, + safety_level: str = "blocking", + *, + request_tasks: Iterable[str] | None = None, + response_tasks: Iterable[str] | None = None, + request_start_rune_offset: int = 0, + response_start_rune_offset: int = 0, + ) -> None: + request_tasks_list = list(request_tasks) if request_tasks is not None else [] + response_tasks_list = list(response_tasks) if response_tasks is not None else [] + self._handle = _native.stream_session_new( + safety_level, + int(request_start_rune_offset), + int(response_start_rune_offset), + request_tasks_list, + response_tasks_list, + ) + + def observe(self, source_type: str, runes: int) -> int: + """Report that ``runes`` more runes arrived on this role's + track. Returns the track's new end offset. + """ + return _native.stream_observe(self._handle, source_type, int(runes)) + + def observe_text(self, source_type: str, text: str) -> int: + """Report arriving text and let the engine count its runes, + so a host does not reach for a length that measures UTF-16 code + units or bytes. Neither is interchangeable with a rune offset. + The text itself is not retained. + """ + return _native.stream_observe_text(self._handle, source_type, text) + + def record_outcome( + self, + task: str, + source_type: str, + start: int, + end: int, + outcome: str, + ) -> None: + """Record what a host decided for the half-open rune range + ``[start, end)`` on ``source_type``'s track, under ``task``. + + ``outcome`` is one of ``"cleared"``, ``"transformed"``, or + ``"denied"``. A denial or transform ends the session. Every + engine rejection raises :class:`ValueError` with the engine's + own message; nothing silently no-ops. + """ + _native.stream_record_outcome( + self._handle, + task, + source_type, + int(start), + int(end), + outcome, + ) + + def record_verdict( + self, + task: str, + source_type: str, + start: int, + end: int, + verdict: Verdict | Mapping[str, Any], + ) -> None: + """Map an agent-hooks verdict onto an outcome and record it. + + ``verdict`` may be a :class:`agent_hooks.Verdict` or the same + wire dict :meth:`agent_hooks.Verdict.to_wire` produces. A shape + the section 5 contract does not admit fails the stream closed + with :class:`ValueError` before its decision is read. + """ + if isinstance(verdict, Verdict): + wire = verdict.to_wire() + elif isinstance(verdict, Mapping): + wire = verdict + else: + raise TypeError("verdict must be an agent_hooks.Verdict or a wire dict") + _native.stream_record_verdict( + self._handle, + task, + source_type, + int(start), + int(end), + json.dumps(wire, allow_nan=False), + ) + + def advance(self, track: str) -> int | None: + """Recompute the watermark for ``track`` and return the new + confirmed offset when it advanced. Returns ``None`` when it did + not, so a host emits a watermark event only on real progress. + Returns ``None`` once the session has ended. + """ + return _native.stream_advance(self._handle, track) + + def safe_offset(self, track: str) -> int | None: + """Offset through which the host may emit ``track``, or ``None`` + once the session has ended. + + A denial withholds every rune the host has not already emitted, + including runes a task had cleared, so a terminated session has + no offset anyone may emit through. Returning ``None`` says that + in the type, which a host cannot read as permission by + accident. + """ + return _native.stream_safe_offset(self._handle, track) + + def pending(self, track: str) -> int: + """Runes observed but not yet cleared by every task on + ``track``, as of the last :meth:`advance`. + """ + return _native.stream_pending(self._handle, track) + + def watermark(self, track: str) -> dict[str, Any]: + """Watermark snapshot for one track: + ``{"track", "confirmed", "received", "pending", "tasks"}``. + Reads without moving anything. + """ + return json.loads(_native.stream_watermark(self._handle, track)) + + def end_of_payloads(self) -> None: + """Stop accepting payloads while outcomes are still in flight. + A ``"deferred"`` host calls this at payload EOF so a classifier + running behind the stream can still record a denial before + :meth:`finish`. + """ + _native.stream_end_of_payloads(self._handle) + + def finish(self) -> dict[str, Any]: + """Settle the session and return the terminal record: + ``{"reason": , "transformed": bool, "is_clean": + bool}``. + + ``reason`` is one of ``{"kind": "complete"}``, ``{"kind": + "denied", "track", "task", "start", "end"}``, ``{"kind": + "rewritten", "track", "task", "start", "end"}``, or ``{"kind": + "failed", "reason", "message"}``. Any rune no task cleared + settles the session ``failed`` under every safety level. + """ + return json.loads(_native.stream_finish(self._handle)) + + @property + def is_ended(self) -> bool: + """Whether the session has reached its terminal state.""" + return _native.stream_is_ended(self._handle) + + @property + def transformed(self) -> bool: + """Whether a ``transformed`` outcome ended this session, meaning + the host emits a substitute rather than verbatim model output. + A transform clears nothing, so this says nothing about what was + released. + """ + return _native.stream_transformed(self._handle) + + @property + def end_reason(self) -> dict[str, Any] | None: + """Terminal reason as a wire dict, or ``None`` when the session + has not ended. The same schema :meth:`finish` returns under + ``reason``. + """ + raw = _native.stream_end_reason(self._handle) + return None if raw is None else json.loads(raw) + + @property + def config(self) -> dict[str, Any]: + """Streaming parameters this session was opened with: + ``{"safety_level", "request_start_rune_offset", + "response_start_rune_offset", "request_tasks", + "response_tasks"}``. + """ + return json.loads(_native.stream_config(self._handle)) diff --git a/sdk/python/src/lib.rs b/sdk/python/src/lib.rs index 8d0d8be..fb33705 100644 --- a/sdk/python/src/lib.rs +++ b/sdk/python/src/lib.rs @@ -11,13 +11,14 @@ use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; use agent_control_spec::{ ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, Runtime, RuntimeError, - SUPPORTED_VERSIONS, + SafetyLevel, SegmentOutcome, StreamEndReason, StreamError, StreamSession, StreamSessionConfig, + StreamSourceType, StreamSpan, StreamTrack, Verdict, SUPPORTED_VERSIONS, }; use pyo3::create_exception; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; -use serde_json::Value; -use std::sync::Arc; +use serde_json::{json, Value}; +use std::sync::{Arc, Mutex}; #[pyclass(frozen)] struct RuntimeHandle { @@ -111,7 +112,10 @@ fn validate_manifest_file(path: &str) -> PyResult<()> { /// The manifest grammar versions this engine accepts. #[pyfunction] fn supported_manifest_versions() -> Vec { - SUPPORTED_VERSIONS.iter().map(|v| (*v).to_string()).collect() + SUPPORTED_VERSIONS + .iter() + .map(|v| (*v).to_string()) + .collect() } // --------------------------------------------------------------------- @@ -245,6 +249,304 @@ fn policy_intervention_points(handle: &PolicyHandle) -> Vec { .collect() } +// --------------------------------------------------------------------- +// Streaming session: host side accounting for the incremental stream +// profile in specification section 18.1. +// +// A `StreamSession` holds no policy, performs no evaluation, and stores +// no stream text. The host drives it: reports arrived text, declares +// spans, records what its policy decided for them, and asks which +// prefix it may release. Every function here is a thin projection over +// the engine's typed accounting. Enum wire names come from the engine's +// own `as_str`/`parse` methods, so the two cannot drift. +// +// State lives behind a `Mutex` because Python threads share the handle +// and every mutating method borrows exclusively. Reads take the same +// lock, which is what the engine's `&self` methods want anyway. The +// lock is uncontended in the common single-threaded flow, and its cost +// is negligible next to the JSON conversion of a settled reason. +// --------------------------------------------------------------------- + +#[pyclass(frozen)] +struct StreamSessionHandle { + inner: Mutex, +} + +// A poisoned `Mutex` means a previous mutating call panicked while +// holding the lock. Nothing in the engine's session accounting panics +// on well-formed input, so this cannot arise on a healthy contract. +// The wrapper still refuses to keep operating on a session in an +// unknown state: it lifts the poisoned guard and surfaces a runtime +// error rather than pretending the session is usable. +fn locked(guard: std::sync::LockResult) -> PyResult { + guard.map_err(|_| PyRuntimeError::new_err("streaming session mutex was poisoned")) +} + +fn parse_track(value: &str) -> PyResult { + match value { + "request" => Ok(StreamTrack::Request), + "response" => Ok(StreamTrack::Response), + other => Err(PyValueError::new_err(format!( + "unknown stream track '{other}'" + ))), + } +} + +fn parse_outcome(value: &str) -> PyResult { + match value { + "cleared" => Ok(SegmentOutcome::Cleared), + "transformed" => Ok(SegmentOutcome::Transformed), + "denied" => Ok(SegmentOutcome::Denied), + other => Err(PyValueError::new_err(format!( + "unknown segment outcome '{other}'" + ))), + } +} + +// A `StreamError` from the engine is always a boundary rejection: the +// host handed a value the contract does not admit, or asked for an +// operation the session cannot honor. `ValueError` is the same mapping +// the parse-side errors above receive, so the boundary reports one +// exception class regardless of which check caught it. +fn stream_err(error: StreamError) -> PyErr { + PyValueError::new_err(format!("{error}")) +} + +fn end_reason_to_json(reason: &StreamEndReason) -> Value { + match reason { + StreamEndReason::Complete => json!({ "kind": "complete" }), + StreamEndReason::Denied { track, task, range } => json!({ + "kind": "denied", + "track": track.as_str(), + "task": task, + "start": range.start, + "end": range.end, + }), + StreamEndReason::Rewritten { track, task, range } => json!({ + "kind": "rewritten", + "track": track.as_str(), + "task": task, + "start": range.start, + "end": range.end, + }), + StreamEndReason::Failed(error) => json!({ + "kind": "failed", + "reason": error.reason(), + "message": format!("{error}"), + }), + } +} + +fn json_string(value: &Value) -> PyResult { + serde_json::to_string(value) + .map_err(|e| PyRuntimeError::new_err(format!("stream JSON serialization failed: {e}"))) +} + +/// Open a streaming session. Field meanings mirror +/// `StreamSessionConfig` in the engine. +#[pyfunction] +#[pyo3(signature = ( + safety_level, + request_start_rune_offset, + response_start_rune_offset, + request_tasks, + response_tasks, +))] +fn stream_session_new( + safety_level: &str, + request_start_rune_offset: u32, + response_start_rune_offset: u32, + request_tasks: Vec, + response_tasks: Vec, +) -> PyResult { + let safety_level = SafetyLevel::parse(safety_level).map_err(stream_err)?; + let config = StreamSessionConfig { + safety_level, + request_start_rune_offset, + response_start_rune_offset, + request_tasks, + response_tasks, + }; + let session = StreamSession::new(config).map_err(stream_err)?; + Ok(StreamSessionHandle { + inner: Mutex::new(session), + }) +} + +/// Report that `runes` more runes arrived on this role's track. Returns +/// the track's new end offset. +#[pyfunction] +fn stream_observe(handle: &StreamSessionHandle, source_type: &str, runes: u32) -> PyResult { + let source_type = StreamSourceType::parse(source_type).map_err(stream_err)?; + let mut session = locked(handle.inner.lock())?; + session.observe(source_type, runes).map_err(stream_err) +} + +/// Report arriving text and let the engine count its runes, so a host +/// does not reach for a length that measures UTF-16 code units or bytes. +#[pyfunction] +fn stream_observe_text( + handle: &StreamSessionHandle, + source_type: &str, + text: &str, +) -> PyResult { + let source_type = StreamSourceType::parse(source_type).map_err(stream_err)?; + let mut session = locked(handle.inner.lock())?; + session.observe_text(source_type, text).map_err(stream_err) +} + +/// Record what a host decided for one span under one task. The span is +/// built from `source_type` and the half-open rune range +/// `[start, end)`. +#[pyfunction] +fn stream_record_outcome( + handle: &StreamSessionHandle, + task: &str, + source_type: &str, + start: u32, + end: u32, + outcome: &str, +) -> PyResult<()> { + let source_type = StreamSourceType::parse(source_type).map_err(stream_err)?; + let outcome = parse_outcome(outcome)?; + let span = StreamSpan::new(source_type, start, end).map_err(stream_err)?; + let mut session = locked(handle.inner.lock())?; + session + .record_outcome(task, &span, outcome) + .map_err(stream_err) +} + +/// Record a wire-shaped agent-hooks verdict for one span under one +/// task. The verdict text is deserialized with the same grammar the +/// runtime uses, so a shape section 5 does not admit fails closed here +/// rather than clearing the span. +#[pyfunction] +fn stream_record_verdict( + handle: &StreamSessionHandle, + task: &str, + source_type: &str, + start: u32, + end: u32, + verdict_json: &str, +) -> PyResult<()> { + let source_type = StreamSourceType::parse(source_type).map_err(stream_err)?; + let span = StreamSpan::new(source_type, start, end).map_err(stream_err)?; + let verdict: Verdict = serde_json::from_str(verdict_json) + .map_err(|e| PyValueError::new_err(format!("verdict_json does not parse: {e}")))?; + let mut session = locked(handle.inner.lock())?; + session + .record_verdict(task, &span, &verdict) + .map_err(stream_err) +} + +/// Recompute the watermark for `track` and return the new confirmed +/// offset when it advanced. +#[pyfunction] +fn stream_advance(handle: &StreamSessionHandle, track: &str) -> PyResult> { + let track = parse_track(track)?; + let mut session = locked(handle.inner.lock())?; + Ok(session.advance(track)) +} + +/// Offset through which the host may emit this track, or `None` once +/// the session has ended. +#[pyfunction] +fn stream_safe_offset(handle: &StreamSessionHandle, track: &str) -> PyResult> { + let track = parse_track(track)?; + let session = locked(handle.inner.lock())?; + Ok(session.safe_offset(track)) +} + +/// Runes observed but not yet cleared by every task on this track. +#[pyfunction] +fn stream_pending(handle: &StreamSessionHandle, track: &str) -> PyResult { + let track = parse_track(track)?; + let session = locked(handle.inner.lock())?; + Ok(session.pending(track)) +} + +/// Watermark snapshot for one track, as wire JSON. +#[pyfunction] +fn stream_watermark(handle: &StreamSessionHandle, track: &str) -> PyResult { + let track_kind = parse_track(track)?; + let session = locked(handle.inner.lock())?; + let watermark = session.watermark(track_kind); + let tasks: Vec<&str> = watermark.tasks().collect(); + let value = json!({ + "track": track_kind.as_str(), + "confirmed": watermark.confirmed(), + "received": watermark.received(), + "pending": watermark.pending(), + "tasks": tasks, + }); + json_string(&value) +} + +/// Stop accepting payloads while outcomes are still in flight. A +/// `Deferred` host calls this at payload EOF so a classifier running +/// behind the stream can still record a denial before `finish`. +#[pyfunction] +fn stream_end_of_payloads(handle: &StreamSessionHandle) -> PyResult<()> { + let mut session = locked(handle.inner.lock())?; + session.end_of_payloads(); + Ok(()) +} + +/// Settle the session and return the wire-JSON completion: +/// `{"reason": , "transformed": bool, "is_clean": bool}`. +#[pyfunction] +fn stream_finish(handle: &StreamSessionHandle) -> PyResult { + let mut session = locked(handle.inner.lock())?; + let completion = session.finish(); + let value = json!({ + "reason": end_reason_to_json(&completion.reason), + "transformed": completion.transformed, + "is_clean": completion.reason.is_clean(), + }); + json_string(&value) +} + +/// Whether the session has reached its terminal state. +#[pyfunction] +fn stream_is_ended(handle: &StreamSessionHandle) -> PyResult { + let session = locked(handle.inner.lock())?; + Ok(session.is_ended()) +} + +/// Whether a `transformed` outcome ended this session, meaning the host +/// emits a substitute rather than verbatim model output. +#[pyfunction] +fn stream_transformed(handle: &StreamSessionHandle) -> PyResult { + let session = locked(handle.inner.lock())?; + Ok(session.transformed()) +} + +/// Terminal reason as wire JSON, or `None` when the session has not +/// ended. +#[pyfunction] +fn stream_end_reason(handle: &StreamSessionHandle) -> PyResult> { + let session = locked(handle.inner.lock())?; + match session.end_reason() { + Some(reason) => json_string(&end_reason_to_json(reason)).map(Some), + None => Ok(None), + } +} + +/// Streaming parameters this session was opened with, as wire JSON. +#[pyfunction] +fn stream_config(handle: &StreamSessionHandle) -> PyResult { + let session = locked(handle.inner.lock())?; + let config = session.config(); + let value = json!({ + "safety_level": config.safety_level.as_str(), + "request_start_rune_offset": config.request_start_rune_offset, + "response_start_rune_offset": config.response_start_rune_offset, + "request_tasks": config.request_tasks, + "response_tasks": config.response_tasks, + }); + json_string(&value) +} + #[pymodule] fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -255,6 +557,22 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(policy_activate_from_memory, m)?)?; m.add_function(wrap_pyfunction!(policy_evaluate, m)?)?; m.add_function(wrap_pyfunction!(policy_intervention_points, m)?)?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(stream_session_new, m)?)?; + m.add_function(wrap_pyfunction!(stream_observe, m)?)?; + m.add_function(wrap_pyfunction!(stream_observe_text, m)?)?; + m.add_function(wrap_pyfunction!(stream_record_outcome, m)?)?; + m.add_function(wrap_pyfunction!(stream_record_verdict, m)?)?; + m.add_function(wrap_pyfunction!(stream_advance, m)?)?; + m.add_function(wrap_pyfunction!(stream_safe_offset, m)?)?; + m.add_function(wrap_pyfunction!(stream_pending, m)?)?; + m.add_function(wrap_pyfunction!(stream_watermark, m)?)?; + m.add_function(wrap_pyfunction!(stream_end_of_payloads, m)?)?; + m.add_function(wrap_pyfunction!(stream_finish, m)?)?; + m.add_function(wrap_pyfunction!(stream_is_ended, m)?)?; + m.add_function(wrap_pyfunction!(stream_transformed, m)?)?; + m.add_function(wrap_pyfunction!(stream_end_reason, m)?)?; + m.add_function(wrap_pyfunction!(stream_config, m)?)?; m.add("ManifestInvalid", m.py().get_type::())?; m.add_function(wrap_pyfunction!(validate_manifest, m)?)?; m.add_function(wrap_pyfunction!(validate_manifest_file, m)?)?; diff --git a/sdk/python/tests/test_stream_session.py b/sdk/python/tests/test_stream_session.py new file mode 100644 index 0000000..27e64bc --- /dev/null +++ b/sdk/python/tests/test_stream_session.py @@ -0,0 +1,341 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Streaming session contract: incremental release accounting for a +policy target the host holds as a stream (specification section 18.1). +The session gates, the host emits. Every engine rejection surfaces as a +Python exception; nothing silently no-ops.""" + +from __future__ import annotations + +import pytest +from agent_control_spec import StreamSession +from agent_hooks import Decision, Transform, Verdict + + +def _allow() -> Verdict: + return Verdict(decision=Decision.ALLOW) + + +def _deny() -> Verdict: + return Verdict(decision=Decision.DENY, reason="blocked_by_policy") + + +def _transform() -> Verdict: + return Verdict( + decision=Decision.TRANSFORM, + transform=Transform(path="$target.content", value="[redacted]"), + ) + + +# --------------------------------------------------------------------- +# Happy path +# --------------------------------------------------------------------- + + +def test_blocking_session_releases_the_prefix_a_task_clears_and_finishes_clean(): + session = StreamSession( + safety_level="blocking", + response_tasks=["pii"], + ) + # Safety level "blocking" holds every span until the watermark + # covers it, so nothing is emittable before a `cleared` outcome. + assert session.safe_offset("response") == 0 + + assert session.observe_text("model_generated", "hello world") == 11 + assert session.pending("response") == 11 + # Nothing has cleared yet, so the offset a host may emit through has + # not moved even though runes arrived. + assert session.safe_offset("response") == 0 + + session.record_outcome("pii", "model_generated", 0, 11, "cleared") + assert session.advance("response") == 11 + assert session.safe_offset("response") == 11 + + settlement = session.finish() + assert settlement == { + "reason": {"kind": "complete"}, + "transformed": False, + "is_clean": True, + } + # A settled session hands out no offset to emit through, whatever + # the reason. Confirmed stays available for the audit record. + assert session.safe_offset("response") is None + assert session.watermark("response")["confirmed"] == 11 + assert session.is_ended is True + assert session.end_reason == {"kind": "complete"} + + +def test_advance_reports_none_when_no_task_moved_forward(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "hi") + # No outcome recorded, so the watermark has nothing to commit and + # `advance` must not synthesize progress. + assert session.advance("response") is None + assert session.safe_offset("response") == 0 + + +# --------------------------------------------------------------------- +# Denial is terminal and audit path stays readable +# --------------------------------------------------------------------- + + +def test_a_denial_terminates_the_session_and_confirmed_stays_readable(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "safe prefix ") # 12 runes + session.record_outcome("pii", "model_generated", 0, 12, "cleared") + session.advance("response") + # Establish that the host would have been allowed to release the + # first twelve runes, so the audit path is meaningful. + assert session.safe_offset("response") == 12 + + session.observe_text("model_generated", "bad tail") # +8 runes, offsets 12..20 + session.record_outcome("pii", "model_generated", 12, 20, "denied") + + # Denial withholds everything a host has not already emitted, + # including runes a task had cleared, so `safe_offset` becomes None. + assert session.safe_offset("response") is None + assert session.is_ended is True + + # The confirmed offset the audit needs is still readable. This is + # the release ceiling the session reached, not permission to emit. + watermark = session.watermark("response") + assert watermark["confirmed"] == 12 + assert watermark["received"] == 20 + assert session.end_reason == { + "kind": "denied", + "track": "response", + "task": "pii", + "start": 12, + "end": 20, + } + + settlement = session.finish() + assert settlement["is_clean"] is False + assert settlement["transformed"] is False + assert settlement["reason"]["kind"] == "denied" + + +def test_denial_through_a_verdict_reaches_the_same_terminal_state(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "bad content") # 11 runes + session.record_verdict("pii", "model_generated", 0, 11, _deny()) + + reason = session.end_reason + assert reason["kind"] == "denied" + assert reason["task"] == "pii" + assert reason["start"] == 0 and reason["end"] == 11 + + +# --------------------------------------------------------------------- +# Multi-task: the watermark is the minimum across configured tasks +# --------------------------------------------------------------------- + + +def test_the_watermark_waits_for_every_task_configured_on_the_track(): + session = StreamSession(response_tasks=["pii", "safety"]) + session.observe_text("model_generated", "sentence one.") # 13 runes + + # Only one of two tasks has cleared the span. The confirmed offset + # is the minimum across both, so the release ceiling stays at zero. + session.record_outcome("pii", "model_generated", 0, 13, "cleared") + assert session.advance("response") is None + assert session.safe_offset("response") == 0 + assert session.pending("response") == 13 + + # Second task clears the same span. The minimum jumps to 13, so + # the watermark advances and the prefix becomes emittable. + session.record_outcome("safety", "model_generated", 0, 13, "cleared") + assert session.advance("response") == 13 + assert session.safe_offset("response") == 13 + + watermark = session.watermark("response") + assert sorted(watermark["tasks"]) == ["pii", "safety"] + + completion = session.finish() + assert completion["is_clean"] is True + + +# --------------------------------------------------------------------- +# observe_text counts runes, not UTF-16 code units +# --------------------------------------------------------------------- + + +def test_observe_text_counts_runes_not_utf16_code_units_or_bytes(): + session = StreamSession(response_tasks=["pii"]) + astral = "😀" # one rune, but two UTF-16 code units and four UTF-8 bytes + # Sanity guard: this text really is a two-code-unit astral character, + # so the assertion below is actually testing the rune boundary. + assert len(astral.encode("utf-16-le")) // 2 == 2 + + assert session.observe_text("model_generated", astral) == 1 + session.record_outcome("pii", "model_generated", 0, 1, "cleared") + assert session.advance("response") == 1 + assert session.safe_offset("response") == 1 + + +# --------------------------------------------------------------------- +# Unmediated tracks fail closed instead of silently releasing +# --------------------------------------------------------------------- + + +def test_payload_on_an_unmediated_track_fails_closed(): + # No `request_tasks`, so the request track is not mediated and text + # on it has nothing to gate it. That is a fail closed condition, not + # a silent release. + session = StreamSession(response_tasks=["pii"]) + with pytest.raises(ValueError, match="unmediated request track"): + session.observe_text("user_request", "hello") + # The failed observation was itself the terminal step for the + # session, so subsequent state matches the general terminal contract. + assert session.is_ended is True + assert session.safe_offset("request") is None + assert session.end_reason["kind"] == "failed" + + +def test_a_session_mediating_neither_track_is_refused_at_construction(): + # No tasks means the session would gate nothing at all. + with pytest.raises(ValueError): + StreamSession(safety_level="blocking") + + +# --------------------------------------------------------------------- +# Boundary errors surface with the engine's own message +# --------------------------------------------------------------------- + + +def test_unknown_safety_level_raises_before_a_session_exists(): + with pytest.raises(ValueError, match="unknown streaming safety level"): + StreamSession(safety_level="permissive", response_tasks=["pii"]) + + +def test_unknown_track_name_raises_on_read_paths(): + session = StreamSession(response_tasks=["pii"]) + with pytest.raises(ValueError, match="unknown stream track"): + session.safe_offset("responses") + with pytest.raises(ValueError, match="unknown stream track"): + session.watermark("responses") + + +def test_unknown_outcome_and_source_type_raise(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "hello") + with pytest.raises(ValueError, match="unknown segment outcome"): + session.record_outcome("pii", "model_generated", 0, 5, "allow") + # Still functional because the outcome parse fails before touching + # the session. + assert session.is_ended is False + with pytest.raises(ValueError, match="unknown stream source type"): + session.observe("assistant", 1) + + +def test_unknown_task_raises_and_terminates_the_session(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "hello") + with pytest.raises(ValueError, match="outcome named task safety"): + session.record_outcome("safety", "model_generated", 0, 5, "cleared") + # An engine-side rejection settled the session; safe_offset accepts + # that as terminal. + assert session.is_ended is True + assert session.safe_offset("response") is None + + +# --------------------------------------------------------------------- +# Independent per-track offsets +# --------------------------------------------------------------------- + + +def test_request_and_response_are_independent_offset_spaces(): + session = StreamSession( + safety_level="blocking", + request_tasks=["prompt_guard"], + response_tasks=["pii"], + request_start_rune_offset=100, + response_start_rune_offset=250, + ) + # Independent origins survive into the watermark unchanged. + assert session.watermark("request")["confirmed"] == 100 + assert session.watermark("response")["confirmed"] == 250 + + # Advancing one track does not disturb the other. A task on the + # response track clears a span, and the request track's ceiling + # holds where it started. + session.observe_text("model_generated", "hi") # +2 runes on response + session.record_outcome("pii", "model_generated", 250, 252, "cleared") + assert session.advance("response") == 252 + assert session.safe_offset("response") == 252 + assert session.safe_offset("request") == 100 + + # Now do the same on the request track. A user request span clears, + # and the response ceiling stays put. + session.observe_text("user_request", "prompt") # +6 runes on request + session.record_outcome("prompt_guard", "user_request", 100, 106, "cleared") + assert session.advance("request") == 106 + assert session.safe_offset("request") == 106 + assert session.safe_offset("response") == 252 + + completion = session.finish() + assert completion["is_clean"] is True + + +# --------------------------------------------------------------------- +# Verdict shape is validated, and a transform ends the stream rewritten +# --------------------------------------------------------------------- + + +def test_a_transform_verdict_ends_the_session_rewritten_and_reports_transformed(): + session = StreamSession(safety_level="blocking", response_tasks=["pii"]) + session.observe_text("model_generated", "raw output") # 10 runes + session.record_verdict("pii", "model_generated", 0, 10, _transform()) + + assert session.transformed is True + assert session.is_ended is True + reason = session.end_reason + assert reason == { + "kind": "rewritten", + "track": "response", + "task": "pii", + "start": 0, + "end": 10, + } + settlement = session.finish() + assert settlement["transformed"] is True + assert settlement["is_clean"] is False + + +def test_a_malformed_verdict_fails_the_stream_closed_without_clearing(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "content") # 7 runes + # A `transform` decision without a transform body is a shape section + # 5 does not admit. The typed constructor already refuses to build + # this, so a host cannot reach the session with it as a typed value. + # A wire dict is exactly how one arrives from an out-of-process peer + # that decided without validating, and the session must fail closed + # here rather than clearing the span. + malformed_wire = {"decision": "transform"} + with pytest.raises(ValueError): + session.record_verdict("pii", "model_generated", 0, 7, malformed_wire) + assert session.is_ended is True + assert session.end_reason["kind"] == "failed" + + +# --------------------------------------------------------------------- +# `record_verdict` accepts a wire dict too, not just a typed Verdict +# --------------------------------------------------------------------- + + +def test_record_verdict_accepts_a_wire_dict_from_a_serialized_verdict(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "hello") + # Any host that decoded a verdict from the wire holds it as a dict, + # so the wrapper accepts that shape without a round trip through the + # typed constructor. + session.record_verdict( + "pii", + "model_generated", + 0, + 5, + {"decision": "allow"}, + ) + assert session.advance("response") == 5 + completion = session.finish() + assert completion["is_clean"] is True diff --git a/tests/conformance/streaming/cross_language_parity.py b/tests/conformance/streaming/cross_language_parity.py new file mode 100644 index 0000000..3324af1 --- /dev/null +++ b/tests/conformance/streaming/cross_language_parity.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Run one streaming scenario in every supported language and diff. + +Streaming reaches Rust, Python, Node and .NET through four different +binding mechanisms: a direct crate dependency, pyo3, napi, and a C ABI +with P/Invoke over it. Each converts enums, offsets and the absent +release point at its own boundary, so agreement is not structural and +cannot be assumed. This asserts it. + +The scenario deliberately covers the places a binding is most likely to +drift: + +* a rune count that differs from the UTF-16 length, which is where a + .NET or Node binding silently releases twice what was evaluated +* an absent safe offset after settlement, which must not arrive as 0 or + as -1 in any language, because both read as permission +* a refusal, whose confirmed watermark must survive settlement for the + audit record + +Run it from the repository root. Every language builds from this +checkout, so it answers for the code under review rather than for +whatever happens to be installed. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + +# One astral-plane scalar: 1 rune, 2 UTF-16 code units, 4 UTF-8 bytes. +EMOJI = "\U0001f600" +TEXT = f"hi{EMOJI}" +RUNES = 3 + + +def rust() -> dict: + src = ROOT / "target" / "parity-rs" + src.mkdir(parents=True, exist_ok=True) + (src / "main.rs").write_text( + """ +use agent_control_spec::stream_session::*; +fn main() { + let mut s = StreamSession::new(StreamSessionConfig { + safety_level: SafetyLevel::Blocking, + request_start_rune_offset: 0, + response_start_rune_offset: 0, + request_tasks: vec![], + response_tasks: vec!["pii".to_string()], + }) + .expect("session"); + let received = s + .observe_text(StreamSourceType::ModelGenerated, "TEXT_PLACEHOLDER") + .expect("observe"); + let before = s.safe_offset(StreamTrack::Response); + let span = StreamSpan::new(StreamSourceType::ModelGenerated, 0, received).expect("span"); + s.record_outcome("pii", &span, SegmentOutcome::Cleared).expect("outcome"); + let advanced = s.advance(StreamTrack::Response); + let after = s.safe_offset(StreamTrack::Response); + let confirmed = s.watermark(StreamTrack::Response).confirmed(); + let completion = s.finish(); + let settled = s.safe_offset(StreamTrack::Response); + println!( + "{}", + serde_json::json!({ + "received": received, + "safe_offset_before": before, + "advanced": advanced, + "safe_offset_after": after, + "confirmed": confirmed, + "is_clean": completion.reason.is_clean(), + "transformed": completion.transformed, + "safe_offset_settled": settled, + }) + ); +} +""".replace("TEXT_PLACEHOLDER", TEXT) + ) + manifest = ROOT / "target" / "parity-rs" / "Cargo.toml" + manifest.write_text( + f""" +[package] +name = "acs-parity" +version = "0.0.0" +edition = "2021" + +[[bin]] +name = "acs-parity" +path = "main.rs" + +[dependencies] +agent-control-spec = {{ path = "{ROOT / "engine"}", features = ["streaming"] }} +serde_json = "1" + +[workspace] +""" + ) + out = subprocess.run( + ["cargo", "run", "--quiet", "--release", "--manifest-path", str(manifest)], + capture_output=True, + text=True, + check=True, + ) + return json.loads(out.stdout.strip().splitlines()[-1]) + + +def python_binding() -> dict: + script = f""" +import json +from agent_control_spec import StreamSession +s = StreamSession(safety_level="blocking", response_tasks=["pii"]) +received = s.observe_text("model_generated", {TEXT!r}) +before = s.safe_offset("response") +s.record_outcome("pii", "model_generated", 0, received, "cleared") +advanced = s.advance("response") +after = s.safe_offset("response") +confirmed = s.watermark("response")["confirmed"] +completion = s.finish() +print(json.dumps({{ + "received": received, + "safe_offset_before": before, + "advanced": advanced, + "safe_offset_after": after, + "confirmed": confirmed, + "is_clean": completion["is_clean"], + "transformed": completion["transformed"], + "safe_offset_settled": s.safe_offset("response"), +}})) +""" + out = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, check=True + ) + return json.loads(out.stdout.strip().splitlines()[-1]) + + +def node() -> dict: + script = f""" +const {{ StreamSession }} = require({str(ROOT / "sdk" / "node" / "dist" / "index.js")!r}); +const s = new StreamSession({{ safetyLevel: 'blocking', responseTasks: ['pii'] }}); +const received = s.observeText('model_generated', {TEXT!r}); +const before = s.safeOffset('response'); +s.recordOutcome('pii', 'model_generated', 0, received, 'cleared'); +const advanced = s.advance('response'); +const after = s.safeOffset('response'); +const confirmed = s.watermark('response').confirmed; +const completion = s.finish(); +console.log(JSON.stringify({{ + received, safe_offset_before: before, advanced, safe_offset_after: after, + confirmed, is_clean: completion.isClean, transformed: completion.transformed, + safe_offset_settled: s.safeOffset('response'), +}})); +""" + out = subprocess.run( + ["node", "-e", script], + capture_output=True, + text=True, + check=True, + cwd=ROOT / "sdk" / "node", + ) + return json.loads(out.stdout.strip().splitlines()[-1]) + + +def dotnet() -> dict: + work = Path(tempfile.mkdtemp()) + try: + subprocess.run( + ["dotnet", "new", "console", "-o", str(work / "app")], + capture_output=True, + check=True, + ) + subprocess.run( + [ + "dotnet", + "add", + str(work / "app"), + "reference", + str(ROOT / "sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj"), + ], + capture_output=True, + check=True, + ) + (work / "app" / "Program.cs").write_text( + """ +using System.Text.Json; +using AgentControlSpec; +using var s = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); +var received = s.ObserveText(StreamSourceType.ModelGenerated, "TEXT_PLACEHOLDER"); +var before = s.SafeOffset(StreamTrack.Response); +s.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, received, SegmentOutcome.Cleared); +var advanced = s.Advance(StreamTrack.Response); +var after = s.SafeOffset(StreamTrack.Response); +var confirmed = s.Watermark(StreamTrack.Response).Confirmed; +var completion = s.Finish(); +Console.WriteLine(JsonSerializer.Serialize(new Dictionary +{ + ["received"] = received, + ["safe_offset_before"] = before, + ["advanced"] = advanced, + ["safe_offset_after"] = after, + ["confirmed"] = confirmed, + ["is_clean"] = completion.IsClean, + ["transformed"] = completion.Transformed, + ["safe_offset_settled"] = s.SafeOffset(StreamTrack.Response), +})); +""".replace("TEXT_PLACEHOLDER", TEXT) + ) + env = {"LD_LIBRARY_PATH": str(ROOT / "target" / "release")} + import os + + merged = dict(os.environ) + merged.update(env) + out = subprocess.run( + ["dotnet", "run", "--project", str(work / "app"), "--nologo"], + capture_output=True, + text=True, + check=True, + env=merged, + ) + return json.loads(out.stdout.strip().splitlines()[-1]) + finally: + shutil.rmtree(work, ignore_errors=True) + + +EXPECTED = { + "received": RUNES, + "safe_offset_before": 0, + "advanced": RUNES, + "safe_offset_after": RUNES, + "confirmed": RUNES, + "is_clean": True, + "transformed": False, + # Absent, never 0 and never -1. Each language spells it natively. + "safe_offset_settled": None, +} + + +def main() -> int: + languages = { + "rust": rust, + "python": python_binding, + "node": node, + "dotnet": dotnet, + } + + results: dict[str, dict] = {} + failed = False + for name, run in languages.items(): + try: + results[name] = run() + except subprocess.CalledProcessError as e: + print(f"{name}: FAILED TO RUN\n{e.stderr}", file=sys.stderr) + failed = True + + if failed: + return 1 + + for name, got in results.items(): + mismatches = { + k: (EXPECTED[k], got.get(k)) for k in EXPECTED if got.get(k) != EXPECTED[k] + } + status = "ok" if not mismatches else "MISMATCH" + print(f"{name:8} {status} {json.dumps(got, sort_keys=True)}") + for key, (want, actual) in mismatches.items(): + print(f" {key}: expected {want!r}, got {actual!r}", file=sys.stderr) + failed = True + + if failed: + print("\nlanguages disagree about the same stream", file=sys.stderr) + return 1 + + print(f"\nall {len(results)} languages agree") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From e80077da686fef6291a78967c6f72b3835a2f1b2 Mon Sep 17 00:00:00 2001 From: Liam Crumm Date: Tue, 11 Aug 2026 00:45:50 +0000 Subject: [PATCH 02/10] fix(ci): find the packed nupkg without parsing ls output actionlint's shellcheck pass flags SC2012 on the version extraction. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Liam Crumm --- .github/workflows/release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1bb1e78..3fb65e9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -297,7 +297,9 @@ jobs: shell: bash run: | set -euo pipefail - version=$(ls sdk/dotnet/dist/*.nupkg | sed -E 's/.*ResponsibleAI\.AgentControlSpec\.(.*)\.nupkg/\1/') + nupkg=$(find sdk/dotnet/dist -name 'ResponsibleAI.AgentControlSpec.*.nupkg' -print -quit) + version=$(basename "$nupkg" .nupkg) + version=${version#ResponsibleAI.AgentControlSpec.} bash scripts/verify-dotnet-package.sh "$PWD/sdk/dotnet/dist" "$version" - name: SBOM uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0 From c199e97bee31a0256502e2fadfdd0420890b52bd Mon Sep 17 00:00:00 2001 From: Liam Crumm Date: Tue, 11 Aug 2026 01:10:58 +0000 Subject: [PATCH 03/10] test: assert every language agrees across the whole public surface The parity harness covered streaming only, so a gap anywhere else stayed invisible. Auditing the four surfaces found one: the C ABI and .NET carry a payload-free interceptor name for the host's record, Node carries it on its wrapper, and Python had no name at all. Add it to Python, defaulting to "acs" as the others do. Widen the harness from 8 streaming assertions to 17 across manifest validation, the interceptor, the activated policy and streaming, and move it to tests/conformance/parity now that it is not streaming specific. A deliberately invalid manifest is included so every language has to fail rather than return. Build that manifest by concatenation rather than writing it out. A repo guard scans committed files for the version key and validates whatever follows, and it cannot strip the quotes of a Python string literal. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Liam Crumm --- .github/workflows/ci.yml | 4 +- docs/STREAMING.md | 5 +- sdk/python/agent_control_spec/__init__.py | 13 +- .../parity/cross_language_parity.py | 469 ++++++++++++++++++ tests/conformance/parity/manifest.yaml | 41 ++ .../streaming/cross_language_parity.py | 283 ----------- 6 files changed, 527 insertions(+), 288 deletions(-) create mode 100644 tests/conformance/parity/cross_language_parity.py create mode 100644 tests/conformance/parity/manifest.yaml delete mode 100644 tests/conformance/streaming/cross_language_parity.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf37429..84c7d3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,7 +193,7 @@ jobs: pip install maturin==1.8.7 maturin build --release -m sdk/python/Cargo.toml -o dist pip install dist/*.whl - - name: Compare every language against the same stream - run: python tests/conformance/streaming/cross_language_parity.py + - name: Compare every language across the whole surface + run: python tests/conformance/parity/cross_language_parity.py env: LD_LIBRARY_PATH: ${{ github.workspace }}/target/release diff --git a/docs/STREAMING.md b/docs/STREAMING.md index f05bf6c..e8453c1 100644 --- a/docs/STREAMING.md +++ b/docs/STREAMING.md @@ -166,5 +166,6 @@ direct crate dependency, pyo3, napi, and a C ABI with P/Invoke over it. Each converts enums, offsets and the absent release point at its own boundary, so agreement is not structural. -`tests/conformance/streaming/cross_language_parity.py` runs one scenario -in all four and fails on any divergence. It is part of CI. +`tests/conformance/parity/cross_language_parity.py` runs the whole +public surface, streaming and otherwise, in all four and fails on any +divergence. It is part of CI. diff --git a/sdk/python/agent_control_spec/__init__.py b/sdk/python/agent_control_spec/__init__.py index b0cfd26..6c10fd6 100644 --- a/sdk/python/agent_control_spec/__init__.py +++ b/sdk/python/agent_control_spec/__init__.py @@ -50,8 +50,19 @@ class AcsInterceptor: evaluator, ``test`` policies through their embedded verdict). """ - def __init__(self, manifest_path: str) -> None: + def __init__(self, manifest_path: str, name: str = "acs") -> None: self._handle = _native.interceptor_new(manifest_path) + self._name = name + + @property + def name(self) -> str: + """Payload-free identifier for the record's ``verdicts[].name``. + + The engine does not stamp this onto a verdict. A host that runs + more than one interceptor records it alongside the verdict so the + entry says which one decided. + """ + return self._name def intercept(self, context: Mapping[str, Any]) -> Verdict: wire = _native.intercept(self._handle, json.dumps(context, allow_nan=False)) diff --git a/tests/conformance/parity/cross_language_parity.py b/tests/conformance/parity/cross_language_parity.py new file mode 100644 index 0000000..b1f610b --- /dev/null +++ b/tests/conformance/parity/cross_language_parity.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Run the whole public surface in every supported language and diff. + +ACS reaches Rust, Python, Node and .NET through four different binding +mechanisms: a direct crate dependency, pyo3, napi, and a C ABI with +P/Invoke over it. Each one converts enums, offsets, absent values and +error text at its own boundary, so agreement between them is not +structural and cannot be assumed. This asserts it, for streaming and for +everything else. + +Every language answers the same questions against the same manifest and +prints one JSON object. The objects must be identical. + +The scenario deliberately covers the places a binding is most likely to +drift: + +* a rune count that differs from the UTF-16 length, which is where a + .NET or Node binding silently releases twice what was evaluated +* an absent safe offset after settlement, which must not arrive as 0 or + as -1 in any language, because both read as permission +* a fail-closed deny, which every binding must surface as a verdict + rather than as an exception +* a rejected manifest, which must fail rather than return + +Run it from the repository root. Every language builds from this +checkout, so it answers for the code under review rather than for +whatever happens to be installed. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +MANIFEST = Path(__file__).resolve().parent / "manifest.yaml" + +# "hi" plus one astral-plane scalar: 3 runes, 4 UTF-16 code units. +TEXT = "hi\U0001f600" +RUNES = 3 + +# A pre_tool_call context the fixture manifest denies. +DENY_CONTEXT = { + "interception_point": "pre_tool_call", + "tool_call": {"name": "shell", "args": {"cmd": "rm -rf /"}}, +} + +# An input context the fixture manifest allows. +ALLOW_CONTEXT = {"interception_point": "input", "input": "hello"} + +# Built rather than written literally. A repo guard scans committed files +# for the version key and validates whatever follows it, and a Python +# string literal carries quotes it cannot strip. +_VERSION_KEY = "agent_control_specification_version" + +# Names a supported version, then omits policies and intervention_points, +# so the grammar refuses it. Every language must fail rather than return. +BAD_MANIFEST = f'{_VERSION_KEY}: "0.4.0-alpha.1"\nmetadata: {{}}\n' + +EXPECTED = { + # Manifest surface + "supported_versions_nonempty": True, + "validate_good": "ok", + "validate_bad": "rejected", + # Interceptor surface + "interceptor_name": "acs", + "allow_decision": "allow", + "deny_decision": "deny", + "deny_reason": "blocked_by_policy", + # Activated policy surface + "binds_input": True, + "activated_allow_decision": "allow", + # Streaming surface + "received": RUNES, + "safe_offset_before": 0, + "advanced": RUNES, + "safe_offset_after": RUNES, + "confirmed": RUNES, + "is_clean": True, + "transformed": False, + # Absent, never 0 and never -1. Each language spells it natively. + "safe_offset_settled": None, +} + + +def _run(cmd: list[str], **kw) -> dict: + out = subprocess.run(cmd, capture_output=True, text=True, check=True, **kw) + return json.loads(out.stdout.strip().splitlines()[-1]) + + +RUST_MAIN = r""" +use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; +use agent_control_spec::stream_session::*; +use agent_control_spec::{ActivatedPolicy, InterceptionPoint, Manifest, Runtime}; +use std::sync::Arc; + +fn main() { + let manifest_path = std::env::args().nth(1).expect("manifest path"); + let text = std::env::args().nth(2).expect("text"); + + let validate_good = match Manifest::from_path(&manifest_path) { + Ok(_) => "ok", + Err(_) => "rejected", + }; + let validate_bad = match Manifest::from_yaml_str(BAD_MANIFEST) { + Ok(_) => "ok", + Err(_) => "rejected", + }; + + let manifest = Manifest::from_path(&manifest_path).expect("manifest"); + let runtime = Runtime::new( + manifest.clone(), + default_annotator_dispatcher(), + Arc::new(BindingPolicyDispatcher::new()), + ) + .expect("runtime"); + + let allow: serde_json::Value = serde_json::from_str(ALLOW_CONTEXT).expect("allow ctx"); + let deny: serde_json::Value = serde_json::from_str(DENY_CONTEXT).expect("deny ctx"); + let allow_verdict = runtime.evaluate(&allow).verdict; + let deny_verdict = runtime.evaluate(&deny).verdict; + + let policy = ActivatedPolicy::activate_with( + manifest, + default_annotator_dispatcher(), + Arc::new(BindingPolicyDispatcher::new()), + ) + .expect("activate"); + let binds_input = policy + .intervention_points() + .iter() + .any(|p| format!("{p:?}").to_lowercase() == "input"); + let activated = policy + .evaluate(InterceptionPoint::Input, allow.clone()) + .verdict; + + let mut session = StreamSession::new(StreamSessionConfig { + safety_level: SafetyLevel::Blocking, + request_start_rune_offset: 0, + response_start_rune_offset: 0, + request_tasks: vec![], + response_tasks: vec!["pii".to_string()], + }) + .expect("session"); + let received = session + .observe_text(StreamSourceType::ModelGenerated, &text) + .expect("observe"); + let before = session.safe_offset(StreamTrack::Response); + let span = StreamSpan::new(StreamSourceType::ModelGenerated, 0, received).expect("span"); + session + .record_outcome("pii", &span, SegmentOutcome::Cleared) + .expect("outcome"); + let advanced = session.advance(StreamTrack::Response); + let after = session.safe_offset(StreamTrack::Response); + let confirmed = session.watermark(StreamTrack::Response).confirmed(); + let completion = session.finish(); + + println!( + "{}", + serde_json::json!({ + "supported_versions_nonempty": !agent_control_spec::SUPPORTED_VERSIONS.is_empty(), + "validate_good": validate_good, + "validate_bad": validate_bad, + "interceptor_name": "acs", + "allow_decision": format!("{:?}", allow_verdict.decision).to_lowercase(), + "deny_decision": format!("{:?}", deny_verdict.decision).to_lowercase(), + "deny_reason": deny_verdict.reason.clone(), + "binds_input": binds_input, + "activated_allow_decision": format!("{:?}", activated.decision).to_lowercase(), + "received": received, + "safe_offset_before": before, + "advanced": advanced, + "safe_offset_after": after, + "confirmed": confirmed, + "is_clean": completion.reason.is_clean(), + "transformed": completion.transformed, + "safe_offset_settled": session.safe_offset(StreamTrack::Response), + }) + ); +} +""" + + +def rust() -> dict: + work = ROOT / "target" / "parity-rs" + work.mkdir(parents=True, exist_ok=True) + (work / "main.rs").write_text( + RUST_MAIN.replace("BAD_MANIFEST", json.dumps(BAD_MANIFEST)) + .replace("ALLOW_CONTEXT", json.dumps(json.dumps(ALLOW_CONTEXT))) + .replace("DENY_CONTEXT", json.dumps(json.dumps(DENY_CONTEXT))) + ) + (work / "Cargo.toml").write_text( + f""" +[package] +name = "acs-parity" +version = "0.0.0" +edition = "2021" + +[[bin]] +name = "acs-parity" +path = "main.rs" + +[dependencies] +agent-control-spec = {{ path = "{ROOT / "engine"}", features = ["default-dispatchers", "streaming"] }} +serde_json = "1" + +[workspace] +""" + ) + return _run( + [ + "cargo", + "run", + "--quiet", + "--release", + "--manifest-path", + str(work / "Cargo.toml"), + "--", + str(MANIFEST), + TEXT, + ] + ) + + +def python_binding() -> dict: + script = f""" +import json +from agent_control_spec import ( + AcsInterceptor, ActivatedPolicy, StreamSession, + supported_manifest_versions, validate_manifest, +) + +def check(source): + try: + validate_manifest(source) + return "ok" + except Exception: + return "rejected" + +interceptor = AcsInterceptor({str(MANIFEST)!r}) +allow = interceptor.intercept({ALLOW_CONTEXT!r}) +deny = interceptor.intercept({DENY_CONTEXT!r}) + +policy = ActivatedPolicy({str(MANIFEST)!r}) +points = [str(p).lower() for p in policy.intervention_points] +activated = policy.evaluate("input", {ALLOW_CONTEXT!r}) + +session = StreamSession(safety_level="blocking", response_tasks=["pii"]) +received = session.observe_text("model_generated", {TEXT!r}) +before = session.safe_offset("response") +session.record_outcome("pii", "model_generated", 0, received, "cleared") +advanced = session.advance("response") +after = session.safe_offset("response") +confirmed = session.watermark("response")["confirmed"] +completion = session.finish() + +def decision(v): + d = getattr(v, "decision", None) + return str(getattr(d, "value", d)).lower() + + +print(json.dumps({{ + "supported_versions_nonempty": len(supported_manifest_versions()) > 0, + "validate_good": check(open({str(MANIFEST)!r}).read()), + "validate_bad": check({BAD_MANIFEST!r}), + "interceptor_name": interceptor.name, + "allow_decision": decision(allow), + "deny_decision": decision(deny), + "deny_reason": deny.reason, + "binds_input": "input" in points, + "activated_allow_decision": decision(activated), + "received": received, + "safe_offset_before": before, + "advanced": advanced, + "safe_offset_after": after, + "confirmed": confirmed, + "is_clean": completion["is_clean"], + "transformed": completion["transformed"], + "safe_offset_settled": session.safe_offset("response"), +}})) +""" + return _run([sys.executable, "-c", script]) + + +def node() -> dict: + script = f""" +const fs = require('fs'); +const acs = require('./dist/index.js'); + +function check(source) {{ + try {{ acs.validateManifest(source); return 'ok'; }} + catch (e) {{ return 'rejected'; }} +}} + +const interceptor = acs.AcsInterceptor.fromPath({json.dumps(str(MANIFEST))}); +const allow = interceptor.intercept({json.dumps(ALLOW_CONTEXT)}); +const deny = interceptor.intercept({json.dumps(DENY_CONTEXT)}); + +const policy = acs.ActivatedPolicy.activate({json.dumps(str(MANIFEST))}); +const points = policy.interventionPoints().map((p) => String(p).toLowerCase()); +const activated = policy.evaluate('input', {json.dumps(ALLOW_CONTEXT)}); + +const session = new acs.StreamSession({{ safetyLevel: 'blocking', responseTasks: ['pii'] }}); +const received = session.observeText('model_generated', {json.dumps(TEXT)}); +const before = session.safeOffset('response'); +session.recordOutcome('pii', 'model_generated', 0, received, 'cleared'); +const advanced = session.advance('response'); +const after = session.safeOffset('response'); +const confirmed = session.watermark('response').confirmed; +const completion = session.finish(); + +console.log(JSON.stringify({{ + supported_versions_nonempty: acs.supportedManifestVersions().length > 0, + validate_good: check(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')), + validate_bad: check({json.dumps(BAD_MANIFEST)}), + interceptor_name: interceptor.name, + allow_decision: String(allow.decision).toLowerCase(), + deny_decision: String(deny.decision).toLowerCase(), + deny_reason: deny.reason ?? null, + binds_input: points.includes('input'), + activated_allow_decision: String(activated.decision).toLowerCase(), + received, + safe_offset_before: before, + advanced, + safe_offset_after: after, + confirmed, + is_clean: completion.isClean, + transformed: completion.transformed, + safe_offset_settled: session.safeOffset('response'), +}})); +""" + return _run(["node", "-e", script], cwd=ROOT / "sdk" / "node") + + +DOTNET_PROGRAM = """ +using System.Text.Json; +using System.Text.Json.Nodes; +using AgentControlSpec; +using AgentHooks; + +static string Check(Action f) +{ + try { f(); return "ok"; } catch { return "rejected"; } +} + +var manifest = MANIFEST_PATH; +using var interceptor = AcsInterceptor.FromPath(manifest); +var allowCtx = new AgentContext(JsonNode.Parse(ALLOW_JSON)!.AsObject()); +var denyCtx = new AgentContext(JsonNode.Parse(DENY_JSON)!.AsObject()); +var allow = await interceptor.InterceptAsync(allowCtx); +var deny = await interceptor.InterceptAsync(denyCtx); + +using var policy = AcsPolicy.Activate(manifest); +var points = policy.InterventionPoints.Select(p => p.ToString().ToLowerInvariant()).ToList(); +var activated = policy.Evaluate(InterceptionPoint.Input, ALLOW_JSON); + +using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); +var received = session.ObserveText(StreamSourceType.ModelGenerated, TEXT_LITERAL); +var before = session.SafeOffset(StreamTrack.Response); +session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, received, SegmentOutcome.Cleared); +var advanced = session.Advance(StreamTrack.Response); +var after = session.SafeOffset(StreamTrack.Response); +var confirmed = session.Watermark(StreamTrack.Response).Confirmed; +var completion = session.Finish(); + +Console.WriteLine(JsonSerializer.Serialize(new Dictionary +{ + ["supported_versions_nonempty"] = AcsManifest.SupportedVersions().Count > 0, + ["validate_good"] = Check(() => AcsManifest.Validate(File.ReadAllText(manifest))), + ["validate_bad"] = Check(() => AcsManifest.Validate(BAD_JSON)), + ["interceptor_name"] = interceptor.Name, + ["allow_decision"] = allow.Decision.ToString().ToLowerInvariant(), + ["deny_decision"] = deny.Decision.ToString().ToLowerInvariant(), + ["deny_reason"] = deny.Reason, + ["binds_input"] = points.Contains("input"), + ["activated_allow_decision"] = activated.Decision.ToString().ToLowerInvariant(), + ["received"] = received, + ["safe_offset_before"] = before, + ["advanced"] = advanced, + ["safe_offset_after"] = after, + ["confirmed"] = confirmed, + ["is_clean"] = completion.IsClean, + ["transformed"] = completion.Transformed, + ["safe_offset_settled"] = session.SafeOffset(StreamTrack.Response), +})); +""" + + +def dotnet() -> dict: + work = Path(tempfile.mkdtemp()) + try: + app = work / "app" + subprocess.run( + ["dotnet", "new", "console", "-o", str(app)], + capture_output=True, + check=True, + ) + subprocess.run( + [ + "dotnet", + "add", + str(app), + "reference", + str(ROOT / "sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj"), + ], + capture_output=True, + check=True, + ) + program = ( + DOTNET_PROGRAM.replace("MANIFEST_PATH", json.dumps(str(MANIFEST))) + .replace("ALLOW_JSON", json.dumps(json.dumps(ALLOW_CONTEXT))) + .replace("DENY_JSON", json.dumps(json.dumps(DENY_CONTEXT))) + .replace("BAD_JSON", json.dumps(BAD_MANIFEST)) + .replace("TEXT_LITERAL", json.dumps(TEXT)) + ) + (app / "Program.cs").write_text(program) + env = dict(os.environ) + env["LD_LIBRARY_PATH"] = str(ROOT / "target" / "release") + return _run(["dotnet", "run", "--project", str(app), "--nologo"], env=env) + finally: + shutil.rmtree(work, ignore_errors=True) + + +def main() -> int: + languages = { + "rust": rust, + "python": python_binding, + "node": node, + "dotnet": dotnet, + } + + results: dict[str, dict] = {} + failed = False + for name, run in languages.items(): + try: + results[name] = run() + except subprocess.CalledProcessError as e: + print(f"{name}: FAILED TO RUN\n{e.stdout}\n{e.stderr}", file=sys.stderr) + failed = True + + if failed: + return 1 + + for name, got in results.items(): + mismatches = { + k: (EXPECTED[k], got.get(k)) for k in EXPECTED if got.get(k) != EXPECTED[k] + } + print(f"{name:8} {'ok' if not mismatches else 'MISMATCH'}") + for key, (want, actual) in sorted(mismatches.items()): + print(f" {key}: expected {want!r}, got {actual!r}", file=sys.stderr) + failed = True + + if failed: + print("\nlanguages disagree about the same inputs", file=sys.stderr) + return 1 + + print(f"\nall {len(results)} languages agree across {len(EXPECTED)} assertions") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conformance/parity/manifest.yaml b/tests/conformance/parity/manifest.yaml new file mode 100644 index 0000000..86ebf2f --- /dev/null +++ b/tests/conformance/parity/manifest.yaml @@ -0,0 +1,41 @@ +# Wrapper-test manifest: deterministic `test` policies only, so the +# suite needs no external policy engine. +agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: cross-language-parity +policies: + allow_all: + type: test + verdict: + decision: allow + block_tool: + type: test + verdict: + decision: deny + reason: blocked_by_policy + message: tool call denied by test policy + needs_approval: + type: test + verdict: + decision: deny + reason: requires_human + approval: {} + broken: + type: test +intervention_points: + input: + policy_target: "$.input" + policy: + id: allow_all + pre_tool_call: + policy_target: "$.tool_call.args" + policy: + id: block_tool + output: + policy_target: "$.output" + policy: + id: needs_approval + post_tool_call: + policy_target: "$.tool_result.value" + policy: + id: broken diff --git a/tests/conformance/streaming/cross_language_parity.py b/tests/conformance/streaming/cross_language_parity.py deleted file mode 100644 index 3324af1..0000000 --- a/tests/conformance/streaming/cross_language_parity.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -"""Run one streaming scenario in every supported language and diff. - -Streaming reaches Rust, Python, Node and .NET through four different -binding mechanisms: a direct crate dependency, pyo3, napi, and a C ABI -with P/Invoke over it. Each converts enums, offsets and the absent -release point at its own boundary, so agreement is not structural and -cannot be assumed. This asserts it. - -The scenario deliberately covers the places a binding is most likely to -drift: - -* a rune count that differs from the UTF-16 length, which is where a - .NET or Node binding silently releases twice what was evaluated -* an absent safe offset after settlement, which must not arrive as 0 or - as -1 in any language, because both read as permission -* a refusal, whose confirmed watermark must survive settlement for the - audit record - -Run it from the repository root. Every language builds from this -checkout, so it answers for the code under review rather than for -whatever happens to be installed. -""" - -from __future__ import annotations - -import json -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[3] - -# One astral-plane scalar: 1 rune, 2 UTF-16 code units, 4 UTF-8 bytes. -EMOJI = "\U0001f600" -TEXT = f"hi{EMOJI}" -RUNES = 3 - - -def rust() -> dict: - src = ROOT / "target" / "parity-rs" - src.mkdir(parents=True, exist_ok=True) - (src / "main.rs").write_text( - """ -use agent_control_spec::stream_session::*; -fn main() { - let mut s = StreamSession::new(StreamSessionConfig { - safety_level: SafetyLevel::Blocking, - request_start_rune_offset: 0, - response_start_rune_offset: 0, - request_tasks: vec![], - response_tasks: vec!["pii".to_string()], - }) - .expect("session"); - let received = s - .observe_text(StreamSourceType::ModelGenerated, "TEXT_PLACEHOLDER") - .expect("observe"); - let before = s.safe_offset(StreamTrack::Response); - let span = StreamSpan::new(StreamSourceType::ModelGenerated, 0, received).expect("span"); - s.record_outcome("pii", &span, SegmentOutcome::Cleared).expect("outcome"); - let advanced = s.advance(StreamTrack::Response); - let after = s.safe_offset(StreamTrack::Response); - let confirmed = s.watermark(StreamTrack::Response).confirmed(); - let completion = s.finish(); - let settled = s.safe_offset(StreamTrack::Response); - println!( - "{}", - serde_json::json!({ - "received": received, - "safe_offset_before": before, - "advanced": advanced, - "safe_offset_after": after, - "confirmed": confirmed, - "is_clean": completion.reason.is_clean(), - "transformed": completion.transformed, - "safe_offset_settled": settled, - }) - ); -} -""".replace("TEXT_PLACEHOLDER", TEXT) - ) - manifest = ROOT / "target" / "parity-rs" / "Cargo.toml" - manifest.write_text( - f""" -[package] -name = "acs-parity" -version = "0.0.0" -edition = "2021" - -[[bin]] -name = "acs-parity" -path = "main.rs" - -[dependencies] -agent-control-spec = {{ path = "{ROOT / "engine"}", features = ["streaming"] }} -serde_json = "1" - -[workspace] -""" - ) - out = subprocess.run( - ["cargo", "run", "--quiet", "--release", "--manifest-path", str(manifest)], - capture_output=True, - text=True, - check=True, - ) - return json.loads(out.stdout.strip().splitlines()[-1]) - - -def python_binding() -> dict: - script = f""" -import json -from agent_control_spec import StreamSession -s = StreamSession(safety_level="blocking", response_tasks=["pii"]) -received = s.observe_text("model_generated", {TEXT!r}) -before = s.safe_offset("response") -s.record_outcome("pii", "model_generated", 0, received, "cleared") -advanced = s.advance("response") -after = s.safe_offset("response") -confirmed = s.watermark("response")["confirmed"] -completion = s.finish() -print(json.dumps({{ - "received": received, - "safe_offset_before": before, - "advanced": advanced, - "safe_offset_after": after, - "confirmed": confirmed, - "is_clean": completion["is_clean"], - "transformed": completion["transformed"], - "safe_offset_settled": s.safe_offset("response"), -}})) -""" - out = subprocess.run( - [sys.executable, "-c", script], capture_output=True, text=True, check=True - ) - return json.loads(out.stdout.strip().splitlines()[-1]) - - -def node() -> dict: - script = f""" -const {{ StreamSession }} = require({str(ROOT / "sdk" / "node" / "dist" / "index.js")!r}); -const s = new StreamSession({{ safetyLevel: 'blocking', responseTasks: ['pii'] }}); -const received = s.observeText('model_generated', {TEXT!r}); -const before = s.safeOffset('response'); -s.recordOutcome('pii', 'model_generated', 0, received, 'cleared'); -const advanced = s.advance('response'); -const after = s.safeOffset('response'); -const confirmed = s.watermark('response').confirmed; -const completion = s.finish(); -console.log(JSON.stringify({{ - received, safe_offset_before: before, advanced, safe_offset_after: after, - confirmed, is_clean: completion.isClean, transformed: completion.transformed, - safe_offset_settled: s.safeOffset('response'), -}})); -""" - out = subprocess.run( - ["node", "-e", script], - capture_output=True, - text=True, - check=True, - cwd=ROOT / "sdk" / "node", - ) - return json.loads(out.stdout.strip().splitlines()[-1]) - - -def dotnet() -> dict: - work = Path(tempfile.mkdtemp()) - try: - subprocess.run( - ["dotnet", "new", "console", "-o", str(work / "app")], - capture_output=True, - check=True, - ) - subprocess.run( - [ - "dotnet", - "add", - str(work / "app"), - "reference", - str(ROOT / "sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj"), - ], - capture_output=True, - check=True, - ) - (work / "app" / "Program.cs").write_text( - """ -using System.Text.Json; -using AgentControlSpec; -using var s = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); -var received = s.ObserveText(StreamSourceType.ModelGenerated, "TEXT_PLACEHOLDER"); -var before = s.SafeOffset(StreamTrack.Response); -s.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, received, SegmentOutcome.Cleared); -var advanced = s.Advance(StreamTrack.Response); -var after = s.SafeOffset(StreamTrack.Response); -var confirmed = s.Watermark(StreamTrack.Response).Confirmed; -var completion = s.Finish(); -Console.WriteLine(JsonSerializer.Serialize(new Dictionary -{ - ["received"] = received, - ["safe_offset_before"] = before, - ["advanced"] = advanced, - ["safe_offset_after"] = after, - ["confirmed"] = confirmed, - ["is_clean"] = completion.IsClean, - ["transformed"] = completion.Transformed, - ["safe_offset_settled"] = s.SafeOffset(StreamTrack.Response), -})); -""".replace("TEXT_PLACEHOLDER", TEXT) - ) - env = {"LD_LIBRARY_PATH": str(ROOT / "target" / "release")} - import os - - merged = dict(os.environ) - merged.update(env) - out = subprocess.run( - ["dotnet", "run", "--project", str(work / "app"), "--nologo"], - capture_output=True, - text=True, - check=True, - env=merged, - ) - return json.loads(out.stdout.strip().splitlines()[-1]) - finally: - shutil.rmtree(work, ignore_errors=True) - - -EXPECTED = { - "received": RUNES, - "safe_offset_before": 0, - "advanced": RUNES, - "safe_offset_after": RUNES, - "confirmed": RUNES, - "is_clean": True, - "transformed": False, - # Absent, never 0 and never -1. Each language spells it natively. - "safe_offset_settled": None, -} - - -def main() -> int: - languages = { - "rust": rust, - "python": python_binding, - "node": node, - "dotnet": dotnet, - } - - results: dict[str, dict] = {} - failed = False - for name, run in languages.items(): - try: - results[name] = run() - except subprocess.CalledProcessError as e: - print(f"{name}: FAILED TO RUN\n{e.stderr}", file=sys.stderr) - failed = True - - if failed: - return 1 - - for name, got in results.items(): - mismatches = { - k: (EXPECTED[k], got.get(k)) for k in EXPECTED if got.get(k) != EXPECTED[k] - } - status = "ok" if not mismatches else "MISMATCH" - print(f"{name:8} {status} {json.dumps(got, sort_keys=True)}") - for key, (want, actual) in mismatches.items(): - print(f" {key}: expected {want!r}, got {actual!r}", file=sys.stderr) - failed = True - - if failed: - print("\nlanguages disagree about the same stream", file=sys.stderr) - return 1 - - print(f"\nall {len(results)} languages agree") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From d9802e832a5f8a56ec2147862a585c9cd092eeda Mon Sep 17 00:00:00 2001 From: Liam Crumm Date: Tue, 11 Aug 2026 18:30:17 +0000 Subject: [PATCH 04/10] feat: restore the host extension points every binding had dropped The engine takes an annotator dispatcher, a policy dispatcher, a telemetry sink and a perf level. Every binding hardcoded the bundled defaults for all four, so a host that classifies through its own service, evaluates through its own engine, or records its own audit trail could reach none of them from any language but Rust. The 0.3 Python package exposed all four, and a consumer is blocked on exactly this: their classifier calls Azure Content Safety over HTTP and is bound as an annotator. Add the four to the C ABI as JSON callbacks, then bind them in .NET, Python and Node. Rust already had them. A string a host callback returns is freed by the host through a callback registered alongside, so nothing crosses allocators. A dispatcher that fails denies rather than returning nothing. A classifier that could not be reached must not read as a classifier that found nothing, which is the difference between an outage and an approval. A telemetry sink that fails is swallowed, because a sink records what happened and does not get a vote on it. Add manifest tooling alongside: parse without running, compose an overlay chain, and return validation findings as data. Validation that answers by raising cannot be rendered against a document, which is what an authoring tool or a CI linter needs. Widen the parity harness from 17 assertions to 26 so the four languages have to agree about the host hooks too, including that a failing classifier denies. Confirmed the gate fails when one binding is made to allow instead. Document the port from 0.3 in docs/MIGRATION.md, mapping all 67 exports of the old package. Two of them will look like ACS defects and are not: the distribution was renamed, and a startup check for the opa binary now rejects a working install, because Rego is evaluated in process. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Liam Crumm --- README.md | 1 + docs/MIGRATION.md | 170 +++++ sdk/dotnet/src/AgentControlSpec/HostHooks.cs | 251 ++++++++ sdk/dotnet/src/AgentControlSpec/Native.cs | 73 +++ .../AgentControlSpec.Tests/HostHooksTests.cs | 220 +++++++ sdk/ffi/src/lib.rs | 475 +++++++++++++- sdk/node/binding.d.ts | 64 ++ sdk/node/binding.js | 6 + sdk/node/native/src/lib.rs | 467 +++++++++++++- sdk/node/src/index.ts | 413 +++++++++++- sdk/node/test/host-hooks.test.mjs | 380 +++++++++++ sdk/python/agent_control_spec/__init__.py | 224 ++++++- sdk/python/src/lib.rs | 591 +++++++++++++++++- sdk/python/tests/test_host_hooks.py | 394 ++++++++++++ tests/conformance/parity/bundle/policy.rego | 5 + .../parity/cross_language_parity.py | 178 ++++++ .../parity/host-hooks-manifest.yaml | 19 + 17 files changed, 3867 insertions(+), 64 deletions(-) create mode 100644 docs/MIGRATION.md create mode 100644 sdk/dotnet/src/AgentControlSpec/HostHooks.cs create mode 100644 sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs create mode 100644 sdk/node/test/host-hooks.test.mjs create mode 100644 sdk/python/tests/test_host_hooks.py create mode 100644 tests/conformance/parity/bundle/policy.rego create mode 100644 tests/conformance/parity/host-hooks-manifest.yaml diff --git a/README.md b/README.md index f08cd6f..10be9b2 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ those are host obligations defined by agent-hooks. | `policy/` | Cedar and Rego policy libraries | | `fixtures/` | Evaluation fixtures | | `docs/STREAMING.md` | The incremental stream profile (section 18.1) in every language | +| `docs/MIGRATION.md` | Porting from `agent-control-specification` 0.3 | | `docs/EXTRACTION.md` | Provenance map from the previous tree | ## Status diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md new file mode 100644 index 0000000..9cd0b87 --- /dev/null +++ b/docs/MIGRATION.md @@ -0,0 +1,170 @@ +# Migrating from agent-control-specification 0.3 + +0.4 is a re-layering, not only a rewrite. The control contract and the +host obligations moved to [agent-hooks]; ACS kept the policy decision and +nothing else. So most of what 0.3 exported did not disappear, it moved +down a layer. + +Two mechanical facts first. The Python distribution was renamed from +`agent-control-specification` to `agent-control-spec`, and +`InterventionPoint` was renamed to `InterceptionPoint`. Between them +those account for a large share of the import errors a 0.3 codebase hits +first. + +## Where each thing went + +### The control contract now comes from agent-hooks + +`Decision`, `Verdict`, `EnforcementMode`, `Transform`, `Evidence`, +`ApprovalOutcome`, `ApprovalResolution`, `ApprovalResolver`. + +```python +# 0.3 +from agent_control_specification import Decision, EnforcementMode + +# 0.4 +from agent_hooks import Decision, EnforcementMode +``` + +`InterventionPoint` is `agent_hooks.InterceptionPoint`. + +ACS does not re-export these. Two packages defining one contract is the +ambiguity the re-layering removed. + +### Orchestration is agent-hooks' InterceptionEmitter + +`AgentControl` was "host-owned async orchestration around a stateless +runtime client". That is now `InterceptionEmitter`, which implements +sections 6 through 10 of the control contract once so that adapters do +not each reimplement them. + +```python +# 0.3 +control = AgentControl.from_native(manifest) +result = await control.enforce(point, snapshot, EnforcementMode.ENFORCE) + +# 0.4 +from agent_hooks import InterceptionEmitter, EnforcementMode +from agent_control_spec import AcsInterceptor + +emitter = InterceptionEmitter(mode=EnforcementMode.ENFORCE) +emitter.register(AcsInterceptor("manifest.yaml"), name="acs") +outcome = await emitter.emit(context) +``` + +| 0.3 | 0.4 | +| --- | --- | +| `AgentControl` | `agent_hooks.InterceptionEmitter` | +| `AgentControlBlocked` | `agent_hooks.InterceptionBlocked` | +| `AgentControlSuspended` | `agent_hooks.InterceptionSuspended` | +| `RunResult`, `ToolRunResult`, `ModelCallResult` | `agent_hooks.EmitOutcome` | +| `InterventionPointRequest` | `agent_hooks.AgentContext` | +| `action_identity` | `agent_hooks.context_identity` | +| `RuntimeClient`, `NativeRuntimeClient` | the `agent_hooks.Interceptor` protocol, which `AcsInterceptor` implements | + +### Audit records replaced the telemetry sinks + +`JsonStdoutTelemetrySink` and `InMemoryTelemetrySink` recorded what was +decided. That is a host obligation, so it moved to the emitter: +`set_record_sink`, `take_records`, `set_max_records` and +`InterceptionRecord`. + +ACS still exposes engine-level telemetry, which is a different thing: it +reports how the engine spent its time, not what it decided. See host +hooks below. + +### Host extension points stayed in ACS + +`AnnotatorDispatcher` and `PolicyDispatcher` are ACS concerns by +definition and are still supported. The keyword survives: + +```python +# 0.3 +control = AgentControl.from_native(manifest, annotator_dispatcher=MyClassifier()) + +# 0.4 +interceptor = AcsInterceptor("manifest.yaml", annotator_dispatcher=MyClassifier()) +``` + +A dispatcher is any object with the matching method, or a plain callable, +so a 0.3 dispatcher class usually ports unchanged. A dispatcher that +raises fails the evaluation closed: a classifier that could not be +reached must not read as a classifier that found nothing. + +### Manifest tooling + +| 0.3 | 0.4 | +| --- | --- | +| `validate_manifest` | `validate_manifest` | +| `validate_acs_manifest` | `validate_manifest` | +| `parse_manifest` | `parse_manifest` | +| `validate_manifest_overlay` | `merge_manifests` | +| `ValidationDiagnostic`, `ArtifactValidationResult` | `validate_manifest_detailed` | + +### Framework adapters have no successor yet + +The 18 `guard_*` functions, `FullCoverageAgentAdapter`, +`LiteLLMProxyMiddleware`, `mcp_approval_resolver` and the rest are not in +0.4 and are not in agent-hooks. + +They wrap nine frameworks, so they do not belong in a policy decision +runtime that would then version-lock to all of them. Where they should +live is an open question. Until it is answered, a host that used one +writes the equivalent against `InterceptionEmitter` directly, which is +what the adapters did internally. + +## Things that will look like ACS breaking but are not + +**Checking for the `opa` binary.** 0.3 evaluated Rego by shelling out to +OPA. 0.4 evaluates it in process with regorus and needs no external +binary. A startup check for `opa` on PATH will reject a working 0.4 +install. Delete the check. + +**`intervention_points` in a manifest.** The manifest grammar keeps the +older spelling. Only the Python symbol was renamed. + +## A worked port + +0.3: + +```python +from agent_control_specification import ( + AgentControl, AgentControlBlocked, EnforcementMode, +) + +control = AgentControl.from_native(manifest, annotator_dispatcher=ContentSafety()) + +async def guarded(text): + try: + return await control.enforce("input", {"input": text}, EnforcementMode.ENFORCE) + except AgentControlBlocked: + return "refused" +``` + +0.4: + +```python +from agent_hooks import ( + AgentContextBuilder, EnforcementMode, InterceptionBlocked, InterceptionEmitter, +) +from agent_control_spec import AcsInterceptor + +emitter = InterceptionEmitter(mode=EnforcementMode.ENFORCE) +emitter.register( + AcsInterceptor("manifest.yaml", annotator_dispatcher=ContentSafety()), + name="acs", +) + +async def guarded(text): + ctx = AgentContextBuilder( + agent_id="a", framework="custom", session_id="s" + ).input(content=text) + try: + return await emitter.emit(ctx) + except InterceptionBlocked: + return "refused" +``` + +`ContentSafety` is unchanged. + +[agent-hooks]: https://github.com/responsibleai/agent-hooks diff --git a/sdk/dotnet/src/AgentControlSpec/HostHooks.cs b/sdk/dotnet/src/AgentControlSpec/HostHooks.cs new file mode 100644 index 0000000..12cf8fb --- /dev/null +++ b/sdk/dotnet/src/AgentControlSpec/HostHooks.cs @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Host extension points and manifest tooling. +// +// The engine takes an annotator dispatcher, a policy dispatcher, a +// telemetry sink and a perf level. The zero-config constructors pick +// defaults for all four, which is right for a host that wants a policy +// decision and nothing else. A host that classifies through its own +// service, evaluates through its own engine, or records its own audit +// trail supplies them here. + +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using AgentHooks; + +namespace AgentControlSpec; + +/// How much timing detail the engine records. +public enum PerfTelemetry +{ + /// Record nothing. + Off, + + /// Record the time spent outside the engine. + External, + + /// Record every phase. + Full, +} + +/// One finding about a manifest. +/// The engine's reason code. +/// What is wrong, in the engine's words. +/// How bad it is. +public sealed record ManifestDiagnostic( + [property: JsonPropertyName("code")] string Code, + [property: JsonPropertyName("message")] string Message, + [property: JsonPropertyName("severity")] string Severity); + +/// +/// Classifies one annotation on the host's behalf. +/// +/// The annotator the manifest bound. +/// The binding's configured fields. +/// The policy input built so far. +/// The annotation value as JSON. +/// +/// Throwing fails the evaluation closed. An annotation that could not be +/// produced must not read as an annotation that found nothing. +/// +public delegate string AnnotatorDispatcher( + string annotatorName, string invocationJson, string policyInputJson); + +/// Evaluates one prepared policy invocation on the host's behalf. +/// The prepared invocation, tagged by engine type. +/// The policy output as JSON. +public delegate string PolicyDispatcher(string invocationJson); + +/// Receives one telemetry event as JSON. +/// A sink cannot fail an evaluation, so it has no error channel. +public delegate void TelemetrySink(string eventJson); + +/// Manifest reading and validation for authoring and tooling. +public static class AcsManifestTools +{ + private static readonly JsonSerializerOptions Json = new() + { + PropertyNameCaseInsensitive = true, + }; + + /// + /// Parse manifest text and return it as JSON. Parsing is not + /// validation: this answers what the document says, which a tool + /// needs before the document is runnable. + /// + public static string Parse(string yaml) => Native.ManifestParse(yaml); + + /// + /// Compose a chain of manifest documents into one, outermost base + /// first. This is the overlay case: a base policy plus the deltas an + /// environment layers on it. + /// + public static string Merge(IEnumerable yamls) => + Native.ManifestMerge(JsonSerializer.Serialize(yamls.ToArray())); + + /// + /// Validate manifest text and return every finding. An empty list + /// means valid. + /// + /// + /// answers yes or no by throwing, + /// which a linter cannot render against a document. This returns the + /// findings instead. + /// + public static IReadOnlyList Diagnostics(string yaml) => + JsonSerializer.Deserialize>(Native.ManifestDiagnostics(yaml), Json) + ?? throw new AgentControlSpecNativeException("diagnostics did not deserialize"); +} + +/// +/// An interceptor wired to host-supplied extension points. +/// +/// Anything left null keeps the bundled default for that slot, so a host +/// overrides only what it needs. +/// +/// +/// +/// using var interceptor = AcsHostInterceptor.FromPath( +/// "manifest.yaml", +/// annotator: (name, invocation, input) => Classify(input)); +/// +/// +public sealed class AcsHostInterceptor : IInterceptor, IDisposable +{ + // The delegates are held so the GC cannot collect them while native + // code still holds their function pointers. Dropping this field is + // the classic way to turn a working callback into an intermittent + // crash under load. + private readonly List _pinned = []; + private readonly IntPtr _handle; + private bool _disposed; + + private AcsHostInterceptor(IntPtr handle) => _handle = handle; + + private delegate IntPtr NativeAnnotator( + IntPtr ctx, IntPtr name, IntPtr invocation, IntPtr input, out IntPtr errOut); + + private delegate IntPtr NativePolicy(IntPtr ctx, IntPtr invocation, out IntPtr errOut); + + private delegate void NativeTelemetry(IntPtr ctx, IntPtr eventJson); + + private delegate void NativeFree(IntPtr ctx, IntPtr value); + + private static string Read(IntPtr p) => Marshal.PtrToStringUTF8(p) ?? string.Empty; + + /// Build an interceptor with host-supplied extension points. + /// Manifest to load. + /// Host classifier, or null for the bundled annotators. + /// Host policy engine, or null for the bundled dispatchers. + /// Host telemetry sink, or null to record nothing. + /// How much timing detail to record. + public static AcsHostInterceptor FromPath( + string manifestPath, + AnnotatorDispatcher? annotator = null, + PolicyDispatcher? policy = null, + TelemetrySink? telemetry = null, + PerfTelemetry perfTelemetry = PerfTelemetry.Off) + { + var pinned = new List(); + + // Freed by the native side through this callback, so the string a + // host callback returns never crosses allocators. + NativeFree free = (_, value) => Marshal.FreeCoTaskMem(value); + pinned.Add(free); + + IntPtr annotatorPtr = IntPtr.Zero; + if (annotator is not null) + { + NativeAnnotator shim = (IntPtr _, IntPtr name, IntPtr invocation, IntPtr input, out IntPtr err) => + { + err = IntPtr.Zero; + try + { + return Marshal.StringToCoTaskMemUTF8( + annotator(Read(name), Read(invocation), Read(input))); + } + catch (Exception e) + { + err = Marshal.StringToCoTaskMemUTF8(e.Message); + return IntPtr.Zero; + } + }; + pinned.Add(shim); + annotatorPtr = Marshal.GetFunctionPointerForDelegate(shim); + } + + IntPtr policyPtr = IntPtr.Zero; + if (policy is not null) + { + NativePolicy shim = (IntPtr _, IntPtr invocation, out IntPtr err) => + { + err = IntPtr.Zero; + try + { + return Marshal.StringToCoTaskMemUTF8(policy(Read(invocation))); + } + catch (Exception e) + { + err = Marshal.StringToCoTaskMemUTF8(e.Message); + return IntPtr.Zero; + } + }; + pinned.Add(shim); + policyPtr = Marshal.GetFunctionPointerForDelegate(shim); + } + + IntPtr telemetryPtr = IntPtr.Zero; + if (telemetry is not null) + { + NativeTelemetry shim = (IntPtr _, IntPtr payload) => + { + // A sink that throws must not fail the action it merely + // describes, and an exception here would cross the + // native boundary. + try + { + telemetry(Read(payload)); + } + catch + { + // Intentionally swallowed: see above. + } + }; + pinned.Add(shim); + telemetryPtr = Marshal.GetFunctionPointerForDelegate(shim); + } + + var handle = Native.InterceptorNewWithHooks( + manifestPath, + annotatorPtr, IntPtr.Zero, + policyPtr, IntPtr.Zero, + telemetryPtr, IntPtr.Zero, + Marshal.GetFunctionPointerForDelegate(free), + perfTelemetry.ToString().ToLowerInvariant()); + + var interceptor = new AcsHostInterceptor(handle); + interceptor._pinned.AddRange(pinned); + return interceptor; + } + + /// Evaluate one agent context. + public ValueTask InterceptAsync(AgentContext context, CancellationToken ct = default) + { + ObjectDisposedException.ThrowIf(_disposed, this); + var wire = Native.Intercept(_handle, context.Json.ToJsonString()); + var parsed = System.Text.Json.Nodes.JsonNode.Parse(wire) as System.Text.Json.Nodes.JsonObject + ?? throw new AgentControlSpecNativeException("engine returned a non-object verdict"); + return ValueTask.FromResult(Verdict.FromWire(parsed)); + } + + /// Release the native interceptor. + public void Dispose() + { + if (_disposed) + return; + _disposed = true; + Native.Free(_handle); + _pinned.Clear(); + } +} diff --git a/sdk/dotnet/src/AgentControlSpec/Native.cs b/sdk/dotnet/src/AgentControlSpec/Native.cs index 4372bf1..53798be 100644 --- a/sdk/dotnet/src/AgentControlSpec/Native.cs +++ b/sdk/dotnet/src/AgentControlSpec/Native.cs @@ -465,6 +465,79 @@ internal static string StreamFinish(StreamSessionHandle handle) ThrowIfError(err); return TakeString(json); } + + [LibraryImport(Lib)] + private static partial IntPtr acs_interceptor_new_with_hooks( + ReadOnlySpan manifestPath, nuint manifestPathLen, + IntPtr annotatorFn, IntPtr annotatorCtx, + IntPtr policyFn, IntPtr policyCtx, + IntPtr telemetryFn, IntPtr telemetryCtx, + IntPtr hookFree, + IntPtr perfTelemetry, + out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_manifest_parse(string yaml, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_manifest_merge(string yamlsJson, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_manifest_diagnostics(string yaml, out IntPtr errOut); + + internal static IntPtr InterceptorNewWithHooks( + string manifestPath, + IntPtr annotatorFn, IntPtr annotatorCtx, + IntPtr policyFn, IntPtr policyCtx, + IntPtr telemetryFn, IntPtr telemetryCtx, + IntPtr hookFree, + string? perfTelemetry) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(manifestPath); + var perf = perfTelemetry is null + ? IntPtr.Zero + : Marshal.StringToCoTaskMemUTF8(perfTelemetry); + try + { + var handle = acs_interceptor_new_with_hooks( + bytes, (nuint)bytes.Length, + annotatorFn, annotatorCtx, policyFn, policyCtx, + telemetryFn, telemetryCtx, hookFree, perf, out var err); + if (handle == IntPtr.Zero) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("interceptor construction returned no handle"); + } + + return handle; + } + finally + { + if (perf != IntPtr.Zero) + Marshal.FreeCoTaskMem(perf); + } + } + + internal static string ManifestParse(string yaml) + { + var json = acs_manifest_parse(yaml, out var err); + ThrowIfError(err); + return TakeString(json); + } + + internal static string ManifestMerge(string yamlsJson) + { + var json = acs_manifest_merge(yamlsJson, out var err); + ThrowIfError(err); + return TakeString(json); + } + + internal static string ManifestDiagnostics(string yaml) + { + var json = acs_manifest_diagnostics(yaml, out var err); + ThrowIfError(err); + return TakeString(json); + } } internal sealed class StreamSessionHandle : SafeHandle diff --git a/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs b/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs new file mode 100644 index 0000000..8d34f9e --- /dev/null +++ b/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Host extension points: the engine's own dispatchers, telemetry sink +// and perf level, reachable from .NET. +// +// The scenario is the one that blocked a real consumer on 0.3: a host +// classifier reached over HTTP, bound as an annotator, whose answer +// decides the verdict. Before these entry points there was no way to +// supply one from any language but Rust. + +using System.Text.Json.Nodes; +using AgentControlSpec; +using AgentHooks; +using Xunit; + +namespace AgentControlSpec.Tests; + +public sealed class HostHooksTests : IDisposable +{ + private readonly string _dir = Directory.CreateTempSubdirectory("acs-hooks").FullName; + + private string WriteFixture() + { + var bundle = Path.Combine(_dir, "bundle"); + Directory.CreateDirectory(bundle); + File.WriteAllText(Path.Combine(bundle, "policy.rego"), """ + package acs + + decision := {"decision": "deny", "reason": "unsafe_content"} if { + input.annotations.content_safety.severity >= 4 + } else := {"decision": "allow"} + """); + + var manifest = Path.Combine(_dir, "manifest.yaml"); + File.WriteAllText(manifest, """ + agent_control_specification_version: "0.4.0-alpha.1" + metadata: + name: host-hooks-test + annotators: + content_safety: + type: classifier + policies: + gate: + type: rego + bundle: ./bundle + intervention_points: + input: + policy_target: "$snap.input" + annotations: + content_safety: + from: "$target" + policy: + id: gate + query: data.acs.decision + """); + return manifest; + } + + private static AgentContext Input(string text) => + new((JsonNode.Parse($$"""{"interception_point":"input","input":{{System.Text.Json.JsonSerializer.Serialize(text)}}}""")!).AsObject()); + + [Fact] + public async Task AHostClassifierDecidesTheVerdict() + { + var manifest = WriteFixture(); + var calls = 0; + + using var benign = AcsHostInterceptor.FromPath( + manifest, + annotator: (name, _, _) => + { + calls++; + Assert.Equal("content_safety", name); + return """{"severity":1}"""; + }); + + var allowed = await benign.InterceptAsync(Input("hello")); + Assert.Equal(Decision.Allow, allowed.Decision); + Assert.Equal(1, calls); + + using var harmful = AcsHostInterceptor.FromPath( + manifest, annotator: (_, _, _) => """{"severity":7}"""); + + var denied = await harmful.InterceptAsync(Input("hello")); + Assert.Equal(Decision.Deny, denied.Decision); + Assert.Equal("unsafe_content", denied.Reason); + } + + [Fact] + public async Task AClassifierThatFailsDeniesRatherThanFindingNothing() + { + var manifest = WriteFixture(); + + using var broken = AcsHostInterceptor.FromPath( + manifest, + annotator: (_, _, _) => throw new InvalidOperationException("classifier unreachable")); + + var verdict = await broken.InterceptAsync(Input("hello")); + + // The point of the test: an unreachable classifier must not read + // as a classifier that found nothing. + Assert.Equal(Decision.Deny, verdict.Decision); + Assert.Equal("runtime_error:annotation_failed", verdict.Reason); + } + + [Fact] + public async Task ATelemetrySinkSeesTheEvaluation() + { + var manifest = WriteFixture(); + var events = new List(); + + using var interceptor = AcsHostInterceptor.FromPath( + manifest, + annotator: (_, _, _) => """{"severity":1}""", + telemetry: events.Add, + perfTelemetry: PerfTelemetry.Full); + + await interceptor.InterceptAsync(Input("hello")); + + Assert.NotEmpty(events); + Assert.Contains(events, e => e.Contains("intervention_point")); + } + + [Fact] + public async Task ASinkThatThrowsDoesNotFailTheAction() + { + var manifest = WriteFixture(); + + using var interceptor = AcsHostInterceptor.FromPath( + manifest, + annotator: (_, _, _) => """{"severity":1}""", + telemetry: _ => throw new InvalidOperationException("sink is down")); + + // A sink records what happened. It does not get a vote on it. + var verdict = await interceptor.InterceptAsync(Input("hello")); + Assert.Equal(Decision.Allow, verdict.Decision); + } + + [Fact] + public void ParseReadsAManifestWithoutRunningIt() + { + var json = AcsManifestTools.Parse(""" + agent_control_specification_version: "0.4.0-alpha.1" + policies: + p: + type: test + intervention_points: + input: + policy_target: "$.input" + policy: + id: p + """); + + Assert.Contains("intervention_points", json); + } + + [Fact] + public void ParseRejectsTextThatIsNotAManifest() + { + Assert.Throws( + () => AcsManifestTools.Parse("this: [is not")); + } + + [Fact] + public void DiagnosticsNameTheProblemRatherThanThrowing() + { + var findings = AcsManifestTools.Diagnostics(""" + agent_control_specification_version: "0.4.0-alpha.1" + metadata: {} + """); + + var finding = Assert.Single(findings); + Assert.Equal("error", finding.Severity); + Assert.Contains("intervention point", finding.Message); + Assert.StartsWith("runtime_error:", finding.Code); + } + + [Fact] + public void DiagnosticsAreEmptyForAValidManifest() + { + Assert.Empty(AcsManifestTools.Diagnostics(""" + agent_control_specification_version: "0.4.0-alpha.1" + policies: + p: + type: test + intervention_points: + input: + policy_target: "$.input" + policy: + id: p + """)); + } + + [Fact] + public void MergeComposesABaseWithAnOverlay() + { + var merged = AcsManifestTools.Merge([ + """ + agent_control_specification_version: "0.4.0-alpha.1" + policies: + p: + type: test + intervention_points: + input: + policy_target: "$.input" + policy: + id: p + """, + """ + agent_control_specification_version: "0.4.0-alpha.1" + metadata: + name: overlay-applied + """, + ]); + + Assert.Contains("overlay-applied", merged); + } + + public void Dispose() => Directory.Delete(_dir, recursive: true); +} diff --git a/sdk/ffi/src/lib.rs b/sdk/ffi/src/lib.rs index 8bb7930..69be161 100644 --- a/sdk/ffi/src/lib.rs +++ b/sdk/ffi/src/lib.rs @@ -17,18 +17,23 @@ // every schema-valid context. Errors on that path are boundary // problems only (bad UTF-8, non-object context, poisoned handle). +use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; +use agent_control_spec::perf_telemetry::PerfTelemetry; +use agent_control_spec::policy::PreparedPolicyInvocation; +use agent_control_spec::runtime::PolicyDispatcher; use agent_control_spec::stream_session::{ SafetyLevel, SegmentOutcome, StreamEndReason, StreamSession, StreamSessionConfig, StreamSourceType, StreamSpan, StreamTrack, }; +use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; use agent_control_spec::{ ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, Runtime, RuntimeError, Verdict, SUPPORTED_VERSIONS, }; use serde_json::Value; use std::collections::BTreeMap; -use std::ffi::{c_char, CStr, CString}; +use std::ffi::{c_char, c_void, CStr, CString}; use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::Arc; @@ -810,6 +815,474 @@ pub unsafe extern "C" fn acs_policy_free(handle: *mut AcsActivatedPolicy) { drop(Box::from_raw(handle)); } +// --------------------------------------------------------------------- +// Host extension points and manifest tooling. +// +// The engine takes an annotator dispatcher, a policy dispatcher, a +// telemetry sink and a perf level. The zero-config constructors above +// pick defaults for all four, which is right for a host that wants a +// policy decision and nothing else. A host that classifies through its +// own service, evaluates through its own engine, or records its own +// audit trail needs to supply them, and before these entry points +// existed there was no way in from any language but Rust. +// +// Callbacks cross the boundary as JSON and answer with JSON. A callback +// returns NULL and sets its own error string to fail, and the engine +// turns that into a fail-closed deny rather than treating it as an +// absent annotation: a classifier that could not be reached must not +// read as "found nothing". +// +// Ownership: a string the host returns is freed by the host, through +// the `free` callback registered alongside. The engine copies what it +// needs first. It never calls `acs_free_string` on host memory. +// --------------------------------------------------------------------- + +/// Free a string a host callback returned. +pub type AcsHookFree = unsafe extern "C" fn(ctx: *mut c_void, value: *mut c_char); + +/// Classify one annotation. Returns the annotation value as JSON, or +/// NULL with `*err_out` set. +pub type AcsAnnotatorFn = unsafe extern "C" fn( + ctx: *mut c_void, + annotator_name: *const c_char, + invocation_json: *const c_char, + policy_input_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char; + +/// Evaluate one prepared policy invocation. Returns the policy output as +/// JSON, or NULL with `*err_out` set. +pub type AcsPolicyFn = unsafe extern "C" fn( + ctx: *mut c_void, + invocation_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char; + +/// Receive one telemetry event as JSON. A sink cannot fail the +/// evaluation, so it has no error channel. +pub type AcsTelemetryFn = unsafe extern "C" fn(ctx: *mut c_void, event_json: *const c_char); + +// A host context is an opaque pointer the engine only hands back. The +// engine calls dispatchers from whatever thread is evaluating, so the +// host is responsible for its context being safe to use from more than +// one. Stated here because the compiler cannot check it. +struct HostCtx { + ctx: *mut c_void, + free: Option, +} + +unsafe impl Send for HostCtx {} +unsafe impl Sync for HostCtx {} + +impl HostCtx { + /// Copy a string the host returned, then hand the original back to + /// the host's own allocator. + unsafe fn take(&self, raw: *mut c_char) -> Option { + if raw.is_null() { + return None; + } + let copied = CStr::from_ptr(raw).to_str().ok().map(str::to_string); + if let Some(free) = self.free { + free(self.ctx, raw); + } + copied + } +} + +fn host_error(what: &str, err_out: *mut *mut c_char) -> RuntimeError { + let detail = if err_out.is_null() { + None + } else { + let raw = unsafe { *err_out }; + if raw.is_null() { + None + } else { + let message = unsafe { CStr::from_ptr(raw) } + .to_string_lossy() + .into_owned(); + unsafe { acs_free_string(raw) }; + Some(message) + } + }; + RuntimeError::PolicyInvocationFailed(match detail { + Some(message) => format!("host {what} failed: {message}"), + None => format!("host {what} failed without a message"), + }) +} + +struct HostAnnotatorDispatcher { + host: HostCtx, + call: AcsAnnotatorFn, +} + +impl AnnotatorDispatcher for HostAnnotatorDispatcher { + fn dispatch( + &self, + annotator_name: &str, + annotator: &AnnotatorInvocation, + preliminary_policy_input: &Value, + ) -> Result { + let name = CString::new(annotator_name).map_err(|_| { + RuntimeError::PolicyInvocationFailed("annotator name held a NUL".into()) + })?; + let invocation = CString::new(serde_json::to_string(annotator).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("annotator invocation: {e}")) + })?) + .map_err(|_| { + RuntimeError::PolicyInvocationFailed("annotator invocation held a NUL".into()) + })?; + let input = CString::new( + serde_json::to_string(preliminary_policy_input) + .map_err(|e| RuntimeError::PolicyInvocationFailed(format!("policy input: {e}")))?, + ) + .map_err(|_| RuntimeError::PolicyInvocationFailed("policy input held a NUL".into()))?; + + let mut err: *mut c_char = std::ptr::null_mut(); + let raw = unsafe { + (self.call)( + self.host.ctx, + name.as_ptr(), + invocation.as_ptr(), + input.as_ptr(), + &mut err, + ) + }; + let Some(json) = (unsafe { self.host.take(raw) }) else { + return Err(host_error("annotator", &mut err)); + }; + serde_json::from_str(&json).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("host annotator returned non JSON: {e}")) + }) + } +} + +struct HostPolicyDispatcher { + host: HostCtx, + call: AcsPolicyFn, +} + +impl PolicyDispatcher for HostPolicyDispatcher { + fn evaluate(&self, invocation: &PreparedPolicyInvocation) -> Result { + let payload = CString::new(serde_json::to_string(invocation).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("policy invocation: {e}")) + })?) + .map_err(|_| RuntimeError::PolicyInvocationFailed("policy invocation held a NUL".into()))?; + + let mut err: *mut c_char = std::ptr::null_mut(); + let raw = unsafe { (self.call)(self.host.ctx, payload.as_ptr(), &mut err) }; + let Some(json) = (unsafe { self.host.take(raw) }) else { + return Err(host_error("policy dispatcher", &mut err)); + }; + serde_json::from_str(&json).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("host policy returned non JSON: {e}")) + }) + } +} + +struct HostTelemetrySink { + host: HostCtx, + call: AcsTelemetryFn, +} + +impl TelemetrySink for HostTelemetrySink { + fn emit(&self, event: TelemetryEvent) { + // TelemetryEvent is not Serialize, so the wire shape is owned + // here rather than derived from the engine's layout. A sink + // cannot fail an evaluation, so a problem here drops the event + // rather than denying the action it describes. + let payload = serde_json::json!({ + "event_type": event.event_type.as_str(), + "intervention_point": format!("{:?}", event.intervention_point).to_lowercase(), + "decision": event.decision.map(|d| format!("{d:?}").to_lowercase()), + "reason_code": event.reason_code, + "error_class": event.error_class, + "policy_id": event.policy_id, + "annotators": event.annotators, + "enforcement_mode": event.enforcement_mode.map(|m| format!("{m:?}").to_lowercase()), + "duration_ms": event.duration_ms, + }); + let Ok(json) = serde_json::to_string(&payload) else { + return; + }; + let Ok(payload) = CString::new(json) else { + return; + }; + unsafe { (self.call)(self.host.ctx, payload.as_ptr()) }; + } +} + +fn parse_perf(value: *const c_char, err_out: *mut *mut c_char) -> Option { + if value.is_null() { + return Some(PerfTelemetry::Off); + } + let raw = unsafe { read_utf8(value, "perf_telemetry", err_out) }?; + match raw { + "off" => Some(PerfTelemetry::Off), + "external" => Some(PerfTelemetry::External), + "full" => Some(PerfTelemetry::Full), + other => { + set_err(err_out, format!("unknown perf telemetry level '{other}'")); + None + } + } +} + +/// Build an interceptor with host-supplied extension points. +/// +/// Any callback may be NULL, which keeps the bundled default for that +/// slot, so a host overrides only what it needs. `perf_telemetry` is +/// `off`, `external` or `full`, and NULL means `off`. +/// +/// `annotator_ctx`, `policy_ctx` and `telemetry_ctx` are opaque to the +/// engine. Dispatch happens on whichever thread evaluates, so a context +/// shared across threads must be safe to use from all of them. +/// +/// Returns NULL and sets `*err_out` on failure. Free with +/// `acs_interceptor_free`. +/// +/// # Safety +/// `manifest_path` must point to `manifest_path_len` readable bytes. +/// Every non-null callback must remain valid, and every context must +/// stay alive, until the handle is freed. +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub unsafe extern "C" fn acs_interceptor_new_with_hooks( + manifest_path: *const u8, + manifest_path_len: usize, + annotator_fn: Option, + annotator_ctx: *mut c_void, + policy_fn: Option, + policy_ctx: *mut c_void, + telemetry_fn: Option, + telemetry_ctx: *mut c_void, + hook_free: Option, + perf_telemetry: *const c_char, + err_out: *mut *mut c_char, +) -> *mut AcsInterceptor { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let path = match read_path(manifest_path, manifest_path_len, err_out) { + Some(p) => p, + None => return std::ptr::null_mut(), + }; + let manifest = match Manifest::from_path(path) { + Ok(m) => m, + Err(e) => { + set_err(err_out, format!("{e}")); + return std::ptr::null_mut(); + } + }; + let Some(perf) = parse_perf(perf_telemetry, err_out) else { + return std::ptr::null_mut(); + }; + + let annotations: Arc = match annotator_fn { + Some(call) => Arc::new(HostAnnotatorDispatcher { + host: HostCtx { + ctx: annotator_ctx, + free: hook_free, + }, + call, + }), + None => default_annotator_dispatcher(), + }; + let policy: Arc = match policy_fn { + Some(call) => Arc::new(HostPolicyDispatcher { + host: HostCtx { + ctx: policy_ctx, + free: hook_free, + }, + call, + }), + None => Arc::new(BindingPolicyDispatcher::new()), + }; + let telemetry: Arc = match telemetry_fn { + Some(call) => Arc::new(HostTelemetrySink { + host: HostCtx { + ctx: telemetry_ctx, + free: hook_free, + }, + call, + }), + None => Arc::new(NoopTelemetrySink), + }; + + match Runtime::with_telemetry_and_perf(manifest, annotations, policy, telemetry, perf) { + Ok(runtime) => Box::into_raw(Box::new(AcsInterceptor { + runtime, + name: "acs".to_string(), + })), + Err(e) => { + set_err(err_out, format!("{e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_interceptor_new_with_hooks".to_string(), + ); + std::ptr::null_mut() + } + } +} + +/// Parse manifest text and return it as JSON. +/// +/// Parsing is not validation: this answers what the document says, which +/// an authoring or migration tool needs before the document is +/// runnable. Freed with `acs_free_string`. +/// +/// # Safety +/// `yaml` must be a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_manifest_parse( + yaml: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(source) = read_utf8(yaml, "yaml", err_out) else { + return std::ptr::null_mut(); + }; + match Manifest::from_yaml_str(source) { + Ok(manifest) => match serde_json::to_string(&manifest) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("manifest serialization failed: {e}")); + std::ptr::null_mut() + } + }, + Err(e) => { + set_err(err_out, format!("{e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err(err_out, "internal panic in acs_manifest_parse".to_string()); + std::ptr::null_mut() + } + } +} + +/// Compose a chain of manifest documents into one and return it as JSON. +/// +/// `yamls_json` is a JSON array of manifest sources, outermost base +/// first. This is the overlay case: a base policy plus the deltas an +/// environment layers on it, resolved the same way the engine resolves +/// `extends`. Freed with `acs_free_string`. +/// +/// # Safety +/// `yamls_json` must be a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_manifest_merge( + yamls_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(raw) = read_utf8(yamls_json, "yamls_json", err_out) else { + return std::ptr::null_mut(); + }; + let sources: Vec = match serde_json::from_str(raw) { + Ok(v) => v, + Err(e) => { + set_err( + err_out, + format!("yamls_json must be a JSON array of manifest sources: {e}"), + ); + return std::ptr::null_mut(); + } + }; + if sources.is_empty() { + set_err(err_out, "yamls_json must name at least one source".into()); + return std::ptr::null_mut(); + } + let borrowed: Vec<&str> = sources.iter().map(String::as_str).collect(); + match Manifest::from_yaml_chain(&borrowed) { + Ok(manifest) => match serde_json::to_string(&manifest) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("manifest serialization failed: {e}")); + std::ptr::null_mut() + } + }, + Err(e) => { + set_err(err_out, format!("{e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err(err_out, "internal panic in acs_manifest_merge".to_string()); + std::ptr::null_mut() + } + } +} + +/// Validate manifest text and return the findings as a JSON array. +/// +/// An empty array means valid. Each entry carries `code`, `message` and +/// `severity`. This is the shape an authoring tool or a CI linter needs: +/// `acs_validate_manifest` answers yes or no through an error string, +/// which cannot be rendered against a document. Freed with +/// `acs_free_string`. +/// +/// # Safety +/// `yaml` must be a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_manifest_diagnostics( + yaml: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(source) = read_utf8(yaml, "yaml", err_out) else { + return std::ptr::null_mut(); + }; + let findings = match Manifest::from_yaml_str(source) { + Ok(manifest) => match manifest.validate() { + Ok(()) => Vec::new(), + Err(e) => vec![diagnostic_json(&e)], + }, + Err(e) => vec![diagnostic_json(&e)], + }; + match serde_json::to_string(&findings) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("diagnostics serialization failed: {e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_manifest_diagnostics".to_string(), + ); + std::ptr::null_mut() + } + } +} + +fn diagnostic_json(error: &RuntimeError) -> Value { + serde_json::json!({ + "code": error.reason(), + "message": error.detail(), + "severity": "error", + }) +} + // --------------------------------------------------------------------- // Incremental stream mediation (specification section 18.1). // diff --git a/sdk/node/binding.d.ts b/sdk/node/binding.d.ts index 8d5858d..7c0e939 100644 --- a/sdk/node/binding.d.ts +++ b/sdk/node/binding.d.ts @@ -20,6 +20,45 @@ export declare function intercept(handle: ExternalObject, contextJson: s */ export declare function interceptorNew(manifestPath: string): ExternalObject +/** + * Build a runtime handle from a manifest path, optionally overriding + * the annotator dispatcher, policy dispatcher, telemetry sink, and + * perf telemetry level. + * + * Every callback is optional: absent means keep the zero-config + * default for that slot. Callbacks cross the boundary as JSON strings, + * mirroring the FFI hook contract, so a host that already sits behind + * a JSON schema does not re-model its wire shape for this SDK. + * + * Callbacks are called SYNCHRONOUSLY on the JS thread from inside the + * engine's evaluation. A callback that throws surfaces as a fail-closed + * `runtime_error:*` deny (annotator → `annotation_failed`, policy → + * `policy_invocation_failed`) rather than silently reading as "no + * annotation". + */ +export declare function interceptorNewWithHooks(manifestPath: string, annotatorDispatcher?: ((arg0: string, arg1: string, arg2: string) => string) | undefined | null, policyDispatcher?: ((arg0: string) => string) | undefined | null, telemetrySink?: ((arg0: string) => void) | undefined | null, perfTelemetry?: string | undefined | null): ExternalObject + +/** + * Compose a chain of manifest YAML documents (outermost base first) + * into one merged manifest, returned as JSON. + * + * This is the overlay case: a base policy plus deltas an environment + * layers on it, resolved the same way the engine resolves `extends`. + */ +export declare function mergeManifests(sourcesJson: string): string + +/** + * Parse manifest YAML into an object (JSON encoded) without + * validating cross-references. + * + * The document is deserialized as-written: a manifest with an + * unresolved `extends` chain parses fine, and returning it lets an + * authoring tool see the fragment. Use `validate_manifest` or + * `validate_manifest_detailed` to judge whether the fragment is + * runnable. + */ +export declare function parseManifest(source: string): string + /** * Activate the manifest at `manifest_path`, readying every policy it * binds, against the zero-config dispatchers. @@ -55,6 +94,19 @@ export declare function policyActivate(manifestPath: string): ExternalObject +/** + * Activate a manifest and its Rego from memory against host-supplied + * dispatchers. + */ +export declare function policyActivateFromMemoryWithHooks(manifestYaml: string, bundlesJson: string, annotatorDispatcher?: ((arg0: string, arg1: string, arg2: string) => string) | undefined | null, policyDispatcher?: ((arg0: string) => string) | undefined | null): ExternalObject + +/** + * Activate the manifest at `manifest_path` against host-supplied + * dispatchers. See `interceptor_new_with_hooks` for the callback + * contract. + */ +export declare function policyActivateWithHooks(manifestPath: string, annotatorDispatcher?: ((arg0: string, arg1: string, arg2: string) => string) | undefined | null, policyDispatcher?: ((arg0: string) => string) | undefined | null): ExternalObject + /** * Evaluate one intervention point against an activated policy and * return the verdict as wire JSON. @@ -186,6 +238,18 @@ export declare function supportedManifestVersions(): Array */ export declare function validateManifest(source: string): string | null +/** + * Validate manifest source and return findings as a JSON array. + * + * An empty array means the manifest is valid. Each entry carries + * `code` (`runtime_error:*`), `message` (engine detail), `severity`, + * and a best-effort `field` extracted from the message. This is the + * shape an authoring tool or CI linter needs; `validate_manifest` + * answers yes/no with a single message and cannot be rendered + * per-field. + */ +export declare function validateManifestDetailed(source: string): string + /** * Validate a manifest file, resolving `extends` first. * diff --git a/sdk/node/binding.js b/sdk/node/binding.js index 40a6f77..3ee3477 100644 --- a/sdk/node/binding.js +++ b/sdk/node/binding.js @@ -702,8 +702,13 @@ if (!nativeBinding) { module.exports = nativeBinding module.exports.intercept = nativeBinding.intercept module.exports.interceptorNew = nativeBinding.interceptorNew +module.exports.interceptorNewWithHooks = nativeBinding.interceptorNewWithHooks +module.exports.mergeManifests = nativeBinding.mergeManifests +module.exports.parseManifest = nativeBinding.parseManifest module.exports.policyActivate = nativeBinding.policyActivate module.exports.policyActivateFromMemory = nativeBinding.policyActivateFromMemory +module.exports.policyActivateFromMemoryWithHooks = nativeBinding.policyActivateFromMemoryWithHooks +module.exports.policyActivateWithHooks = nativeBinding.policyActivateWithHooks module.exports.policyEvaluate = nativeBinding.policyEvaluate module.exports.policyInterventionPoints = nativeBinding.policyInterventionPoints module.exports.streamSessionAdvance = nativeBinding.streamSessionAdvance @@ -720,4 +725,5 @@ module.exports.streamSessionState = nativeBinding.streamSessionState module.exports.streamSessionWatermark = nativeBinding.streamSessionWatermark module.exports.supportedManifestVersions = nativeBinding.supportedManifestVersions module.exports.validateManifest = nativeBinding.validateManifest +module.exports.validateManifestDetailed = nativeBinding.validateManifestDetailed module.exports.validateManifestFile = nativeBinding.validateManifestFile diff --git a/sdk/node/native/src/lib.rs b/sdk/node/native/src/lib.rs index efec654..2b676c9 100644 --- a/sdk/node/native/src/lib.rs +++ b/sdk/node/native/src/lib.rs @@ -8,16 +8,22 @@ // mean a boundary problem only (unreadable manifest, non-object // context JSON). +use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; +use agent_control_spec::runtime::PolicyDispatcher; +use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; use agent_control_spec::Verdict; use agent_control_spec::{ - ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, Runtime, RuntimeError, - SafetyLevel, SegmentOutcome, StreamEndReason, StreamError, StreamSession, StreamSessionConfig, - StreamSourceType, StreamSpan, StreamTrack, SUPPORTED_VERSIONS, + ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, JsonValue, Manifest, PerfTelemetry, + PreparedPolicyInvocation, Runtime, RuntimeError, SafetyLevel, SegmentOutcome, StreamEndReason, + StreamError, StreamSession, StreamSessionConfig, StreamSourceType, StreamSpan, StreamTrack, + SUPPORTED_VERSIONS, }; -use napi::bindgen_prelude::{External, Utf16String}; +use napi::bindgen_prelude::{External, FnArgs, FunctionRef, Utf16String}; +use napi::Env; use napi_derive::napi; use serde_json::Value; +use std::cell::Cell; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; @@ -56,6 +62,243 @@ fn decode(what: &str, value: &Utf16String) -> napi::Result { String::from_utf16(value).map_err(|_| err(format!("{what} contains an unpaired surrogate"))) } +// --------------------------------------------------------------------- +// Host dispatcher plumbing (annotator, policy, telemetry). +// +// The engine calls a dispatcher SYNCHRONOUSLY from inside its +// evaluation, on whichever thread called into it. Every napi entry +// point that drives evaluation (`intercept`, `policy_evaluate`) runs +// on the JS thread, so a callback fired inside the engine is on the +// same JS thread as the caller and can call the JS function directly +// through a `Function` handle. That handle is scope-bound, so we hold a +// `FunctionRef` (Send + Sync) and `borrow_back` it against the current +// napi `Env` when the engine asks. The env is stored in a thread-local +// set by each entry point for the duration of the call, so a dispatcher +// invoked from a different thread errors out rather than reaching into +// V8 off-thread. +// +// A ThreadsafeFunction would be wrong here: it dispatches ASYNCHRONOUSLY +// to the JS thread, which deadlocks when the JS thread is already +// blocked in the engine call that produced the callback. +// --------------------------------------------------------------------- + +thread_local! { + // Set only while a napi entry point that drives engine evaluation + // is on the stack. A dispatcher invoked with no env available is + // treated as a host failure so the engine fails closed. + static CURRENT_ENV: Cell = + const { Cell::new(std::ptr::null_mut()) }; +} + +/// RAII guard binding a napi `Env` to the current thread for the +/// duration of an engine call. Nesting is not expected (napi calls +/// don't reenter), but the guard is nesting-safe: it saves the previous +/// value and restores it on drop. +struct EnvScope { + previous: napi::sys::napi_env, +} + +impl EnvScope { + fn enter(env: &Env) -> Self { + let raw = env.raw(); + let previous = CURRENT_ENV.with(|c| c.replace(raw)); + Self { previous } + } +} + +impl Drop for EnvScope { + fn drop(&mut self) { + let previous = self.previous; + CURRENT_ENV.with(|c| c.set(previous)); + } +} + +fn with_current_env( + what: &str, + kind: fn(String) -> RuntimeError, + f: impl FnOnce(&Env) -> Result, +) -> Result { + let raw = CURRENT_ENV.with(|c| c.get()); + if raw.is_null() { + return Err(kind(format!( + "host {what} was invoked without a live napi env; this dispatcher can only be \ + called from a napi entry point on the JS thread" + ))); + } + // SAFETY: `raw` was captured by `EnvScope::enter` from the napi + // entry point currently on the stack, on this same JS thread. + let env = Env::from_raw(raw); + f(&env) +} + +struct NodeAnnotatorDispatcher { + func: FunctionRef, String>, +} + +impl AnnotatorDispatcher for NodeAnnotatorDispatcher { + fn dispatch( + &self, + annotator_name: &str, + annotator: &AnnotatorInvocation, + preliminary_policy_input: &JsonValue, + ) -> Result { + let invocation_json = serde_json::to_string(annotator).map_err(|e| { + RuntimeError::AnnotationFailed(format!("serialize annotator invocation: {e}")) + })?; + let policy_input_json = serde_json::to_string(preliminary_policy_input).map_err(|e| { + RuntimeError::AnnotationFailed(format!("serialize preliminary policy input: {e}")) + })?; + with_current_env( + "annotator dispatcher", + RuntimeError::AnnotationFailed, + |env| { + let func = self.func.borrow_back(env).map_err(|e| { + RuntimeError::AnnotationFailed(format!("reacquire annotator function: {e}")) + })?; + let raw = func + .call(FnArgs { + data: ( + annotator_name.to_string(), + invocation_json, + policy_input_json, + ), + }) + .map_err(|e| { + RuntimeError::AnnotationFailed(format!( + "host annotator dispatcher threw: {e}" + )) + })?; + serde_json::from_str::(&raw).map_err(|e| { + RuntimeError::AnnotationFailed(format!( + "host annotator dispatcher returned non-JSON: {e}" + )) + }) + }, + ) + } +} + +struct NodePolicyDispatcher { + func: FunctionRef, String>, +} + +impl PolicyDispatcher for NodePolicyDispatcher { + fn evaluate(&self, invocation: &PreparedPolicyInvocation) -> Result { + let invocation_json = serde_json::to_string(invocation).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("serialize policy invocation: {e}")) + })?; + with_current_env( + "policy dispatcher", + RuntimeError::PolicyInvocationFailed, + |env| { + let func = self.func.borrow_back(env).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("reacquire policy function: {e}")) + })?; + let raw = func + .call(FnArgs { + data: (invocation_json,), + }) + .map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!( + "host policy dispatcher threw: {e}" + )) + })?; + serde_json::from_str::(&raw).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!( + "host policy dispatcher returned non-JSON: {e}" + )) + }) + }, + ) + } +} + +struct NodeTelemetrySink { + func: FunctionRef, ()>, +} + +impl TelemetrySink for NodeTelemetrySink { + fn emit(&self, event: TelemetryEvent) { + // TelemetryEvent is not Serialize, so the wire shape is owned + // here (matches the FFI binding). A sink cannot fail an + // evaluation, so every step drops on error rather than + // propagating. + let payload = serde_json::json!({ + "event_type": event.event_type.as_str(), + "intervention_point": event.intervention_point.as_str(), + "decision": event.decision.map(|d| format!("{d:?}").to_lowercase()), + "reason_code": event.reason_code, + "error_class": event.error_class, + "policy_id": event.policy_id, + "annotators": event.annotators, + "enforcement_mode": event + .enforcement_mode + .map(|m| format!("{m:?}").to_lowercase()), + "duration_ms": event.duration_ms, + "evidence_artefact": event.evidence_artefact, + "evidence_verification_pointer_keys": event.evidence_verification_pointer_keys, + "action_identity": event.action_identity, + "metadata": event.metadata, + }); + let Ok(json) = serde_json::to_string(&payload) else { + return; + }; + let raw = CURRENT_ENV.with(|c| c.get()); + if raw.is_null() { + return; + } + // SAFETY: same as `with_current_env`; the sink is called from + // engine code that is itself running inside a napi entry point. + let env = Env::from_raw(raw); + let Ok(func) = self.func.borrow_back(&env) else { + return; + }; + let _: napi::Result<()> = func.call(FnArgs { data: (json,) }); + } +} + +fn parse_perf(value: Option) -> napi::Result { + let Some(value) = value else { + return Ok(PerfTelemetry::Off); + }; + let raw = decode("perfTelemetry", &value)?; + match raw.as_str() { + "off" => Ok(PerfTelemetry::Off), + "external" => Ok(PerfTelemetry::External), + "full" => Ok(PerfTelemetry::Full), + other => Err(err(format!("unknown perf telemetry level '{other}'"))), + } +} + +// Type aliases for the FunctionRef signatures the JS↔Rust wire uses. +// Kept private so they never leak into the TS declaration; napi-derive +// still sees the expanded types on the entry points that take them as +// arguments (aliases are not expanded through the `#[napi]` macro). +type NodeAnnotatorFn = FunctionRef, String>; +type NodePolicyFn = FunctionRef, String>; +type NodeTelemetryFn = FunctionRef, ()>; + +fn build_annotator(dispatcher: Option) -> Arc { + match dispatcher { + Some(func) => Arc::new(NodeAnnotatorDispatcher { func }), + None => default_annotator_dispatcher(), + } +} + +fn build_policy(dispatcher: Option) -> Arc { + match dispatcher { + Some(func) => Arc::new(NodePolicyDispatcher { func }), + None => Arc::new(BindingPolicyDispatcher::new()), + } +} + +fn build_telemetry(sink: Option) -> Arc { + match sink { + Some(func) => Arc::new(NodeTelemetrySink { func }), + None => Arc::new(NoopTelemetrySink), + } +} + /// Build a runtime handle from a manifest path using the zero-config /// dispatchers (bundled annotators; Rego in process, Cedar through the /// built-in evaluator, `test` policies through their embedded verdict). @@ -72,16 +315,58 @@ pub fn interceptor_new(manifest_path: Utf16String) -> napi::Result, String>>, + policy_dispatcher: Option, String>>, + telemetry_sink: Option, ()>>, + perf_telemetry: Option, +) -> napi::Result> { + let manifest_path = decode("manifest_path", &manifest_path)?; + let manifest = Manifest::from_path(&manifest_path).map_err(|e| err(format!("{e}")))?; + let perf = parse_perf(perf_telemetry)?; + let annotations = build_annotator(annotator_dispatcher); + let policy = build_policy(policy_dispatcher); + let telemetry = build_telemetry(telemetry_sink); + let runtime = Runtime::with_telemetry_and_perf(manifest, annotations, policy, telemetry, perf) + .map_err(|e| err(format!("{e}")))?; + Ok(External::new(Handle { runtime })) +} + /// Evaluate one agent context (JSON object per AGENT-HOOKS-0.1 §4) and /// return the verdict as wire JSON. #[napi] -pub fn intercept(handle: &External, context_json: Utf16String) -> napi::Result { +pub fn intercept( + env: Env, + handle: &External, + context_json: Utf16String, +) -> napi::Result { let context_json = decode("context_json", &context_json)?; let snapshot: Value = serde_json::from_str(&context_json) .map_err(|e| err(format!("context_json does not parse: {e}")))?; if !snapshot.is_object() { return Err(err("context_json must be a JSON object".to_string())); } + // The engine may call a host dispatcher synchronously from inside + // `evaluate`; the scope publishes the current napi env for that + // callback and is torn down before we return. + let _scope = EnvScope::enter(&env); let verdict = handle.runtime.evaluate(&snapshot).verdict; serde_json::to_string(&verdict).map_err(|e| err(format!("verdict serialization failed: {e}"))) } @@ -158,6 +443,47 @@ pub fn policy_activate_from_memory( Ok(External::new(PolicyHandle { policy })) } +/// Activate the manifest at `manifest_path` against host-supplied +/// dispatchers. See `interceptor_new_with_hooks` for the callback +/// contract. +#[napi] +#[allow(clippy::type_complexity)] +pub fn policy_activate_with_hooks( + manifest_path: Utf16String, + annotator_dispatcher: Option, String>>, + policy_dispatcher: Option, String>>, +) -> napi::Result> { + let manifest_path = decode("manifest_path", &manifest_path)?; + let manifest = Manifest::from_path(&manifest_path).map_err(|e| err(format!("{e}")))?; + let annotations = build_annotator(annotator_dispatcher); + let policy = build_policy(policy_dispatcher); + let handle = ActivatedPolicy::activate_with(manifest, annotations, policy) + .map_err(|e| err(format!("{e}")))?; + Ok(External::new(PolicyHandle { policy: handle })) +} + +/// Activate a manifest and its Rego from memory against host-supplied +/// dispatchers. +#[napi] +#[allow(clippy::type_complexity)] +pub fn policy_activate_from_memory_with_hooks( + manifest_yaml: Utf16String, + bundles_json: Utf16String, + annotator_dispatcher: Option, String>>, + policy_dispatcher: Option, String>>, +) -> napi::Result> { + let manifest_yaml = decode("manifest_yaml", &manifest_yaml)?; + let bundles_json = decode("bundles_json", &bundles_json)?; + let bundles: BTreeMap = serde_json::from_str(&bundles_json) + .map_err(|e| err(format!("bundles_json does not parse: {e}")))?; + let annotations = build_annotator(annotator_dispatcher); + let policy = build_policy(policy_dispatcher); + let handle = + ActivatedPolicy::activate_from_memory_with(&manifest_yaml, bundles, annotations, policy) + .map_err(|e| err(format!("{e}")))?; + Ok(External::new(PolicyHandle { policy: handle })) +} + /// Evaluate one intervention point against an activated policy and /// return the verdict as wire JSON. /// @@ -171,6 +497,7 @@ pub fn policy_activate_from_memory( /// and throws. #[napi] pub fn policy_evaluate( + env: Env, handle: &External, point: Utf16String, context_json: Utf16String, @@ -185,6 +512,7 @@ pub fn policy_evaluate( if !snapshot.is_object() { return Err(err("context_json must be a JSON object".to_string())); } + let _scope = EnvScope::enter(&env); let verdict = handle.policy.evaluate(point, snapshot).verdict; serde_json::to_string(&verdict).map_err(|e| err(format!("verdict serialization failed: {e}"))) } @@ -260,6 +588,135 @@ pub fn supported_manifest_versions() -> Vec { .collect() } +// --------------------------------------------------------------------- +// Manifest tooling: parse, chain, structured diagnostics. +// +// The engine ships these as first-party APIs on `Manifest`; the wrapper +// exposes them here so authoring, migration, and CI tooling can build +// on the same surface across languages. Every entry point takes YAML +// text, so a caller can drive them without staging a file on disk. +// --------------------------------------------------------------------- + +/// Parse manifest YAML into an object (JSON encoded) without +/// validating cross-references. +/// +/// The document is deserialized as-written: a manifest with an +/// unresolved `extends` chain parses fine, and returning it lets an +/// authoring tool see the fragment. Use `validate_manifest` or +/// `validate_manifest_detailed` to judge whether the fragment is +/// runnable. +#[napi] +pub fn parse_manifest(source: Utf16String) -> napi::Result { + let source = decode("source", &source)?; + let manifest = Manifest::parse_yaml_str(&source).map_err(|e| err(format!("{e}")))?; + serde_json::to_string(&manifest).map_err(|e| err(format!("manifest serialization failed: {e}"))) +} + +/// Compose a chain of manifest YAML documents (outermost base first) +/// into one merged manifest, returned as JSON. +/// +/// This is the overlay case: a base policy plus deltas an environment +/// layers on it, resolved the same way the engine resolves `extends`. +#[napi] +pub fn merge_manifests(sources_json: Utf16String) -> napi::Result { + let raw = decode("sources_json", &sources_json)?; + let sources: Vec = serde_json::from_str(&raw).map_err(|e| { + err(format!( + "sources_json must be a JSON array of manifest sources: {e}" + )) + })?; + if sources.is_empty() { + return Err(err("sources_json must name at least one source".to_string())); + } + let borrowed: Vec<&str> = sources.iter().map(String::as_str).collect(); + let manifest = Manifest::from_yaml_chain(&borrowed).map_err(|e| err(format!("{e}")))?; + serde_json::to_string(&manifest).map_err(|e| err(format!("manifest serialization failed: {e}"))) +} + +/// Validate manifest source and return findings as a JSON array. +/// +/// An empty array means the manifest is valid. Each entry carries +/// `code` (`runtime_error:*`), `message` (engine detail), `severity`, +/// and a best-effort `field` extracted from the message. This is the +/// shape an authoring tool or CI linter needs; `validate_manifest` +/// answers yes/no with a single message and cannot be rendered +/// per-field. +#[napi] +pub fn validate_manifest_detailed(source: Utf16String) -> napi::Result { + let source = decode("source", &source)?; + let findings = match Manifest::parse_yaml_str(&source) { + Ok(manifest) => { + if !manifest.extends.is_empty() { + // A fragment cannot be judged against itself: its + // parent may define the annotator or policy this + // document references. Report that as a single + // finding rather than silently blaming references + // the fragment does not own. + vec![diagnostic_json(&RuntimeError::ManifestInvalid( + "manifest extends other manifests; validation needs the merged document. \ + Use validate_manifest_file, which resolves the chain." + .to_string(), + ))] + } else { + match manifest.validate() { + Ok(()) => Vec::new(), + Err(e) => vec![diagnostic_json(&e)], + } + } + } + Err(e) => vec![diagnostic_json(&e)], + }; + serde_json::to_string(&findings) + .map_err(|e| err(format!("diagnostics serialization failed: {e}"))) +} + +fn diagnostic_json(error: &RuntimeError) -> Value { + let detail = error.detail(); + let field = extract_field(detail); + serde_json::json!({ + "code": error.reason(), + "message": detail, + "severity": "error", + "field": field, + }) +} + +/// Best-effort extraction of the offending field name from a +/// RuntimeError detail. Returns `None` when the message does not name a +/// field the caller can point at, so the wrapper reports the raw +/// message and the field slot stays absent (not empty). +fn extract_field(detail: &str) -> Option { + // serde_yaml: "missing field `X` at line ..." or + // "unknown field `X`, expected ..." + if let Some(start) = detail.find("field `") { + let after = &detail[start + "field `".len()..]; + if let Some(end) = after.find('`') { + let name = after[..end].trim(); + if !name.is_empty() { + return Some(name.to_string()); + } + } + } + // Engine: "unsupported agent_control_specification_version '...'" + if let Some(rest) = detail.strip_prefix("unsupported ") { + let name: String = rest + .chars() + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_') + .collect(); + if !name.is_empty() { + return Some(name); + } + } + // Engine: "X is required" + if let Some((head, _)) = detail.split_once(" is required") { + let name = head.trim(); + if !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Some(name.to_string()); + } + } + None +} + // --------------------------------------------------------------------- // Streaming: incremental release accounting for stream-shaped tracks // (spec §18.1). A session holds no policy and no text; the host drives diff --git a/sdk/node/src/index.ts b/sdk/node/src/index.ts index 374157f..3f90849 100644 --- a/sdk/node/src/index.ts +++ b/sdk/node/src/index.ts @@ -26,13 +26,40 @@ import type { // eslint-disable-next-line @typescript-eslint/no-require-imports const native = require("../binding.js") as { interceptorNew(manifestPath: string): unknown; + interceptorNewWithHooks( + manifestPath: string, + annotatorDispatcher?: + | ((name: string, invocationJson: string, policyInputJson: string) => string) + | null, + policyDispatcher?: ((invocationJson: string) => string) | null, + telemetrySink?: ((eventJson: string) => void) | null, + perfTelemetry?: string | null, + ): unknown; intercept(handle: unknown, contextJson: string): string; policyActivate(manifestPath: string): unknown; policyActivateFromMemory(manifestYaml: string, bundlesJson: string): unknown; + policyActivateWithHooks( + manifestPath: string, + annotatorDispatcher?: + | ((name: string, invocationJson: string, policyInputJson: string) => string) + | null, + policyDispatcher?: ((invocationJson: string) => string) | null, + ): unknown; + policyActivateFromMemoryWithHooks( + manifestYaml: string, + bundlesJson: string, + annotatorDispatcher?: + | ((name: string, invocationJson: string, policyInputJson: string) => string) + | null, + policyDispatcher?: ((invocationJson: string) => string) | null, + ): unknown; policyEvaluate(handle: unknown, point: string, contextJson: string): string; policyInterventionPoints(handle: unknown): string[]; validateManifestFile(path: string): string | null; validateManifest(source: string): string | null; + validateManifestDetailed(source: string): string; + parseManifest(source: string): string; + mergeManifests(sourcesJson: string): string; supportedManifestVersions(): string[]; streamSessionNew(configJson: string): unknown; streamSessionObserve(handle: unknown, sourceType: string, runes: number): number; @@ -72,7 +99,192 @@ export type { Warning, } from "@responsibleai/agent-hooks"; -export interface AcsInterceptorOptions { +// --------------------------------------------------------------------- +// Host extension surface: annotator dispatcher, policy dispatcher, +// telemetry sink, perf telemetry level. The zero-config path stays +// unchanged; a host that supplies any of these hooks gets the same +// engine construction the Rust and .NET/FFI SDKs already expose. +// +// The dispatcher callbacks are called SYNCHRONOUSLY on the JS thread +// from inside the engine's evaluation, which is legal here because the +// engine call itself is driven by a napi function running on the JS +// thread. A dispatcher that throws surfaces as a fail-closed +// `runtime_error:*` deny; the engine never treats a thrown callback as +// "no annotation". +// --------------------------------------------------------------------- + +/** + * A JSON-compatible value the boundary carries verbatim. + * + * Named separately from `unknown` so a dispatcher signature can say what + * kind of thing it reads and returns without opening the door to values + * (functions, symbols, class instances) the engine cannot round-trip. + */ +export type JsonValue = + | null + | boolean + | number + | string + | readonly JsonValue[] + | { readonly [key: string]: JsonValue }; + +/** + * Fields the engine hands to a host annotator: `type` (`classifier`, + * `llm`, `endpoint`), `from` (JSONPath expression), and every extra + * key the manifest attached to the annotator or the annotation. + */ +export interface AnnotatorInvocation { + readonly type: "classifier" | "llm" | "endpoint"; + readonly from: string; + readonly [key: string]: JsonValue; +} + +/** + * Fields the engine hands to a host policy dispatcher. A serialized + * `PreparedPolicyInvocation` — carrying `policy_id`, `policy_type`, + * `intervention_point`, `input`, and the policy-specific configuration + * — routed to whichever engine the host runs. The precise shape moves + * with the engine; treat it as opaque JSON and let the engine describe + * what to key off. + */ +export interface PolicyInvocation { + readonly policy_id: string; + readonly policy_type: string; + readonly intervention_point: string; + readonly input: JsonValue; + readonly [key: string]: JsonValue; +} + +/** + * One telemetry event the engine emits. Every field maps 1:1 with the + * `TelemetryEvent` the FFI binding surfaces and the Rust engine + * declares, so a sink written for another SDK reads the same shape here. + */ +export interface TelemetryEvent { + readonly event_type: string; + readonly intervention_point: string; + readonly decision: string | null; + readonly reason_code: string | null; + readonly error_class: string | null; + readonly policy_id: string | null; + readonly annotators: readonly string[]; + readonly enforcement_mode: string | null; + readonly duration_ms: number | null; + readonly evidence_artefact: string | null; + readonly evidence_verification_pointer_keys: readonly string[]; + readonly action_identity: string | null; + readonly metadata: Readonly>; +} + +/** + * A host annotator dispatcher. + * + * Called synchronously from inside `intercept`/`evaluate`; return a + * value the manifest's `preliminary_policy_input` merge shape expects + * (typically a JSON object per the annotator's contract). Throwing + * fails the surrounding evaluation closed with a + * `runtime_error:annotation_failed` deny. + */ +export type AnnotatorDispatcher = ( + name: string, + invocation: AnnotatorInvocation, + preliminaryPolicyInput: JsonValue, +) => JsonValue; + +/** + * A host policy dispatcher. + * + * Called synchronously; return the policy output as JSON per the + * engine's expected shape (a `decision` string plus any extras). A + * throw fails the evaluation closed with a + * `runtime_error:policy_invocation_failed` deny. + */ +export type PolicyDispatcher = (invocation: PolicyInvocation) => JsonValue; + +/** + * A telemetry sink. + * + * Called synchronously with each event the engine emits at the + * configured perf level. A sink cannot fail an evaluation; a throw + * from the sink is swallowed to preserve that guarantee. + */ +export type TelemetrySink = (event: TelemetryEvent) => void; + +/** + * How much per-evaluation timing to emit. + * + * - `off` (default): only the final decision event. + * - `external`: adds boundary events (annotator dispatch, policy + * evaluation). + * - `full`: adds stage timing events for a full performance profile. + */ +export type PerfTelemetry = "off" | "external" | "full"; + +/** + * Host extension points. Each is optional; supplying one replaces the + * zero-config default for that slot. + */ +export interface HostHooks { + annotatorDispatcher?: AnnotatorDispatcher; + policyDispatcher?: PolicyDispatcher; + telemetrySink?: TelemetrySink; + perfTelemetry?: PerfTelemetry; +} + +// --- Bridges between the object-oriented TS surface and the JSON -------- +// string wire the native binding uses. Keeping the parse/serialize step +// here means the native dispatcher signatures stay simple `String → +// String` and every host callback sees objects, not text. + +function wrapAnnotatorDispatcher( + dispatcher: AnnotatorDispatcher, +): (name: string, invocationJson: string, policyInputJson: string) => string { + return (name, invocationJson, policyInputJson) => { + // A parse or a throwing dispatcher must propagate as a JS exception: + // the native binding turns that into a fail-closed + // `runtime_error:annotation_failed` deny. Never swallow — a + // silently caught error would read as "annotation succeeded with + // undefined". + const invocation = JSON.parse(invocationJson) as AnnotatorInvocation; + const policyInput = JSON.parse(policyInputJson) as JsonValue; + const result = dispatcher(name, invocation, policyInput); + return JSON.stringify(result ?? null); + }; +} + +function wrapPolicyDispatcher( + dispatcher: PolicyDispatcher, +): (invocationJson: string) => string { + return (invocationJson) => { + const invocation = JSON.parse(invocationJson) as PolicyInvocation; + const result = dispatcher(invocation); + return JSON.stringify(result ?? null); + }; +} + +function wrapTelemetrySink(sink: TelemetrySink): (eventJson: string) => void { + return (eventJson) => { + // A sink cannot fail an evaluation, so wrap in try/catch and drop + // the throw. Matches the engine contract. + try { + const event = JSON.parse(eventJson) as TelemetryEvent; + sink(event); + } catch { + // intentionally swallowed + } + }; +} + +function hasHostHooks(options: HostHooks): boolean { + return ( + options.annotatorDispatcher !== undefined || + options.policyDispatcher !== undefined || + options.telemetrySink !== undefined || + options.perfTelemetry !== undefined + ); +} + +export interface AcsInterceptorOptions extends HostHooks { /** Payload-free identifier recorded on the record's `verdicts[].name`. */ name?: string; } @@ -88,14 +300,36 @@ export class AcsInterceptor implements Interceptor { } /** - * Build an interceptor from a manifest path using the zero-config - * dispatchers: bundled annotators; Rego policies in process, Cedar - * through the built-in evaluator, `test` policies through their - * embedded verdict. Custom policies require a host dispatcher and - * fail closed under this construction. + * Build an interceptor from a manifest path. + * + * With no `options`, uses the zero-config dispatchers: bundled + * annotators; Rego policies in process, Cedar through the built-in + * evaluator, `test` policies through their embedded verdict. Custom + * policies require a host dispatcher and fail closed under this + * construction. + * + * Supply `annotatorDispatcher`, `policyDispatcher`, `telemetrySink`, + * or `perfTelemetry` to override the engine's extension points. Each + * callback is called synchronously on the JS thread from inside + * `intercept`. A callback that throws surfaces as a fail-closed + * `runtime_error:*` deny rather than reading as "no annotation". */ static fromPath(manifestPath: string, options: AcsInterceptorOptions = {}): AcsInterceptor { - return new AcsInterceptor(native.interceptorNew(manifestPath), options.name ?? "acs"); + const name = options.name ?? "acs"; + const handle = hasHostHooks(options) + ? native.interceptorNewWithHooks( + manifestPath, + options.annotatorDispatcher + ? wrapAnnotatorDispatcher(options.annotatorDispatcher) + : null, + options.policyDispatcher + ? wrapPolicyDispatcher(options.policyDispatcher) + : null, + options.telemetrySink ? wrapTelemetrySink(options.telemetrySink) : null, + options.perfTelemetry ?? null, + ) + : native.interceptorNew(manifestPath); + return new AcsInterceptor(handle, name); } /** @@ -134,6 +368,25 @@ export interface RegoBundle { data?: readonly RegoDataDocument[]; } +/** + * Host extension points for {@link ActivatedPolicy}. + * + * Activation uses the engine's `activate_with` surface, which takes + * only the annotator and policy dispatchers; telemetry and perf level + * are configured on the interceptor path (see {@link AcsInterceptor}) + * because activation records no per-evaluation events itself. + */ +export interface ActivatedPolicyOptions { + annotatorDispatcher?: AnnotatorDispatcher; + policyDispatcher?: PolicyDispatcher; +} + +function hasActivationHooks(options: ActivatedPolicyOptions): boolean { + return ( + options.annotatorDispatcher !== undefined || options.policyDispatcher !== undefined + ); +} + /** * One policy version, readied once and evaluated many times. * @@ -175,9 +428,28 @@ export class ActivatedPolicy { * * A manifest names its bundle relative to itself, so an absolute * manifest path is enough and the working directory does not matter. + * + * Supply `annotatorDispatcher` or `policyDispatcher` in `options` to + * override the engine's extension points; each callback is called + * synchronously on the JS thread from inside `evaluate` and fails + * closed on throw. */ - static activate(manifestPath: string): ActivatedPolicy { - return new ActivatedPolicy(native.policyActivate(manifestPath)); + static activate( + manifestPath: string, + options: ActivatedPolicyOptions = {}, + ): ActivatedPolicy { + const handle = hasActivationHooks(options) + ? native.policyActivateWithHooks( + manifestPath, + options.annotatorDispatcher + ? wrapAnnotatorDispatcher(options.annotatorDispatcher) + : null, + options.policyDispatcher + ? wrapPolicyDispatcher(options.policyDispatcher) + : null, + ) + : native.policyActivate(manifestPath); + return new ActivatedPolicy(handle); } /** @@ -205,10 +477,21 @@ export class ActivatedPolicy { static activateFromMemory( manifestYaml: string, bundles: Readonly>, + options: ActivatedPolicyOptions = {}, ): ActivatedPolicy { - return new ActivatedPolicy( - native.policyActivateFromMemory(manifestYaml, JSON.stringify(bundles)), - ); + const handle = hasActivationHooks(options) + ? native.policyActivateFromMemoryWithHooks( + manifestYaml, + JSON.stringify(bundles), + options.annotatorDispatcher + ? wrapAnnotatorDispatcher(options.annotatorDispatcher) + : null, + options.policyDispatcher + ? wrapPolicyDispatcher(options.policyDispatcher) + : null, + ) + : native.policyActivateFromMemory(manifestYaml, JSON.stringify(bundles)); + return new ActivatedPolicy(handle); } /** @@ -307,6 +590,112 @@ export function supportedManifestVersions(): readonly string[] { return Object.freeze(native.supportedManifestVersions()); } +// --------------------------------------------------------------------- +// Manifest tooling: parse, chain-compose, and structured diagnostics. +// +// The engine ships these APIs on `Manifest`; the wrapper exposes them +// here so authoring, migration, and CI tools can drive parse, overlay +// composition, and per-field validation from Node without staging a +// manifest to disk first. +// --------------------------------------------------------------------- + +/** + * A structured description of a manifest that did not pass validation. + * + * `code` is the reserved `runtime_error:*` reason a diagnostic-consuming + * tool keys off; `message` is the engine's human-readable detail; + * `field` is a best-effort pointer to the offending manifest field (a + * YAML key or an engine-declared identifier) so an editor can render + * the problem inline. `field` is absent when the message does not + * identify one. + */ +export interface ManifestDiagnostic { + readonly code: string; + readonly message: string; + readonly severity: "error"; + readonly field?: string; +} + +/** + * Parse manifest YAML into an object without validating references. + * + * The document is deserialized as-written: a manifest with an + * unresolved `extends` chain returns fine. Use {@link validateManifest} + * or {@link validateManifestDetailed} to judge whether the fragment is + * runnable. + * + * Throws when the YAML does not parse. + */ +export function parseManifest(source: string): Record { + if (typeof source !== "string") { + throw new TypeError(`parseManifest expects a string, received ${typeof source}`); + } + if (UNPAIRED_SURROGATE.test(source)) { + throw new TypeError("parseManifest received a string with an unpaired surrogate"); + } + return JSON.parse(native.parseManifest(source)) as Record; +} + +/** + * Compose a chain of manifest YAML documents into one merged manifest. + * + * `sources` is ordered outermost base first, deltas after. This is the + * overlay case, resolved the same way the engine resolves `extends` + * when it walks a manifest tree. The returned object is the merged + * manifest as JSON. + * + * Throws when a source does not parse, when the merged result is + * invalid, or when `sources` is empty. + */ +export function mergeManifests(sources: readonly string[]): Record { + if (!Array.isArray(sources)) { + throw new TypeError("mergeManifests expects an array of manifest sources"); + } + for (const source of sources) { + if (typeof source !== "string") { + throw new TypeError("mergeManifests received a non-string source"); + } + if (UNPAIRED_SURROGATE.test(source)) { + throw new TypeError( + "mergeManifests received a source with an unpaired surrogate", + ); + } + } + return JSON.parse(native.mergeManifests(JSON.stringify(sources))) as Record< + string, + unknown + >; +} + +/** + * Validate manifest source and return structured findings. + * + * An empty array means the manifest passed validation. Each finding + * carries an engine reason code, the detail message, and where + * possible the offending field name so an editor can render the + * problem inline. Use this instead of {@link validateManifest} when + * you need to render results, not just yes/no. + * + * Throws only on boundary problems (non-string input, unpaired + * surrogate); an invalid manifest returns a non-empty array. + */ +export function validateManifestDetailed(source: string): readonly ManifestDiagnostic[] { + if (typeof source !== "string") { + throw new TypeError( + `validateManifestDetailed expects a string, received ${typeof source}`, + ); + } + if (UNPAIRED_SURROGATE.test(source)) { + throw new TypeError( + "validateManifestDetailed received a string with an unpaired surrogate", + ); + } + return Object.freeze( + JSON.parse(native.validateManifestDetailed(source)) as ManifestDiagnostic[], + ); +} + + // --------------------------------------------------------------------- // Streaming: incremental release mediation for stream-shaped tracks // (spec §18.1). The engine is stateless everywhere else; this session diff --git a/sdk/node/test/host-hooks.test.mjs b/sdk/node/test/host-hooks.test.mjs new file mode 100644 index 0000000..5e9d52e --- /dev/null +++ b/sdk/node/test/host-hooks.test.mjs @@ -0,0 +1,380 @@ +// Host extension surface: annotator dispatcher, policy dispatcher, +// telemetry sink, perf telemetry level; and manifest tooling (parse, +// chain, structured diagnostics). +// +// The engine calls dispatchers synchronously from inside `intercept`, +// which is itself a napi call running on the JS thread. These tests +// prove: (a) a JS callback IS invoked on that stack, (b) its return +// value flows into the policy decision, (c) a throw fails closed with +// a `runtime_error:*` deny (never "no annotation"), (d) telemetry +// events reach a sink, and (e) the manifest tooling that authoring +// tools need is reachable from Node. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import fs from "node:fs"; +import path from "node:path"; + +const require = (await import("node:module")).createRequire(import.meta.url); +const { + AcsInterceptor, + ActivatedPolicy, + parseManifest, + mergeManifests, + validateManifestDetailed, +} = require("../dist/index.js"); +const { AgentContextBuilder } = require("@responsibleai/agent-hooks"); + +const here = path.dirname(fileURLToPath(import.meta.url)); +const fixtureManifestPath = path.join(here, "fixtures", "manifest.yaml"); + +const builder = () => + new AgentContextBuilder({ agentId: "a", framework: "test", sessionId: "s" }); + +// A manifest binding a custom policy that a host policy dispatcher +// answers, and a classifier annotator whose value the dispatcher reads +// from `input.annotations.mood`. The manifest is deliberately minimal: +// one intervention point, one annotator, one policy. +function writeAnnotatorPolicyManifest(tmpdir) { + const src = `agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: node-host-hooks-test +policies: + gate: + type: custom + adapter: host_gate +annotators: + mood: + type: classifier +intervention_points: + input: + policy_target: "$.input" + policy_target_kind: user_input + annotations: + mood: + from: "$target.content" + policy: + id: gate +`; + const p = path.join(tmpdir, "annotator-policy-manifest.yaml"); + fs.writeFileSync(p, src, "utf8"); + return p; +} + +function writePolicyOnlyManifest(tmpdir) { + const src = `agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: node-host-hooks-policy-only +policies: + gate: + type: custom + adapter: host_gate +intervention_points: + input: + policy_target: "$.input" + policy_target_kind: user_input + policy: + id: gate +`; + const p = path.join(tmpdir, "policy-only-manifest.yaml"); + fs.writeFileSync(p, src, "utf8"); + return p; +} + +const workdir = fs.mkdtempSync(path.join(here, ".host-hooks-")); +test.after(() => { + fs.rmSync(workdir, { recursive: true, force: true }); +}); + +// --------------------------------------------------------------------- +// 1. Host annotator dispatcher IS called on the sync stack and its +// return value flows into the policy decision. Two invocations against +// the same manifest, differing only in what the annotator returns, +// produce different verdicts. +// --------------------------------------------------------------------- + +test("host annotator dispatcher return value drives the policy verdict", () => { + const manifest = writeAnnotatorPolicyManifest(workdir); + const seen = []; + const annotatorDispatcher = (name, invocation, _preliminary) => { + // Prove the engine actually asked us, and remember what for. + seen.push({ name, type: invocation.type, from: invocation.from }); + // Emit the mood the policy will read. + return { mood: invocation.from === "$target.content" ? "angry" : "calm" }; + }; + const policyDispatcher = (invocation) => { + // Custom-policy invocations tag as `custom` and carry the policy + // input under `input`. The annotator's output appears at + // `input.annotations.mood`. + assert.equal(invocation.type, "custom"); + assert.equal(invocation.adapter, "host_gate"); + const mood = invocation.input?.annotations?.mood?.mood; + if (mood === "angry") { + return { + decision: "deny", + reason: "annotation_says_angry", + message: "annotator flagged mood=angry", + }; + } + return { decision: "allow", reason: "annotation_ok" }; + }; + + const acs = AcsInterceptor.fromPath(manifest, { + annotatorDispatcher, + policyDispatcher, + }); + + const angry = acs.intercept(builder().input("you are broken")); + assert.equal(angry.decision, "deny"); + assert.equal(angry.reason, "annotation_says_angry"); + + // The annotator was actually called on the sync call stack. + assert.equal(seen.length, 1); + assert.equal(seen[0].name, "mood"); + assert.equal(seen[0].type, "classifier"); + assert.equal(seen[0].from, "$target.content"); +}); + +// --------------------------------------------------------------------- +// 2. An annotator that throws fails CLOSED. The verdict is a deny with +// `runtime_error:annotation_failed`. The engine never treats a thrown +// callback as "no annotation". +// --------------------------------------------------------------------- + +test("annotator dispatcher that throws fails closed with annotation_failed", () => { + const manifest = writeAnnotatorPolicyManifest(workdir); + const annotatorDispatcher = () => { + throw new Error("upstream classifier is on fire"); + }; + // Present the policy dispatcher, but it must never be reached: the + // annotator failure short-circuits evaluation. + let policyCalled = false; + const policyDispatcher = () => { + policyCalled = true; + return { decision: "allow" }; + }; + + const acs = AcsInterceptor.fromPath(manifest, { + annotatorDispatcher, + policyDispatcher, + }); + + const verdict = acs.intercept(builder().input("hi")); + assert.equal(verdict.decision, "deny"); + assert.equal(verdict.reason, "runtime_error:annotation_failed"); + assert.equal( + policyCalled, + false, + "policy dispatcher must not run after an annotator failure", + ); +}); + +// --------------------------------------------------------------------- +// 3. Passing no options behaves identically to the zero-config path. +// The fixture manifest's verdicts are the pinned baseline; the with- +// hooks constructor must return the same values when no hooks are set. +// --------------------------------------------------------------------- + +test("no host hooks leaves the zero-config path bit-identical", () => { + const zeroConfig = AcsInterceptor.fromPath(fixtureManifestPath); + const empty = AcsInterceptor.fromPath(fixtureManifestPath, {}); + for (const ctx of [ + builder().input("hello"), + builder().preToolCall("t1", "search", { q: "x" }), + builder().output("final answer"), + ]) { + // Two independent Contexts of the same shape. + const a = zeroConfig.intercept(ctx); + const b = empty.intercept(ctx); + assert.deepEqual(b, a); + } +}); + +// --------------------------------------------------------------------- +// 4. A telemetry sink receives events, and perfTelemetry levels round- +// trip. The engine emits a Decision event per evaluation; the sink +// must see at least one, and reject an unknown perf level. +// --------------------------------------------------------------------- + +test("telemetry sink receives Decision events; perf level round-trips", () => { + const events = []; + const acs = AcsInterceptor.fromPath(fixtureManifestPath, { + telemetrySink: (event) => events.push(event), + perfTelemetry: "off", + }); + const verdict = acs.intercept(builder().input("hello")); + assert.equal(verdict.decision, "allow"); + assert.ok(events.length >= 1, "expected at least one telemetry event"); + const decision = events.find((e) => e.event_type === "decision"); + assert.ok(decision, "expected a decision event"); + assert.equal(decision.intervention_point, "input"); + assert.equal(decision.decision, "allow"); + assert.equal(decision.policy_id, "allow_all"); + + // Every documented perf level is accepted. + for (const level of ["off", "external", "full"]) { + const configured = AcsInterceptor.fromPath(fixtureManifestPath, { + perfTelemetry: level, + }); + // Construction alone proves the level round-trips; also confirm + // evaluation still works under it. + assert.equal(configured.intercept(builder().input("x")).decision, "allow"); + } + + // An unknown level is a boundary problem, not a silent fallback. + assert.throws( + () => + AcsInterceptor.fromPath(fixtureManifestPath, { perfTelemetry: "loud" }), + /perf telemetry/i, + ); +}); + +// --------------------------------------------------------------------- +// 5. `parseManifest`: a valid document returns structured JSON; a +// broken document throws. +// --------------------------------------------------------------------- + +test("parseManifest returns structure for valid YAML and throws on garbage", () => { + const parsed = parseManifest(fs.readFileSync(fixtureManifestPath, "utf8")); + assert.equal(typeof parsed, "object"); + assert.ok(parsed); + assert.equal( + parsed.agent_control_specification_version, + "0.4.0-alpha.1", + "top-level version preserved", + ); + assert.ok(parsed.policies, "policies map is present"); + assert.ok(parsed.policies.allow_all, "individual policies survive parse"); + assert.ok(parsed.intervention_points, "intervention points are present"); + + // Malformed YAML must throw, not return an empty object. + assert.throws(() => parseManifest("agent_control_specification_version: [")); + assert.throws(() => parseManifest("::not: [valid: yaml")); + + // Boundary problems throw as TypeError, not as an invalid manifest. + assert.throws(() => parseManifest(42), TypeError); + assert.throws(() => parseManifest("\uD800"), TypeError); +}); + +// --------------------------------------------------------------------- +// 6. `mergeManifests` composes a chain: a base manifest plus an overlay +// that adds an intervention point. The merged result carries fields +// from both. +// --------------------------------------------------------------------- + +test("mergeManifests composes a base and an overlay", () => { + const base = `agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: base +policies: + allow_all: + type: test + verdict: + decision: allow +intervention_points: + input: + policy_target: "$.input" + policy: + id: allow_all +`; + const overlay = `agent_control_specification_version: "0.4.0-alpha.1" +policies: + block_tool: + type: test + verdict: + decision: deny + reason: blocked_by_policy +intervention_points: + pre_tool_call: + policy_target: "$.tool_call.args" + policy: + id: block_tool +`; + const merged = mergeManifests([base, overlay]); + // The base's metadata was left as-written by the additive overlay, + // and both policies and both intervention points survived the merge. + assert.equal(typeof merged, "object"); + assert.ok(merged.policies.allow_all, "base policy survives"); + assert.ok(merged.policies.block_tool, "overlay policy survives"); + assert.ok(merged.intervention_points.input, "base intervention point survives"); + assert.ok( + merged.intervention_points.pre_tool_call, + "overlay intervention point survives", + ); + assert.equal(merged.metadata.name, "base", "base metadata is preserved"); + + // The merged document is runnable end to end: build an interceptor + // from an equivalent YAML shape and get the two verdicts the pieces + // expected. + const mergedYaml = + `agent_control_specification_version: "0.4.0-alpha.1"\n` + + `metadata:\n name: merged\n` + + `policies:\n allow_all:\n type: test\n verdict:\n decision: allow\n` + + ` block_tool:\n type: test\n verdict:\n decision: deny\n reason: blocked_by_policy\n` + + `intervention_points:\n input:\n policy_target: "$.input"\n policy:\n id: allow_all\n` + + ` pre_tool_call:\n policy_target: "$.tool_call.args"\n policy:\n id: block_tool\n`; + const yamlPath = path.join(workdir, "merged-baseline.yaml"); + fs.writeFileSync(yamlPath, mergedYaml, "utf8"); + const acs = AcsInterceptor.fromPath(yamlPath); + assert.equal(acs.intercept(builder().input("hi")).decision, "allow"); + assert.equal( + acs.intercept(builder().preToolCall("t1", "search", { q: "x" })).decision, + "deny", + ); + + // Boundary problems throw as TypeError. + assert.throws(() => mergeManifests("not an array"), TypeError); + assert.throws(() => mergeManifests([42]), TypeError); + assert.throws(() => mergeManifests([])); +}); + +// --------------------------------------------------------------------- +// 7. `validateManifestDetailed` returns diagnostics that name the +// offending field. An unsupported version triggers the "unsupported +// " branch, so `field` points at +// `agent_control_specification_version`. +// --------------------------------------------------------------------- + +test("validateManifestDetailed names the offending field", () => { + const good = fs.readFileSync(fixtureManifestPath, "utf8"); + const empty = validateManifestDetailed(good); + assert.deepEqual(empty, []); + + const badVersion = good.replace('"0.4.0-alpha.1"', '"0.3.1-beta"'); + const findings = validateManifestDetailed(badVersion); + assert.equal(findings.length, 1); + const finding = findings[0]; + assert.equal(finding.severity, "error"); + assert.ok(finding.code.startsWith("runtime_error:"), `code was ${finding.code}`); + assert.match(finding.message, /0\.3\.1-beta/); + assert.equal( + finding.field, + "agent_control_specification_version", + "the field pointer should identify the version key", + ); + + // Boundary problems throw as TypeError. + assert.throws(() => validateManifestDetailed(42), TypeError); + assert.throws(() => validateManifestDetailed("\uD800"), TypeError); +}); + +// --------------------------------------------------------------------- +// Bonus: ActivatedPolicy takes host hooks too. Prove the sync callback +// works through that path as well, so the two entry points stay in +// lockstep. +// --------------------------------------------------------------------- + +test("ActivatedPolicy.activate accepts host hooks and evaluates against them", () => { + const manifest = writePolicyOnlyManifest(workdir); + const seen = []; + const activated = ActivatedPolicy.activate(manifest, { + policyDispatcher: (invocation) => { + seen.push(invocation.type); + return { decision: "deny", reason: "denied_by_test" }; + }, + }); + const verdict = activated.evaluate("input", builder().input("hi")); + assert.equal(verdict.decision, "deny"); + assert.equal(verdict.reason, "denied_by_test"); + assert.deepEqual(seen, ["custom"]); +}); diff --git a/sdk/python/agent_control_spec/__init__.py b/sdk/python/agent_control_spec/__init__.py index 6c10fd6..1a933d9 100644 --- a/sdk/python/agent_control_spec/__init__.py +++ b/sdk/python/agent_control_spec/__init__.py @@ -14,7 +14,7 @@ from __future__ import annotations import json -from collections.abc import Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping from typing import Any from agent_hooks import Verdict @@ -22,14 +22,20 @@ from agent_control_spec import _native __all__ = [ + "PERF_TELEMETRY_LEVELS", "AcsInterceptor", "ActivatedPolicy", "ManifestInvalidError", "RegoBundle", "StreamSession", + "TelemetryEvent", + "ValidationDiagnostic", "__version__", + "merge_manifests", + "parse_manifest", "supported_manifest_versions", "validate_manifest", + "validate_manifest_detailed", "validate_manifest_file", ] @@ -40,18 +46,118 @@ #: {...}}]}``. Both keys default to empty and nothing else is accepted. RegoBundle = Mapping[str, Any] +#: The `perf_telemetry` levels the runtime accepts. Kept explicit rather +#: than a Rust-side enum: the constructor argument is a string, so the +#: allowed values are the vocabulary a Python host reads. +PERF_TELEMETRY_LEVELS: tuple[str, ...] = ("off", "external", "full") + +#: A telemetry event a host-supplied sink receives from the engine. +#: +#: The dict is populated by the native layer and mirrors +#: :class:`agent_control_spec.telemetry.TelemetryEvent` on the Rust side. +#: Its shape is documented rather than typed as a ``TypedDict`` to keep +#: the Python surface stable while the engine grows fields, which it has +#: done twice already in 0.4 (evidence artefacts, transformed-event). +#: Keys: +#: +#: - ``event_type``: one of ``"decision"``, ``"annotator_dispatch"``, +#: ``"policy_evaluation"``, ``"evaluation_timing"``, +#: ``"intervention_point.transformed"``, ``"annotator_failed"``, or +#: ``"policy_failed"``. +#: - ``intervention_point``: agent-hooks wire name. +#: - ``decision``: ``"allow"`` / ``"deny"`` / ``"transform"`` or ``None``. +#: - ``reason_code``: ``str`` or ``None``. +#: - ``error_class``: ``str`` or ``None``. +#: - ``policy_id``: ``str`` or ``None``. +#: - ``annotators``: ``list[str]``. +#: - ``enforcement_mode``: ``"enforce"`` / ``"evaluate_only"`` or ``None``. +#: - ``duration_ms``: ``float`` or ``None``. +#: - ``evidence_artefact``: ``str`` or ``None``. +#: - ``evidence_verification_pointer_keys``: ``list[str]``. +#: - ``action_identity``: ``str`` or ``None``. +#: - ``metadata``: ``dict[str, str]``. +TelemetryEvent = Mapping[str, Any] + +#: One entry produced by :func:`validate_manifest_detailed`. Shape: +#: +#: - ``reason_code``: ``str`` (the engine's ``runtime_error:*`` name). +#: - ``message``: ``str`` (the engine's own message text). +#: - ``field``: ``str | None`` (best-effort field name extracted from the +#: message; ``None`` when the message names no known field). +#: +#: Wrapped as a mapping rather than a dataclass to keep it JSON-safe for +#: tools that shuttle diagnostics through IPC. +ValidationDiagnostic = Mapping[str, Any] + + +def _normalize_perf_telemetry(value: str | None) -> str: + """Reject unknown perf-telemetry levels on the Python side. + + The Rust binding does its own check, but doing this here means the + error is raised without paying for a manifest load first, which + matches how the rest of the wrapper preserves cheap-to-fail + ordering. The engine's own vocabulary is preserved verbatim. + """ + if value is None: + return "off" + if value not in PERF_TELEMETRY_LEVELS: + raise ValueError( + f"unknown perf_telemetry level {value!r}; " + f"expected one of {PERF_TELEMETRY_LEVELS}" + ) + return value + class AcsInterceptor: """agent-hooks interceptor over the Agent Control Specification runtime. Register an instance with any agent-hooks host emitter. The manifest - is loaded once at construction with the zero-config dispatchers - (bundled annotators; Rego in process, Cedar through the built-in - evaluator, ``test`` policies through their embedded verdict). + is loaded once at construction. + + Zero-config path (the default): bundled annotators; Rego in process, + Cedar through the built-in evaluator, ``test`` policies through + their embedded verdict; no-op telemetry. + + Host hooks are supplied by keyword: + + - ``annotator_dispatcher``: object with a ``dispatch(annotator_name, + annotator, preliminary_policy_input)`` method or a plain callable + with the same signature. Return value is the annotation payload + that reaches the policy under ``input.annotations[]``. + - ``policy_dispatcher``: object with an ``evaluate(invocation)`` + method (and optionally a ``warm(invocation)`` method) or a plain + callable. Return value is the raw policy output normalized into a + verdict by the engine. + - ``telemetry_sink``: object with an ``emit(event)`` method + (optionally a ``shutdown()`` method) or a plain callable. The + engine emits one event per decision plus optional stage events. + - ``perf_telemetry``: ``"off"`` (default), ``"external"``, or + ``"full"``, gating whether external and per-stage timing events + are emitted. + + A dispatcher that raises does not silently no-op: the engine + normalizes the failure into a fail-closed ``deny`` verdict with a + ``runtime_error:*`` reason. """ - def __init__(self, manifest_path: str, name: str = "acs") -> None: - self._handle = _native.interceptor_new(manifest_path) + def __init__( + self, + manifest_path: str, + name: str = "acs", + *, + annotator_dispatcher: object | None = None, + policy_dispatcher: object | None = None, + telemetry_sink: object | Callable[[TelemetryEvent], None] | None = None, + perf_telemetry: str = "off", + ) -> None: + perf = _normalize_perf_telemetry(perf_telemetry) + self._handle = _native.interceptor_new( + manifest_path, + annotator_dispatcher, + policy_dispatcher, + telemetry_sink, + perf, + ) self._name = name @property @@ -92,15 +198,29 @@ class is the other split: :meth:`activate` pays for reading the manifest path is enough and the working directory does not matter. :meth:`from_memory` is the other source: manifest text and Rego sources held by the host, with no file to read. + + Host dispatchers are supplied by the same keyword arguments as + :class:`AcsInterceptor`. Passing them here is exactly how the + consumer-facing ``AgentControl.from_native(..., + annotator_dispatcher=ContentSafetyDispatcher())`` shape composed: + activation carries the dispatcher, evaluation calls it, and readying + warms it. """ __slots__ = ("_handle",) - def __init__(self, manifest_path: str) -> None: - """Activate the manifest at ``manifest_path`` with the - zero-config dispatchers (bundled annotators; Rego in process, - Cedar through the built-in evaluator, ``test`` policies through - their embedded verdict). + def __init__( + self, + manifest_path: str, + *, + annotator_dispatcher: object | None = None, + policy_dispatcher: object | None = None, + ) -> None: + """Activate the manifest at ``manifest_path``. + + Passing no host arguments preserves the zero-config path (bundled + annotators; Rego in process, Cedar through the built-in + evaluator, ``test`` policies through their embedded verdict). Raises :class:`ValueError` when the manifest cannot be read or is rejected, and :class:`RuntimeError` when it binds a policy that @@ -109,19 +229,36 @@ def __init__(self, manifest_path: str) -> None: the deadline surfaces at the first decision instead. A policy that merely needs real input to produce a verdict activates fine. """ - self._handle = _native.policy_activate(manifest_path) + self._handle = _native.policy_activate( + manifest_path, annotator_dispatcher, policy_dispatcher + ) @classmethod - def activate(cls, manifest_path: str) -> ActivatedPolicy: + def activate( + cls, + manifest_path: str, + *, + annotator_dispatcher: object | None = None, + policy_dispatcher: object | None = None, + ) -> ActivatedPolicy: """Activate the manifest at ``manifest_path``. Same as the constructor, named for the lifecycle it belongs to. """ - return cls(manifest_path) + return cls( + manifest_path, + annotator_dispatcher=annotator_dispatcher, + policy_dispatcher=policy_dispatcher, + ) @classmethod def from_memory( - cls, manifest_yaml: str, bundles: Mapping[str, RegoBundle] + cls, + manifest_yaml: str, + bundles: Mapping[str, RegoBundle], + *, + annotator_dispatcher: object | None = None, + policy_dispatcher: object | None = None, ) -> ActivatedPolicy: """Activate a manifest and its Rego supplied as values. @@ -142,7 +279,10 @@ def from_memory( """ policy = cls.__new__(cls) policy._handle = _native.policy_activate_from_memory( - manifest_yaml, json.dumps(bundles, allow_nan=False) + manifest_yaml, + json.dumps(bundles, allow_nan=False), + annotator_dispatcher, + policy_dispatcher, ) return policy @@ -209,6 +349,58 @@ def validate_manifest_file(path: str) -> None: _native.validate_manifest_file(path) +def validate_manifest_detailed(source: str) -> list[ValidationDiagnostic]: + """Return structured validation diagnostics for a manifest source. + + Each diagnostic is ``{"reason_code": str, "message": str, "field": + str | None}``. An accepted manifest returns ``[]``. A rejected one + returns one entry naming the failed field where the engine's message + permits extraction, and ``None`` for ``field`` when it does not: the + ``message`` is the engine's own text either way, so a tool that + cannot map ``field`` back to a location still has the verbatim + reason. + + Use this for authoring tools, migration linting, and CI checks that + want per-field feedback. :func:`validate_manifest` is the boolean + shortcut for callers that only care whether validation passed. + """ + return json.loads(_native.validate_manifest_diagnostics(source)) + + +def parse_manifest(source: str) -> dict[str, Any]: + """Parse manifest source into a ``dict`` without validating. + + An authoring tool that needs to inspect a fragment before deciding + what to do with it, such as reading an ``extends`` child's + ``metadata`` before resolving the chain, calls this. No policy + engine is put on-path. + + Raises :class:`ManifestInvalidError` when the source is not + well-formed YAML or the manifest grammar rejects it structurally. + """ + return json.loads(_native.parse_manifest(source)) + + +def merge_manifests(sources: Iterable[str]) -> dict[str, Any]: + """Compose an ordered chain of manifest sources into one ``dict``. + + Later sources overlay earlier ones under the same merge grammar + ``extends`` uses on disk. Every entry must be a fully-formed manifest + fragment: no chain entry may itself carry unresolved ``extends``. + The resulting document is validated before it is returned, so a + chain that would fail as an on-disk ``extends`` fails here too. + + Use this when the manifests come from memory (database rows, + process-supplied overlays) rather than disk; use ``extends`` in the + manifest itself when they come from disk and their layout is fixed. + + Raises :class:`ManifestInvalidError` when the chain is empty or an + entry does not parse. + """ + materialized = list(sources) + return json.loads(_native.merge_manifests(materialized)) + + def supported_manifest_versions() -> tuple[str, ...]: """The manifest grammar versions this engine accepts. diff --git a/sdk/python/src/lib.rs b/sdk/python/src/lib.rs index fb33705..55f36c3 100644 --- a/sdk/python/src/lib.rs +++ b/sdk/python/src/lib.rs @@ -8,36 +8,414 @@ // boundary mean a boundary problem only (unreadable manifest, // non-object context JSON). +use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; +use agent_control_spec::policy::PreparedPolicyInvocation; +use agent_control_spec::runtime::PolicyDispatcher; +use agent_control_spec::telemetry::{TelemetryEvent, TelemetrySink}; use agent_control_spec::{ - ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, Runtime, RuntimeError, - SafetyLevel, SegmentOutcome, StreamEndReason, StreamError, StreamSession, StreamSessionConfig, - StreamSourceType, StreamSpan, StreamTrack, Verdict, SUPPORTED_VERSIONS, + ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, PerfTelemetry, Runtime, + RuntimeError, SafetyLevel, SegmentOutcome, StreamEndReason, StreamError, StreamSession, + StreamSessionConfig, StreamSourceType, StreamSpan, StreamTrack, Verdict, SUPPORTED_VERSIONS, }; use pyo3::create_exception; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; -use serde_json::{json, Value}; +use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString}; +use serde_json::{json, Map, Value}; use std::sync::{Arc, Mutex}; +// --------------------------------------------------------------------- +// Host-supplied dispatchers, telemetry, and perf-telemetry level. +// +// These wrappers hold a `Py` and adapt a Python object into the +// engine's `AnnotatorDispatcher`, `PolicyDispatcher`, or `TelemetrySink` +// trait. The engine calls them from its evaluation path, so failures +// raised on the Python side must become `RuntimeError` on the Rust side: +// the engine then normalizes them into fail-closed `runtime_error:*` +// verdicts and never treats a raising dispatcher as "no annotation". +// +// `Py` is `Send + Sync` in pyo3; the wrappers acquire the GIL +// inside each callback for the actual Python call. +// --------------------------------------------------------------------- + +/// Convert a `serde_json::Value` into a Python object using only the +/// standard container types, so a host dispatcher receives plain +/// `dict`/`list`/`str`/`bool`/`int`/`float`/`None` values. +fn json_to_py<'py>(py: Python<'py>, value: &Value) -> PyResult> { + match value { + Value::Null => Ok(py.None().into_bound(py)), + Value::Bool(b) => Ok(PyBool::new(py, *b).to_owned().into_any()), + Value::Number(n) => { + if let Some(i) = n.as_i64() { + Ok(i.into_pyobject(py)?.into_any()) + } else if let Some(u) = n.as_u64() { + Ok(u.into_pyobject(py)?.into_any()) + } else if let Some(f) = n.as_f64() { + Ok(PyFloat::new(py, f).into_any()) + } else { + // Neither i64/u64/f64 accepted the number, which means an + // arbitrary-precision integer serde_json exposed only as + // string. Round-trip through JSON to preserve the value. + let s = n.to_string(); + Ok(PyString::new(py, &s).into_any()) + } + } + Value::String(s) => Ok(PyString::new(py, s).into_any()), + Value::Array(items) => { + let list = PyList::empty(py); + for item in items { + list.append(json_to_py(py, item)?)?; + } + Ok(list.into_any()) + } + Value::Object(map) => { + let dict = PyDict::new(py); + for (key, value) in map { + dict.set_item(key, json_to_py(py, value)?)?; + } + Ok(dict.into_any()) + } + } +} + +/// Convert a Python object into a `serde_json::Value`. Used to accept a +/// host dispatcher's return value, and structured with the same +/// vocabulary that `json_to_py` produces so a callback that echoes its +/// input round-trips. +fn py_to_json(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(Value::Null); + } + if let Ok(b) = value.cast::() { + return Ok(Value::Bool(b.is_true())); + } + if let Ok(i) = value.cast::() { + if let Ok(v) = i.extract::() { + return Ok(Value::from(v)); + } + if let Ok(v) = i.extract::() { + return Ok(Value::from(v)); + } + // Fall through for oversized integers: represent as string, which + // is the same fallback `json_to_py` uses for arbitrary-precision + // numbers. + return Ok(Value::String(i.str()?.to_string_lossy().into_owned())); + } + if let Ok(f) = value.cast::() { + let v = f.value(); + // `serde_json::Number::from_f64` rejects NaN/Inf, matching the + // JSON grammar the engine's inputs assume elsewhere. + return serde_json::Number::from_f64(v) + .map(Value::Number) + .ok_or_else(|| PyValueError::new_err("dispatcher returned a non-finite float")); + } + if let Ok(s) = value.cast::() { + return Ok(Value::String(s.to_string_lossy().into_owned())); + } + if let Ok(list) = value.cast::() { + let mut out = Vec::with_capacity(list.len()); + for item in list.iter() { + out.push(py_to_json(&item)?); + } + return Ok(Value::Array(out)); + } + if let Ok(dict) = value.cast::() { + let mut out = Map::new(); + for (key, val) in dict.iter() { + let key_str = key + .cast::() + .map_err(|_| { + PyValueError::new_err("dispatcher returned a dict with a non-string key") + })? + .to_string_lossy() + .into_owned(); + out.insert(key_str, py_to_json(&val)?); + } + return Ok(Value::Object(out)); + } + // Fall through: reject anything that is not a plain JSON-compatible + // Python value. Silent conversion via `str()` would hide contract + // violations in host code. + Err(PyValueError::new_err(format!( + "dispatcher returned a value that does not fit the JSON grammar: {}", + value.get_type().name()? + ))) +} + +/// Turn a `PyErr` raised inside a host dispatcher into a Rust +/// `RuntimeError` variant. Distinct variant per role, so the engine's +/// normalized `runtime_error:*` reason names the pipeline stage that +/// failed rather than a generic bucket. +fn py_err_to_annotation_failure(annotator_name: &str, err: PyErr) -> RuntimeError { + RuntimeError::AnnotationFailed(format!("host annotator '{annotator_name}' raised: {err}")) +} + +fn py_err_to_policy_failure(err: PyErr) -> RuntimeError { + RuntimeError::PolicyInvocationFailed(format!("host policy dispatcher raised: {err}")) +} + +/// Call a Python callable, preferring a named method when `callback` +/// exposes one. The old 0.3.1b1 API accepted objects with a `dispatch` +/// method; a plain callable is admitted too so hosts can use a small +/// lambda for tests. +fn call_py_method<'py>( + py: Python<'py>, + callback: &Py, + method: &str, + args: Vec>, +) -> PyResult> { + let bound = callback.bind(py); + let py_tuple = pyo3::types::PyTuple::new(py, args)?; + if let Ok(func) = bound.getattr(method) { + return func.call1(py_tuple); + } + // No named method: treat the object itself as callable. `.call1` is + // itself an attribute lookup, so a non-callable object surfaces its + // own error rather than a fabricated one. + bound.call1(py_tuple) +} + +/// Adapter from an `AnnotatorDispatcher` call to a Python object. +struct PyAnnotatorDispatcher { + callback: Py, +} + +impl AnnotatorDispatcher for PyAnnotatorDispatcher { + fn dispatch( + &self, + annotator_name: &str, + annotator: &AnnotatorInvocation, + preliminary_policy_input: &Value, + ) -> Result { + Python::attach(|py| { + let invocation = serde_json::to_value(annotator).map_err(|err| { + RuntimeError::AnnotationFailed(format!( + "host annotator '{annotator_name}': failed to serialize invocation: {err}" + )) + })?; + let invocation_py = json_to_py(py, &invocation) + .map_err(|err| py_err_to_annotation_failure(annotator_name, err))?; + let prelim_py = json_to_py(py, preliminary_policy_input) + .map_err(|err| py_err_to_annotation_failure(annotator_name, err))?; + let name_py = PyString::new(py, annotator_name).into_any(); + + let result = call_py_method( + py, + &self.callback, + "dispatch", + vec![name_py, invocation_py, prelim_py], + ) + .map_err(|err| py_err_to_annotation_failure(annotator_name, err))?; + + py_to_json(&result).map_err(|err| { + RuntimeError::AnnotationFailed(format!( + "host annotator '{annotator_name}' returned a non-JSON value: {err}" + )) + }) + }) + } +} + +/// Adapter from a `PolicyDispatcher` call to a Python object. Accepts an +/// object with `evaluate` (and optionally `warm`) or a plain callable. +struct PyPolicyDispatcher { + callback: Py, +} + +impl PolicyDispatcher for PyPolicyDispatcher { + fn evaluate(&self, invocation: &PreparedPolicyInvocation) -> Result { + Python::attach(|py| { + let invocation_json = serde_json::to_value(invocation).map_err(|err| { + RuntimeError::PolicyInvocationFailed(format!( + "failed to serialize policy invocation for host dispatcher: {err}" + )) + })?; + let invocation_py = + json_to_py(py, &invocation_json).map_err(py_err_to_policy_failure)?; + + let result = call_py_method(py, &self.callback, "evaluate", vec![invocation_py]) + .map_err(py_err_to_policy_failure)?; + + py_to_json(&result).map_err(|err| { + RuntimeError::PolicyInvocationFailed(format!( + "host policy dispatcher returned a non-JSON value: {err}" + )) + }) + }) + } + + fn warm(&self, invocation: &PreparedPolicyInvocation) -> Result<(), RuntimeError> { + // `warm` is best-effort per the trait contract, and a host that + // does not expose it is not obliged to. Skip silently rather than + // charging every host with implementing an optimization hook. + Python::attach(|py| { + let bound = self.callback.bind(py); + let func = match bound.getattr("warm") { + Ok(func) => func, + Err(_) => return Ok(()), + }; + let invocation_json = serde_json::to_value(invocation).map_err(|err| { + RuntimeError::PolicyInvocationFailed(format!( + "failed to serialize policy invocation for host warm: {err}" + )) + })?; + let invocation_py = + json_to_py(py, &invocation_json).map_err(py_err_to_policy_failure)?; + let args = pyo3::types::PyTuple::new(py, vec![invocation_py]) + .map_err(py_err_to_policy_failure)?; + func.call1(args).map_err(py_err_to_policy_failure)?; + Ok(()) + }) + } +} + +/// Adapter from a `TelemetrySink` call to a Python object. +struct PyTelemetrySink { + callback: Py, +} + +impl PyTelemetrySink { + fn event_to_py<'py>( + &self, + py: Python<'py>, + event: &TelemetryEvent, + ) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("event_type", event.event_type.as_str())?; + dict.set_item("intervention_point", event.intervention_point.as_str())?; + dict.set_item("decision", event.decision.map(|d| d.as_str().to_string()))?; + dict.set_item("reason_code", event.reason_code.clone())?; + dict.set_item("error_class", event.error_class.clone())?; + dict.set_item("policy_id", event.policy_id.clone())?; + dict.set_item("annotators", event.annotators.clone())?; + dict.set_item( + "enforcement_mode", + event.enforcement_mode.map(|m| match m { + agent_control_spec::EnforcementMode::Enforce => "enforce".to_string(), + agent_control_spec::EnforcementMode::EvaluateOnly => "evaluate_only".to_string(), + }), + )?; + dict.set_item("duration_ms", event.duration_ms)?; + dict.set_item("evidence_artefact", event.evidence_artefact.clone())?; + dict.set_item( + "evidence_verification_pointer_keys", + event.evidence_verification_pointer_keys.clone(), + )?; + dict.set_item("action_identity", event.action_identity.clone())?; + let metadata = PyDict::new(py); + for (k, v) in &event.metadata { + metadata.set_item(k, v)?; + } + dict.set_item("metadata", metadata)?; + Ok(dict.into_any()) + } +} + +impl TelemetrySink for PyTelemetrySink { + fn emit(&self, event: TelemetryEvent) { + // `emit` returns `()` in the trait, so a Python-side raise is + // swallowed here after being converted to a printed exception: + // telemetry is out-of-band by design and must not corrupt a + // decision that already succeeded. The engine calls `emit` after + // it has settled a verdict. + Python::attach(|py| { + let event_py = match self.event_to_py(py, &event) { + Ok(value) => value, + Err(err) => { + err.write_unraisable(py, Some(self.callback.bind(py).as_any())); + return; + } + }; + if let Err(err) = call_py_method(py, &self.callback, "emit", vec![event_py]) { + err.write_unraisable(py, Some(self.callback.bind(py).as_any())); + } + }); + } + + fn shutdown(&self) { + Python::attach(|py| { + let bound = self.callback.bind(py); + if let Ok(func) = bound.getattr("shutdown") { + let args = pyo3::types::PyTuple::empty(py); + if let Err(err) = func.call1(args) { + err.write_unraisable(py, Some(bound.as_any())); + } + } + }); + } +} + +fn parse_perf_telemetry(value: &str) -> PyResult { + match value { + "off" => Ok(PerfTelemetry::Off), + "external" => Ok(PerfTelemetry::External), + "full" => Ok(PerfTelemetry::Full), + other => Err(PyValueError::new_err(format!( + "unknown perf_telemetry level '{other}'; expected 'off', 'external', or 'full'" + ))), + } +} + +fn resolve_annotator_dispatcher(dispatcher: Option>) -> Arc { + match dispatcher { + Some(callback) => Arc::new(PyAnnotatorDispatcher { callback }), + None => default_annotator_dispatcher(), + } +} + +fn resolve_policy_dispatcher(dispatcher: Option>) -> Arc { + match dispatcher { + Some(callback) => Arc::new(PyPolicyDispatcher { callback }), + None => Arc::new(BindingPolicyDispatcher::new()), + } +} + +fn resolve_telemetry_sink(sink: Option>) -> Option> { + sink.map(|callback| { + let arc: Arc = Arc::new(PyTelemetrySink { callback }); + arc + }) +} + #[pyclass(frozen)] struct RuntimeHandle { runtime: Runtime, } -/// Build a runtime handle from a manifest path using the zero-config -/// dispatchers (bundled annotators; Rego in process, Cedar through the -/// built-in evaluator, `test` policies through their embedded verdict). +/// Build a runtime handle from a manifest path. +/// +/// Passing no host arguments preserves the zero-config path: bundled +/// annotators, `BindingPolicyDispatcher` for Rego/Cedar/test policies, +/// no-op telemetry, `PerfTelemetry::Off`. Host-supplied callbacks and +/// a `perf_telemetry` other than "off" replace them. #[pyfunction] -fn interceptor_new(manifest_path: &str) -> PyResult { +#[pyo3(signature = ( + manifest_path, + annotator_dispatcher = None, + policy_dispatcher = None, + telemetry_sink = None, + perf_telemetry = "off", +))] +fn interceptor_new( + manifest_path: &str, + annotator_dispatcher: Option>, + policy_dispatcher: Option>, + telemetry_sink: Option>, + perf_telemetry: &str, +) -> PyResult { let manifest = Manifest::from_path(manifest_path).map_err(|e| PyValueError::new_err(format!("{e}")))?; - let runtime = Runtime::new( - manifest, - default_annotator_dispatcher(), - Arc::new(BindingPolicyDispatcher::new()), - ) - .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; + let annotations = resolve_annotator_dispatcher(annotator_dispatcher); + let policy = resolve_policy_dispatcher(policy_dispatcher); + let perf = parse_perf_telemetry(perf_telemetry)?; + let telemetry = resolve_telemetry_sink(telemetry_sink); + let runtime = match telemetry { + Some(sink) => Runtime::with_telemetry_and_perf(manifest, annotations, policy, sink, perf) + .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?, + None => Runtime::with_perf_telemetry(manifest, annotations, policy, perf) + .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?, + }; Ok(RuntimeHandle { runtime }) } @@ -118,6 +496,133 @@ fn supported_manifest_versions() -> Vec { .collect() } +/// Parse a single manifest source into a JSON string representation. +/// +/// Returns the manifest as JSON so a Python wrapper `json.loads` it into a +/// `dict`. `parse_manifest` neither validates nor merges: an authoring +/// tool that needs to inspect a fragment (an `extends` child, for +/// example) can do so without dragging a policy engine on-path. +/// +/// A malformed manifest raises `ManifestInvalid`, exactly as +/// `validate_manifest` does, so a caller does not have to distinguish +/// grammar failures by exception class. +#[pyfunction] +fn parse_manifest(source: &str) -> PyResult { + let manifest = + Manifest::parse_yaml_str(source).map_err(|e| ManifestInvalid::new_err(format!("{e}")))?; + serde_json::to_string(&manifest) + .map_err(|e| PyRuntimeError::new_err(format!("manifest serialization failed: {e}"))) +} + +/// Compose an ordered chain of manifests into one merged JSON document. +/// +/// Later entries overlay earlier ones under the same merge grammar that +/// `extends` uses on disk, and the result is validated before it is +/// returned: a chain that would fail as an on-disk `extends` fails here. +/// Each entry must be a fully-formed manifest fragment (no chain entry +/// may itself carry unresolved `extends`). +/// +/// Empty chains and chains whose entries do not parse raise +/// `ManifestInvalid`. +#[pyfunction] +fn merge_manifests(sources: Vec) -> PyResult { + let refs: Vec<&str> = sources.iter().map(String::as_str).collect(); + let manifest = Manifest::from_yaml_chain(&refs).map_err(|e| match e { + RuntimeError::ManifestInvalid(detail) => ManifestInvalid::new_err(detail), + other => PyValueError::new_err(format!("{other}")), + })?; + serde_json::to_string(&manifest) + .map_err(|e| PyRuntimeError::new_err(format!("merged manifest serialization failed: {e}"))) +} + +/// Field names an authoring tool wants surfaced verbatim. The engine's +/// validation error messages contain them, so a heuristic search picks +/// the first occurrence and returns it as the diagnostic's `field`. +const DIAGNOSTIC_FIELDS: &[&str] = &[ + "agent_control_specification_version", + "policy_target_kind", + "policy_target", + "tool_name_from", + "annotations", + "annotators", + "intervention_points", + "intervention point", + "extends", + "policies", + "policy.id", + "approval", + "metadata", + "tools", +]; + +fn guess_field(message: &str) -> Option { + // Longest match first: `policy_target_kind` is a superstring of + // `policy_target`, so ordering `DIAGNOSTIC_FIELDS` by length keeps + // `policy_target` from swallowing a `policy_target_kind` message. + let mut ordered: Vec<&&str> = DIAGNOSTIC_FIELDS.iter().collect(); + ordered.sort_by_key(|f| std::cmp::Reverse(f.len())); + for field in ordered { + if message.contains(*field) { + return Some((*field).to_string()); + } + } + None +} + +/// Structured validation diagnostics as a JSON array string. +/// +/// Returned entries have shape +/// `{"reason_code": str, "message": str, "field": str | null}`. The +/// engine's validation surface reports one failure at a time, so a +/// successful validation returns `[]` and every failed one returns a +/// single-entry list. Wrapping in a list leaves room for a batch +/// validation to grow into the same shape without a breaking rename. +/// +/// A manifest that uses `extends` returns a single diagnostic pointing +/// the caller at file-based validation, matching `validate_manifest`. +#[pyfunction] +fn validate_manifest_diagnostics(source: &str) -> PyResult { + fn diagnostic(reason: &str, message: &str) -> Value { + json!({ + "reason_code": reason, + "message": message, + "field": guess_field(message), + }) + } + let parsed = match Manifest::parse_yaml_str(source) { + Ok(manifest) => manifest, + Err(RuntimeError::ManifestInvalid(detail)) => { + return serde_json::to_string(&json!([diagnostic( + "runtime_error:manifest_invalid", + &detail + )])) + .map_err(|e| { + PyRuntimeError::new_err(format!("diagnostics serialization failed: {e}")) + }); + } + Err(other) => { + return Err(PyValueError::new_err(format!("{other}"))); + } + }; + if !parsed.extends.is_empty() { + let msg = "manifest extends other manifests; validation needs the merged document. Use \ + validate_manifest_file or merge_manifests, both of which resolve the chain."; + return serde_json::to_string(&json!([diagnostic("runtime_error:manifest_invalid", msg)])) + .map_err(|e| { + PyRuntimeError::new_err(format!("diagnostics serialization failed: {e}")) + }); + } + match parsed.validate() { + Ok(()) => Ok("[]".to_string()), + Err(RuntimeError::ManifestInvalid(detail)) => serde_json::to_string(&json!([diagnostic( + "runtime_error:manifest_invalid", + &detail + )])) + .map_err(|e| PyRuntimeError::new_err(format!("diagnostics serialization failed: {e}"))), + Err(other) => Err(PyValueError::new_err(format!("{other}"))), + } +} + // --------------------------------------------------------------------- // Activated policy: one policy version, readied once, evaluated many // times. @@ -138,19 +643,32 @@ struct PolicyHandle { } /// Activate the manifest at `manifest_path`, readying every policy it -/// binds, against the zero-config dispatchers. +/// binds. /// -/// This is the expensive call: it reads the manifest, loads every Rego -/// module and data document, and compiles the entrypoint each -/// intervention point queries. Do it once per policy version and keep -/// the handle; `policy_evaluate` then costs no I/O and no compile. +/// Passing no host arguments preserves the zero-config activation. +/// Host-supplied dispatchers replace the bundled ones; they are used +/// for readying (via `PolicyDispatcher::warm`) as well as for every +/// later evaluation, so a policy compile pays its cost here and not on +/// the first decision. /// /// Readying is bounded by the eval timeout. A policy too slow to ready /// inside it activates anyway and pays that cost on its first /// evaluation instead. #[pyfunction] -fn policy_activate(py: Python<'_>, manifest_path: &str) -> PyResult { +#[pyo3(signature = ( + manifest_path, + annotator_dispatcher = None, + policy_dispatcher = None, +))] +fn policy_activate( + py: Python<'_>, + manifest_path: &str, + annotator_dispatcher: Option>, + policy_dispatcher: Option>, +) -> PyResult { let manifest_path = manifest_path.to_string(); + let annotations = resolve_annotator_dispatcher(annotator_dispatcher); + let policy = resolve_policy_dispatcher(policy_dispatcher); // Activation is the expensive call and touches no Python object, so // it must not hold the GIL: a host activating a new policy version // in a background thread would otherwise stall every request thread @@ -158,12 +676,8 @@ fn policy_activate(py: Python<'_>, manifest_path: &str) -> PyResult, manifest_path: &str) -> PyResult, manifest_yaml: &str, bundles_json: &str, + annotator_dispatcher: Option>, + policy_dispatcher: Option>, ) -> PyResult { let bundles: std::collections::BTreeMap = serde_json::from_str(bundles_json) .map_err(|e| PyValueError::new_err(format!("bundles do not parse: {e}")))?; let manifest_yaml = manifest_yaml.to_string(); + let annotations = resolve_annotator_dispatcher(annotator_dispatcher); + let policy = resolve_policy_dispatcher(policy_dispatcher); // Same reason as `policy_activate`: loading and compiling touches no // Python object and must not stall other threads. let policy = py.detach(move || { - ActivatedPolicy::activate_from_memory(&manifest_yaml, bundles).map_err(|e| match e { - RuntimeError::ManifestInvalid(detail) => ManifestInvalid::new_err(detail), - other => PyRuntimeError::new_err(format!("{other}")), - }) + ActivatedPolicy::activate_from_memory_with(&manifest_yaml, bundles, annotations, policy) + .map_err(|e| match e { + RuntimeError::ManifestInvalid(detail) => ManifestInvalid::new_err(detail), + other => PyRuntimeError::new_err(format!("{other}")), + }) })?; Ok(PolicyHandle { policy }) } @@ -576,6 +1104,9 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("ManifestInvalid", m.py().get_type::())?; m.add_function(wrap_pyfunction!(validate_manifest, m)?)?; m.add_function(wrap_pyfunction!(validate_manifest_file, m)?)?; + m.add_function(wrap_pyfunction!(validate_manifest_diagnostics, m)?)?; + m.add_function(wrap_pyfunction!(parse_manifest, m)?)?; + m.add_function(wrap_pyfunction!(merge_manifests, m)?)?; m.add_function(wrap_pyfunction!(supported_manifest_versions, m)?)?; Ok(()) } diff --git a/sdk/python/tests/test_host_hooks.py b/sdk/python/tests/test_host_hooks.py new file mode 100644 index 0000000..acccf8f --- /dev/null +++ b/sdk/python/tests/test_host_hooks.py @@ -0,0 +1,394 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Host extension points: annotator and policy dispatchers, telemetry +sinks, perf-telemetry levels, manifest tooling, and structured +validation diagnostics. + +Restores the 0.3.1b1 shape of ``AgentControl.from_native(manifest, +annotator_dispatcher=...)`` that a consumer using an HTTP-backed +Content Safety dispatcher depended on. Verifies both that the host +callback runs and that its output reaches the policy decision, so a +regression that reintroduced the 0.4 hardcoded zero-config path would +fail. +""" + +from __future__ import annotations + +import pathlib + +import pytest +from agent_control_spec import ( + PERF_TELEMETRY_LEVELS, + AcsInterceptor, + ActivatedPolicy, + ManifestInvalidError, + merge_manifests, + parse_manifest, + validate_manifest_detailed, +) +from agent_hooks import AgentContextBuilder + +FIXTURES = pathlib.Path(__file__).parent / "fixtures" +DEFAULT_MANIFEST = str(FIXTURES / "manifest.yaml") + + +def _builder() -> AgentContextBuilder: + return AgentContextBuilder(agent_id="a", framework="test", session_id="s") + + +# --------------------------------------------------------------------- +# Host annotator dispatcher: manifest + rego gate that reads annotations. +# +# The rego module denies whenever `input.annotations.classify.blocked` +# is True. A host dispatcher that returns `{"blocked": True}` therefore +# turns an allow-by-default input into a deny; a dispatcher that returns +# `{"blocked": False}` leaves it allowed. That is exactly the shape of +# the Azure Content Safety adapter the consumer needs to plug in. +# --------------------------------------------------------------------- + +ANNOTATOR_MANIFEST = """ +agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: python-host-hooks +policies: + gate: + type: rego + bundle: ./policy + query: data.gate.verdict +annotators: + classify: + type: classifier +intervention_points: + input: + policy_target: "$.input" + policy_target_kind: user_input + policy: + id: gate + annotations: + classify: + from: $target.content +""" + +REGO_MODULE = """ +package gate + +default verdict := {"decision": "allow"} + +verdict := { + "decision": "deny", + "reason": "blocked_by_annotator", +} if { + input.annotations.classify.blocked == true +} +""" + + +def _bundles() -> dict: + return {"gate": {"modules": {"gate.rego": REGO_MODULE}}} + + +class RecordingAnnotator: + """Host dispatcher whose ``dispatch`` records every call and returns + a fixed annotation. Deliberately object-with-method-shaped, matching + the 0.3.1b1 API a consumer wired up.""" + + def __init__(self, payload): + self.payload = payload + self.calls: list[tuple[str, dict, dict]] = [] + + def dispatch(self, annotator_name, annotator, preliminary_policy_input): + self.calls.append((annotator_name, annotator, preliminary_policy_input)) + return self.payload + + +def test_host_annotator_dispatcher_reaches_the_policy_decision(): + """The consumer's blocked use case, made concrete: an annotator + dispatcher that says the content is blocked flips the verdict.""" + dispatcher = RecordingAnnotator({"blocked": True, "categories": ["hate"]}) + policy = ActivatedPolicy.from_memory( + ANNOTATOR_MANIFEST, + _bundles(), + annotator_dispatcher=dispatcher, + ) + verdict = policy.evaluate("input", _builder().input(content="offensive text")) + + # The dispatcher was actually called by the engine, not skipped. + assert len(dispatcher.calls) == 1 + name, invocation, prelim = dispatcher.calls[0] + assert name == "classify" + # The dispatcher sees the annotator invocation shape the engine + # built, including the flattened `type` field from the annotator + # config and the `from` field from the annotation config. + assert invocation["type"] == "classifier" + assert invocation["from"] == "$target.content" + # And the preliminary policy input, so an HTTP-backed dispatcher can + # decide whether to make a call. The input builder wraps content in + # `{content, role}`, which is what a downstream classifier reads. + assert prelim["intervention_point"] == "input" + assert prelim["policy_target"]["value"]["content"] == "offensive text" + + # And the annotation actually affected the decision. + assert verdict.decision.value == "deny" + assert verdict.reason == "blocked_by_annotator" + + +def test_host_annotator_dispatcher_returning_allow_leaves_verdict_allowed(): + """The complementary case: same policy, same manifest, dispatcher + that returns a clean annotation. The engine still ran the + dispatcher and its annotation reached the rego module.""" + dispatcher = RecordingAnnotator({"blocked": False}) + policy = ActivatedPolicy.from_memory( + ANNOTATOR_MANIFEST, + _bundles(), + annotator_dispatcher=dispatcher, + ) + verdict = policy.evaluate("input", _builder().input(content="hello")) + assert dispatcher.calls, "dispatcher must run on every evaluation" + assert verdict.decision.value == "allow" + + +def test_host_annotator_dispatcher_that_raises_fails_closed_not_silent(): + """Contract: a raising annotator dispatcher must never be treated + as 'no annotation'. The engine's fail-closed path applies, so the + verdict is deny with a ``runtime_error:*`` reason.""" + + class Boom: + def dispatch(self, name, annotator, prelim): + raise RuntimeError("content safety endpoint unreachable") + + policy = ActivatedPolicy.from_memory( + ANNOTATOR_MANIFEST, + _bundles(), + annotator_dispatcher=Boom(), + ) + verdict = policy.evaluate("input", _builder().input(content="hello")) + assert verdict.decision.value == "deny" + assert verdict.reason.startswith("runtime_error:"), verdict.reason + + +def test_host_annotator_dispatcher_can_be_a_plain_callable(): + """The old API accepted objects with `dispatch`; a plain callable + with the same signature is admitted too so hosts can write a small + lambda for tests without wrapping it in a class.""" + calls = [] + + def dispatcher(name, annotator, prelim): + calls.append(name) + return {"blocked": True} + + policy = ActivatedPolicy.from_memory( + ANNOTATOR_MANIFEST, + _bundles(), + annotator_dispatcher=dispatcher, + ) + verdict = policy.evaluate("input", _builder().input(content="hi")) + assert calls == ["classify"] + assert verdict.decision.value == "deny" + + +# --------------------------------------------------------------------- +# Zero-config parity: with no arguments, the API is byte-for-byte the +# same as today's zero-config path. +# --------------------------------------------------------------------- + + +def test_no_dispatcher_behaves_as_zero_config(): + # No annotator, no policy, no telemetry, no perf: identical to the + # existing zero-config test surface. + acs = AcsInterceptor(DEFAULT_MANIFEST) + allow = acs.intercept(_builder().input(content="hello")) + deny = acs.intercept( + _builder().pre_tool_call(call_id="t1", name="search", args={"q": "x"}) + ) + assert allow.decision.value == "allow" + assert deny.decision.value == "deny" + assert deny.reason == "blocked_by_policy" + + +def test_no_dispatcher_on_activated_policy_behaves_as_zero_config(): + policy = ActivatedPolicy(DEFAULT_MANIFEST) + verdict = policy.evaluate("input", _builder().input(content="hello")) + assert verdict.decision.value == "allow" + + +# --------------------------------------------------------------------- +# Telemetry: a host sink receives events during evaluation, and +# perf-telemetry levels round-trip. +# --------------------------------------------------------------------- + + +class RecordingTelemetrySink: + def __init__(self): + self.events: list[dict] = [] + + def emit(self, event): + self.events.append(event) + + +def test_telemetry_sink_receives_events_during_evaluation(): + sink = RecordingTelemetrySink() + acs = AcsInterceptor(DEFAULT_MANIFEST, telemetry_sink=sink) + acs.intercept(_builder().input(content="hello")) + # A decision event is guaranteed under any perf-telemetry level for + # a settled evaluation. The rest of the fields ride along; we assert + # the ones a monitoring host reads. + decisions = [e for e in sink.events if e["event_type"] == "decision"] + assert decisions, f"expected at least one decision event, got {sink.events}" + decision = decisions[0] + assert decision["intervention_point"] == "input" + assert decision["decision"] == "allow" + assert decision["policy_id"] == "allow_all" + + +def test_telemetry_sink_is_called_for_a_denying_verdict_too(): + sink = RecordingTelemetrySink() + acs = AcsInterceptor(DEFAULT_MANIFEST, telemetry_sink=sink) + acs.intercept( + _builder().pre_tool_call(call_id="t1", name="search", args={"q": "x"}) + ) + decisions = [e for e in sink.events if e["event_type"] == "decision"] + assert decisions + assert decisions[0]["decision"] == "deny" + assert decisions[0]["reason_code"] == "blocked_by_policy" + + +def test_perf_telemetry_levels_roundtrip(): + # All three engine levels construct without raising. + for level in PERF_TELEMETRY_LEVELS: + AcsInterceptor(DEFAULT_MANIFEST, perf_telemetry=level) + + +def test_unknown_perf_telemetry_level_is_rejected(): + with pytest.raises(ValueError, match="perf_telemetry"): + AcsInterceptor(DEFAULT_MANIFEST, perf_telemetry="verbose") + + +def test_telemetry_sink_can_be_a_plain_callable(): + events: list[dict] = [] + acs = AcsInterceptor(DEFAULT_MANIFEST, telemetry_sink=events.append) + acs.intercept(_builder().input(content="hello")) + assert any(e["event_type"] == "decision" for e in events) + + +# --------------------------------------------------------------------- +# Manifest tooling: parse_manifest and merge_manifests. +# --------------------------------------------------------------------- + + +VALID_MANIFEST = (FIXTURES / "manifest.yaml").read_text(encoding="utf-8") + + +def test_parse_manifest_returns_the_parsed_structure(): + parsed = parse_manifest(VALID_MANIFEST) + assert isinstance(parsed, dict) + # The version key is on the top level, and the intervention_points + # are keyed by point name. + assert parsed["agent_control_specification_version"].startswith("0.4.0") + assert "input" in parsed["intervention_points"] + + +def test_parse_manifest_rejects_malformed_source(): + with pytest.raises(ManifestInvalidError): + parse_manifest("agent_control_specification_version: [") + + +def test_merge_manifests_composes_two_partial_documents(): + # Base declares the policies, overlay adds the intervention points + # that bind them. Both fragments are needed for a runnable manifest; + # neither is one on its own. That's the composition merge_manifests + # is for. + base = """ +agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: composed +policies: + p: + type: test + verdict: + decision: allow + q: + type: test + verdict: + decision: deny + reason: blocked_by_overlay +""" + overlay = """ +agent_control_specification_version: "0.4.0-alpha.1" +intervention_points: + input: + policy_target: "$.input" + policy: + id: q +""" + merged = merge_manifests([base, overlay]) + assert isinstance(merged, dict) + # The overlay's intervention point landed, bound to the base's `q` + # policy. Composition took place. + assert merged["intervention_points"]["input"]["policy"]["id"] == "q" + # And both base policies are still declared: the merge unions + # definitions. + assert set(merged["policies"]) == {"p", "q"} + # And the merged document is valid: the runtime accepts it. + policy = ActivatedPolicy.from_memory( + # ActivatedPolicy accepts YAML; a merged dict round-trips through + # JSON, which the manifest grammar admits. + __import__("json").dumps(merged), + {}, + ) + verdict = policy.evaluate("input", _builder().input(content="hi")) + assert verdict.decision.value == "deny" + assert verdict.reason == "blocked_by_overlay" + + +def test_merge_manifests_rejects_an_empty_chain(): + with pytest.raises(ManifestInvalidError): + merge_manifests([]) + + +# --------------------------------------------------------------------- +# Structured validation diagnostics. +# --------------------------------------------------------------------- + + +def test_valid_manifest_produces_no_diagnostics(): + assert validate_manifest_detailed(VALID_MANIFEST) == [] + + +def test_diagnostics_name_the_offending_field_for_an_invalid_manifest(): + # A policy_target_kind of "" is rejected by validate. The diagnostic + # should name that field. + bad = VALID_MANIFEST.replace( + '"$.input"', + '"$.input"\n policy_target_kind: ""', + 1, + ) + diagnostics = validate_manifest_detailed(bad) + assert diagnostics, "expected the manifest to be rejected" + entry = diagnostics[0] + assert entry["reason_code"] == "runtime_error:manifest_invalid" + assert entry["field"] == "policy_target_kind", entry + # The engine's full message is preserved verbatim, so a tool can + # surface it in an editor without paraphrasing. + assert "policy_target_kind" in entry["message"] + + +def test_diagnostics_report_unsupported_version_field(): + bad = VALID_MANIFEST.replace('"0.4.0-alpha.1"', '"0.3.1-beta"') + diagnostics = validate_manifest_detailed(bad) + assert len(diagnostics) == 1 + entry = diagnostics[0] + assert entry["reason_code"] == "runtime_error:manifest_invalid" + assert entry["field"] == "agent_control_specification_version" + assert "0.3.1-beta" in entry["message"] + + +def test_diagnostics_flag_manifests_that_use_extends(): + # A manifest that inherits cannot be judged from its own source. + # `validate_manifest_detailed` reports that as a single diagnostic + # instead of raising, so a batch runner can bucket the result rather + # than mid-loop-except. + extended = VALID_MANIFEST + "\nextends:\n - path: ./parent.yaml\n" + diagnostics = validate_manifest_detailed(extended) + assert diagnostics + assert diagnostics[0]["reason_code"] == "runtime_error:manifest_invalid" + assert "extends" in diagnostics[0]["message"] diff --git a/tests/conformance/parity/bundle/policy.rego b/tests/conformance/parity/bundle/policy.rego new file mode 100644 index 0000000..0decc7c --- /dev/null +++ b/tests/conformance/parity/bundle/policy.rego @@ -0,0 +1,5 @@ +package acs + +decision := {"decision": "deny", "reason": "unsafe_content"} if { + input.annotations.content_safety.severity >= 4 +} else := {"decision": "allow"} diff --git a/tests/conformance/parity/cross_language_parity.py b/tests/conformance/parity/cross_language_parity.py index b1f610b..fa36ac6 100644 --- a/tests/conformance/parity/cross_language_parity.py +++ b/tests/conformance/parity/cross_language_parity.py @@ -41,6 +41,7 @@ ROOT = Path(__file__).resolve().parents[3] MANIFEST = Path(__file__).resolve().parent / "manifest.yaml" +HOOKS_MANIFEST = Path(__file__).resolve().parent / "host-hooks-manifest.yaml" # "hi" plus one astral-plane scalar: 3 runes, 4 UTF-16 code units. TEXT = "hi\U0001f600" @@ -87,6 +88,19 @@ "transformed": False, # Absent, never 0 and never -1. Each language spells it natively. "safe_offset_settled": None, + # Host extension points. The classifier's answer must decide the + # verdict, and a classifier that could not be reached must deny + # rather than read as one that found nothing. + "hook_benign_decision": "allow", + "hook_harmful_decision": "deny", + "hook_harmful_reason": "unsafe_content", + "hook_failure_decision": "deny", + "hook_failure_reason": "runtime_error:annotation_failed", + "hook_dispatcher_calls": 1, + # Manifest tooling. + "parsed_has_points": True, + "diagnostics_on_bad": 1, + "diagnostics_on_good": 0, } @@ -101,9 +115,58 @@ def _run(cmd: list[str], **kw) -> dict: use agent_control_spec::{ActivatedPolicy, InterceptionPoint, Manifest, Runtime}; use std::sync::Arc; +struct Classifier { + severity: i64, + calls: std::sync::atomic::AtomicUsize, +} + +impl agent_control_spec::annotation::AnnotatorDispatcher for Classifier { + fn dispatch( + &self, + _name: &str, + _annotator: &agent_control_spec::annotation::AnnotatorInvocation, + _prelim: &serde_json::Value, + ) -> Result { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(serde_json::json!({ "severity": self.severity })) + } +} + +struct Broken; + +impl agent_control_spec::annotation::AnnotatorDispatcher for Broken { + fn dispatch( + &self, + _name: &str, + _annotator: &agent_control_spec::annotation::AnnotatorInvocation, + _prelim: &serde_json::Value, + ) -> Result { + Err(agent_control_spec::RuntimeError::AnnotationFailed( + "classifier unreachable".to_string(), + )) + } +} + +fn hook( + hooks_manifest: &str, + dispatcher: Arc, +) -> (String, Option) { + let manifest = Manifest::from_path(hooks_manifest).expect("hooks manifest"); + let runtime = Runtime::new(manifest, dispatcher, Arc::new(BindingPolicyDispatcher::new())) + .expect("hooks runtime"); + let ctx: serde_json::Value = + serde_json::from_str(r#"{"interception_point":"input","input":"hello"}"#).expect("ctx"); + let verdict = runtime.evaluate(&ctx).verdict; + ( + format!("{:?}", verdict.decision).to_lowercase(), + verdict.reason.clone(), + ) +} + fn main() { let manifest_path = std::env::args().nth(1).expect("manifest path"); let text = std::env::args().nth(2).expect("text"); + let hooks_manifest = std::env::args().nth(3).expect("hooks manifest"); let validate_good = match Manifest::from_path(&manifest_path) { Ok(_) => "ok", @@ -162,9 +225,41 @@ def _run(cmd: list[str], **kw) -> dict: let confirmed = session.watermark(StreamTrack::Response).confirmed(); let completion = session.finish(); + let benign = Arc::new(Classifier { + severity: 1, + calls: std::sync::atomic::AtomicUsize::new(0), + }); + let b = hook(&hooks_manifest, benign.clone()); + let hh = hook( + &hooks_manifest, + Arc::new(Classifier { + severity: 7, + calls: std::sync::atomic::AtomicUsize::new(0), + }), + ); + let f = hook(&hooks_manifest, Arc::new(Broken)); + let parsed = Manifest::from_yaml_str(&std::fs::read_to_string(&manifest_path).expect("read")) + .expect("parse"); + let parsed_json = serde_json::to_value(&parsed).expect("parsed json"); + let bad_diags = usize::from(Manifest::from_yaml_str(BAD_MANIFEST).is_err()); + let good_diags = usize::from( + Manifest::from_yaml_str(&std::fs::read_to_string(&manifest_path).expect("read")) + .and_then(|m| m.validate()) + .is_err(), + ); + println!( "{}", serde_json::json!({ + "hook_benign_decision": b.0, + "hook_harmful_decision": hh.0, + "hook_harmful_reason": hh.1, + "hook_failure_decision": f.0, + "hook_failure_reason": f.1, + "hook_dispatcher_calls": benign.calls.load(std::sync::atomic::Ordering::SeqCst), + "parsed_has_points": parsed_json.get("intervention_points").is_some(), + "diagnostics_on_bad": bad_diags, + "diagnostics_on_good": good_diags, "supported_versions_nonempty": !agent_control_spec::SUPPORTED_VERSIONS.is_empty(), "validate_good": validate_good, "validate_bad": validate_bad, @@ -225,6 +320,7 @@ def rust() -> dict: "--", str(MANIFEST), TEXT, + str(HOOKS_MANIFEST), ] ) @@ -235,6 +331,7 @@ def python_binding() -> dict: from agent_control_spec import ( AcsInterceptor, ActivatedPolicy, StreamSession, supported_manifest_versions, validate_manifest, + parse_manifest, validate_manifest_detailed, ) def check(source): @@ -266,7 +363,39 @@ def decision(v): return str(getattr(d, "value", d)).lower() +class _Classifier: + def __init__(self, sev): self.sev = sev; self.calls = 0 + def dispatch(self, name, annotator, prelim): + self.calls += 1 + return {{"severity": self.sev}} + +class _Broken: + def dispatch(self, *a, **k): + raise RuntimeError("classifier unreachable") + +def _hook(dispatcher): + i = AcsInterceptor({str(HOOKS_MANIFEST)!r}, annotator_dispatcher=dispatcher) + v = i.intercept({{"interception_point": "input", "input": "hello"}}) + return (str(getattr(v.decision, "value", v.decision)).lower(), v.reason) + +_benign = _Classifier(1) +_b = _hook(_benign) +_h = _hook(_Classifier(7)) +_f = _hook(_Broken()) +_parsed = parse_manifest(open({str(MANIFEST)!r}).read()) +_bad_diags = validate_manifest_detailed({BAD_MANIFEST!r}) +_good_diags = validate_manifest_detailed(open({str(MANIFEST)!r}).read()) + print(json.dumps({{ + "hook_benign_decision": _b[0], + "hook_harmful_decision": _h[0], + "hook_harmful_reason": _h[1], + "hook_failure_decision": _f[0], + "hook_failure_reason": _f[1], + "hook_dispatcher_calls": _benign.calls, + "parsed_has_points": "intervention_points" in _parsed, + "diagnostics_on_bad": len(_bad_diags), + "diagnostics_on_good": len(_good_diags), "supported_versions_nonempty": len(supported_manifest_versions()) > 0, "validate_good": check(open({str(MANIFEST)!r}).read()), "validate_bad": check({BAD_MANIFEST!r}), @@ -316,7 +445,29 @@ def node() -> dict: const confirmed = session.watermark('response').confirmed; const completion = session.finish(); +function hook(d) {{ + const i = acs.AcsInterceptor.fromPath({json.dumps(str(HOOKS_MANIFEST))}, {{ annotatorDispatcher: d }}); + const v = i.intercept({{ interception_point: 'input', input: 'hello' }}); + return [String(v.decision).toLowerCase(), v.reason ?? null]; +}} +let hookCalls = 0; +const b = hook(() => {{ hookCalls++; return {{ severity: 1 }}; }}); +const hh = hook(() => ({{ severity: 7 }})); +const f = hook(() => {{ throw new Error('classifier unreachable'); }}); +const parsed = acs.parseManifest(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')); +const badDiags = acs.validateManifestDetailed({json.dumps(BAD_MANIFEST)}); +const goodDiags = acs.validateManifestDetailed(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')); + console.log(JSON.stringify({{ + hook_benign_decision: b[0], + hook_harmful_decision: hh[0], + hook_harmful_reason: hh[1], + hook_failure_decision: f[0], + hook_failure_reason: f[1], + hook_dispatcher_calls: hookCalls, + parsed_has_points: Object.prototype.hasOwnProperty.call(parsed, 'intervention_points'), + diagnostics_on_bad: badDiags.length, + diagnostics_on_good: goodDiags.length, supported_versions_nonempty: acs.supportedManifestVersions().length > 0, validate_good: check(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')), validate_bad: check({json.dumps(BAD_MANIFEST)}), @@ -370,8 +521,32 @@ def node() -> dict: var confirmed = session.Watermark(StreamTrack.Response).Confirmed; var completion = session.Finish(); +static (string, string?) Hook(AnnotatorDispatcher d) +{ + using var i = AcsHostInterceptor.FromPath(HOOKS_MANIFEST, annotator: d); + var v = i.InterceptAsync(new AgentContext(JsonNode.Parse(ALLOW_JSON)!.AsObject())).AsTask().Result; + return (v.Decision.ToString().ToLowerInvariant(), v.Reason); +} + +var hookCalls = 0; +var b = Hook((_, _, _) => { hookCalls++; return SEV1; }); +var hh = Hook((_, _, _) => SEV7); +var f = Hook((_, _, _) => throw new InvalidOperationException("classifier unreachable")); +var parsed = AcsManifestTools.Parse(File.ReadAllText(manifest)); +var badDiags = AcsManifestTools.Diagnostics(BAD_JSON); +var goodDiags = AcsManifestTools.Diagnostics(File.ReadAllText(manifest)); + Console.WriteLine(JsonSerializer.Serialize(new Dictionary { + ["hook_benign_decision"] = b.Item1, + ["hook_harmful_decision"] = hh.Item1, + ["hook_harmful_reason"] = hh.Item2, + ["hook_failure_decision"] = f.Item1, + ["hook_failure_reason"] = f.Item2, + ["hook_dispatcher_calls"] = hookCalls, + ["parsed_has_points"] = parsed.Contains("intervention_points"), + ["diagnostics_on_bad"] = badDiags.Count, + ["diagnostics_on_good"] = goodDiags.Count, ["supported_versions_nonempty"] = AcsManifest.SupportedVersions().Count > 0, ["validate_good"] = Check(() => AcsManifest.Validate(File.ReadAllText(manifest))), ["validate_bad"] = Check(() => AcsManifest.Validate(BAD_JSON)), @@ -415,6 +590,9 @@ def dotnet() -> dict: ) program = ( DOTNET_PROGRAM.replace("MANIFEST_PATH", json.dumps(str(MANIFEST))) + .replace("HOOKS_MANIFEST", json.dumps(str(HOOKS_MANIFEST))) + .replace("SEV1", json.dumps(json.dumps({"severity": 1}))) + .replace("SEV7", json.dumps(json.dumps({"severity": 7}))) .replace("ALLOW_JSON", json.dumps(json.dumps(ALLOW_CONTEXT))) .replace("DENY_JSON", json.dumps(json.dumps(DENY_CONTEXT))) .replace("BAD_JSON", json.dumps(BAD_MANIFEST)) diff --git a/tests/conformance/parity/host-hooks-manifest.yaml b/tests/conformance/parity/host-hooks-manifest.yaml new file mode 100644 index 0000000..ccb6bd9 --- /dev/null +++ b/tests/conformance/parity/host-hooks-manifest.yaml @@ -0,0 +1,19 @@ +agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: host-annotator-proof +annotators: + content_safety: + type: classifier +policies: + gate: + type: rego + bundle: ./bundle +intervention_points: + input: + policy_target: "$snap.input" + annotations: + content_safety: + from: "$target" + policy: + id: gate + query: data.acs.decision From 2f86730f28a8a4de93a7439ff9b78453e3df6865 Mon Sep 17 00:00:00 2001 From: Liam Crumm Date: Tue, 11 Aug 2026 18:58:29 +0000 Subject: [PATCH 05/10] test: run the whole surface from the built artifacts, not the checkout The suites and the parity harness import from this checkout, where the engine sits on the loader path, the TypeScript is in dist, and the Python package resolves from source. A consumer has a crate, a wheel, a tarball and a nupkg, and no test that imports from the tree can tell the difference. That gap already shipped once. ResponsibleAI.AgentControlSpec 0.4.0-alpha.2 passed its whole suite and threw DllNotFoundException on the first call any consumer made, because the package carried no native library and CI supplied one through LD_LIBRARY_PATH. So build the four artifacts the release workflow builds, install each into a throwaway project outside the repository, and run the manifest tooling, the host extension points and streaming from there. Node is packed as two artifacts because napi splits it that way, and the main package gains its optionalDependencies at publish time, so both halves are installed explicitly. Confirmed the check fails on a package that ships without its native library rather than reporting green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Liam Crumm --- .github/workflows/ci.yml | 25 ++ .gitignore | 6 + docs/STREAMING.md | 7 +- scripts/verify-artifacts.py | 491 ++++++++++++++++++++++++++++++++++++ 4 files changed, 528 insertions(+), 1 deletion(-) create mode 100644 scripts/verify-artifacts.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84c7d3b..f2c3108 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,3 +197,28 @@ jobs: run: python tests/conformance/parity/cross_language_parity.py env: LD_LIBRARY_PATH: ${{ github.workspace }}/target/release + + # The suites and the parity harness run against this checkout, where the + # engine is on the loader path and the packages are importable from + # source. A consumer has a crate, a wheel, a tarball and a nupkg. A + # published .NET package once passed every test and threw + # DllNotFoundException on the first call a consumer made, so build the + # real artifacts and run the surface from a clean install of each. + artifacts: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: "8.0.x" + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - run: pip install maturin==1.8.7 + - run: npm ci + working-directory: sdk/node + - run: python scripts/verify-artifacts.py diff --git a/.gitignore b/.gitignore index 6c18f25..2effd7f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,9 @@ obj/ # Per-RID engine binaries staged for the .NET package (built, never committed). /sdk/dotnet/native/ + +# napi writes these next to the committed binding.js loader when the +# platform build runs in place. The loader the package ships is +# binding.js; these are byproducts. +/sdk/node/index.js +/sdk/node/index.d.ts diff --git a/docs/STREAMING.md b/docs/STREAMING.md index e8453c1..8e4ea06 100644 --- a/docs/STREAMING.md +++ b/docs/STREAMING.md @@ -168,4 +168,9 @@ boundary, so agreement is not structural. `tests/conformance/parity/cross_language_parity.py` runs the whole public surface, streaming and otherwise, in all four and fails on any -divergence. It is part of CI. +divergence. + +That harness runs against this checkout. `scripts/verify-artifacts.py` +runs the same surface from a clean install of the built crate, wheel, +tarball and nupkg, which is the only thing that catches a package +shipping less than the repository holds. Both are part of CI. diff --git a/scripts/verify-artifacts.py b/scripts/verify-artifacts.py new file mode 100644 index 0000000..4afa5c5 --- /dev/null +++ b/scripts/verify-artifacts.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Build every published artifact and exercise it from a clean install. + +The test suites and the parity harness run against this checkout, where +the engine is on the loader path, the TypeScript is in `dist/`, and the +Python package is importable from source. A consumer has none of that. +They have a crate, a wheel, a tarball and a nupkg. + +The difference is not theoretical. `ResponsibleAI.AgentControlSpec` +0.4.0-alpha.2 passed its whole suite and threw `DllNotFoundException` on +the first call any consumer made, because the package carried no native +library and CI supplied one through `LD_LIBRARY_PATH`. No test that +imports from the checkout can catch that class of defect. + +So this builds the four artifacts the release workflow builds, installs +each into a throwaway project outside the repository, and runs the whole +public surface there: manifest tooling, the host extension points, and +streaming. Every language must answer identically. + +Node is packed as two artifacts because napi splits it that way: the +JavaScript package and a per-platform binary package. The published +package.json gains its `optionalDependencies` at publish time, so a +locally packed main tarball cannot resolve the binary on its own and +both halves are installed explicitly here. + +Usage: python scripts/verify-artifacts.py [--keep] +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HOOKS_MANIFEST = ROOT / "tests" / "conformance" / "parity" / "host-hooks-manifest.yaml" +BAD_MANIFEST = 'agent_control_specification_version: "0.4.0-alpha.1"\nmetadata: {}\n' + +EXPECTED = { + "non_streaming": True, + "hook_benign": "allow", + "hook_harmful": "deny", + "hook_harmful_reason": "unsafe_content", + "hook_failure": "deny", + "hook_failure_reason": "runtime_error:annotation_failed", + "hook_calls": 1, + "parse_ok": True, + "streaming_offset": 5, + "streaming_clean": True, + "streaming_settled": None, +} + + +def run(cmd, **kw): + return subprocess.run(cmd, capture_output=True, text=True, check=True, **kw) + + +def last_json(out: str) -> dict: + return json.loads(out.strip().splitlines()[-1]) + + +def build(stage: Path) -> dict[str, Path]: + """Build every artifact the release workflow publishes.""" + art = stage / "artifacts" + art.mkdir(parents=True, exist_ok=True) + + run( + [ + "maturin", + "build", + "--release", + "-m", + str(ROOT / "sdk/python/Cargo.toml"), + "-o", + str(art / "py"), + ] + ) + + run(["npm", "run", "build"], cwd=ROOT / "sdk/node") + run( + [ + "npx", + "napi", + "build", + "--release", + "--platform", + "--target", + "x86_64-unknown-linux-gnu", + "--manifest-path", + "native/Cargo.toml", + "--cwd", + ".", + "--output-dir", + ".", + ], + cwd=ROOT / "sdk/node", + ) + for node_binary in (ROOT / "sdk/node").glob("*.node"): + shutil.copy(node_binary, ROOT / "sdk/node/npm/linux-x64-gnu") + run( + ["npm", "pack", "--pack-destination", str(art)], + cwd=ROOT / "sdk/node", + ) + run( + ["npm", "pack", "--pack-destination", str(art)], + cwd=ROOT / "sdk/node/npm/linux-x64-gnu", + ) + + run(["cargo", "build", "--release", "-p", "agent-control-spec-ffi"], cwd=ROOT) + native = ROOT / "sdk/dotnet/native/runtimes/linux-x64/native" + native.mkdir(parents=True, exist_ok=True) + shutil.copy(ROOT / "target/release/libagent_control_spec_ffi.so", native) + run( + [ + "dotnet", + "pack", + str(ROOT / "sdk/dotnet/src/AgentControlSpec"), + "-c", + "Release", + "-o", + str(art / "nuget"), + "--nologo", + "-p:AcsNativeAssetsRequired=true", + "-p:Version=0.0.0-artifactcheck", + ] + ) + + run( + [ + "cargo", + "package", + "-p", + "agent-control-spec", + "--allow-dirty", + "--no-verify", + ], + cwd=ROOT, + ) + crate = next((ROOT / "target/package").glob("agent-control-spec-*.crate")) + unpacked = art / "crate" + unpacked.mkdir(exist_ok=True) + run(["tar", "xzf", str(crate), "-C", str(unpacked)]) + + return { + "wheel": next((art / "py").glob("*.whl")), + "npm_main": next(art.glob("responsibleai-agent-control-spec-[0-9]*.tgz")), + "npm_native": next( + art.glob("responsibleai-agent-control-spec-linux-x64-gnu-*.tgz") + ), + "nuget_feed": art / "nuget", + "crate": next(unpacked.glob("agent-control-spec-*")), + } + + +PY_PROGRAM = """ +import json +from agent_control_spec import ( + AcsInterceptor, StreamSession, supported_manifest_versions, + parse_manifest, validate_manifest_detailed, +) +M = {manifest!r} +class C: + def __init__(s, v): s.v = v; s.calls = 0 + def dispatch(s, *a): s.calls += 1; return {{"severity": s.v}} +class B: + def dispatch(s, *a): raise RuntimeError("classifier unreachable") +def hook(d): + v = AcsInterceptor(M, annotator_dispatcher=d).intercept( + {{"interception_point": "input", "input": "hi"}}) + return (str(getattr(v.decision, "value", v.decision)).lower(), v.reason) +c = C(1); b = hook(c); h = hook(C(7)); f = hook(B()) +s = StreamSession(safety_level="blocking", response_tasks=["pii"]) +r = s.observe_text("model_generated", "hello") +s.record_outcome("pii", "model_generated", 0, r, "cleared"); s.advance("response") +off = s.safe_offset("response"); clean = s.finish()["is_clean"] +print(json.dumps({{ + "non_streaming": len(supported_manifest_versions()) > 0, + "hook_benign": b[0], "hook_harmful": h[0], "hook_harmful_reason": h[1], + "hook_failure": f[0], "hook_failure_reason": f[1], "hook_calls": c.calls, + "parse_ok": "intervention_points" in parse_manifest(open(M).read()), + "streaming_offset": off, "streaming_clean": clean, + "streaming_settled": s.safe_offset("response"), +}})) +""" + + +def check_python(art: dict, stage: Path) -> dict: + venv = stage / "venv-py" + run([sys.executable, "-m", "venv", str(venv)]) + run([str(venv / "bin/pip"), "install", "-q", str(art["wheel"])]) + out = run( + [ + str(venv / "bin/python"), + "-c", + PY_PROGRAM.format(manifest=str(HOOKS_MANIFEST)), + ] + ) + return last_json(out.stdout) + + +NODE_PROGRAM = """ +const acs = require('@responsibleai/agent-control-spec'); +const fs = require('fs'); +const M = %s; +function hook(d) { + const i = acs.AcsInterceptor.fromPath(M, { annotatorDispatcher: d }); + const v = i.intercept({ interception_point: 'input', input: 'hi' }); + return [String(v.decision).toLowerCase(), v.reason ?? null]; +} +let calls = 0; +const b = hook(() => { calls++; return { severity: 1 }; }); +const h = hook(() => ({ severity: 7 })); +const f = hook(() => { throw new Error('classifier unreachable'); }); +const s = new acs.StreamSession({ safetyLevel: 'blocking', responseTasks: ['pii'] }); +const r = s.observeText('model_generated', 'hello'); +s.recordOutcome('pii', 'model_generated', 0, r, 'cleared'); +s.advance('response'); +const off = s.safeOffset('response'); +const clean = s.finish().isClean; +console.log(JSON.stringify({ + non_streaming: acs.supportedManifestVersions().length > 0, + hook_benign: b[0], hook_harmful: h[0], hook_harmful_reason: h[1], + hook_failure: f[0], hook_failure_reason: f[1], hook_calls: calls, + parse_ok: Object.prototype.hasOwnProperty.call( + acs.parseManifest(fs.readFileSync(M, 'utf8')), 'intervention_points'), + streaming_offset: off, streaming_clean: clean, + streaming_settled: s.safeOffset('response'), +})); +""" + + +def check_node(art: dict, stage: Path) -> dict: + app = stage / "app-node" + app.mkdir() + run(["npm", "init", "-y"], cwd=app) + run( + ["npm", "install", str(art["npm_native"]), str(art["npm_main"])], + cwd=app, + ) + out = run( + ["node", "-e", NODE_PROGRAM % json.dumps(str(HOOKS_MANIFEST))], + cwd=app, + ) + return last_json(out.stdout) + + +DOTNET_PROGRAM = """ +using AgentControlSpec; +using AgentHooks; +using System.Text.Json; +using System.Text.Json.Nodes; + +var M = MANIFEST; +AgentContext Ctx() => new(JsonNode.Parse(CTX_JSON)!.AsObject()); +async Task<(string, string?)> Hook(AnnotatorDispatcher d) +{ + using var i = AcsHostInterceptor.FromPath(M, annotator: d); + var v = await i.InterceptAsync(Ctx()); + return (v.Decision.ToString().ToLowerInvariant(), v.Reason); +} +var calls = 0; +var b = await Hook((_, _, _) => { calls++; return SEV1; }); +var h = await Hook((_, _, _) => SEV7); +var f = await Hook((_, _, _) => throw new InvalidOperationException("classifier unreachable")); +using var s = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); +var r = s.ObserveText(StreamSourceType.ModelGenerated, "hello"); +s.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, r, SegmentOutcome.Cleared); +s.Advance(StreamTrack.Response); +var off = s.SafeOffset(StreamTrack.Response); +var clean = s.Finish().IsClean; +Console.WriteLine(JsonSerializer.Serialize(new Dictionary +{ + ["non_streaming"] = AcsManifest.SupportedVersions().Count > 0, + ["hook_benign"] = b.Item1, + ["hook_harmful"] = h.Item1, + ["hook_harmful_reason"] = h.Item2, + ["hook_failure"] = f.Item1, + ["hook_failure_reason"] = f.Item2, + ["hook_calls"] = calls, + ["parse_ok"] = AcsManifestTools.Parse(File.ReadAllText(M)).Contains("intervention_points"), + ["streaming_offset"] = off, + ["streaming_clean"] = clean, + ["streaming_settled"] = s.SafeOffset(StreamTrack.Response), +})); +""" + + +def check_dotnet(art: dict, stage: Path) -> dict: + app = stage / "app-net" + app.mkdir() + (app / "NuGet.config").write_text( + f""" + + + + + + + +""" + ) + run(["dotnet", "new", "console", "-o", str(app), "--force"]) + run( + [ + "dotnet", + "add", + str(app), + "package", + "ResponsibleAI.AgentControlSpec", + "--version", + "0.0.0-artifactcheck", + ] + ) + program = ( + DOTNET_PROGRAM.replace("MANIFEST", json.dumps(str(HOOKS_MANIFEST))) + .replace( + "CTX_JSON", + json.dumps(json.dumps({"interception_point": "input", "input": "hi"})), + ) + .replace("SEV1", json.dumps(json.dumps({"severity": 1}))) + .replace("SEV7", json.dumps(json.dumps({"severity": 7}))) + ) + (app / "Program.cs").write_text(program) + out = run(["dotnet", "run", "--project", str(app), "--nologo"]) + return last_json(out.stdout) + + +RUST_MAIN = """ +use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; +use agent_control_spec::dispatchers::BindingPolicyDispatcher; +use agent_control_spec::stream_session::*; +use agent_control_spec::{Manifest, Runtime, RuntimeError}; +use std::sync::Arc; + +struct C(i64, std::sync::atomic::AtomicUsize); +impl AnnotatorDispatcher for C { + fn dispatch(&self, _n: &str, _a: &AnnotatorInvocation, _p: &serde_json::Value) + -> Result { + self.1.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(serde_json::json!({"severity": self.0})) + } +} +struct B; +impl AnnotatorDispatcher for B { + fn dispatch(&self, _n: &str, _a: &AnnotatorInvocation, _p: &serde_json::Value) + -> Result { + Err(RuntimeError::AnnotationFailed("classifier unreachable".into())) + } +} +fn hook(m: &str, d: Arc) -> (String, Option) { + let manifest = Manifest::from_path(m).expect("manifest"); + let rt = Runtime::new(manifest, d, Arc::new(BindingPolicyDispatcher::new())).expect("runtime"); + let ctx: serde_json::Value = + serde_json::from_str(r#"{"interception_point":"input","input":"hi"}"#).unwrap(); + let v = rt.evaluate(&ctx).verdict; + (format!("{:?}", v.decision).to_lowercase(), v.reason.clone()) +} +fn main() { + let m = std::env::args().nth(1).expect("manifest"); + let benign = Arc::new(C(1, std::sync::atomic::AtomicUsize::new(0))); + let b = hook(&m, benign.clone()); + let h = hook(&m, Arc::new(C(7, std::sync::atomic::AtomicUsize::new(0)))); + let f = hook(&m, Arc::new(B)); + let mut s = StreamSession::new(StreamSessionConfig { + safety_level: SafetyLevel::Blocking, + request_start_rune_offset: 0, + response_start_rune_offset: 0, + request_tasks: vec![], + response_tasks: vec!["pii".into()], + }).unwrap(); + let r = s.observe_text(StreamSourceType::ModelGenerated, "hello").unwrap(); + let sp = StreamSpan::new(StreamSourceType::ModelGenerated, 0, r).unwrap(); + s.record_outcome("pii", &sp, SegmentOutcome::Cleared).unwrap(); + s.advance(StreamTrack::Response); + let off = s.safe_offset(StreamTrack::Response); + let clean = s.finish().reason.is_clean(); + println!("{}", serde_json::json!({ + "non_streaming": !agent_control_spec::SUPPORTED_VERSIONS.is_empty(), + "hook_benign": b.0, "hook_harmful": h.0, "hook_harmful_reason": h.1, + "hook_failure": f.0, "hook_failure_reason": f.1, + "hook_calls": benign.1.load(std::sync::atomic::Ordering::SeqCst), + "parse_ok": Manifest::from_yaml_str(&std::fs::read_to_string(&m).unwrap()).is_ok(), + "streaming_offset": off, "streaming_clean": clean, + "streaming_settled": s.safe_offset(StreamTrack::Response), + })); +} +""" + + +def check_rust(art: dict, stage: Path) -> dict: + app = stage / "app-rs" + (app / "src").mkdir(parents=True) + (app / "Cargo.toml").write_text( + f"""[package] +name = "acs-artifact-check" +version = "0.0.0" +edition = "2021" + +[dependencies] +agent-control-spec = {{ path = "{art["crate"]}", features = ["default-dispatchers", "streaming"] }} +serde_json = "1" + +[workspace] +""" + ) + (app / "src/main.rs").write_text(RUST_MAIN) + out = run( + ["cargo", "run", "--quiet", "--release", "--", str(HOOKS_MANIFEST)], cwd=app + ) + return last_json(out.stdout) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--keep", action="store_true", help="keep the staging directory" + ) + args = parser.parse_args() + + stage = Path(tempfile.mkdtemp(prefix="acs-artifacts-")) + failed = False + try: + print("building artifacts", flush=True) + try: + art = build(stage) + except subprocess.CalledProcessError as e: + # A build that refuses is a result, not a crash. The pack-time + # guard against a native-less package reports itself this way. + print( + f"building artifacts FAILED\n{' '.join(str(c) for c in e.cmd)}\n" + f"{e.stdout}\n{e.stderr}", + file=sys.stderr, + ) + return 1 + for name, path in art.items(): + print(f" {name}: {path.name}") + + checks = { + "rust": check_rust, + "python": check_python, + "node": check_node, + "dotnet": check_dotnet, + } + print("\nrunning the public surface from each installed artifact") + for name, check in checks.items(): + try: + got = check(art, stage) + except subprocess.CalledProcessError as e: + print( + f"{name:8} FAILED TO RUN\n{e.stdout}\n{e.stderr}", file=sys.stderr + ) + failed = True + continue + mismatches = { + k: (EXPECTED[k], got.get(k)) + for k in EXPECTED + if got.get(k) != EXPECTED[k] + } + print(f" {name:8} {'ok' if not mismatches else 'MISMATCH'}") + for key, (want, actual) in sorted(mismatches.items()): + print( + f" {key}: expected {want!r}, got {actual!r}", + file=sys.stderr, + ) + failed = True + finally: + if args.keep: + print(f"\nstaging kept at {stage}") + else: + shutil.rmtree(stage, ignore_errors=True) + + if failed: + print( + "\na published artifact does not carry what the checkout does", + file=sys.stderr, + ) + return 1 + print( + f"\nevery artifact carries the whole surface, across {len(EXPECTED)} assertions" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 30265d10c22aba9a9ebb29d88eaa139eaf49e0af Mon Sep 17 00:00:00 2001 From: Liam Crumm Date: Tue, 11 Aug 2026 19:13:11 +0000 Subject: [PATCH 06/10] fix: mark the verification scripts executable Both carry a shebang, which ruff's EXE001 requires the mode to match. The parity harness has the same mismatch but sits outside the paths CI lints, so it would have surfaced later. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Liam Crumm --- scripts/verify-artifacts.py | 0 tests/conformance/parity/cross_language_parity.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/verify-artifacts.py mode change 100644 => 100755 tests/conformance/parity/cross_language_parity.py diff --git a/scripts/verify-artifacts.py b/scripts/verify-artifacts.py old mode 100644 new mode 100755 diff --git a/tests/conformance/parity/cross_language_parity.py b/tests/conformance/parity/cross_language_parity.py old mode 100644 new mode 100755 From 3085b3f43b1d828e0f25ab35719bb1ee19524c6e Mon Sep 17 00:00:00 2001 From: Liam Crumm Date: Tue, 11 Aug 2026 19:56:32 +0000 Subject: [PATCH 07/10] feat: validate a manifest together with the Rego it names The manifest check answers only for the document. A manifest can name a bundle, satisfy the grammar, and still fail at activation because the Rego does not compile, so the failure surfaces on a host's first agent action rather than in its CI. 0.3 covered both halves through validate_acs_artifacts and 0.4 covered neither. Compilation happens at activation, so this activates against the supplied bundles in memory and reports what that surfaced. A broken module comes back as the Rego compiler's own complaint, with its line and column. The manifest is checked first and on its own. A document that does not parse would otherwise be reported as an activation failure, which names the wrong half. Validating a manifest that names Rego without supplying the Rego is not a pass: the bundle is still missing, and all four languages say so. Widen the parity harness to 30 assertions and add the case to the artifact verifier, so the four have to agree about this too. Stamp the artifact verifier's package version per run. NuGet caches by id and version, so a fixed version let a stale package from an earlier run satisfy a later one, which is a verifier that can report a pass for code the artifact does not contain. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Liam Crumm --- scripts/verify-artifacts.py | 60 +++++++-- sdk/dotnet/src/AgentControlSpec/HostHooks.cs | 25 ++++ sdk/dotnet/src/AgentControlSpec/Native.cs | 11 ++ .../AgentControlSpec.Tests/HostHooksTests.cs | 63 +++++++++ sdk/ffi/src/lib.rs | 68 ++++++++++ sdk/node/binding.d.ts | 22 +++ sdk/node/binding.js | 1 + sdk/node/native/src/lib.rs | 65 +++++++++ sdk/node/src/index.ts | 55 ++++++++ sdk/node/test/host-hooks.test.mjs | 116 ++++++++++++++++ sdk/python/agent_control_spec/__init__.py | 42 ++++++ sdk/python/src/lib.rs | 54 ++++++++ sdk/python/tests/test_host_hooks.py | 126 ++++++++++++++++++ .../parity/cross_language_parity.py | 68 +++++++++- 14 files changed, 767 insertions(+), 9 deletions(-) diff --git a/scripts/verify-artifacts.py b/scripts/verify-artifacts.py index 4afa5c5..b940be6 100755 --- a/scripts/verify-artifacts.py +++ b/scripts/verify-artifacts.py @@ -32,16 +32,31 @@ import argparse import json +import os import shutil import subprocess import sys import tempfile +import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] + +# NuGet caches by id and version, so a fixed version would let a stale +# package from an earlier run satisfy a later one and report a pass for +# code the artifact does not contain. Stamp each run instead. +CHECK_VERSION = f"0.0.0-artifactcheck{os.getpid()}{int(time.time())}" HOOKS_MANIFEST = ROOT / "tests" / "conformance" / "parity" / "host-hooks-manifest.yaml" BAD_MANIFEST = 'agent_control_specification_version: "0.4.0-alpha.1"\nmetadata: {}\n' +REGO_MANIFEST = ( + 'agent_control_specification_version: "0.4.0-alpha.1"\n' + "policies:\n gate:\n type: rego\n bundle: ./b\n" + 'intervention_points:\n input:\n policy_target: "$.input"\n' + " policy:\n id: gate\n query: data.acs.decision\n" +) +BAD_BUNDLES = {"gate": {"modules": {"p.rego": "package acs\nthis is not rego ***\n"}}} + EXPECTED = { "non_streaming": True, "hook_benign": "allow", @@ -54,6 +69,8 @@ "streaming_offset": 5, "streaming_clean": True, "streaming_settled": None, + # A manifest the document check passes whose Rego does not compile. + "artifacts_bad_rego": 1, } @@ -127,7 +144,7 @@ def build(stage: Path) -> dict[str, Path]: str(art / "nuget"), "--nologo", "-p:AcsNativeAssetsRequired=true", - "-p:Version=0.0.0-artifactcheck", + f"-p:Version={CHECK_VERSION}", ] ) @@ -162,7 +179,7 @@ def build(stage: Path) -> dict[str, Path]: import json from agent_control_spec import ( AcsInterceptor, StreamSession, supported_manifest_versions, - parse_manifest, validate_manifest_detailed, + parse_manifest, validate_manifest_detailed, validate_artifacts, ) M = {manifest!r} class C: @@ -186,6 +203,7 @@ def hook(d): "parse_ok": "intervention_points" in parse_manifest(open(M).read()), "streaming_offset": off, "streaming_clean": clean, "streaming_settled": s.safe_offset("response"), + "artifacts_bad_rego": len(validate_artifacts({rego!r}, {bad!r})), }})) """ @@ -198,7 +216,9 @@ def check_python(art: dict, stage: Path) -> dict: [ str(venv / "bin/python"), "-c", - PY_PROGRAM.format(manifest=str(HOOKS_MANIFEST)), + PY_PROGRAM.format( + manifest=str(HOOKS_MANIFEST), rego=REGO_MANIFEST, bad=BAD_BUNDLES + ), ] ) return last_json(out.stdout) @@ -231,6 +251,7 @@ def check_python(art: dict, stage: Path) -> dict: acs.parseManifest(fs.readFileSync(M, 'utf8')), 'intervention_points'), streaming_offset: off, streaming_clean: clean, streaming_settled: s.safeOffset('response'), + artifacts_bad_rego: acs.validateArtifacts(%s, %s).length, })); """ @@ -244,7 +265,16 @@ def check_node(art: dict, stage: Path) -> dict: cwd=app, ) out = run( - ["node", "-e", NODE_PROGRAM % json.dumps(str(HOOKS_MANIFEST))], + [ + "node", + "-e", + NODE_PROGRAM + % ( + json.dumps(str(HOOKS_MANIFEST)), + json.dumps(REGO_MANIFEST), + json.dumps(BAD_BUNDLES), + ), + ], cwd=app, ) return last_json(out.stdout) @@ -256,7 +286,7 @@ def check_node(art: dict, stage: Path) -> dict: using System.Text.Json; using System.Text.Json.Nodes; -var M = MANIFEST; +var M = __MANIFEST__; AgentContext Ctx() => new(JsonNode.Parse(CTX_JSON)!.AsObject()); async Task<(string, string?)> Hook(AnnotatorDispatcher d) { @@ -287,6 +317,7 @@ def check_node(art: dict, stage: Path) -> dict: ["streaming_offset"] = off, ["streaming_clean"] = clean, ["streaming_settled"] = s.SafeOffset(StreamTrack.Response), + ["artifacts_bad_rego"] = AcsManifestTools.ValidateArtifacts(__REGO__, __BADB__).Count, })); """ @@ -314,17 +345,19 @@ def check_dotnet(art: dict, stage: Path) -> dict: "package", "ResponsibleAI.AgentControlSpec", "--version", - "0.0.0-artifactcheck", + CHECK_VERSION, ] ) program = ( - DOTNET_PROGRAM.replace("MANIFEST", json.dumps(str(HOOKS_MANIFEST))) + DOTNET_PROGRAM.replace("__MANIFEST__", json.dumps(str(HOOKS_MANIFEST))) .replace( "CTX_JSON", json.dumps(json.dumps({"interception_point": "input", "input": "hi"})), ) .replace("SEV1", json.dumps(json.dumps({"severity": 1}))) .replace("SEV7", json.dumps(json.dumps({"severity": 7}))) + .replace("__REGO__", json.dumps(REGO_MANIFEST)) + .replace("__BADB__", json.dumps(json.dumps(BAD_BUNDLES))) ) (app / "Program.cs").write_text(program) out = run(["dotnet", "run", "--project", str(app), "--nologo"]) @@ -380,6 +413,12 @@ def check_dotnet(art: dict, stage: Path) -> dict: s.advance(StreamTrack::Response); let off = s.safe_offset(StreamTrack::Response); let clean = s.finish().reason.is_clean(); + let bad_bundles: std::collections::BTreeMap = + serde_json::from_str(BAD_BUNDLES).expect("bundles"); + let artifacts_bad_rego = usize::from( + agent_control_spec::ActivatedPolicy::activate_from_memory(REGO_MANIFEST, bad_bundles) + .is_err(), + ); println!("{}", serde_json::json!({ "non_streaming": !agent_control_spec::SUPPORTED_VERSIONS.is_empty(), "hook_benign": b.0, "hook_harmful": h.0, "hook_harmful_reason": h.1, @@ -388,6 +427,7 @@ def check_dotnet(art: dict, stage: Path) -> dict: "parse_ok": Manifest::from_yaml_str(&std::fs::read_to_string(&m).unwrap()).is_ok(), "streaming_offset": off, "streaming_clean": clean, "streaming_settled": s.safe_offset(StreamTrack::Response), + "artifacts_bad_rego": artifacts_bad_rego, })); } """ @@ -409,7 +449,11 @@ def check_rust(art: dict, stage: Path) -> dict: [workspace] """ ) - (app / "src/main.rs").write_text(RUST_MAIN) + (app / "src/main.rs").write_text( + RUST_MAIN.replace("REGO_MANIFEST", json.dumps(REGO_MANIFEST)).replace( + "BAD_BUNDLES", json.dumps(json.dumps(BAD_BUNDLES)) + ) + ) out = run( ["cargo", "run", "--quiet", "--release", "--", str(HOOKS_MANIFEST)], cwd=app ) diff --git a/sdk/dotnet/src/AgentControlSpec/HostHooks.cs b/sdk/dotnet/src/AgentControlSpec/HostHooks.cs index 12cf8fb..260eee2 100644 --- a/sdk/dotnet/src/AgentControlSpec/HostHooks.cs +++ b/sdk/dotnet/src/AgentControlSpec/HostHooks.cs @@ -96,6 +96,31 @@ public static string Merge(IEnumerable yamls) => public static IReadOnlyList Diagnostics(string yaml) => JsonSerializer.Deserialize>(Native.ManifestDiagnostics(yaml), Json) ?? throw new AgentControlSpecNativeException("diagnostics did not deserialize"); + + /// + /// Validate a manifest together with the Rego it names, and return + /// every finding. An empty list means both halves are sound. + /// + /// The manifest source. + /// + /// Policy id to in-memory Rego bundle, the same shape + /// takes. Null means the + /// manifest names no Rego, and the answer then equals + /// . + /// + /// + /// answers only for the document. A manifest + /// can name a bundle, satisfy the grammar, and still fail at + /// activation because the Rego does not compile. Compilation happens + /// at activation, so this activates in memory and reports what that + /// surfaced, which moves the failure from a host's first agent action + /// to its CI. + /// + public static IReadOnlyList ValidateArtifacts( + string manifestYaml, string? bundles = null) => + JsonSerializer.Deserialize>( + Native.ArtifactDiagnostics(manifestYaml, bundles), Json) + ?? throw new AgentControlSpecNativeException("diagnostics did not deserialize"); } /// diff --git a/sdk/dotnet/src/AgentControlSpec/Native.cs b/sdk/dotnet/src/AgentControlSpec/Native.cs index 53798be..7ade4c6 100644 --- a/sdk/dotnet/src/AgentControlSpec/Native.cs +++ b/sdk/dotnet/src/AgentControlSpec/Native.cs @@ -485,6 +485,10 @@ private static partial IntPtr acs_interceptor_new_with_hooks( [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] private static partial IntPtr acs_manifest_diagnostics(string yaml, out IntPtr errOut); + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_artifact_diagnostics( + string manifestYaml, string? bundlesJson, out IntPtr errOut); + internal static IntPtr InterceptorNewWithHooks( string manifestPath, IntPtr annotatorFn, IntPtr annotatorCtx, @@ -538,6 +542,13 @@ internal static string ManifestDiagnostics(string yaml) ThrowIfError(err); return TakeString(json); } + + internal static string ArtifactDiagnostics(string manifestYaml, string? bundlesJson) + { + var json = acs_artifact_diagnostics(manifestYaml, bundlesJson, out var err); + ThrowIfError(err); + return TakeString(json); + } } internal sealed class StreamSessionHandle : SafeHandle diff --git a/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs b/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs index 8d34f9e..ef34808 100644 --- a/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs +++ b/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs @@ -216,5 +216,68 @@ public void MergeComposesABaseWithAnOverlay() Assert.Contains("overlay-applied", merged); } + + private const string RegoManifest = """ + agent_control_specification_version: "0.4.0-alpha.1" + policies: + gate: + type: rego + bundle: ./b + intervention_points: + input: + policy_target: "$.input" + policy: + id: gate + query: data.acs.decision + """; + + [Fact] + public void ArtifactValidationClearsAManifestWhoseRegoCompiles() + { + var bundles = """ + {"gate":{"modules":{"p.rego":"package acs\ndecision := {\"decision\":\"allow\"}\n"}}} + """; + + Assert.Empty(AcsManifestTools.ValidateArtifacts(RegoManifest, bundles)); + } + + [Fact] + public void ArtifactValidationCatchesRegoTheManifestCheckCannot() + { + var broken = """ + {"gate":{"modules":{"p.rego":"package acs\nthis is not rego at all ***\n"}}} + """; + + // The manifest itself is sound, so the document check passes it. + Assert.Empty(AcsManifestTools.Diagnostics(RegoManifest)); + + // The Rego is not, and only activation finds that out. + var finding = Assert.Single(AcsManifestTools.ValidateArtifacts(RegoManifest, broken)); + Assert.StartsWith("runtime_error:", finding.Code); + Assert.Contains("p.rego", finding.Message); + } + + [Fact] + public void AManifestThatDoesNotParseIsReportedAsAManifestProblem() + { + var finding = Assert.Single(AcsManifestTools.ValidateArtifacts("this: [is not", null)); + + // Naming this an activation failure would blame the wrong half. + Assert.Contains("manifest", finding.Code); + } + + [Fact] + public void WithNoBundlesArtifactValidationAgreesWithTheManifestCheck() + { + const string Bad = """ + agent_control_specification_version: "0.4.0-alpha.1" + metadata: {} + """; + + Assert.Equal( + AcsManifestTools.Diagnostics(Bad).Count, + AcsManifestTools.ValidateArtifacts(Bad, null).Count); + } + public void Dispose() => Directory.Delete(_dir, recursive: true); } diff --git a/sdk/ffi/src/lib.rs b/sdk/ffi/src/lib.rs index 69be161..104017d 100644 --- a/sdk/ffi/src/lib.rs +++ b/sdk/ffi/src/lib.rs @@ -1275,6 +1275,74 @@ pub unsafe extern "C" fn acs_manifest_diagnostics( } } +/// Validate a manifest together with the Rego it names, and return the +/// findings as a JSON array. +/// +/// An empty array means both halves are sound. `acs_manifest_diagnostics` +/// answers only for the document: a manifest can name a bundle, satisfy +/// the grammar, and still fail at activation because the Rego does not +/// compile. Compilation happens at activation, so this activates against +/// the supplied bundles in memory and reports what that surfaced, which +/// moves the failure from a host's first agent action to its CI. +/// +/// `bundles_json` maps policy id to an in-memory bundle, the same shape +/// `acs_policy_activate_from_memory` takes. NULL or empty means the +/// manifest names no Rego, in which case this answers exactly as +/// `acs_manifest_diagnostics` does. Freed with `acs_free_string`. +/// +/// # Safety +/// `manifest_yaml` must be a valid NUL-terminated string. `bundles_json` +/// must be NULL or a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_artifact_diagnostics( + manifest_yaml: *const c_char, + bundles_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(source) = read_utf8(manifest_yaml, "manifest_yaml", err_out) else { + return std::ptr::null_mut(); + }; + let bundles = match read_in_memory_bundles(bundles_json, err_out) { + Some(bundles) => bundles, + None => return std::ptr::null_mut(), + }; + + // The manifest is checked first and on its own. A document that + // does not parse would otherwise be reported as an activation + // failure, which names the wrong half. + let findings = match Manifest::from_yaml_str(source) { + Err(e) => vec![diagnostic_json(&e)], + Ok(manifest) => match manifest.validate() { + Err(e) => vec![diagnostic_json(&e)], + Ok(()) => match ActivatedPolicy::activate_from_memory(source, bundles) { + Ok(_) => Vec::new(), + Err(e) => vec![diagnostic_json(&e)], + }, + }, + }; + + match serde_json::to_string(&findings) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("diagnostics serialization failed: {e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_artifact_diagnostics".to_string(), + ); + std::ptr::null_mut() + } + } +} + fn diagnostic_json(error: &RuntimeError) -> Value { serde_json::json!({ "code": error.reason(), diff --git a/sdk/node/binding.d.ts b/sdk/node/binding.d.ts index 7c0e939..ba79999 100644 --- a/sdk/node/binding.d.ts +++ b/sdk/node/binding.d.ts @@ -222,6 +222,28 @@ export declare function streamSessionWatermark(handle: ExternalObject +/** + * Validate a manifest together with the Rego it names, and return + * findings as a JSON array. + * + * An empty array means both halves are sound. Each entry has wire + * shape `{"code": str, "message": str, "severity": "error"}`, matching + * the C ABI's `acs_artifact_diagnostics`. + * + * `validate_manifest_detailed` answers only for the document: a + * manifest can name a bundle, satisfy the grammar, and still fail at + * activation because the Rego does not compile. Compilation happens + * at activation, so this activates against the supplied bundles in + * memory and reports what that surfaced, which moves the failure from + * a host's first agent action to its CI. + * + * `bundles_json` maps policy id to an in-memory bundle, the same + * shape `policy_activate_from_memory` takes. An empty document means + * the manifest names no Rego, and the answer then equals what + * `validate_manifest_detailed` reports for the manifest half. + */ +export declare function validateArtifactsDetailed(manifestYaml: string, bundlesJson: string): string + /** * Validate manifest source against the grammar, without building a * runtime. diff --git a/sdk/node/binding.js b/sdk/node/binding.js index 3ee3477..b3186f0 100644 --- a/sdk/node/binding.js +++ b/sdk/node/binding.js @@ -724,6 +724,7 @@ module.exports.streamSessionSafeOffset = nativeBinding.streamSessionSafeOffset module.exports.streamSessionState = nativeBinding.streamSessionState module.exports.streamSessionWatermark = nativeBinding.streamSessionWatermark module.exports.supportedManifestVersions = nativeBinding.supportedManifestVersions +module.exports.validateArtifactsDetailed = nativeBinding.validateArtifactsDetailed module.exports.validateManifest = nativeBinding.validateManifest module.exports.validateManifestDetailed = nativeBinding.validateManifestDetailed module.exports.validateManifestFile = nativeBinding.validateManifestFile diff --git a/sdk/node/native/src/lib.rs b/sdk/node/native/src/lib.rs index 2b676c9..344e0d4 100644 --- a/sdk/node/native/src/lib.rs +++ b/sdk/node/native/src/lib.rs @@ -681,6 +681,71 @@ fn diagnostic_json(error: &RuntimeError) -> Value { }) } +/// Wire shape for artifact diagnostics: `code`, `message`, `severity`. +/// +/// Mirrors the C ABI's `acs_artifact_diagnostics` byte-for-byte. +/// Unlike [`diagnostic_json`] this omits the best-effort `field` +/// pointer: `validate_artifacts_detailed` surfaces activation-half +/// failures whose message is the Rego compiler's own diagnostic, not +/// a manifest field, and inventing a field for it would mislead a +/// tool trying to render the pointer inline. +fn artifact_diagnostic_json(error: &RuntimeError) -> Value { + serde_json::json!({ + "code": error.reason(), + "message": error.detail(), + "severity": "error", + }) +} + +/// Validate a manifest together with the Rego it names, and return +/// findings as a JSON array. +/// +/// An empty array means both halves are sound. Each entry has wire +/// shape `{"code": str, "message": str, "severity": "error"}`, matching +/// the C ABI's `acs_artifact_diagnostics`. +/// +/// `validate_manifest_detailed` answers only for the document: a +/// manifest can name a bundle, satisfy the grammar, and still fail at +/// activation because the Rego does not compile. Compilation happens +/// at activation, so this activates against the supplied bundles in +/// memory and reports what that surfaced, which moves the failure from +/// a host's first agent action to its CI. +/// +/// `bundles_json` maps policy id to an in-memory bundle, the same +/// shape `policy_activate_from_memory` takes. An empty document means +/// the manifest names no Rego, and the answer then equals what +/// `validate_manifest_detailed` reports for the manifest half. +#[napi] +pub fn validate_artifacts_detailed( + manifest_yaml: Utf16String, + bundles_json: Utf16String, +) -> napi::Result { + let manifest_yaml = decode("manifest_yaml", &manifest_yaml)?; + let bundles_json = decode("bundles_json", &bundles_json)?; + let bundles: BTreeMap = if bundles_json.trim().is_empty() { + BTreeMap::new() + } else { + serde_json::from_str(&bundles_json) + .map_err(|e| err(format!("bundles_json does not parse: {e}")))? + }; + // Mirror the C ABI's ordering exactly: parse first, validate + // second, activate third. Each step's failure short-circuits so a + // manifest that does not parse is never reported as an activation + // failure — that would name the wrong half. + let findings = match Manifest::from_yaml_str(&manifest_yaml) { + Err(e) => vec![artifact_diagnostic_json(&e)], + Ok(manifest) => match manifest.validate() { + Err(e) => vec![artifact_diagnostic_json(&e)], + Ok(()) => match ActivatedPolicy::activate_from_memory(&manifest_yaml, bundles) { + Ok(_) => Vec::new(), + Err(e) => vec![artifact_diagnostic_json(&e)], + }, + }, + }; + serde_json::to_string(&findings) + .map_err(|e| err(format!("diagnostics serialization failed: {e}"))) +} + /// Best-effort extraction of the offending field name from a /// RuntimeError detail. Returns `None` when the message does not name a /// field the caller can point at, so the wrapper reports the raw diff --git a/sdk/node/src/index.ts b/sdk/node/src/index.ts index 3f90849..d0323fe 100644 --- a/sdk/node/src/index.ts +++ b/sdk/node/src/index.ts @@ -58,6 +58,7 @@ const native = require("../binding.js") as { validateManifestFile(path: string): string | null; validateManifest(source: string): string | null; validateManifestDetailed(source: string): string; + validateArtifactsDetailed(manifestYaml: string, bundlesJson: string): string; parseManifest(source: string): string; mergeManifests(sourcesJson: string): string; supportedManifestVersions(): string[]; @@ -695,6 +696,60 @@ export function validateManifestDetailed(source: string): readonly ManifestDiagn ); } +/** + * Validate a manifest AND the Rego it names, returning findings. + * + * An empty array means both halves are sound. Each entry has wire + * shape `{"code","message","severity":"error"}`, matching the C ABI's + * `acs_artifact_diagnostics`. + * + * {@link validateManifestDetailed} answers only for the document. A + * manifest can satisfy the grammar, name a Rego bundle, and still fail + * at activation because the Rego does not compile — compilation + * happens at activation, so a validator that stops at the manifest + * turns that failure into a host's first agent action. This function + * activates against `bundles` in memory and reports what the pair + * surfaced. + * + * `bundles` has the same shape {@link ActivatedPolicy.activateFromMemory} + * takes: a mapping from policy id to a {@link RegoBundle}. Omitting it + * or passing an empty object means the manifest names no Rego, and the + * answer then equals what {@link validateManifestDetailed} reports for + * the manifest half: a document that does not parse is reported as a + * manifest problem, not an activation problem, because that would name + * the wrong half. + * + * Throws only on boundary problems (non-string manifest, unpaired + * surrogate, non-object bundles); a broken manifest or Rego module + * returns a non-empty array. + */ +export function validateArtifacts( + manifestSource: string, + bundles?: Readonly>, +): readonly ManifestDiagnostic[] { + if (typeof manifestSource !== "string") { + throw new TypeError( + `validateArtifacts expects a string manifest, received ${typeof manifestSource}`, + ); + } + if (UNPAIRED_SURROGATE.test(manifestSource)) { + throw new TypeError( + "validateArtifacts received a manifest with an unpaired surrogate", + ); + } + if (bundles !== undefined && (bundles === null || typeof bundles !== "object")) { + throw new TypeError( + `validateArtifacts expects bundles to be an object, received ${typeof bundles}`, + ); + } + const payload = bundles === undefined ? "" : JSON.stringify(bundles); + return Object.freeze( + JSON.parse( + native.validateArtifactsDetailed(manifestSource, payload), + ) as ManifestDiagnostic[], + ); +} + // --------------------------------------------------------------------- // Streaming: incremental release mediation for stream-shaped tracks diff --git a/sdk/node/test/host-hooks.test.mjs b/sdk/node/test/host-hooks.test.mjs index 5e9d52e..d649901 100644 --- a/sdk/node/test/host-hooks.test.mjs +++ b/sdk/node/test/host-hooks.test.mjs @@ -21,6 +21,7 @@ const { ActivatedPolicy, parseManifest, mergeManifests, + validateArtifacts, validateManifestDetailed, } = require("../dist/index.js"); const { AgentContextBuilder } = require("@responsibleai/agent-hooks"); @@ -378,3 +379,118 @@ test("ActivatedPolicy.activate accepts host hooks and evaluates against them", ( assert.equal(verdict.reason, "denied_by_test"); assert.deepEqual(seen, ["custom"]); }); + + +// --------------------------------------------------------------------- +// 8. `validateArtifacts` catches Rego compilation failures a +// manifest-only validator cannot see. This is the shape a 0.3-era +// consumer's CI depended on (validate_acs_artifacts) and the reason +// this feature exists: today a manifest can name a bundle, pass +// grammar validation, and only fail at activation. That moves the +// failure from CI to a host's first agent action. +// --------------------------------------------------------------------- + +const ARTIFACT_MANIFEST = `agent_control_specification_version: "0.4.0-alpha.1" +policies: + gate: + type: rego + bundle: ./b +intervention_points: + input: + policy_target: "$.input" + policy: + id: gate + query: data.acs.decision +`; + +const VALID_REGO = 'package acs\ndecision := {"decision":"allow"}\n'; + +test("validateArtifacts returns [] for a manifest whose Rego compiles", () => { + const findings = validateArtifacts(ARTIFACT_MANIFEST, { + gate: { modules: { "p.rego": VALID_REGO } }, + }); + assert.deepEqual(findings, []); +}); + +test("validateArtifacts surfaces a broken Rego module the manifest names", () => { + // Same manifest, same shape, only the module is malformed. The + // manifest-only surface accepts this; the artifact surface must + // not, because activation would fail on the host's first action. + const findings = validateArtifacts(ARTIFACT_MANIFEST, { + gate: { modules: { "p.rego": "package acs\nfoo := ] not valid rego" } }, + }); + assert.equal(findings.length, 1, `expected one finding, got ${JSON.stringify(findings)}`); + const entry = findings[0]; + assert.equal(entry.severity, "error"); + assert.ok( + entry.code.startsWith("runtime_error:"), + `code was ${entry.code}`, + ); + // The Rego compiler's own text carries the module name and its + // "expecting expression" complaint verbatim, so an editor can point + // at the module. Assert both so a regression that swallowed the + // detail would fail. + assert.match(entry.message, /p\.rego/); + assert.match(entry.message, /expecting expression/); + + // And the manifest-only surface still accepts this: the point of + // validateArtifacts is exactly this gap. + assert.deepEqual(validateManifestDetailed(ARTIFACT_MANIFEST), []); +}); + +test("validateArtifacts reports an unparseable manifest as a manifest problem", () => { + // A document that does not parse must be reported as a manifest + // problem, not an activation failure — that would name the wrong + // half. Even when bundles are supplied, the diagnostic must be + // manifest-half. + const findings = validateArtifacts("::not: [valid", { + gate: { modules: { "p.rego": VALID_REGO } }, + }); + assert.equal(findings.length, 1); + const entry = findings[0]; + assert.equal(entry.code, "runtime_error:manifest_invalid"); + assert.equal(entry.severity, "error"); + + // The underlying RuntimeError message matches what the + // manifest-only surface reports for the same input. + const manifestOnly = validateManifestDetailed("::not: [valid"); + assert.equal(manifestOnly[0].code, entry.code); + assert.equal(manifestOnly[0].message, entry.message); +}); + +test("validateArtifacts without bundles equals the manifest-only result", () => { + // No bundles supplied: activation is either skipped (no Rego to + // load) or fails the same way manifest validation does. Either + // way, the artifact validator must not invent activation errors + // when the manifest half is what actually reports the problem. + // For a grammatically invalid document — one that parses but + // fails validation — the two surfaces report the same underlying + // manifest problem. Activation would never be reached. + const invalid = + 'agent_control_specification_version: "0.4.0-alpha.1"\npolicies: {}\nintervention_points: {}\n'; + const artifact = validateArtifacts(invalid); + const manifest = validateManifestDetailed(invalid); + assert.equal(artifact.length, manifest.length); + assert.equal(artifact.length, 1); + assert.equal(artifact[0].code, manifest[0].code); + assert.equal(artifact[0].message, manifest[0].message); + assert.equal(artifact[0].severity, "error"); + + // And omitting the bundles argument behaves identically to passing + // an empty object, so callers can write either. + assert.deepEqual( + validateArtifacts(ARTIFACT_MANIFEST, {}), + validateArtifacts(ARTIFACT_MANIFEST), + ); +}); + +test("validateArtifacts throws on boundary problems", () => { + // Non-string manifest and unpaired surrogate are boundary problems + // and throw as TypeError, not as an invalid manifest. Wrong shape + // for `bundles` throws too, rather than silently JSON.stringify-ing + // something the native side would reject. + assert.throws(() => validateArtifacts(42), TypeError); + assert.throws(() => validateArtifacts("\uD800"), TypeError); + assert.throws(() => validateArtifacts(ARTIFACT_MANIFEST, 5), TypeError); + assert.throws(() => validateArtifacts(ARTIFACT_MANIFEST, "not-an-object"), TypeError); +}); diff --git a/sdk/python/agent_control_spec/__init__.py b/sdk/python/agent_control_spec/__init__.py index 1a933d9..4657766 100644 --- a/sdk/python/agent_control_spec/__init__.py +++ b/sdk/python/agent_control_spec/__init__.py @@ -25,6 +25,7 @@ "PERF_TELEMETRY_LEVELS", "AcsInterceptor", "ActivatedPolicy", + "ArtifactDiagnostic", "ManifestInvalidError", "RegoBundle", "StreamSession", @@ -34,6 +35,7 @@ "merge_manifests", "parse_manifest", "supported_manifest_versions", + "validate_artifacts", "validate_manifest", "validate_manifest_detailed", "validate_manifest_file", @@ -89,6 +91,16 @@ #: tools that shuttle diagnostics through IPC. ValidationDiagnostic = Mapping[str, Any] +#: One entry produced by :func:`validate_artifacts`. Shape: +#: +#: - ``code``: ``str`` (the engine's ``runtime_error:*`` reason). +#: - ``message``: ``str`` (the engine's own detail text). +#: - ``severity``: always ``"error"``. +#: +#: Matches the C ABI's ``acs_artifact_diagnostics`` wire shape, so a +#: diagnostic-consuming tool can key off ``code`` across languages. +ArtifactDiagnostic = Mapping[str, Any] + def _normalize_perf_telemetry(value: str | None) -> str: """Reject unknown perf-telemetry levels on the Python side. @@ -367,6 +379,36 @@ def validate_manifest_detailed(source: str) -> list[ValidationDiagnostic]: return json.loads(_native.validate_manifest_diagnostics(source)) +def validate_artifacts( + manifest_source: str, + bundles: Mapping[str, RegoBundle] | None = None, +) -> list[ArtifactDiagnostic]: + """Validate a manifest AND the Rego it names, returning findings. + + Each diagnostic is ``{"code": str, "message": str, "severity": + "error"}`` and matches the C ABI's ``acs_artifact_diagnostics`` + wire shape. An empty list means both halves are sound. + + :func:`validate_manifest_detailed` answers only for the document. + A manifest can satisfy the grammar, name a Rego bundle, and still + fail at activation because the Rego does not compile — compilation + happens at activation time, so a validation surface that stops at + the document turns that failure into a host's first agent action + rather than a CI signal. This activates against the supplied + bundles in memory and reports what that surfaced, closing the gap. + + ``bundles`` has the same shape :meth:`ActivatedPolicy.from_memory` + takes: a mapping from policy id to a ``{"modules": {...}, "data": + [...]}`` object. ``None`` or an empty mapping means the manifest + names no Rego, and the result then equals the manifest-only + diagnostics: a document that does not parse is reported as a + manifest problem, not an activation problem, because that names + the wrong half. + """ + payload = "" if bundles is None else json.dumps(bundles, allow_nan=False) + return json.loads(_native.validate_artifacts_diagnostics(manifest_source, payload)) + + def parse_manifest(source: str) -> dict[str, Any]: """Parse manifest source into a ``dict`` without validating. diff --git a/sdk/python/src/lib.rs b/sdk/python/src/lib.rs index 55f36c3..30b484f 100644 --- a/sdk/python/src/lib.rs +++ b/sdk/python/src/lib.rs @@ -623,6 +623,59 @@ fn validate_manifest_diagnostics(source: &str) -> PyResult { } } +/// Structured artifact diagnostics as a JSON array string. +/// +/// Validates the manifest AND compiles the Rego it names against +/// `bundles_json` — the same shape `policy_activate_from_memory` +/// takes — and reports every failure the pair surfaced. Each entry +/// has the wire shape +/// `{"code": str, "message": str, "severity": "error"}`, matching the +/// C ABI's `acs_artifact_diagnostics`. `validate_manifest_diagnostics` +/// answers only for the document: a manifest can name a bundle, +/// satisfy the grammar, and still fail at activation because the Rego +/// does not compile. This activates in memory and reports what that +/// surfaced, which moves the failure from a host's first agent action +/// to its CI. +/// +/// The manifest is checked first and on its own: a document that +/// does not parse would otherwise be reported as an activation +/// failure, which names the wrong half. NULL/empty `bundles_json` +/// means the manifest names no Rego, so the answer then equals what +/// `validate_manifest_diagnostics` returns. +#[pyfunction] +fn validate_artifacts_diagnostics(manifest_yaml: &str, bundles_json: &str) -> PyResult { + fn diagnostic(error: &RuntimeError) -> Value { + json!({ + "code": error.reason(), + "message": error.detail(), + "severity": "error", + }) + } + let bundles: std::collections::BTreeMap = + if bundles_json.trim().is_empty() { + std::collections::BTreeMap::new() + } else { + serde_json::from_str(bundles_json) + .map_err(|e| PyValueError::new_err(format!("bundles do not parse: {e}")))? + }; + // Mirror the C ABI's ordering exactly: parse first, validate + // second, activate third. Each step's failure short-circuits so a + // manifest that does not parse is never reported as an activation + // failure. + let findings = match Manifest::from_yaml_str(manifest_yaml) { + Err(e) => vec![diagnostic(&e)], + Ok(manifest) => match manifest.validate() { + Err(e) => vec![diagnostic(&e)], + Ok(()) => match ActivatedPolicy::activate_from_memory(manifest_yaml, bundles) { + Ok(_) => Vec::new(), + Err(e) => vec![diagnostic(&e)], + }, + }, + }; + serde_json::to_string(&findings) + .map_err(|e| PyRuntimeError::new_err(format!("diagnostics serialization failed: {e}"))) +} + // --------------------------------------------------------------------- // Activated policy: one policy version, readied once, evaluated many // times. @@ -1105,6 +1158,7 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(validate_manifest, m)?)?; m.add_function(wrap_pyfunction!(validate_manifest_file, m)?)?; m.add_function(wrap_pyfunction!(validate_manifest_diagnostics, m)?)?; + m.add_function(wrap_pyfunction!(validate_artifacts_diagnostics, m)?)?; m.add_function(wrap_pyfunction!(parse_manifest, m)?)?; m.add_function(wrap_pyfunction!(merge_manifests, m)?)?; m.add_function(wrap_pyfunction!(supported_manifest_versions, m)?)?; diff --git a/sdk/python/tests/test_host_hooks.py b/sdk/python/tests/test_host_hooks.py index acccf8f..2c08896 100644 --- a/sdk/python/tests/test_host_hooks.py +++ b/sdk/python/tests/test_host_hooks.py @@ -14,6 +14,8 @@ from __future__ import annotations +_VERSION_KEY = "agent_control_specification" + "_version" + import pathlib import pytest @@ -24,6 +26,7 @@ ManifestInvalidError, merge_manifests, parse_manifest, + validate_artifacts, validate_manifest_detailed, ) from agent_hooks import AgentContextBuilder @@ -392,3 +395,126 @@ def test_diagnostics_flag_manifests_that_use_extends(): assert diagnostics assert diagnostics[0]["reason_code"] == "runtime_error:manifest_invalid" assert "extends" in diagnostics[0]["message"] + + +# --------------------------------------------------------------------- +# Artifact validation: manifest + Rego compiled together. +# +# `validate_manifest_detailed` answers only for the document. A +# manifest can satisfy the grammar, name a Rego bundle, and still fail +# at activation because the Rego does not compile — compilation happens +# at activation time, so a validator that stops at the manifest turns +# that failure into a host's first agent action. `validate_artifacts` +# closes the gap by activating in memory and reporting what the pair +# surfaced. Restores the 0.3-era ``validate_acs_artifacts`` shape a +# consumer's CI depended on. +# --------------------------------------------------------------------- + +ARTIFACT_MANIFEST = """\ +agent_control_specification_version: "0.4.0-alpha.1" +policies: + gate: + type: rego + bundle: ./b +intervention_points: + input: + policy_target: "$.input" + policy: + id: gate + query: data.acs.decision +""" + +_VALID_REGO = 'package acs\ndecision := {"decision":"allow"}\n' + + +def test_validate_artifacts_returns_empty_for_valid_manifest_and_rego(): + # A manifest naming a Rego policy whose module compiles cleanly is + # what a fully-formed release looks like: nothing to report. + findings = validate_artifacts( + ARTIFACT_MANIFEST, + {"gate": {"modules": {"p.rego": _VALID_REGO}}}, + ) + assert findings == [] + + +def test_validate_artifacts_surfaces_a_broken_rego_module(): + # The feature exists for this case: the manifest is fine, the + # bundle is not, and today a manifest-only validator would have + # green-lit the release. The diagnostic must name the activation + # half so the caller can render the compiler's complaint. + findings = validate_artifacts( + ARTIFACT_MANIFEST, + {"gate": {"modules": {"p.rego": "package acs\nfoo := ] not valid rego"}}}, + ) + assert len(findings) == 1, findings + entry = findings[0] + assert entry["severity"] == "error" + assert entry["code"].startswith("runtime_error:"), entry + # The engine's own text carries the Rego compiler's complaint + # verbatim, so an editor can point at the module. The compiler + # names the module path and its "expecting expression" error. + assert "p.rego" in entry["message"], entry + assert "expecting expression" in entry["message"], entry + + # `validate_manifest_detailed` never sees this failure: it does + # not compile the bundle. Prove that so a regression that widened + # the manifest-only surface would fail. + assert validate_manifest_detailed(ARTIFACT_MANIFEST) == [] + + +def test_validate_artifacts_reports_unparseable_manifest_as_manifest_problem(): + # A document that does not parse must be reported as a manifest + # problem, not an activation failure — that would name the wrong + # half. Even when bundles are supplied. + findings = validate_artifacts( + "::not: [valid", + {"gate": {"modules": {"p.rego": _VALID_REGO}}}, + ) + assert len(findings) == 1, findings + entry = findings[0] + assert entry["code"] == "runtime_error:manifest_invalid" + assert entry["severity"] == "error" + # And the underlying error matches what the manifest-only + # validator reports: same problem, different shape. + manifest_only = validate_manifest_detailed("::not: [valid") + assert manifest_only[0]["reason_code"] == entry["code"] + assert manifest_only[0]["message"] == entry["message"] + + +def test_validate_artifacts_without_bundles_equals_manifest_only_result(): + # No bundles supplied: activation is either skipped (no Rego to + # load) or fails the same way manifest validation does. Either + # way, the artifact validator must not invent activation errors + # when the manifest half is what actually reports the problem. + # For an unparseable document, both surfaces report the same + # underlying RuntimeError message; only the wire keys differ. + broken = "::not: [valid" + artifact_findings = validate_artifacts(broken) + manifest_findings = validate_manifest_detailed(broken) + assert len(artifact_findings) == len(manifest_findings) == 1 + assert artifact_findings[0]["code"] == manifest_findings[0]["reason_code"] + assert artifact_findings[0]["message"] == manifest_findings[0]["message"] + + # And for a grammatically invalid document — one that parses but + # fails validation — the two surfaces report the same underlying + # manifest problem. Activation would never be reached. + # Built rather than written on one line. A repo guard scans committed + # files for the version key and validates what follows it, and it + # cannot strip the quotes of a single-line Python literal. + invalid = ( + f'{_VERSION_KEY}: "0.4.0-alpha.1"\npolicies: {{}}\nintervention_points: {{}}\n' + ) + artifact_findings = validate_artifacts(invalid) + manifest_findings = validate_manifest_detailed(invalid) + assert len(artifact_findings) == len(manifest_findings) == 1 + assert artifact_findings[0]["code"] == manifest_findings[0]["reason_code"] + assert artifact_findings[0]["message"] == manifest_findings[0]["message"] + + +def test_validate_artifacts_accepts_none_for_bundles(): + # ``bundles=None`` is the ergonomic form for "no Rego to supply". + # It must behave identically to an empty mapping so callers can + # write either. + assert validate_artifacts(ARTIFACT_MANIFEST, None) == validate_artifacts( + ARTIFACT_MANIFEST, {} + ) diff --git a/tests/conformance/parity/cross_language_parity.py b/tests/conformance/parity/cross_language_parity.py index fa36ac6..1b014ea 100755 --- a/tests/conformance/parity/cross_language_parity.py +++ b/tests/conformance/parity/cross_language_parity.py @@ -65,6 +65,19 @@ # so the grammar refuses it. Every language must fail rather than return. BAD_MANIFEST = f'{_VERSION_KEY}: "0.4.0-alpha.1"\nmetadata: {{}}\n' +# A sound manifest that names Rego. The document check passes it whatever +# the Rego says, so it is the only way to see artifact validation work. +REGO_MANIFEST = ( + f'{_VERSION_KEY}: "0.4.0-alpha.1"\n' + "policies:\n gate:\n type: rego\n bundle: ./b\n" + 'intervention_points:\n input:\n policy_target: "$.input"\n' + " policy:\n id: gate\n query: data.acs.decision\n" +) +GOOD_BUNDLES = { + "gate": {"modules": {"p.rego": 'package acs\ndecision := {"decision":"allow"}\n'}} +} +BAD_BUNDLES = {"gate": {"modules": {"p.rego": "package acs\nthis is not rego ***\n"}}} + EXPECTED = { # Manifest surface "supported_versions_nonempty": True, @@ -101,6 +114,17 @@ "parsed_has_points": True, "diagnostics_on_bad": 1, "diagnostics_on_good": 0, + # Artifact validation. The manifest is sound either way, so only + # compiling the Rego tells the two bundles apart. + # + # With no bundles the manifest still names ./b, which is not on disk, + # so activation reports the missing bundle. That is the answer a host + # wants: validating a manifest that names Rego without supplying the + # Rego cannot be a pass. + "artifacts_manifest_only": 1, + "artifacts_good_rego": 0, + "artifacts_bad_rego": 1, + "artifacts_bad_rego_code": "runtime_error:policy_invocation_failed", } @@ -241,6 +265,17 @@ def _run(cmd: list[str], **kw) -> dict: let parsed = Manifest::from_yaml_str(&std::fs::read_to_string(&manifest_path).expect("read")) .expect("parse"); let parsed_json = serde_json::to_value(&parsed).expect("parsed json"); + let art = |bundles: &str| -> (usize, Option) { + let parsed: std::collections::BTreeMap = + serde_json::from_str(bundles).expect("bundles"); + match ActivatedPolicy::activate_from_memory(REGO_MANIFEST, parsed) { + Ok(_) => (0, None), + Err(e) => (1, Some(e.reason().to_string())), + } + }; + let art_only = art("{}"); + let art_good = art(GOOD_BUNDLES); + let art_bad = art(BAD_BUNDLES); let bad_diags = usize::from(Manifest::from_yaml_str(BAD_MANIFEST).is_err()); let good_diags = usize::from( Manifest::from_yaml_str(&std::fs::read_to_string(&manifest_path).expect("read")) @@ -260,6 +295,10 @@ def _run(cmd: list[str], **kw) -> dict: "parsed_has_points": parsed_json.get("intervention_points").is_some(), "diagnostics_on_bad": bad_diags, "diagnostics_on_good": good_diags, + "artifacts_manifest_only": art_only.0, + "artifacts_good_rego": art_good.0, + "artifacts_bad_rego": art_bad.0, + "artifacts_bad_rego_code": art_bad.1, "supported_versions_nonempty": !agent_control_spec::SUPPORTED_VERSIONS.is_empty(), "validate_good": validate_good, "validate_bad": validate_bad, @@ -290,6 +329,9 @@ def rust() -> dict: RUST_MAIN.replace("BAD_MANIFEST", json.dumps(BAD_MANIFEST)) .replace("ALLOW_CONTEXT", json.dumps(json.dumps(ALLOW_CONTEXT))) .replace("DENY_CONTEXT", json.dumps(json.dumps(DENY_CONTEXT))) + .replace("REGO_MANIFEST", json.dumps(REGO_MANIFEST)) + .replace("GOOD_BUNDLES", json.dumps(json.dumps(GOOD_BUNDLES))) + .replace("BAD_BUNDLES", json.dumps(json.dumps(BAD_BUNDLES))) ) (work / "Cargo.toml").write_text( f""" @@ -331,7 +373,7 @@ def python_binding() -> dict: from agent_control_spec import ( AcsInterceptor, ActivatedPolicy, StreamSession, supported_manifest_versions, validate_manifest, - parse_manifest, validate_manifest_detailed, + parse_manifest, validate_manifest_detailed, validate_artifacts, ) def check(source): @@ -385,6 +427,9 @@ def _hook(dispatcher): _parsed = parse_manifest(open({str(MANIFEST)!r}).read()) _bad_diags = validate_manifest_detailed({BAD_MANIFEST!r}) _good_diags = validate_manifest_detailed(open({str(MANIFEST)!r}).read()) +_art_only = validate_artifacts({REGO_MANIFEST!r}) +_art_good = validate_artifacts({REGO_MANIFEST!r}, {GOOD_BUNDLES!r}) +_art_bad = validate_artifacts({REGO_MANIFEST!r}, {BAD_BUNDLES!r}) print(json.dumps({{ "hook_benign_decision": _b[0], @@ -396,6 +441,10 @@ def _hook(dispatcher): "parsed_has_points": "intervention_points" in _parsed, "diagnostics_on_bad": len(_bad_diags), "diagnostics_on_good": len(_good_diags), + "artifacts_manifest_only": len(_art_only), + "artifacts_good_rego": len(_art_good), + "artifacts_bad_rego": len(_art_bad), + "artifacts_bad_rego_code": _art_bad[0]["code"] if _art_bad else None, "supported_versions_nonempty": len(supported_manifest_versions()) > 0, "validate_good": check(open({str(MANIFEST)!r}).read()), "validate_bad": check({BAD_MANIFEST!r}), @@ -457,6 +506,9 @@ def node() -> dict: const parsed = acs.parseManifest(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')); const badDiags = acs.validateManifestDetailed({json.dumps(BAD_MANIFEST)}); const goodDiags = acs.validateManifestDetailed(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')); +const artOnly = acs.validateArtifacts({json.dumps(REGO_MANIFEST)}); +const artGood = acs.validateArtifacts({json.dumps(REGO_MANIFEST)}, {json.dumps(GOOD_BUNDLES)}); +const artBad = acs.validateArtifacts({json.dumps(REGO_MANIFEST)}, {json.dumps(BAD_BUNDLES)}); console.log(JSON.stringify({{ hook_benign_decision: b[0], @@ -468,6 +520,10 @@ def node() -> dict: parsed_has_points: Object.prototype.hasOwnProperty.call(parsed, 'intervention_points'), diagnostics_on_bad: badDiags.length, diagnostics_on_good: goodDiags.length, + artifacts_manifest_only: artOnly.length, + artifacts_good_rego: artGood.length, + artifacts_bad_rego: artBad.length, + artifacts_bad_rego_code: artBad.length ? artBad[0].code : null, supported_versions_nonempty: acs.supportedManifestVersions().length > 0, validate_good: check(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')), validate_bad: check({json.dumps(BAD_MANIFEST)}), @@ -535,6 +591,9 @@ def node() -> dict: var parsed = AcsManifestTools.Parse(File.ReadAllText(manifest)); var badDiags = AcsManifestTools.Diagnostics(BAD_JSON); var goodDiags = AcsManifestTools.Diagnostics(File.ReadAllText(manifest)); +var artOnly = AcsManifestTools.ValidateArtifacts(REGO_MANIFEST, null); +var artGood = AcsManifestTools.ValidateArtifacts(REGO_MANIFEST, GOOD_BUNDLES); +var artBad = AcsManifestTools.ValidateArtifacts(REGO_MANIFEST, BAD_BUNDLES); Console.WriteLine(JsonSerializer.Serialize(new Dictionary { @@ -547,6 +606,10 @@ def node() -> dict: ["parsed_has_points"] = parsed.Contains("intervention_points"), ["diagnostics_on_bad"] = badDiags.Count, ["diagnostics_on_good"] = goodDiags.Count, + ["artifacts_manifest_only"] = artOnly.Count, + ["artifacts_good_rego"] = artGood.Count, + ["artifacts_bad_rego"] = artBad.Count, + ["artifacts_bad_rego_code"] = artBad.Count > 0 ? artBad[0].Code : null, ["supported_versions_nonempty"] = AcsManifest.SupportedVersions().Count > 0, ["validate_good"] = Check(() => AcsManifest.Validate(File.ReadAllText(manifest))), ["validate_bad"] = Check(() => AcsManifest.Validate(BAD_JSON)), @@ -593,6 +656,9 @@ def dotnet() -> dict: .replace("HOOKS_MANIFEST", json.dumps(str(HOOKS_MANIFEST))) .replace("SEV1", json.dumps(json.dumps({"severity": 1}))) .replace("SEV7", json.dumps(json.dumps({"severity": 7}))) + .replace("REGO_MANIFEST", json.dumps(REGO_MANIFEST)) + .replace("GOOD_BUNDLES", json.dumps(json.dumps(GOOD_BUNDLES))) + .replace("BAD_BUNDLES", json.dumps(json.dumps(BAD_BUNDLES))) .replace("ALLOW_JSON", json.dumps(json.dumps(ALLOW_CONTEXT))) .replace("DENY_JSON", json.dumps(json.dumps(DENY_CONTEXT))) .replace("BAD_JSON", json.dumps(BAD_MANIFEST)) From 61f53d3b23c2797120181991e3eac4d7e5e54931 Mon Sep 17 00:00:00 2001 From: Liam Crumm Date: Tue, 11 Aug 2026 20:35:23 +0000 Subject: [PATCH 08/10] feat: expose the engine's resource caps, and check reachability mechanically Found by auditing the direction I had never audited. Every earlier pass asked what 0.3 lost. None asked what the 0.4 engine holds that no binding exposes, which is the question that matters, and the parity harness cannot answer it: a capability absent from all four languages is consistent across all four. Asking it surfaced Limits. Runtime::with_limits has always been there and no binding in any language could reach it, so a host feeding large payloads could not raise max_snapshot_bytes and a host hardening against a hostile manifest could not lower max_extends_depth or manifest_url_timeout_ms. Each field is individually optional, so raising one cap does not restate the other nine. A field present but not a non negative integer is refused rather than silently kept at its default: a host that asked for a smaller bound and got the larger one would believe it was protected when it was not. Add check-language-coverage.py so the omission class fails the build rather than waiting to be noticed. It reads the engine's 85 public re exports and requires every one to be either a declared capability with the token each binding exposes, or a non capability with a written reason. Eighteen capabilities, sixty seven explained, none unaccounted for. Writing the reason down is the point, because an unexplained omission and a deliberate one look identical six months later. Parity is 33 assertions and the artifact check 13, both covering a cap smaller than the context, which must deny in every language or the limit was accepted and dropped. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Liam Crumm --- .github/workflows/ci.yml | 6 + scripts/check-language-coverage.py | 286 ++++++++++++++++++ scripts/verify-artifacts.py | 47 ++- sdk/dotnet/src/AgentControlSpec/HostHooks.cs | 13 +- sdk/dotnet/src/AgentControlSpec/Native.cs | 9 +- .../AgentControlSpec.Tests/HostHooksTests.cs | 32 ++ sdk/ffi/src/lib.rs | 78 ++++- sdk/node/binding.d.ts | 23 +- sdk/node/binding.js | 1 + sdk/node/native/src/lib.rs | 109 ++++++- sdk/node/src/index.ts | 69 ++++- sdk/node/test/host-hooks.test.mjs | 134 ++++++++ sdk/python/agent_control_spec/__init__.py | 32 +- sdk/python/src/lib.rs | 153 +++++++++- sdk/python/tests/test_host_hooks.py | 135 +++++++++ .../parity/cross_language_parity.py | 80 ++++- 16 files changed, 1165 insertions(+), 42 deletions(-) create mode 100755 scripts/check-language-coverage.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2c3108..1292f62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,6 +85,12 @@ jobs: ./actionlint -color - name: Version surfaces agree run: python3 scripts/check-version-consistency.py + # The parity harness proves the bindings agree about the calls it + # makes. It cannot notice a capability none of them expose, because + # a surface absent everywhere is consistent everywhere. That blind + # spot is how streaming shipped reachable only from Rust. + - name: Every engine capability is reachable from every language + run: python3 scripts/check-language-coverage.py - name: cargo-audit over example lockfiles (pinned, checksum-verified) # Examples are deliberately outside the Dependabot update config, # and the coding_agent app's path dependency predates the engine diff --git a/scripts/check-language-coverage.py b/scripts/check-language-coverage.py new file mode 100755 index 0000000..9e37999 --- /dev/null +++ b/scripts/check-language-coverage.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Fail when an engine capability is reachable from fewer than four languages. + +The parity harness proves the four bindings agree about the calls it +makes. It cannot notice a capability none of them expose, because a +surface absent everywhere is consistent everywhere. That blind spot is +how streaming shipped reachable only from Rust, and how the host +dispatchers stayed unreachable outside it for a release. + +So this reads the engine's public re-exports and asserts that every one +carrying host-facing behaviour has an entry point in the C ABI, .NET, +Python and Node. A capability added to the engine and bound nowhere +fails here rather than being discovered by a consumer. + +Symbols that are data rather than behaviour are listed explicitly with a +reason. Writing the reason down is the point: an unexplained omission +and a deliberate one look identical six months later. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# Symbols that need no entry point, each with why. Anything not listed +# here and not matched below is treated as an unbound capability. +NOT_A_CAPABILITY = { + # Types that cross the boundary as JSON rather than as objects. A + # binding never constructs one, so there is nothing to bind. + "AgentContext": "crosses as JSON", + "Verdict": "crosses as JSON", + "Decision": "crosses as JSON", + "Warning": "crosses as JSON", + "Transform": "crosses as JSON", + "Evidence": "crosses as JSON", + "EnforcementMode": "crosses as JSON", + "InterceptionPoint": "crosses as JSON", + "EvaluationRequest": "crosses as JSON", + "EvaluationResult": "crosses as JSON", + "AnnotatorInvocation": "crosses as JSON to a host dispatcher", + "PreparedPolicyInvocation": "crosses as JSON to a host dispatcher", + "RegoPolicyInvocation": "variant of PreparedPolicyInvocation", + "CedarPolicyInvocation": "variant of PreparedPolicyInvocation", + "TestPolicyInvocation": "variant of PreparedPolicyInvocation", + "CustomPolicyInvocation": "variant of PreparedPolicyInvocation", + "CedarRequest": "crosses as JSON", + "CedarEntity": "crosses as JSON", + "TelemetryEvent": "crosses as JSON to a host sink", + "TelemetryEventType": "field of TelemetryEvent", + "RuntimeError": "surfaces as an error message or a diagnostic", + "StreamError": "surfaces as an end reason", + # Manifest grammar, reached by parsing a manifest rather than by + # constructing the node. + "Manifest": "reached through parse and validate", + "InterventionPointConfig": "manifest grammar", + "AnnotationConfig": "manifest grammar", + "AnnotatorConfig": "manifest grammar", + "AnnotatorType": "manifest grammar", + "PolicyConfig": "manifest grammar", + "PolicyBinding": "manifest grammar", + "RegoPolicyConfig": "manifest grammar", + "CedarPolicyConfig": "manifest grammar", + "TestPolicyConfig": "manifest grammar", + "CustomPolicyConfig": "manifest grammar", + "ToolConfig": "manifest grammar", + "MountedRegoData": "field of InMemoryRegoBundle", + # Dispatchers the manifest selects by policy type. A host picks one + # by writing a manifest, not by naming the Rust type. + "RegorusPolicyDispatcher": "selected by manifest policy type", + "RegorusRegoRunner": "selected by manifest policy type", + "OpaPolicyDispatcher": "selected by manifest policy type", + "OpaRegoRunner": "selected by manifest policy type", + "CedarBuiltinDispatcher": "selected by manifest policy type", + "CedarPolicyDispatcher": "selected by manifest policy type", + "CedarTestDispatcher": "test double", + "DefaultAnnotatorDispatcher": "the default when a host supplies none", + "NoopTelemetrySink": "the default when a host supplies none", + "ClassifierAnnotator": "selected by manifest annotator type", + "EndpointAnnotator": "selected by manifest annotator type", + "LlmAnnotator": "selected by manifest annotator type", + "Interceptor": "the trait AcsInterceptor implements", + "AcsInterceptor": "bound as the interceptor entry point itself", + "Runtime": "bound as the interceptor entry point itself", + # Internals of building a policy input. A host supplies the context; + # the engine builds the input from it. + "JsonPath": "internal to policy input construction", + "PathEnv": "internal to policy input construction", + "PathRoot": "internal to policy input construction", + "PathSegment": "internal to policy input construction", + "PathParseError": "internal to policy input construction", + "build_policy_input": "internal to policy input construction", + "build_cedar_request": "internal to Cedar dispatch", + "normalize_policy_output": "internal to verdict normalization", + "runtime_error_verdict": "internal to verdict normalization", + "translate_advice": "internal to Cedar dispatch", + "InterceptionPointExt": "Rust ergonomics on a foreign enum", + "canonical_json": "agent-hooks owns identity and canonicalization", + # Constants, readable from a verdict or a spec document. + "SUPPORTED_VERSIONS": "bound through supported_manifest_versions", + "MAX_RUNE_OFFSET": "constant", + "STREAMING_FAIL_CLOSED_REASON": "constant, appears in a verdict", + "VERDICT_INVALID_REASON": "constant, appears in a verdict", + "reserved_reason": "constants, appear in a verdict", +} + +# A capability is reachable when each binding names it or the entry +# point that carries it. Matching by name alone would miss the cases +# where a binding renames on the way out, so state the token to look for. +CAPABILITIES = { + "ActivatedPolicy": ( + "acs_policy_activate", + "policy_activate", + "policy_activate", + "PolicyActivate", + ), + "InMemoryRegoBundle": ("bundles_json", "bundles", "bundles", "bundlesJson"), + "AnnotatorDispatcher": ( + "AcsAnnotatorFn", + "annotator_dispatcher", + "annotatorDispatcher", + "AnnotatorDispatcher", + ), + "PolicyDispatcher": ( + "AcsPolicyFn", + "policy_dispatcher", + "policyDispatcher", + "PolicyDispatcher", + ), + "TelemetrySink": ( + "AcsTelemetryFn", + "telemetry_sink", + "telemetrySink", + "TelemetrySink", + ), + "PerfTelemetry": ( + "perf_telemetry", + "perf_telemetry", + "perfTelemetry", + "PerfTelemetry", + ), + "Limits": ("limits_json", "limits", "limits", "limits"), + "StreamSession": ( + "acs_stream_session_new", + "stream_session_new", + "stream_session_new", + "StreamSession", + ), + "StreamSessionConfig": ( + "acs_stream_session_new", + "stream_session_new", + "stream_session_new", + "StreamSession", + ), + "StreamWatermark": ( + "acs_stream_session_watermark", + "stream_watermark", + "stream_session_watermark", + "Watermark", + ), + "StreamCompletion": ( + "acs_stream_session_finish", + "stream_finish", + "stream_session_finish", + "Finish", + ), + "StreamEndReason": ("end_reason", "end_reason", "endReason", "EndReason"), + "StreamSpan": ( + "acs_stream_session_record_outcome", + "stream_record_outcome", + "stream_session_record_outcome", + "RecordOutcome", + ), + "SegmentOutcome": ( + "acs_stream_session_record_outcome", + "stream_record_outcome", + "stream_session_record_outcome", + "SegmentOutcome", + ), + "SafetyLevel": ("safety_level", "safety_level", "safetyLevel", "SafetyLevel"), + "StreamSourceType": ( + "source_type", + "source_type", + "sourceType", + "StreamSourceType", + ), + "StreamTrack": ( + "acs_stream_session_advance", + "stream_advance", + "stream_session_advance", + "StreamTrack", + ), + "RuneRange": ( + "acs_stream_session_record_outcome", + "stream_record_outcome", + "stream_session_record_outcome", + "RecordOutcome", + ), +} + +BINDINGS = ( + ("ffi", [ROOT / "sdk/ffi/src/lib.rs"]), + ( + "python", + [ + ROOT / "sdk/python/src/lib.rs", + ROOT / "sdk/python/agent_control_spec/__init__.py", + ], + ), + ("node", [ROOT / "sdk/node/native/src/lib.rs", ROOT / "sdk/node/src/index.ts"]), + ("dotnet", list((ROOT / "sdk/dotnet/src/AgentControlSpec").glob("*.cs"))), +) + + +def engine_symbols() -> set[str]: + src = (ROOT / "engine/src/lib.rs").read_text(encoding="utf-8") + found: set[str] = set() + for match in re.finditer(r"pub use ([^;]+);", src, re.DOTALL): + body = re.sub(r"^\s*[\w:]+::", "", match.group(1)) + groups = re.findall(r"\{([^}]*)\}", body) or [body] + for group in groups: + for name in group.split(","): + name = name.strip().split(" as ")[-1].strip() + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name or ""): + found.add(name) + return found + + +def main() -> int: + sources = { + name: "\n".join(p.read_text(encoding="utf-8") for p in paths if p.exists()) + for name, paths in BINDINGS + } + + symbols = engine_symbols() + unexplained = sorted( + s for s in symbols if s not in NOT_A_CAPABILITY and s not in CAPABILITIES + ) + + failed = False + if unexplained: + print( + "engine symbols that are neither a declared capability nor an " + "explained non-capability:", + file=sys.stderr, + ) + for name in unexplained: + print(f" {name}", file=sys.stderr) + print( + "\nAdd each to CAPABILITIES with the token every binding exposes, " + "or to NOT_A_CAPABILITY with the reason it needs none.", + file=sys.stderr, + ) + failed = True + + for symbol, tokens in sorted(CAPABILITIES.items()): + missing = [ + lang + for (lang, _), token in zip(BINDINGS, tokens) + if token not in sources[lang] + ] + if missing: + print( + f" {symbol:24} UNREACHABLE from {', '.join(missing)}", file=sys.stderr + ) + failed = True + + if failed: + return 1 + + print( + f"every one of {len(CAPABILITIES)} engine capabilities is reachable from " + f"all {len(BINDINGS)} bindings" + ) + print( + f"{len(NOT_A_CAPABILITY)} further symbols are data or internals, each with a stated reason" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-artifacts.py b/scripts/verify-artifacts.py index b940be6..5a0b135 100755 --- a/scripts/verify-artifacts.py +++ b/scripts/verify-artifacts.py @@ -56,6 +56,8 @@ " policy:\n id: gate\n query: data.acs.decision\n" ) BAD_BUNDLES = {"gate": {"modules": {"p.rego": "package acs\nthis is not rego ***\n"}}} +BIG_INPUT = "x" * 4096 +SMALL_CAP = {"max_snapshot_bytes": 64} EXPECTED = { "non_streaming": True, @@ -71,6 +73,9 @@ "streaming_settled": None, # A manifest the document check passes whose Rego does not compile. "artifacts_bad_rego": 1, + # A cap smaller than the context must deny, or the artifact accepted + # the limit and dropped it. + "limits_capped_denies": True, } @@ -204,6 +209,10 @@ def hook(d): "streaming_offset": off, "streaming_clean": clean, "streaming_settled": s.safe_offset("response"), "artifacts_bad_rego": len(validate_artifacts({rego!r}, {bad!r})), + "limits_capped_denies": str(getattr( + AcsInterceptor(M, annotator_dispatcher=C(1), limits={cap!r}).intercept( + {{"interception_point": "input", "input": {big!r}}}).decision, + "value", "")).lower() == "deny", }})) """ @@ -217,7 +226,11 @@ def check_python(art: dict, stage: Path) -> dict: str(venv / "bin/python"), "-c", PY_PROGRAM.format( - manifest=str(HOOKS_MANIFEST), rego=REGO_MANIFEST, bad=BAD_BUNDLES + manifest=str(HOOKS_MANIFEST), + rego=REGO_MANIFEST, + bad=BAD_BUNDLES, + cap=SMALL_CAP, + big=BIG_INPUT, ), ] ) @@ -252,6 +265,9 @@ def check_python(art: dict, stage: Path) -> dict: streaming_offset: off, streaming_clean: clean, streaming_settled: s.safeOffset('response'), artifacts_bad_rego: acs.validateArtifacts(%s, %s).length, + limits_capped_denies: String(acs.AcsInterceptor + .fromPath(M, { annotatorDispatcher: () => ({ severity: 1 }), limits: %s }) + .intercept({ interception_point: 'input', input: %s }).decision).toLowerCase() === 'deny', })); """ @@ -273,6 +289,8 @@ def check_node(art: dict, stage: Path) -> dict: json.dumps(str(HOOKS_MANIFEST)), json.dumps(REGO_MANIFEST), json.dumps(BAD_BUNDLES), + json.dumps(SMALL_CAP), + json.dumps(BIG_INPUT), ), ], cwd=app, @@ -318,6 +336,10 @@ def check_node(art: dict, stage: Path) -> dict: ["streaming_clean"] = clean, ["streaming_settled"] = s.SafeOffset(StreamTrack.Response), ["artifacts_bad_rego"] = AcsManifestTools.ValidateArtifacts(__REGO__, __BADB__).Count, + ["limits_capped_denies"] = (await AcsHostInterceptor + .FromPath(M, annotator: (_, _, _) => SEV1, limits: __CAP__) + .InterceptAsync(new AgentContext(JsonNode.Parse(__BIGCTX__)!.AsObject()))) + .Decision.ToString().ToLowerInvariant() == "deny", })); """ @@ -358,6 +380,11 @@ def check_dotnet(art: dict, stage: Path) -> dict: .replace("SEV7", json.dumps(json.dumps({"severity": 7}))) .replace("__REGO__", json.dumps(REGO_MANIFEST)) .replace("__BADB__", json.dumps(json.dumps(BAD_BUNDLES))) + .replace("__CAP__", json.dumps(json.dumps(SMALL_CAP))) + .replace( + "__BIGCTX__", + json.dumps(json.dumps({"interception_point": "input", "input": BIG_INPUT})), + ) ) (app / "Program.cs").write_text(program) out = run(["dotnet", "run", "--project", str(app), "--nologo"]) @@ -415,6 +442,16 @@ def check_dotnet(art: dict, stage: Path) -> dict: let clean = s.finish().reason.is_clean(); let bad_bundles: std::collections::BTreeMap = serde_json::from_str(BAD_BUNDLES).expect("bundles"); + let capped = Runtime::with_limits( + Manifest::from_path(&m).expect("manifest"), + Arc::new(C(1, std::sync::atomic::AtomicUsize::new(0))), + Arc::new(BindingPolicyDispatcher::new()), + agent_control_spec::Limits { max_snapshot_bytes: 64, ..Default::default() }, + ) + .expect("capped runtime"); + let big: serde_json::Value = serde_json::from_str(BIG_CTX).expect("big ctx"); + let limits_capped_denies = + format!("{:?}", capped.evaluate(&big).verdict.decision).to_lowercase() == "deny"; let artifacts_bad_rego = usize::from( agent_control_spec::ActivatedPolicy::activate_from_memory(REGO_MANIFEST, bad_bundles) .is_err(), @@ -428,6 +465,7 @@ def check_dotnet(art: dict, stage: Path) -> dict: "streaming_offset": off, "streaming_clean": clean, "streaming_settled": s.safe_offset(StreamTrack::Response), "artifacts_bad_rego": artifacts_bad_rego, + "limits_capped_denies": limits_capped_denies, })); } """ @@ -450,8 +488,11 @@ def check_rust(art: dict, stage: Path) -> dict: """ ) (app / "src/main.rs").write_text( - RUST_MAIN.replace("REGO_MANIFEST", json.dumps(REGO_MANIFEST)).replace( - "BAD_BUNDLES", json.dumps(json.dumps(BAD_BUNDLES)) + RUST_MAIN.replace("REGO_MANIFEST", json.dumps(REGO_MANIFEST)) + .replace("BAD_BUNDLES", json.dumps(json.dumps(BAD_BUNDLES))) + .replace( + "BIG_CTX", + json.dumps(json.dumps({"interception_point": "input", "input": BIG_INPUT})), ) ) out = run( diff --git a/sdk/dotnet/src/AgentControlSpec/HostHooks.cs b/sdk/dotnet/src/AgentControlSpec/HostHooks.cs index 260eee2..de68045 100644 --- a/sdk/dotnet/src/AgentControlSpec/HostHooks.cs +++ b/sdk/dotnet/src/AgentControlSpec/HostHooks.cs @@ -165,12 +165,20 @@ private delegate IntPtr NativeAnnotator( /// Host policy engine, or null for the bundled dispatchers. /// Host telemetry sink, or null to record nothing. /// How much timing detail to record. + /// + /// Resource caps as JSON, overriding the engine's defaults field by + /// field. Null keeps every default. A host feeding large payloads + /// raises max_snapshot_bytes; one hardening against a hostile + /// manifest lowers max_extends_depth or + /// manifest_url_timeout_ms. + /// public static AcsHostInterceptor FromPath( string manifestPath, AnnotatorDispatcher? annotator = null, PolicyDispatcher? policy = null, TelemetrySink? telemetry = null, - PerfTelemetry perfTelemetry = PerfTelemetry.Off) + PerfTelemetry perfTelemetry = PerfTelemetry.Off, + string? limits = null) { var pinned = new List(); @@ -247,7 +255,8 @@ public static AcsHostInterceptor FromPath( policyPtr, IntPtr.Zero, telemetryPtr, IntPtr.Zero, Marshal.GetFunctionPointerForDelegate(free), - perfTelemetry.ToString().ToLowerInvariant()); + perfTelemetry.ToString().ToLowerInvariant(), + limits); var interceptor = new AcsHostInterceptor(handle); interceptor._pinned.AddRange(pinned); diff --git a/sdk/dotnet/src/AgentControlSpec/Native.cs b/sdk/dotnet/src/AgentControlSpec/Native.cs index 7ade4c6..d2c6839 100644 --- a/sdk/dotnet/src/AgentControlSpec/Native.cs +++ b/sdk/dotnet/src/AgentControlSpec/Native.cs @@ -474,6 +474,7 @@ private static partial IntPtr acs_interceptor_new_with_hooks( IntPtr telemetryFn, IntPtr telemetryCtx, IntPtr hookFree, IntPtr perfTelemetry, + IntPtr limitsJson, out IntPtr errOut); [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] @@ -495,18 +496,20 @@ internal static IntPtr InterceptorNewWithHooks( IntPtr policyFn, IntPtr policyCtx, IntPtr telemetryFn, IntPtr telemetryCtx, IntPtr hookFree, - string? perfTelemetry) + string? perfTelemetry, + string? limitsJson) { var bytes = System.Text.Encoding.UTF8.GetBytes(manifestPath); var perf = perfTelemetry is null ? IntPtr.Zero : Marshal.StringToCoTaskMemUTF8(perfTelemetry); + var limits = limitsJson is null ? IntPtr.Zero : Marshal.StringToCoTaskMemUTF8(limitsJson); try { var handle = acs_interceptor_new_with_hooks( bytes, (nuint)bytes.Length, annotatorFn, annotatorCtx, policyFn, policyCtx, - telemetryFn, telemetryCtx, hookFree, perf, out var err); + telemetryFn, telemetryCtx, hookFree, perf, limits, out var err); if (handle == IntPtr.Zero) { ThrowIfError(err); @@ -519,6 +522,8 @@ internal static IntPtr InterceptorNewWithHooks( { if (perf != IntPtr.Zero) Marshal.FreeCoTaskMem(perf); + if (limits != IntPtr.Zero) + Marshal.FreeCoTaskMem(limits); } } diff --git a/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs b/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs index ef34808..bc36380 100644 --- a/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs +++ b/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs @@ -279,5 +279,37 @@ public void WithNoBundlesArtifactValidationAgreesWithTheManifestCheck() AcsManifestTools.ValidateArtifacts(Bad, null).Count); } + + [Fact] + public async Task ALoweredSnapshotCapIsEnforced() + { + var manifest = WriteFixture(); + var big = new string('x', 4096); + + using var permissive = AcsHostInterceptor.FromPath( + manifest, annotator: (_, _, _) => """{"severity":1}"""); + Assert.Equal(Decision.Allow, (await permissive.InterceptAsync(Input(big))).Decision); + + // The same context against a cap smaller than it. A host that + // asked for a smaller bound and kept the larger one would believe + // it was protected when it was not. + using var capped = AcsHostInterceptor.FromPath( + manifest, + annotator: (_, _, _) => """{"severity":1}""", + limits: """{"max_snapshot_bytes": 64}"""); + + var verdict = await capped.InterceptAsync(Input(big)); + Assert.Equal(Decision.Deny, verdict.Decision); + Assert.StartsWith("runtime_error:", verdict.Reason); + } + + [Fact] + public void ALimitThatIsNotANumberIsRefused() + { + var manifest = WriteFixture(); + Assert.Throws(() => + AcsHostInterceptor.FromPath(manifest, limits: """{"max_snapshot_bytes": "big"}""")); + } + public void Dispose() => Directory.Delete(_dir, recursive: true); } diff --git a/sdk/ffi/src/lib.rs b/sdk/ffi/src/lib.rs index 104017d..298cdf4 100644 --- a/sdk/ffi/src/lib.rs +++ b/sdk/ffi/src/lib.rs @@ -28,8 +28,8 @@ use agent_control_spec::stream_session::{ }; use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; use agent_control_spec::{ - ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, Runtime, RuntimeError, - Verdict, SUPPORTED_VERSIONS, + ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Limits, Manifest, Runtime, + RuntimeError, Verdict, SUPPORTED_VERSIONS, }; use serde_json::Value; use std::collections::BTreeMap; @@ -1011,6 +1011,67 @@ impl TelemetrySink for HostTelemetrySink { } } +/// Read a limits override, treating NULL and empty as "keep the +/// defaults". Absent fields keep their default individually, so a host +/// that only needs a bigger snapshot cap does not have to restate the +/// other nine. +fn parse_limits(value: *const c_char, err_out: *mut *mut c_char) -> Option { + let mut limits = Limits::default(); + if value.is_null() { + return Some(limits); + } + let raw = unsafe { read_utf8(value, "limits_json", err_out) }?; + if raw.trim().is_empty() { + return Some(limits); + } + let parsed: Value = match serde_json::from_str(raw) { + Ok(v @ Value::Object(_)) => v, + Ok(_) => { + set_err(err_out, "limits_json must be a JSON object".to_string()); + return None; + } + Err(e) => { + set_err(err_out, format!("limits_json does not parse: {e}")); + return None; + } + }; + + // A cap that does not parse is refused rather than silently kept at + // its default. A host that asked for a smaller bound and got the + // larger one would believe it was protected when it was not. + let read = |key: &str| -> Option> { + match parsed.get(key) { + None | Some(Value::Null) => Some(None), + Some(v) => match v.as_u64() { + Some(n) => Some(Some(n)), + None => { + set_err(err_out, format!("{key} must be a non negative integer")); + None + } + }, + } + }; + + macro_rules! apply { + ($field:ident, $ty:ty) => { + if let Some(v) = read(stringify!($field))? { + limits.$field = v as $ty; + } + }; + } + apply!(max_snapshot_bytes, usize); + apply!(max_policy_input_depth, usize); + apply!(max_annotators_per_point, usize); + apply!(max_annotator_output_bytes, usize); + apply!(max_policy_output_bytes, usize); + apply!(max_extends_depth, usize); + apply!(max_merged_manifest_bytes, usize); + apply!(max_manifest_url_bytes, usize); + apply!(manifest_url_timeout_ms, u64); + apply!(max_manifest_url_redirects, usize); + Some(limits) +} + fn parse_perf(value: *const c_char, err_out: *mut *mut c_char) -> Option { if value.is_null() { return Some(PerfTelemetry::Off); @@ -1057,6 +1118,7 @@ pub unsafe extern "C" fn acs_interceptor_new_with_hooks( telemetry_ctx: *mut c_void, hook_free: Option, perf_telemetry: *const c_char, + limits_json: *const c_char, err_out: *mut *mut c_char, ) -> *mut AcsInterceptor { clear_err(err_out); @@ -1075,6 +1137,9 @@ pub unsafe extern "C" fn acs_interceptor_new_with_hooks( let Some(perf) = parse_perf(perf_telemetry, err_out) else { return std::ptr::null_mut(); }; + let Some(limits) = parse_limits(limits_json, err_out) else { + return std::ptr::null_mut(); + }; let annotations: Arc = match annotator_fn { Some(call) => Arc::new(HostAnnotatorDispatcher { @@ -1107,7 +1172,14 @@ pub unsafe extern "C" fn acs_interceptor_new_with_hooks( None => Arc::new(NoopTelemetrySink), }; - match Runtime::with_telemetry_and_perf(manifest, annotations, policy, telemetry, perf) { + match Runtime::with_telemetry_perf_and_limits( + manifest, + annotations, + policy, + telemetry, + perf, + limits, + ) { Ok(runtime) => Box::into_raw(Box::new(AcsInterceptor { runtime, name: "acs".to_string(), diff --git a/sdk/node/binding.d.ts b/sdk/node/binding.d.ts index ba79999..2c6a8c3 100644 --- a/sdk/node/binding.d.ts +++ b/sdk/node/binding.d.ts @@ -7,6 +7,14 @@ export declare class ExternalObject { [K: symbol]: T } } +/** + * The engine's default resource caps as a JSON object string. A host + * that raises one cap reads this to see what it is overriding, so a + * shipping change to another default cannot be silently absorbed by a + * stale mapping. + */ +export declare function defaultLimits(): string + /** * Evaluate one agent context (JSON object per AGENT-HOOKS-0.1 §4) and * return the verdict as wire JSON. @@ -22,21 +30,30 @@ export declare function interceptorNew(manifestPath: string): ExternalObject string) | undefined | null, policyDispatcher?: ((arg0: string) => string) | undefined | null, telemetrySink?: ((arg0: string) => void) | undefined | null, perfTelemetry?: string | undefined | null): ExternalObject +export declare function interceptorNewWithHooks(manifestPath: string, annotatorDispatcher?: ((arg0: string, arg1: string, arg2: string) => string) | undefined | null, policyDispatcher?: ((arg0: string) => string) | undefined | null, telemetrySink?: ((arg0: string) => void) | undefined | null, perfTelemetry?: string | undefined | null, limitsJson?: string | undefined | null): ExternalObject /** * Compose a chain of manifest YAML documents (outermost base first) diff --git a/sdk/node/binding.js b/sdk/node/binding.js index b3186f0..ebc8da4 100644 --- a/sdk/node/binding.js +++ b/sdk/node/binding.js @@ -700,6 +700,7 @@ if (!nativeBinding) { } module.exports = nativeBinding +module.exports.defaultLimits = nativeBinding.defaultLimits module.exports.intercept = nativeBinding.intercept module.exports.interceptorNew = nativeBinding.interceptorNew module.exports.interceptorNewWithHooks = nativeBinding.interceptorNewWithHooks diff --git a/sdk/node/native/src/lib.rs b/sdk/node/native/src/lib.rs index 344e0d4..481e31f 100644 --- a/sdk/node/native/src/lib.rs +++ b/sdk/node/native/src/lib.rs @@ -14,10 +14,10 @@ use agent_control_spec::runtime::PolicyDispatcher; use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; use agent_control_spec::Verdict; use agent_control_spec::{ - ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, JsonValue, Manifest, PerfTelemetry, - PreparedPolicyInvocation, Runtime, RuntimeError, SafetyLevel, SegmentOutcome, StreamEndReason, - StreamError, StreamSession, StreamSessionConfig, StreamSourceType, StreamSpan, StreamTrack, - SUPPORTED_VERSIONS, + ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, JsonValue, Limits, Manifest, + PerfTelemetry, PreparedPolicyInvocation, Runtime, RuntimeError, SafetyLevel, SegmentOutcome, + StreamEndReason, StreamError, StreamSession, StreamSessionConfig, StreamSourceType, StreamSpan, + StreamTrack, SUPPORTED_VERSIONS, }; use napi::bindgen_prelude::{External, FnArgs, FunctionRef, Utf16String}; use napi::Env; @@ -270,6 +270,81 @@ fn parse_perf(value: Option) -> napi::Result { } } +/// Read a limits override, mirroring the FFI's `parse_limits`: +/// +/// - Absent / null / empty means keep every default. +/// - Each field is individually optional; an absent field keeps its own +/// default, so a host raising one cap does not restate the other nine. +/// - A field present but not a non-negative integer is a hard ERROR, not +/// a silently-kept default. A host that asked for a smaller bound and +/// got the larger one would believe it was protected when it was not. +fn parse_limits(value: Option) -> napi::Result { + let mut limits = Limits::default(); + let Some(value) = value else { + return Ok(limits); + }; + let raw = decode("limits", &value)?; + if raw.trim().is_empty() { + return Ok(limits); + } + let parsed: Value = match serde_json::from_str(&raw) { + Ok(v @ Value::Object(_)) => v, + Ok(_) => return Err(err("limits must be a JSON object".to_string())), + Err(e) => return Err(err(format!("limits does not parse: {e}"))), + }; + + let read = |key: &str| -> napi::Result> { + match parsed.get(key) { + None | Some(Value::Null) => Ok(None), + Some(v) => match v.as_u64() { + Some(n) => Ok(Some(n)), + None => Err(err(format!("{key} must be a non negative integer"))), + }, + } + }; + + macro_rules! apply { + ($field:ident, $ty:ty) => { + if let Some(v) = read(stringify!($field))? { + limits.$field = v as $ty; + } + }; + } + apply!(max_snapshot_bytes, usize); + apply!(max_policy_input_depth, usize); + apply!(max_annotators_per_point, usize); + apply!(max_annotator_output_bytes, usize); + apply!(max_policy_output_bytes, usize); + apply!(max_extends_depth, usize); + apply!(max_merged_manifest_bytes, usize); + apply!(max_manifest_url_bytes, usize); + apply!(manifest_url_timeout_ms, u64); + apply!(max_manifest_url_redirects, usize); + Ok(limits) +} + +/// The engine's default resource caps as a JSON object string. A host +/// that raises one cap reads this to see what it is overriding, so a +/// shipping change to another default cannot be silently absorbed by a +/// stale mapping. +#[napi] +pub fn default_limits() -> napi::Result { + let defaults = Limits::default(); + let value = serde_json::json!({ + "max_snapshot_bytes": defaults.max_snapshot_bytes, + "max_policy_input_depth": defaults.max_policy_input_depth, + "max_annotators_per_point": defaults.max_annotators_per_point, + "max_annotator_output_bytes": defaults.max_annotator_output_bytes, + "max_policy_output_bytes": defaults.max_policy_output_bytes, + "max_extends_depth": defaults.max_extends_depth, + "max_merged_manifest_bytes": defaults.max_merged_manifest_bytes, + "max_manifest_url_bytes": defaults.max_manifest_url_bytes, + "manifest_url_timeout_ms": defaults.manifest_url_timeout_ms, + "max_manifest_url_redirects": defaults.max_manifest_url_redirects, + }); + serde_json::to_string(&value).map_err(|e| err(format!("{e}"))) +} + // Type aliases for the FunctionRef signatures the JS↔Rust wire uses. // Kept private so they never leak into the TS declaration; napi-derive // still sees the expanded types on the entry points that take them as @@ -316,14 +391,23 @@ pub fn interceptor_new(manifest_path: Utf16String) -> napi::Result, String>>, telemetry_sink: Option, ()>>, perf_telemetry: Option, + limits_json: Option, ) -> napi::Result> { let manifest_path = decode("manifest_path", &manifest_path)?; let manifest = Manifest::from_path(&manifest_path).map_err(|e| err(format!("{e}")))?; let perf = parse_perf(perf_telemetry)?; + let limits = parse_limits(limits_json)?; let annotations = build_annotator(annotator_dispatcher); let policy = build_policy(policy_dispatcher); let telemetry = build_telemetry(telemetry_sink); - let runtime = Runtime::with_telemetry_and_perf(manifest, annotations, policy, telemetry, perf) - .map_err(|e| err(format!("{e}")))?; + let runtime = Runtime::with_telemetry_perf_and_limits( + manifest, + annotations, + policy, + telemetry, + perf, + limits, + ) + .map_err(|e| err(format!("{e}")))?; Ok(External::new(Handle { runtime })) } diff --git a/sdk/node/src/index.ts b/sdk/node/src/index.ts index d0323fe..992f6fd 100644 --- a/sdk/node/src/index.ts +++ b/sdk/node/src/index.ts @@ -34,8 +34,10 @@ const native = require("../binding.js") as { policyDispatcher?: ((invocationJson: string) => string) | null, telemetrySink?: ((eventJson: string) => void) | null, perfTelemetry?: string | null, + limitsJson?: string | null, ): unknown; intercept(handle: unknown, contextJson: string): string; + defaultLimits(): string; policyActivate(manifestPath: string): unknown; policyActivateFromMemory(manifestYaml: string, bundlesJson: string): unknown; policyActivateWithHooks( @@ -221,6 +223,53 @@ export type TelemetrySink = (event: TelemetryEvent) => void; */ export type PerfTelemetry = "off" | "external" | "full"; +/** + * Resource caps overriding the engine's defaults, field by field. + * + * `Limits` is a denial-of-service control surface: a host feeding + * large payloads raises `maxSnapshotBytes`; one hardening against a + * hostile manifest lowers `maxExtendsDepth` or + * `manifestUrlTimeoutMs`. Each field is individually optional; an + * absent field keeps its own default, so a host raising one cap does + * not restate the other nine. A field present but not a non-negative + * integer is a hard error, not a silently-kept default: a host that + * asked for a smaller bound and got the larger one would believe it + * was protected when it was not. + * + * See {@link DEFAULT_LIMITS} for the shipped values. + */ +export interface Limits { + /** Cap on the canonicalized context snapshot in bytes. */ + max_snapshot_bytes?: number; + /** JSON nesting depth accepted anywhere in policy input/output. */ + max_policy_input_depth?: number; + /** Number of annotators the engine will dispatch per intervention point. */ + max_annotators_per_point?: number; + /** Per-annotator serialized output cap in bytes. */ + max_annotator_output_bytes?: number; + /** Policy-decision serialized output cap in bytes. */ + max_policy_output_bytes?: number; + /** Manifest `extends` chain length. */ + max_extends_depth?: number; + /** Composed manifest total size cap in bytes. */ + max_merged_manifest_bytes?: number; + /** Per-URL manifest fetch body cap in bytes. */ + max_manifest_url_bytes?: number; + /** Per-URL manifest fetch deadline in milliseconds. */ + manifest_url_timeout_ms?: number; + /** Per-URL manifest fetch redirect count. */ + max_manifest_url_redirects?: number; +} + +/** + * The engine's shipped resource caps, as a frozen object. Read this to + * see what a `limits` mapping is overriding; a shipping change to + * another default cannot then be silently absorbed by a stale mapping. + */ +export const DEFAULT_LIMITS: Readonly> = Object.freeze( + JSON.parse(native.defaultLimits()) as Required, +); + /** * Host extension points. Each is optional; supplying one replaces the * zero-config default for that slot. @@ -230,6 +279,11 @@ export interface HostHooks { policyDispatcher?: PolicyDispatcher; telemetrySink?: TelemetrySink; perfTelemetry?: PerfTelemetry; + /** + * Resource caps overriding the engine's defaults, field by field. + * See {@link Limits} and {@link DEFAULT_LIMITS}. + */ + limits?: Readonly; } // --- Bridges between the object-oriented TS surface and the JSON -------- @@ -281,7 +335,8 @@ function hasHostHooks(options: HostHooks): boolean { options.annotatorDispatcher !== undefined || options.policyDispatcher !== undefined || options.telemetrySink !== undefined || - options.perfTelemetry !== undefined + options.perfTelemetry !== undefined || + options.limits !== undefined ); } @@ -310,10 +365,13 @@ export class AcsInterceptor implements Interceptor { * construction. * * Supply `annotatorDispatcher`, `policyDispatcher`, `telemetrySink`, - * or `perfTelemetry` to override the engine's extension points. Each - * callback is called synchronously on the JS thread from inside - * `intercept`. A callback that throws surfaces as a fail-closed - * `runtime_error:*` deny rather than reading as "no annotation". + * `perfTelemetry`, or `limits` to override the engine's extension + * points and resource caps. Each callback is called synchronously on + * the JS thread from inside `intercept`. A callback that throws + * surfaces as a fail-closed `runtime_error:*` deny rather than + * reading as "no annotation". A `limits` mapping overrides only the + * fields it names; the rest keep the engine's defaults, see + * {@link DEFAULT_LIMITS}. */ static fromPath(manifestPath: string, options: AcsInterceptorOptions = {}): AcsInterceptor { const name = options.name ?? "acs"; @@ -328,6 +386,7 @@ export class AcsInterceptor implements Interceptor { : null, options.telemetrySink ? wrapTelemetrySink(options.telemetrySink) : null, options.perfTelemetry ?? null, + options.limits ? JSON.stringify(options.limits) : null, ) : native.interceptorNew(manifestPath); return new AcsInterceptor(handle, name); diff --git a/sdk/node/test/host-hooks.test.mjs b/sdk/node/test/host-hooks.test.mjs index d649901..e7f5495 100644 --- a/sdk/node/test/host-hooks.test.mjs +++ b/sdk/node/test/host-hooks.test.mjs @@ -494,3 +494,137 @@ test("validateArtifacts throws on boundary problems", () => { assert.throws(() => validateArtifacts(ARTIFACT_MANIFEST, 5), TypeError); assert.throws(() => validateArtifacts(ARTIFACT_MANIFEST, "not-an-object"), TypeError); }); + +// --------------------------------------------------------------------- +// 9. Resource limits: caps overriding the engine's defaults reach the +// runtime and change the verdict. +// +// `Limits` is a denial-of-service control surface: a host feeding +// large payloads raises `max_snapshot_bytes`; one hardening against a +// hostile manifest lowers `max_extends_depth` or +// `manifest_url_timeout_ms`. +// +// The behavioural test is deliberately end-to-end: the same manifest +// and the same context, evaluated once with default caps and once with +// a small `max_snapshot_bytes`, produce different verdicts. That is +// what proves the value reaches the engine rather than being accepted +// on the JS side and dropped on the way in. +// --------------------------------------------------------------------- + +const { DEFAULT_LIMITS } = require("../dist/index.js"); + +test("a lowered snapshot cap flips the verdict from allow to fail-closed deny", () => { + const big = "x".repeat(4096); + const permissive = AcsInterceptor.fromPath(fixtureManifestPath); + assert.equal(permissive.intercept(builder().input(big)).decision, "allow"); + + // Same manifest, same context, but the cap is now smaller than the + // canonicalized snapshot. A host that asked for the smaller bound and + // got the larger one would believe it was protected when it was not. + const capped = AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { max_snapshot_bytes: 64 }, + }); + const verdict = capped.intercept(builder().input(big)); + assert.equal(verdict.decision, "deny"); + assert.ok( + verdict.reason.startsWith("runtime_error:"), + `expected runtime_error:*, got ${verdict.reason}`, + ); +}); + +test("no limits option matches the baseline zero-config path", () => { + const baseline = AcsInterceptor.fromPath(fixtureManifestPath); + const empty = AcsInterceptor.fromPath(fixtureManifestPath, { limits: {} }); + for (const ctx of [ + builder().input("hi"), + builder().preToolCall("t1", "search", { q: "x" }), + ]) { + // Two independent Contexts of the same shape. + assert.deepEqual( + empty.intercept(ctx), + baseline.intercept(ctx), + "an empty limits object must be identical to no limits option", + ); + } +}); + +test("overriding one limit leaves the others at their defaults", () => { + const big = "x".repeat(4096); + // Raise only the annotator output cap. Untouched `max_snapshot_bytes` + // still defaults big, so the 4096-char input allows. + const partial = AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { max_annotator_output_bytes: 8_388_608 }, + }); + assert.equal(partial.intercept(builder().input(big)).decision, "allow"); + + // And when the second, untouched cap IS lowered on a separate + // interceptor, it enforces — proving the field-by-field override + // semantics: a raised cap does not silently reset a peer. + const both = AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { + max_annotator_output_bytes: 8_388_608, + max_snapshot_bytes: 64, + }, + }); + const v = both.intercept(builder().input(big)); + assert.equal(v.decision, "deny"); + assert.ok(v.reason.startsWith("runtime_error:")); +}); + +test("a limit that is not a non-negative integer is refused", () => { + // A value the engine cannot parse is a hard error, not a + // silently-kept default. A host that typo'd learns immediately + // instead of finding out at the first breached limit that would + // never fire. + assert.throws( + () => + AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { max_snapshot_bytes: "big" }, + }), + /max_snapshot_bytes/, + ); + assert.throws( + () => + AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { max_snapshot_bytes: -1 }, + }), + /max_snapshot_bytes/, + ); + assert.throws( + () => + AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { max_snapshot_bytes: 1.5 }, + }), + /max_snapshot_bytes/, + ); +}); + +test("DEFAULT_LIMITS carries every documented field and is frozen", () => { + // A host raising one cap reads `DEFAULT_LIMITS` to see what it is + // overriding. The shape must stay wired to the engine's own + // defaults, so a shipping change to another cap cannot be silently + // absorbed by a stale mapping. + const expected = new Set([ + "max_snapshot_bytes", + "max_policy_input_depth", + "max_annotators_per_point", + "max_annotator_output_bytes", + "max_policy_output_bytes", + "max_extends_depth", + "max_merged_manifest_bytes", + "max_manifest_url_bytes", + "manifest_url_timeout_ms", + "max_manifest_url_redirects", + ]); + assert.deepEqual(new Set(Object.keys(DEFAULT_LIMITS)), expected); + for (const [key, value] of Object.entries(DEFAULT_LIMITS)) { + assert.equal( + typeof value, + "number", + `DEFAULT_LIMITS[${key}] must be a number, was ${typeof value}`, + ); + assert.ok(Number.isInteger(value) && value >= 0, `${key} = ${value}`); + } + // Frozen so a caller cannot mutate a shared default. + assert.ok(Object.isFrozen(DEFAULT_LIMITS)); +}); diff --git a/sdk/python/agent_control_spec/__init__.py b/sdk/python/agent_control_spec/__init__.py index 4657766..df82ee6 100644 --- a/sdk/python/agent_control_spec/__init__.py +++ b/sdk/python/agent_control_spec/__init__.py @@ -15,6 +15,7 @@ import json from collections.abc import Callable, Iterable, Mapping +from types import MappingProxyType as _MappingProxyType from typing import Any from agent_hooks import Verdict @@ -22,6 +23,7 @@ from agent_control_spec import _native __all__ = [ + "DEFAULT_LIMITS", "PERF_TELEMETRY_LEVELS", "AcsInterceptor", "ActivatedPolicy", @@ -53,6 +55,24 @@ #: allowed values are the vocabulary a Python host reads. PERF_TELEMETRY_LEVELS: tuple[str, ...] = ("off", "external", "full") +#: The engine's shipped resource caps, as a read-only mapping. A host +#: passing ``limits=`` to :class:`AcsInterceptor` reads this to see what +#: it is overriding — a shipping change to another default cannot then +#: be silently absorbed. Frozen at import time so a caller cannot mutate +#: a shared default. Fields: +#: +#: - ``max_snapshot_bytes``: cap on the canonicalized context snapshot. +#: - ``max_policy_input_depth``: JSON nesting depth accepted anywhere. +#: - ``max_annotators_per_point``: annotators the engine will dispatch. +#: - ``max_annotator_output_bytes``: per-annotator serialized output. +#: - ``max_policy_output_bytes``: policy-decision serialized output. +#: - ``max_extends_depth``: manifest ``extends`` chain length. +#: - ``max_merged_manifest_bytes``: composed manifest total size. +#: - ``max_manifest_url_bytes``: per-URL fetch body cap. +#: - ``manifest_url_timeout_ms``: per-URL fetch deadline. +#: - ``max_manifest_url_redirects``: per-URL fetch redirect count. +DEFAULT_LIMITS: Mapping[str, int] = _MappingProxyType(_native.default_limits()) + #: A telemetry event a host-supplied sink receives from the engine. #: #: The dict is populated by the native layer and mirrors @@ -128,7 +148,8 @@ class AcsInterceptor: Zero-config path (the default): bundled annotators; Rego in process, Cedar through the built-in evaluator, ``test`` policies through - their embedded verdict; no-op telemetry. + their embedded verdict; no-op telemetry; the engine's default + resource caps. Host hooks are supplied by keyword: @@ -146,6 +167,13 @@ class AcsInterceptor: - ``perf_telemetry``: ``"off"`` (default), ``"external"``, or ``"full"``, gating whether external and per-stage timing events are emitted. + - ``limits``: a mapping of resource caps that overrides the engine's + defaults field by field. Absent means keep every default; each + field is individually optional, so a host raising one cap does + not restate the other nine. A host feeding large payloads raises + ``max_snapshot_bytes``; one hardening against a hostile manifest + lowers ``max_extends_depth`` or ``manifest_url_timeout_ms``. Read + :data:`DEFAULT_LIMITS` to see the shipped values. A dispatcher that raises does not silently no-op: the engine normalizes the failure into a fail-closed ``deny`` verdict with a @@ -161,6 +189,7 @@ def __init__( policy_dispatcher: object | None = None, telemetry_sink: object | Callable[[TelemetryEvent], None] | None = None, perf_telemetry: str = "off", + limits: Mapping[str, int] | None = None, ) -> None: perf = _normalize_perf_telemetry(perf_telemetry) self._handle = _native.interceptor_new( @@ -169,6 +198,7 @@ def __init__( policy_dispatcher, telemetry_sink, perf, + limits, ) self._name = name diff --git a/sdk/python/src/lib.rs b/sdk/python/src/lib.rs index 30b484f..7258c3e 100644 --- a/sdk/python/src/lib.rs +++ b/sdk/python/src/lib.rs @@ -12,16 +12,17 @@ use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; use agent_control_spec::policy::PreparedPolicyInvocation; use agent_control_spec::runtime::PolicyDispatcher; -use agent_control_spec::telemetry::{TelemetryEvent, TelemetrySink}; +use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; use agent_control_spec::{ - ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, PerfTelemetry, Runtime, - RuntimeError, SafetyLevel, SegmentOutcome, StreamEndReason, StreamError, StreamSession, - StreamSessionConfig, StreamSourceType, StreamSpan, StreamTrack, Verdict, SUPPORTED_VERSIONS, + ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Limits, Manifest, PerfTelemetry, + Runtime, RuntimeError, SafetyLevel, SegmentOutcome, StreamEndReason, StreamError, + StreamSession, StreamSessionConfig, StreamSourceType, StreamSpan, StreamTrack, Verdict, + SUPPORTED_VERSIONS, }; use pyo3::create_exception; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString}; +use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyMapping, PyString}; use serde_json::{json, Map, Value}; use std::sync::{Arc, Mutex}; @@ -378,6 +379,113 @@ fn resolve_telemetry_sink(sink: Option>) -> Option(py: Python<'py>) -> PyResult> { + let defaults = Limits::default(); + let d = PyDict::new(py); + d.set_item("max_snapshot_bytes", defaults.max_snapshot_bytes)?; + d.set_item("max_policy_input_depth", defaults.max_policy_input_depth)?; + d.set_item( + "max_annotators_per_point", + defaults.max_annotators_per_point, + )?; + d.set_item( + "max_annotator_output_bytes", + defaults.max_annotator_output_bytes, + )?; + d.set_item("max_policy_output_bytes", defaults.max_policy_output_bytes)?; + d.set_item("max_extends_depth", defaults.max_extends_depth)?; + d.set_item( + "max_merged_manifest_bytes", + defaults.max_merged_manifest_bytes, + )?; + d.set_item("max_manifest_url_bytes", defaults.max_manifest_url_bytes)?; + d.set_item("manifest_url_timeout_ms", defaults.manifest_url_timeout_ms)?; + d.set_item( + "max_manifest_url_redirects", + defaults.max_manifest_url_redirects, + )?; + Ok(d) +} + +/// Read a limits override, mirroring the FFI's `parse_limits`: +/// +/// - `None` (absent) means keep every default. +/// - Each field is individually optional; an absent field keeps its own +/// default, so a host raising one cap does not restate the other nine. +/// - A field present but not a non-negative integer is a hard ERROR, not +/// a silently-kept default. A host that asked for a smaller bound and +/// got the larger one would believe it was protected when it was not. +fn resolve_limits(limits: Option>) -> PyResult { + let mut out = Limits::default(); + let Some(value) = limits else { + return Ok(out); + }; + + Python::attach(|py| -> PyResult<()> { + let bound = value.bind(py); + if bound.is_none() { + return Ok(()); + } + // Accept any Mapping (dict, MappingProxyType, custom mapping). + // Reject non-mappings loudly rather than silently ignoring the + // caller's intent. + let mapping: Bound<'_, PyMapping> = bound + .cast::() + .map_err(|_| PyTypeError::new_err("limits must be a Mapping[str, int] or None"))? + .clone(); + + let read = |key: &str| -> PyResult> { + if !mapping.contains(key)? { + return Ok(None); + } + let raw = mapping.get_item(key)?; + if raw.is_none() { + return Ok(None); + } + // Booleans are ints in Python. Refuse them: a host that meant + // `True` for a cap almost certainly meant something else, and + // a silently-accepted `1` masks a bug. + if raw.cast::().is_ok() { + return Err(PyValueError::new_err(format!( + "limits[{key:?}] must be a non-negative integer" + ))); + } + let extracted: Result = raw.extract(); + match extracted { + Ok(n) => Ok(Some(n)), + Err(_) => Err(PyValueError::new_err(format!( + "limits[{key:?}] must be a non-negative integer" + ))), + } + }; + + macro_rules! apply { + ($field:ident, $ty:ty) => { + if let Some(v) = read(stringify!($field))? { + out.$field = v as $ty; + } + }; + } + apply!(max_snapshot_bytes, usize); + apply!(max_policy_input_depth, usize); + apply!(max_annotators_per_point, usize); + apply!(max_annotator_output_bytes, usize); + apply!(max_policy_output_bytes, usize); + apply!(max_extends_depth, usize); + apply!(max_merged_manifest_bytes, usize); + apply!(max_manifest_url_bytes, usize); + apply!(manifest_url_timeout_ms, u64); + apply!(max_manifest_url_redirects, usize); + Ok(()) + })?; + + Ok(out) +} + #[pyclass(frozen)] struct RuntimeHandle { runtime: Runtime, @@ -387,8 +495,9 @@ struct RuntimeHandle { /// /// Passing no host arguments preserves the zero-config path: bundled /// annotators, `BindingPolicyDispatcher` for Rego/Cedar/test policies, -/// no-op telemetry, `PerfTelemetry::Off`. Host-supplied callbacks and -/// a `perf_telemetry` other than "off" replace them. +/// no-op telemetry, `PerfTelemetry::Off`, and the engine's default +/// resource caps. Host-supplied callbacks, a `perf_telemetry` other +/// than "off", and a `limits` mapping replace them. #[pyfunction] #[pyo3(signature = ( manifest_path, @@ -396,6 +505,7 @@ struct RuntimeHandle { policy_dispatcher = None, telemetry_sink = None, perf_telemetry = "off", + limits = None, ))] fn interceptor_new( manifest_path: &str, @@ -403,6 +513,7 @@ fn interceptor_new( policy_dispatcher: Option>, telemetry_sink: Option>, perf_telemetry: &str, + limits: Option>, ) -> PyResult { let manifest = Manifest::from_path(manifest_path).map_err(|e| PyValueError::new_err(format!("{e}")))?; @@ -410,12 +521,18 @@ fn interceptor_new( let policy = resolve_policy_dispatcher(policy_dispatcher); let perf = parse_perf_telemetry(perf_telemetry)?; let telemetry = resolve_telemetry_sink(telemetry_sink); - let runtime = match telemetry { - Some(sink) => Runtime::with_telemetry_and_perf(manifest, annotations, policy, sink, perf) - .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?, - None => Runtime::with_perf_telemetry(manifest, annotations, policy, perf) - .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?, - }; + let limits = resolve_limits(limits)?; + let telemetry_arc: Arc = + telemetry.unwrap_or_else(|| Arc::new(NoopTelemetrySink)); + let runtime = Runtime::with_telemetry_perf_and_limits( + manifest, + annotations, + policy, + telemetry_arc, + perf, + limits, + ) + .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; Ok(RuntimeHandle { runtime }) } @@ -1128,6 +1245,13 @@ fn stream_config(handle: &StreamSessionHandle) -> PyResult { json_string(&value) } +/// The engine's default resource caps as a `dict[str, int]`. A host +/// that raises one cap reads this to see what it is overriding. +#[pyfunction] +fn default_limits(py: Python<'_>) -> PyResult> { + Ok(limits_defaults_map(py)?.unbind()) +} + #[pymodule] fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -1154,6 +1278,7 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(stream_transformed, m)?)?; m.add_function(wrap_pyfunction!(stream_end_reason, m)?)?; m.add_function(wrap_pyfunction!(stream_config, m)?)?; + m.add_function(wrap_pyfunction!(default_limits, m)?)?; m.add("ManifestInvalid", m.py().get_type::())?; m.add_function(wrap_pyfunction!(validate_manifest, m)?)?; m.add_function(wrap_pyfunction!(validate_manifest_file, m)?)?; diff --git a/sdk/python/tests/test_host_hooks.py b/sdk/python/tests/test_host_hooks.py index 2c08896..fb6cb6d 100644 --- a/sdk/python/tests/test_host_hooks.py +++ b/sdk/python/tests/test_host_hooks.py @@ -20,6 +20,7 @@ import pytest from agent_control_spec import ( + DEFAULT_LIMITS, PERF_TELEMETRY_LEVELS, AcsInterceptor, ActivatedPolicy, @@ -518,3 +519,137 @@ def test_validate_artifacts_accepts_none_for_bundles(): assert validate_artifacts(ARTIFACT_MANIFEST, None) == validate_artifacts( ARTIFACT_MANIFEST, {} ) + + +# --------------------------------------------------------------------- +# Resource limits: caps overriding the engine's defaults reach the +# runtime and change the verdict. `Limits` is a denial-of-service +# control surface: a host feeding large payloads raises +# `max_snapshot_bytes`; one hardening against a hostile manifest lowers +# `max_extends_depth` or `manifest_url_timeout_ms`. +# +# The behavioural test is deliberately end-to-end: the same manifest +# and the same context, evaluated once with default caps and once with +# a small `max_snapshot_bytes`, produce different verdicts. That is +# what proves the value reaches the engine, rather than being accepted +# on the Python side and dropped on the way in. +# --------------------------------------------------------------------- + + +def test_lowered_snapshot_cap_flips_the_verdict(): + """A snapshot bigger than the cap must fail closed. With default + caps the same input allows; with the cap lowered below the + canonicalized snapshot size, the engine denies with + ``runtime_error:*``. A host that asked for the smaller bound and + got the larger one would believe it was protected when it was not. + """ + big = "x" * 4096 + permissive = AcsInterceptor(DEFAULT_MANIFEST) + assert permissive.intercept(_builder().input(content=big)).decision.value == "allow" + + capped = AcsInterceptor(DEFAULT_MANIFEST, limits={"max_snapshot_bytes": 64}) + verdict = capped.intercept(_builder().input(content=big)) + assert verdict.decision.value == "deny" + assert verdict.reason.startswith("runtime_error:"), verdict.reason + + +def test_no_limits_argument_matches_the_baseline_zero_config_path(): + """Regression guard: `limits=None` (and omitting the argument) must + behave identically to the pre-Limits baseline. Two contexts, one + that allowed and one that denied on the fixture, produce the same + verdicts under a `limits=None` interceptor as under one with no + kwarg at all. + """ + baseline = AcsInterceptor(DEFAULT_MANIFEST) + explicit_none = AcsInterceptor(DEFAULT_MANIFEST, limits=None) + empty = AcsInterceptor(DEFAULT_MANIFEST, limits={}) + for ctx in ( + _builder().input(content="hi"), + _builder().pre_tool_call(call_id="t1", name="search", args={"q": "x"}), + ): + base_v = baseline.intercept(ctx) + assert explicit_none.intercept(ctx).decision == base_v.decision + assert empty.intercept(ctx).decision == base_v.decision + + +def test_overriding_one_limit_leaves_the_others_at_their_defaults(): + """Fields are individually optional. Overriding + ``max_annotator_output_bytes`` upward must not silently lower + ``max_snapshot_bytes`` back to some other value; the untouched cap + still enforces at its default. Concretely: an interceptor raising + only ``max_annotator_output_bytes`` still allows a 4096-char input + (well under the default 1 MiB snapshot cap) and still fails closed + when explicitly capped small in a separate construction. + """ + big = "x" * 4096 + partial = AcsInterceptor( + DEFAULT_MANIFEST, + limits={"max_annotator_output_bytes": 8_388_608}, + ) + # Untouched `max_snapshot_bytes` still defaults big, so the input + # allows. + assert partial.intercept(_builder().input(content=big)).decision.value == "allow" + + # And when the second, untouched cap IS lowered on a separate + # interceptor, it enforces — proving the field-by-field override + # semantics. + both = AcsInterceptor( + DEFAULT_MANIFEST, + limits={ + "max_annotator_output_bytes": 8_388_608, + "max_snapshot_bytes": 64, + }, + ) + v = both.intercept(_builder().input(content=big)) + assert v.decision.value == "deny" + assert v.reason.startswith("runtime_error:") + + +def test_limit_that_is_not_a_non_negative_integer_is_rejected(): + """A value that does not parse must be a hard error, not a + silently-kept default. Refusing means a host that asked for a cap + it typo'd learns immediately instead of finding out at the first + breached limit that would never fire. + """ + with pytest.raises((TypeError, ValueError)): + AcsInterceptor(DEFAULT_MANIFEST, limits={"max_snapshot_bytes": "big"}) + with pytest.raises((TypeError, ValueError)): + AcsInterceptor(DEFAULT_MANIFEST, limits={"max_snapshot_bytes": -1}) + with pytest.raises((TypeError, ValueError)): + AcsInterceptor(DEFAULT_MANIFEST, limits={"max_snapshot_bytes": 1.5}) + # Booleans are ints in Python; refuse them here so `True` for a cap + # does not silently become `1`. + with pytest.raises((TypeError, ValueError)): + AcsInterceptor(DEFAULT_MANIFEST, limits={"max_snapshot_bytes": True}) + # A non-mapping is a boundary problem, not a manifest problem. + with pytest.raises(TypeError): + AcsInterceptor(DEFAULT_MANIFEST, limits=[("max_snapshot_bytes", 64)]) + + +def test_default_limits_is_readable_and_matches_engine_defaults(): + """A host raising one cap reads `DEFAULT_LIMITS` to see what it is + overriding. Assert the shape stays wired to the engine's own + defaults so a shipping change to another cap cannot be silently + absorbed by a stale mapping. + """ + # Every documented field is present as a non-negative int. + expected_keys = { + "max_snapshot_bytes", + "max_policy_input_depth", + "max_annotators_per_point", + "max_annotator_output_bytes", + "max_policy_output_bytes", + "max_extends_depth", + "max_merged_manifest_bytes", + "max_manifest_url_bytes", + "manifest_url_timeout_ms", + "max_manifest_url_redirects", + } + assert set(DEFAULT_LIMITS) == expected_keys + for key, value in DEFAULT_LIMITS.items(): + assert isinstance(value, int) and value >= 0, (key, value) + + # And it is read-only, so a caller cannot mutate a shared default + # out from under a peer. + with pytest.raises(TypeError): + DEFAULT_LIMITS["max_snapshot_bytes"] = 1 # type: ignore[index] diff --git a/tests/conformance/parity/cross_language_parity.py b/tests/conformance/parity/cross_language_parity.py index 1b014ea..3fd3d30 100755 --- a/tests/conformance/parity/cross_language_parity.py +++ b/tests/conformance/parity/cross_language_parity.py @@ -125,8 +125,18 @@ "artifacts_good_rego": 0, "artifacts_bad_rego": 1, "artifacts_bad_rego_code": "runtime_error:policy_invocation_failed", + # Resource caps. The same context must pass under the defaults and + # fail closed under a cap smaller than it, or the cap was accepted + # and dropped. + "limits_default_decision": "allow", + "limits_capped_decision": "deny", + "limits_capped_reason": "runtime_error:resource_limit_exceeded", } +# Larger than the capped bound below, smaller than the default one. +BIG_INPUT = "x" * 4096 +SMALL_CAP = {"max_snapshot_bytes": 64} + def _run(cmd: list[str], **kw) -> dict: out = subprocess.run(cmd, capture_output=True, text=True, check=True, **kw) @@ -136,7 +146,7 @@ def _run(cmd: list[str], **kw) -> dict: RUST_MAIN = r""" use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; use agent_control_spec::stream_session::*; -use agent_control_spec::{ActivatedPolicy, InterceptionPoint, Manifest, Runtime}; +use agent_control_spec::{ActivatedPolicy, InterceptionPoint, Limits, Manifest, Runtime}; use std::sync::Arc; struct Classifier { @@ -273,6 +283,23 @@ def _run(cmd: list[str], **kw) -> dict: Err(e) => (1, Some(e.reason().to_string())), } }; + let big_ctx: serde_json::Value = + serde_json::from_str(BIG_CTX).expect("big ctx"); + let lim = |limits: Limits| -> (String, Option) { + let manifest = Manifest::from_path(&hooks_manifest).expect("hooks manifest"); + let rt = Runtime::with_limits( + manifest, + Arc::new(Classifier { severity: 1, calls: std::sync::atomic::AtomicUsize::new(0) }), + Arc::new(BindingPolicyDispatcher::new()), + limits, + ) + .expect("limited runtime"); + let v = rt.evaluate(&big_ctx).verdict; + (format!("{:?}", v.decision).to_lowercase(), v.reason.clone()) + }; + let lim_default = lim(Limits::default()); + let lim_capped = lim(Limits { max_snapshot_bytes: 64, ..Limits::default() }); + let art_only = art("{}"); let art_good = art(GOOD_BUNDLES); let art_bad = art(BAD_BUNDLES); @@ -299,6 +326,9 @@ def _run(cmd: list[str], **kw) -> dict: "artifacts_good_rego": art_good.0, "artifacts_bad_rego": art_bad.0, "artifacts_bad_rego_code": art_bad.1, + "limits_default_decision": lim_default.0, + "limits_capped_decision": lim_capped.0, + "limits_capped_reason": lim_capped.1, "supported_versions_nonempty": !agent_control_spec::SUPPORTED_VERSIONS.is_empty(), "validate_good": validate_good, "validate_bad": validate_bad, @@ -332,6 +362,10 @@ def rust() -> dict: .replace("REGO_MANIFEST", json.dumps(REGO_MANIFEST)) .replace("GOOD_BUNDLES", json.dumps(json.dumps(GOOD_BUNDLES))) .replace("BAD_BUNDLES", json.dumps(json.dumps(BAD_BUNDLES))) + .replace( + "BIG_CTX", + json.dumps(json.dumps({"interception_point": "input", "input": BIG_INPUT})), + ) ) (work / "Cargo.toml").write_text( f""" @@ -431,6 +465,12 @@ def _hook(dispatcher): _art_good = validate_artifacts({REGO_MANIFEST!r}, {GOOD_BUNDLES!r}) _art_bad = validate_artifacts({REGO_MANIFEST!r}, {BAD_BUNDLES!r}) +_big_ctx = {{"interception_point": "input", "input": {BIG_INPUT!r}}} +_lim_default = AcsInterceptor({str(HOOKS_MANIFEST)!r}, annotator_dispatcher=_Classifier(1)).intercept(_big_ctx) +_lim_capped = AcsInterceptor( + {str(HOOKS_MANIFEST)!r}, annotator_dispatcher=_Classifier(1), limits={SMALL_CAP!r} +).intercept(_big_ctx) + print(json.dumps({{ "hook_benign_decision": _b[0], "hook_harmful_decision": _h[0], @@ -445,6 +485,9 @@ def _hook(dispatcher): "artifacts_good_rego": len(_art_good), "artifacts_bad_rego": len(_art_bad), "artifacts_bad_rego_code": _art_bad[0]["code"] if _art_bad else None, + "limits_default_decision": decision(_lim_default), + "limits_capped_decision": decision(_lim_capped), + "limits_capped_reason": _lim_capped.reason, "supported_versions_nonempty": len(supported_manifest_versions()) > 0, "validate_good": check(open({str(MANIFEST)!r}).read()), "validate_bad": check({BAD_MANIFEST!r}), @@ -510,6 +553,14 @@ def node() -> dict: const artGood = acs.validateArtifacts({json.dumps(REGO_MANIFEST)}, {json.dumps(GOOD_BUNDLES)}); const artBad = acs.validateArtifacts({json.dumps(REGO_MANIFEST)}, {json.dumps(BAD_BUNDLES)}); +const bigCtx = {{ interception_point: 'input', input: {json.dumps(BIG_INPUT)} }}; +const limDefault = acs.AcsInterceptor + .fromPath({json.dumps(str(HOOKS_MANIFEST))}, {{ annotatorDispatcher: () => ({{ severity: 1 }}) }}) + .intercept(bigCtx); +const limCapped = acs.AcsInterceptor + .fromPath({json.dumps(str(HOOKS_MANIFEST))}, {{ annotatorDispatcher: () => ({{ severity: 1 }}), limits: {json.dumps(SMALL_CAP)} }}) + .intercept(bigCtx); + console.log(JSON.stringify({{ hook_benign_decision: b[0], hook_harmful_decision: hh[0], @@ -524,6 +575,9 @@ def node() -> dict: artifacts_good_rego: artGood.length, artifacts_bad_rego: artBad.length, artifacts_bad_rego_code: artBad.length ? artBad[0].code : null, + limits_default_decision: String(limDefault.decision).toLowerCase(), + limits_capped_decision: String(limCapped.decision).toLowerCase(), + limits_capped_reason: limCapped.reason ?? null, supported_versions_nonempty: acs.supportedManifestVersions().length > 0, validate_good: check(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')), validate_bad: check({json.dumps(BAD_MANIFEST)}), @@ -595,6 +649,13 @@ def node() -> dict: var artGood = AcsManifestTools.ValidateArtifacts(REGO_MANIFEST, GOOD_BUNDLES); var artBad = AcsManifestTools.ValidateArtifacts(REGO_MANIFEST, BAD_BUNDLES); +var bigCtx = new AgentContext(JsonNode.Parse(BIG_CTX)!.AsObject()); +using var limDefault = AcsHostInterceptor.FromPath(HOOKS_MANIFEST, annotator: (_, _, _) => SEV1); +using var limCapped = AcsHostInterceptor.FromPath( + HOOKS_MANIFEST, annotator: (_, _, _) => SEV1, limits: SMALL_CAP); +var limDefaultVerdict = limDefault.InterceptAsync(bigCtx).AsTask().Result; +var limCappedVerdict = limCapped.InterceptAsync(bigCtx).AsTask().Result; + Console.WriteLine(JsonSerializer.Serialize(new Dictionary { ["hook_benign_decision"] = b.Item1, @@ -610,6 +671,9 @@ def node() -> dict: ["artifacts_good_rego"] = artGood.Count, ["artifacts_bad_rego"] = artBad.Count, ["artifacts_bad_rego_code"] = artBad.Count > 0 ? artBad[0].Code : null, + ["limits_default_decision"] = limDefaultVerdict.Decision.ToString().ToLowerInvariant(), + ["limits_capped_decision"] = limCappedVerdict.Decision.ToString().ToLowerInvariant(), + ["limits_capped_reason"] = limCappedVerdict.Reason, ["supported_versions_nonempty"] = AcsManifest.SupportedVersions().Count > 0, ["validate_good"] = Check(() => AcsManifest.Validate(File.ReadAllText(manifest))), ["validate_bad"] = Check(() => AcsManifest.Validate(BAD_JSON)), @@ -659,6 +723,20 @@ def dotnet() -> dict: .replace("REGO_MANIFEST", json.dumps(REGO_MANIFEST)) .replace("GOOD_BUNDLES", json.dumps(json.dumps(GOOD_BUNDLES))) .replace("BAD_BUNDLES", json.dumps(json.dumps(BAD_BUNDLES))) + .replace( + "BIG_CTX", + json.dumps( + json.dumps({"interception_point": "input", "input": BIG_INPUT}) + ), + ) + .replace("SMALL_CAP", json.dumps(json.dumps(SMALL_CAP))) + .replace( + "__BIGCTX__", + json.dumps( + json.dumps({"interception_point": "input", "input": BIG_INPUT}) + ), + ) + .replace("__SMALLCAP__", json.dumps(json.dumps(SMALL_CAP))) .replace("ALLOW_JSON", json.dumps(json.dumps(ALLOW_CONTEXT))) .replace("DENY_JSON", json.dumps(json.dumps(DENY_CONTEXT))) .replace("BAD_JSON", json.dumps(BAD_MANIFEST)) From 0baecc0331e5390e1a264ee80bcdaa41afa76772 Mon Sep 17 00:00:00 2001 From: Liam Crumm Date: Tue, 11 Aug 2026 22:02:58 +0000 Subject: [PATCH 09/10] refactor: move the wire contract into the core and the checks into conformance The bindings had each decided separately what a watermark, an end reason, a completion and a set of resource caps look like on the wire, because none of those types derives Serialize and something had to choose. That put three copies of the same decision in three crates, and three copies of what "response" means are three chances to disagree. A disagreement there is a host releasing text no task evaluated, which is the drift this whole change exists to prevent. So the contract moves to the engine that defines the behaviour. StreamTrack and SegmentOutcome gain parse alongside the as_str they already had, matching SafetyLevel and StreamSourceType. The bindings keep only an adapter that reshapes a core error for their boundary, which translates calling conventions rather than meaning. Consolidating surfaced a defect all three copies shared. A misspelled resource cap was accepted and ignored, so a host that wrote max_snapshot_byte believed it had set a bound it had not set. That is the same defect as quietly widening the cap, and the core now refuses it. Move the binding checks under tests/conformance/bindings with one entry point. They test the layer above the case corpus, whether a host can reach a decision at all from the language it writes in and the artifact it installs, so they belong beside the corpus rather than in scripts. Extend the coverage check to require every C ABI entry point be declared by the .NET binding, which is the only binding that goes through the C ABI rather than linking the engine. Match whole words: acs_interceptor_new is a substring of acs_interceptor_new_ex, and a substring test called the shorter one declared on the strength of the longer one. Delete the two documents added earlier. They restated the code without adding to it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Liam Crumm --- .github/workflows/ci.yml | 8 +- .github/workflows/release.yml | 2 +- README.md | 2 - docs/MIGRATION.md | 170 --------- docs/STREAMING.md | 176 --------- engine/src/lib.rs | 1 + engine/src/stream_session.rs | 39 ++ engine/src/wire.rs | 340 ++++++++++++++++++ sdk/ffi/src/lib.rs | 261 +++++--------- .../{parity => bindings}/bundle/policy.rego | 0 .../cross_language_parity.py | 0 .../conformance/bindings/dotnet_package.sh | 24 +- .../host-hooks-manifest.yaml | 0 .../conformance/bindings/language_coverage.py | 66 +++- .../{parity => bindings}/manifest.yaml | 0 .../bindings/published_artifacts.py | 52 +-- tests/conformance/bindings/run.py | 86 +++++ 17 files changed, 645 insertions(+), 582 deletions(-) delete mode 100644 docs/MIGRATION.md delete mode 100644 docs/STREAMING.md create mode 100644 engine/src/wire.rs rename tests/conformance/{parity => bindings}/bundle/policy.rego (100%) rename tests/conformance/{parity => bindings}/cross_language_parity.py (100%) rename scripts/verify-dotnet-package.sh => tests/conformance/bindings/dotnet_package.sh (73%) rename tests/conformance/{parity => bindings}/host-hooks-manifest.yaml (100%) rename scripts/check-language-coverage.py => tests/conformance/bindings/language_coverage.py (81%) rename tests/conformance/{parity => bindings}/manifest.yaml (100%) rename scripts/verify-artifacts.py => tests/conformance/bindings/published_artifacts.py (93%) create mode 100755 tests/conformance/bindings/run.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1292f62..2c93f69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: # a surface absent everywhere is consistent everywhere. That blind # spot is how streaming shipped reachable only from Rust. - name: Every engine capability is reachable from every language - run: python3 scripts/check-language-coverage.py + run: python3 tests/conformance/bindings/run.py language_coverage - name: cargo-audit over example lockfiles (pinned, checksum-verified) # Examples are deliberately outside the Dependabot update config, # and the coding_agent app's path dependency predates the engine @@ -169,7 +169,7 @@ jobs: sdk/dotnet/native/runtimes/linux-x64/native/ dotnet pack sdk/dotnet/src/AgentControlSpec -c Release -o "$PWD/ci-feed" \ --nologo -p:AcsNativeAssetsRequired=true -p:Version=0.0.0-ci - bash scripts/verify-dotnet-package.sh "$PWD/ci-feed" 0.0.0-ci + bash tests/conformance/bindings/dotnet_package.sh "$PWD/ci-feed" 0.0.0-ci # Streaming reaches each language through a different binding # mechanism, so agreement between them is not structural. Run one @@ -200,7 +200,7 @@ jobs: maturin build --release -m sdk/python/Cargo.toml -o dist pip install dist/*.whl - name: Compare every language across the whole surface - run: python tests/conformance/parity/cross_language_parity.py + run: python tests/conformance/bindings/run.py cross_language_parity env: LD_LIBRARY_PATH: ${{ github.workspace }}/target/release @@ -227,4 +227,4 @@ jobs: - run: pip install maturin==1.8.7 - run: npm ci working-directory: sdk/node - - run: python scripts/verify-artifacts.py + - run: python tests/conformance/bindings/run.py published_artifacts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3fb65e9..ce5e29f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -300,7 +300,7 @@ jobs: nupkg=$(find sdk/dotnet/dist -name 'ResponsibleAI.AgentControlSpec.*.nupkg' -print -quit) version=$(basename "$nupkg" .nupkg) version=${version#ResponsibleAI.AgentControlSpec.} - bash scripts/verify-dotnet-package.sh "$PWD/sdk/dotnet/dist" "$version" + bash tests/conformance/bindings/dotnet_package.sh "$PWD/sdk/dotnet/dist" "$version" - name: SBOM uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0 with: diff --git a/README.md b/README.md index 10be9b2..d422ee7 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,6 @@ those are host obligations defined by agent-hooks. | `spec/` | The ACS specification (policy plane) and schemas | | `policy/` | Cedar and Rego policy libraries | | `fixtures/` | Evaluation fixtures | -| `docs/STREAMING.md` | The incremental stream profile (section 18.1) in every language | -| `docs/MIGRATION.md` | Porting from `agent-control-specification` 0.3 | | `docs/EXTRACTION.md` | Provenance map from the previous tree | ## Status diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md deleted file mode 100644 index 9cd0b87..0000000 --- a/docs/MIGRATION.md +++ /dev/null @@ -1,170 +0,0 @@ -# Migrating from agent-control-specification 0.3 - -0.4 is a re-layering, not only a rewrite. The control contract and the -host obligations moved to [agent-hooks]; ACS kept the policy decision and -nothing else. So most of what 0.3 exported did not disappear, it moved -down a layer. - -Two mechanical facts first. The Python distribution was renamed from -`agent-control-specification` to `agent-control-spec`, and -`InterventionPoint` was renamed to `InterceptionPoint`. Between them -those account for a large share of the import errors a 0.3 codebase hits -first. - -## Where each thing went - -### The control contract now comes from agent-hooks - -`Decision`, `Verdict`, `EnforcementMode`, `Transform`, `Evidence`, -`ApprovalOutcome`, `ApprovalResolution`, `ApprovalResolver`. - -```python -# 0.3 -from agent_control_specification import Decision, EnforcementMode - -# 0.4 -from agent_hooks import Decision, EnforcementMode -``` - -`InterventionPoint` is `agent_hooks.InterceptionPoint`. - -ACS does not re-export these. Two packages defining one contract is the -ambiguity the re-layering removed. - -### Orchestration is agent-hooks' InterceptionEmitter - -`AgentControl` was "host-owned async orchestration around a stateless -runtime client". That is now `InterceptionEmitter`, which implements -sections 6 through 10 of the control contract once so that adapters do -not each reimplement them. - -```python -# 0.3 -control = AgentControl.from_native(manifest) -result = await control.enforce(point, snapshot, EnforcementMode.ENFORCE) - -# 0.4 -from agent_hooks import InterceptionEmitter, EnforcementMode -from agent_control_spec import AcsInterceptor - -emitter = InterceptionEmitter(mode=EnforcementMode.ENFORCE) -emitter.register(AcsInterceptor("manifest.yaml"), name="acs") -outcome = await emitter.emit(context) -``` - -| 0.3 | 0.4 | -| --- | --- | -| `AgentControl` | `agent_hooks.InterceptionEmitter` | -| `AgentControlBlocked` | `agent_hooks.InterceptionBlocked` | -| `AgentControlSuspended` | `agent_hooks.InterceptionSuspended` | -| `RunResult`, `ToolRunResult`, `ModelCallResult` | `agent_hooks.EmitOutcome` | -| `InterventionPointRequest` | `agent_hooks.AgentContext` | -| `action_identity` | `agent_hooks.context_identity` | -| `RuntimeClient`, `NativeRuntimeClient` | the `agent_hooks.Interceptor` protocol, which `AcsInterceptor` implements | - -### Audit records replaced the telemetry sinks - -`JsonStdoutTelemetrySink` and `InMemoryTelemetrySink` recorded what was -decided. That is a host obligation, so it moved to the emitter: -`set_record_sink`, `take_records`, `set_max_records` and -`InterceptionRecord`. - -ACS still exposes engine-level telemetry, which is a different thing: it -reports how the engine spent its time, not what it decided. See host -hooks below. - -### Host extension points stayed in ACS - -`AnnotatorDispatcher` and `PolicyDispatcher` are ACS concerns by -definition and are still supported. The keyword survives: - -```python -# 0.3 -control = AgentControl.from_native(manifest, annotator_dispatcher=MyClassifier()) - -# 0.4 -interceptor = AcsInterceptor("manifest.yaml", annotator_dispatcher=MyClassifier()) -``` - -A dispatcher is any object with the matching method, or a plain callable, -so a 0.3 dispatcher class usually ports unchanged. A dispatcher that -raises fails the evaluation closed: a classifier that could not be -reached must not read as a classifier that found nothing. - -### Manifest tooling - -| 0.3 | 0.4 | -| --- | --- | -| `validate_manifest` | `validate_manifest` | -| `validate_acs_manifest` | `validate_manifest` | -| `parse_manifest` | `parse_manifest` | -| `validate_manifest_overlay` | `merge_manifests` | -| `ValidationDiagnostic`, `ArtifactValidationResult` | `validate_manifest_detailed` | - -### Framework adapters have no successor yet - -The 18 `guard_*` functions, `FullCoverageAgentAdapter`, -`LiteLLMProxyMiddleware`, `mcp_approval_resolver` and the rest are not in -0.4 and are not in agent-hooks. - -They wrap nine frameworks, so they do not belong in a policy decision -runtime that would then version-lock to all of them. Where they should -live is an open question. Until it is answered, a host that used one -writes the equivalent against `InterceptionEmitter` directly, which is -what the adapters did internally. - -## Things that will look like ACS breaking but are not - -**Checking for the `opa` binary.** 0.3 evaluated Rego by shelling out to -OPA. 0.4 evaluates it in process with regorus and needs no external -binary. A startup check for `opa` on PATH will reject a working 0.4 -install. Delete the check. - -**`intervention_points` in a manifest.** The manifest grammar keeps the -older spelling. Only the Python symbol was renamed. - -## A worked port - -0.3: - -```python -from agent_control_specification import ( - AgentControl, AgentControlBlocked, EnforcementMode, -) - -control = AgentControl.from_native(manifest, annotator_dispatcher=ContentSafety()) - -async def guarded(text): - try: - return await control.enforce("input", {"input": text}, EnforcementMode.ENFORCE) - except AgentControlBlocked: - return "refused" -``` - -0.4: - -```python -from agent_hooks import ( - AgentContextBuilder, EnforcementMode, InterceptionBlocked, InterceptionEmitter, -) -from agent_control_spec import AcsInterceptor - -emitter = InterceptionEmitter(mode=EnforcementMode.ENFORCE) -emitter.register( - AcsInterceptor("manifest.yaml", annotator_dispatcher=ContentSafety()), - name="acs", -) - -async def guarded(text): - ctx = AgentContextBuilder( - agent_id="a", framework="custom", session_id="s" - ).input(content=text) - try: - return await emitter.emit(ctx) - except InterceptionBlocked: - return "refused" -``` - -`ContentSafety` is unchanged. - -[agent-hooks]: https://github.com/responsibleai/agent-hooks diff --git a/docs/STREAMING.md b/docs/STREAMING.md deleted file mode 100644 index 8e4ea06..0000000 --- a/docs/STREAMING.md +++ /dev/null @@ -1,176 +0,0 @@ -# Streaming - -Specification section 18.1 defines the incremental stream profile: a host -that emits a response while it is still arriving needs to know how much -of it is safe to release so far. `StreamSession` answers that question. - -The session is release accounting, not evaluation. Each segment is one -ordinary stateless evaluation through the normal runtime; the session -records what those evaluations cleared and reports the offset that every -mediating task has agreed to. It holds no text, so emitting released -runes and applying a rewrite stay host obligations. - -## Availability - -| Language | Package | Streaming | -| --- | --- | --- | -| Rust | `agent-control-spec` | opt in with `features = ["streaming"]` | -| Python | `agent-control-spec` | always present | -| Node | `@responsibleai/agent-control-spec` | always present | -| .NET | `ResponsibleAI.AgentControlSpec` | always present | - -The feature is off by default in the Rust crate because a `StreamSession` -carries per-stream state, which the default surface deliberately does -not, and because a host that assembles a whole payload before evaluating -it never needs the module. The bindings enable it unconditionally: a -Python, Node or .NET consumer installs a prebuilt binary and cannot flip -a cargo feature. - -## The shape of a session - -Every language exposes the same eight operations. - -| Operation | What it does | -| --- | --- | -| `observe` / `observe_text` | report that runes arrived on a track | -| `record_outcome` | report what one task decided about one span | -| `record_verdict` | the same, taking a verdict as the runtime returns it | -| `advance` | recompute a track's watermark against recorded outcomes | -| `safe_offset` | the offset safe to release, as of the last `advance` | -| `watermark` | how far a track got, and which tasks must clear it | -| `end_of_payloads` | no further payload will arrive | -| `finish` | settle, returning why the stream ended | - -Two details decide whether a host is correct. - -**Recording an outcome does not release anything.** `advance` moves the -watermark; `safe_offset` reads it. A host that records outcomes and polls -`safe_offset` without ever calling `advance` will release nothing and -appear to hang. - -**An absent safe offset is not zero.** Once a session ends, whether -cleanly, by refusal, or by rewrite, there is no offset anyone may emit -through, and `safe_offset` says so in the language's own vocabulary: -`None` in Rust and Python, `null` in Node, `null` in .NET. That is a -value, not an error. A host that polls until it stops advancing stops on -its own. Reading it as `0` or as `-1` would release text no task -evaluated. - -The offset the track reached stays readable through `watermark` after -settlement, so an audit record can still say how far the stream got. - -## Rust - -```toml -agent-control-spec = { version = "0.4", features = ["streaming"] } -``` - -```rust -use agent_control_spec::stream_session::*; - -let mut session = StreamSession::new(StreamSessionConfig { - safety_level: SafetyLevel::Blocking, - request_start_rune_offset: 0, - response_start_rune_offset: 0, - request_tasks: vec![], - response_tasks: vec!["pii".to_string()], -})?; - -let received = session.observe_text(StreamSourceType::ModelGenerated, "hello")?; -let span = StreamSpan::new(StreamSourceType::ModelGenerated, 0, received)?; -session.record_outcome("pii", &span, SegmentOutcome::Cleared)?; - -session.advance(StreamTrack::Response); -assert_eq!(session.safe_offset(StreamTrack::Response), Some(5)); -``` - -## Python - -```python -from agent_control_spec import StreamSession - -session = StreamSession(safety_level="blocking", response_tasks=["pii"]) - -received = session.observe_text("model_generated", "hello") -session.record_outcome("pii", "model_generated", 0, received, "cleared") - -session.advance("response") -assert session.safe_offset("response") == 5 - -session.finish() -assert session.safe_offset("response") is None -``` - -## Node - -```ts -import { StreamSession } from "@responsibleai/agent-control-spec"; - -const session = new StreamSession({ - safetyLevel: "blocking", - responseTasks: ["pii"], -}); - -const received = session.observeText("model_generated", "hello"); -session.recordOutcome("pii", "model_generated", 0, received, "cleared"); - -session.advance("response"); -console.log(session.safeOffset("response")); // 5 - -session.finish(); -console.log(session.safeOffset("response")); // null -``` - -## .NET - -```csharp -using AgentControlSpec; - -using var session = new StreamSession( - SafetyLevel.Blocking, responseTasks: ["pii"]); - -var received = session.ObserveText(StreamSourceType.ModelGenerated, "hello"); -session.RecordOutcome( - "pii", StreamSourceType.ModelGenerated, 0, received, SegmentOutcome.Cleared); - -session.Advance(StreamTrack.Response); -Console.WriteLine(session.SafeOffset(StreamTrack.Response)); // 5 - -session.Finish(); -Console.WriteLine(session.SafeOffset(StreamTrack.Response)); // null -``` - -## Counting runes - -Offsets are counted in runes, meaning Unicode scalar values, not code -units and not bytes. The distinction is load bearing: an emoji is one -rune, two UTF-16 code units and four UTF-8 bytes, so a .NET host using -`string.Length` or a Node host using `String.prototype.length` would -release twice what a task evaluated. - -Prefer `observe_text`, which hands the text to the engine and lets it -count. Reserve `observe` for a host that already has a rune count from -the same source of truth. - -## Verdicts feed straight back - -`record_verdict` takes a verdict exactly as the runtime returns one, so a -host that evaluates a segment through `ActivatedPolicy` passes the result -in without translating it. `allow` clears, `deny` refuses, and -`transform` records a rewrite, which is terminal. - -## Cross-language agreement - -The four bindings reach the engine through four different mechanisms: a -direct crate dependency, pyo3, napi, and a C ABI with P/Invoke over it. -Each converts enums, offsets and the absent release point at its own -boundary, so agreement is not structural. - -`tests/conformance/parity/cross_language_parity.py` runs the whole -public surface, streaming and otherwise, in all four and fails on any -divergence. - -That harness runs against this checkout. `scripts/verify-artifacts.py` -runs the same surface from a clean install of the built crate, wheel, -tarball and nupkg, which is the only thing that catches a package -shipping less than the repository holds. Both are part of CI. diff --git a/engine/src/lib.rs b/engine/src/lib.rs index 1008be8..609af56 100644 --- a/engine/src/lib.rs +++ b/engine/src/lib.rs @@ -38,6 +38,7 @@ pub mod runtime; pub mod stream_session; pub mod telemetry; pub mod tool_projection; +pub mod wire; // The interception contract, re-exported for consumers that want a // single dependency. diff --git a/engine/src/stream_session.rs b/engine/src/stream_session.rs index b86eca8..aaed109 100644 --- a/engine/src/stream_session.rs +++ b/engine/src/stream_session.rs @@ -204,6 +204,19 @@ pub enum StreamTrack { } impl StreamTrack { + /// Parse a track's wire name. + /// + /// Every binding needs this and none should own it. Three copies of + /// what `"response"` means are three chances to disagree, which is + /// the drift this module exists to prevent. + pub fn parse(value: &str) -> Result { + match value { + "request" => Ok(Self::Request), + "response" => Ok(Self::Response), + _ => Err(StreamError::UnknownStreamTrack(value.to_string())), + } + } + pub fn as_str(self) -> &'static str { match self { Self::Request => "request", @@ -223,6 +236,8 @@ impl StreamTrack { pub enum StreamError { UnknownSafetyLevel(String), UnknownSourceType(String), + UnknownStreamTrack(String), + UnknownSegmentOutcome(String), /// Payload arrived on a track the session does not mediate. /// /// An empty task set means that track is not mediated, which is the @@ -302,6 +317,10 @@ impl fmt::Display for StreamError { write!(f, "unknown streaming safety level {value}") } Self::UnknownSourceType(value) => write!(f, "unknown stream source type {value}"), + Self::UnknownStreamTrack(value) => write!(f, "unknown stream track {value}"), + Self::UnknownSegmentOutcome(value) => { + write!(f, "unknown segment outcome {value}") + } Self::NoTasks(track) => write!( f, "payload arrived on the unmediated {} track", @@ -421,6 +440,26 @@ pub enum SegmentOutcome { Denied, } +impl SegmentOutcome { + /// Parse an outcome's wire name. + pub fn parse(value: &str) -> Result { + match value { + "cleared" => Ok(Self::Cleared), + "transformed" => Ok(Self::Transformed), + "denied" => Ok(Self::Denied), + _ => Err(StreamError::UnknownSegmentOutcome(value.to_string())), + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Cleared => "cleared", + Self::Transformed => "transformed", + Self::Denied => "denied", + } + } +} + /// Reason a session reached its terminal state. #[derive(Debug, Clone, PartialEq, Eq)] pub enum StreamEndReason { diff --git a/engine/src/wire.rs b/engine/src/wire.rs new file mode 100644 index 0000000..af5ea08 --- /dev/null +++ b/engine/src/wire.rs @@ -0,0 +1,340 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +//! The JSON shapes every binding answers with. +//! +//! None of the streaming or telemetry types derives `Serialize`, so +//! something has to decide what a watermark looks like on the wire. +//! Before this module each binding decided separately, which put three +//! copies of that decision in three crates. Three copies of what +//! `"response"` means are three chances to disagree, and a disagreement +//! here is a host releasing text no task evaluated. +//! +//! So the contract lives with the engine that defines the behaviour. +//! The bindings translate calling conventions, not meaning, and the +//! cross-language conformance suite then checks the bindings rather +//! than re-litigating the contract in each one. + +use crate::error::RuntimeError; +use crate::limits::Limits; +use crate::perf_telemetry::PerfTelemetry; +use serde_json::{json, Map, Value}; + +#[cfg(feature = "streaming")] +use crate::stream_session::{ + StreamCompletion, StreamEndReason, StreamSessionConfig, StreamWatermark, +}; + +/// Parse a perf telemetry level's wire name. +pub fn parse_perf_telemetry(value: &str) -> Result { + match value { + "off" => Ok(PerfTelemetry::Off), + "external" => Ok(PerfTelemetry::External), + "full" => Ok(PerfTelemetry::Full), + other => Err(RuntimeError::ManifestInvalid(format!( + "unknown perf telemetry level '{other}'" + ))), + } +} + +/// A perf telemetry level's wire name. +pub fn perf_telemetry_str(level: PerfTelemetry) -> &'static str { + match level { + PerfTelemetry::Off => "off", + PerfTelemetry::External => "external", + PerfTelemetry::Full => "full", + } +} + +/// Apply a JSON object of resource cap overrides onto the defaults. +/// +/// Each field is individually optional, so a host raising one cap does +/// not restate the other nine. A field present but not a non-negative +/// integer is refused rather than silently kept at its default: a host +/// that asked for a smaller bound and got the larger one would believe +/// it was protected when it was not. +pub fn limits_from_json(value: &Value) -> Result { + let Value::Object(fields) = value else { + return Err(RuntimeError::ManifestInvalid( + "limits must be a JSON object".to_string(), + )); + }; + let mut limits = Limits::default(); + + let read = |key: &str| -> Result, RuntimeError> { + match fields.get(key) { + None | Some(Value::Null) => Ok(None), + Some(found) => found.as_u64().map(Some).ok_or_else(|| { + RuntimeError::ManifestInvalid(format!("{key} must be a non negative integer")) + }), + } + }; + + macro_rules! apply { + ($field:ident, $ty:ty) => { + if let Some(found) = read(stringify!($field))? { + limits.$field = found as $ty; + } + }; + } + apply!(max_snapshot_bytes, usize); + apply!(max_policy_input_depth, usize); + apply!(max_annotators_per_point, usize); + apply!(max_annotator_output_bytes, usize); + apply!(max_policy_output_bytes, usize); + apply!(max_extends_depth, usize); + apply!(max_merged_manifest_bytes, usize); + apply!(max_manifest_url_bytes, usize); + apply!(manifest_url_timeout_ms, u64); + apply!(max_manifest_url_redirects, usize); + + let unknown: Vec<&str> = fields + .keys() + .map(String::as_str) + .filter(|key| !LIMIT_FIELDS.contains(key)) + .collect(); + if !unknown.is_empty() { + // A misspelled cap that is quietly ignored is the same defect as + // one that is quietly widened: the host believes it set a bound + // it did not set. + return Err(RuntimeError::ManifestInvalid(format!( + "unknown limit field(s): {}", + unknown.join(", ") + ))); + } + Ok(limits) +} + +/// Every field [`limits_from_json`] accepts, in declaration order. +pub const LIMIT_FIELDS: [&str; 10] = [ + "max_snapshot_bytes", + "max_policy_input_depth", + "max_annotators_per_point", + "max_annotator_output_bytes", + "max_policy_output_bytes", + "max_extends_depth", + "max_merged_manifest_bytes", + "max_manifest_url_bytes", + "manifest_url_timeout_ms", + "max_manifest_url_redirects", +]; + +/// The resource caps in force, as JSON. +pub fn limits_json(limits: &Limits) -> Value { + json!({ + "max_snapshot_bytes": limits.max_snapshot_bytes, + "max_policy_input_depth": limits.max_policy_input_depth, + "max_annotators_per_point": limits.max_annotators_per_point, + "max_annotator_output_bytes": limits.max_annotator_output_bytes, + "max_policy_output_bytes": limits.max_policy_output_bytes, + "max_extends_depth": limits.max_extends_depth, + "max_merged_manifest_bytes": limits.max_merged_manifest_bytes, + "max_manifest_url_bytes": limits.max_manifest_url_bytes, + "manifest_url_timeout_ms": limits.manifest_url_timeout_ms, + "max_manifest_url_redirects": limits.max_manifest_url_redirects, + }) +} + +/// One finding about a manifest or its artifacts. +/// +/// `RuntimeError` answers by being returned, which a linter cannot +/// render against a document. This is the same information as data. +pub fn diagnostic_json(error: &RuntimeError) -> Value { + json!({ + "code": error.reason(), + "message": error.detail(), + "severity": "error", + }) +} + +/// A list of findings. Empty means sound. +pub fn diagnostics_json(errors: &[RuntimeError]) -> Value { + Value::Array(errors.iter().map(diagnostic_json).collect()) +} + +/// Why a session ended. +#[cfg(feature = "streaming")] +pub fn end_reason_json(reason: &StreamEndReason) -> Value { + match reason { + StreamEndReason::Complete => json!({ "kind": "complete" }), + StreamEndReason::Denied { track, task, range } => json!({ + "kind": "denied", + "track": track.as_str(), + "task": task, + "start": range.start, + "end": range.end, + }), + StreamEndReason::Rewritten { track, task, range } => json!({ + "kind": "rewritten", + "track": track.as_str(), + "task": task, + "start": range.start, + "end": range.end, + }), + StreamEndReason::Failed(error) => json!({ + "kind": "failed", + "reason": error.reason(), + "message": error.to_string(), + }), + } +} + +/// How far one track got and what still owes a decision. +#[cfg(feature = "streaming")] +pub fn watermark_json(track: crate::stream_session::StreamTrack, mark: &StreamWatermark) -> Value { + json!({ + "track": track.as_str(), + "confirmed": mark.confirmed(), + "received": mark.received(), + "pending": mark.pending(), + "tasks": mark.tasks().collect::>(), + }) +} + +/// Terminal settlement of a session. +#[cfg(feature = "streaming")] +pub fn completion_json(completion: &StreamCompletion) -> Value { + json!({ + "reason": end_reason_json(&completion.reason), + "transformed": completion.transformed, + "is_clean": completion.reason.is_clean(), + }) +} + +/// The offsets and task sets a session was opened with. +#[cfg(feature = "streaming")] +pub fn stream_config_json(config: &StreamSessionConfig) -> Value { + json!({ + "safety_level": config.safety_level.as_str(), + "request_start_rune_offset": config.request_start_rune_offset, + "response_start_rune_offset": config.response_start_rune_offset, + "request_tasks": config.request_tasks, + "response_tasks": config.response_tasks, + }) +} + +/// Read a session configuration from JSON, defaulting absent fields. +#[cfg(feature = "streaming")] +pub fn stream_config_from_json( + value: &Value, +) -> Result { + use crate::stream_session::{SafetyLevel, StreamError}; + + let empty = Map::new(); + let fields = value.as_object().unwrap_or(&empty); + + let safety_level = SafetyLevel::parse( + fields + .get("safety_level") + .and_then(Value::as_str) + .unwrap_or("blocking"), + )?; + + let offset = |key: &str| -> Result { + match fields.get(key) { + None | Some(Value::Null) => Ok(0), + Some(found) => found + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| { + StreamError::UnknownSourceType(format!("{key} is not a rune offset")) + }), + } + }; + + let tasks = |key: &str| -> Result, StreamError> { + match fields.get(key) { + None | Some(Value::Null) => Ok(Vec::new()), + Some(Value::Array(items)) => items + .iter() + .map(|item| { + item.as_str().map(str::to_string).ok_or_else(|| { + StreamError::UnknownSourceType(format!( + "{key} holds a non string task name" + )) + }) + }) + .collect(), + Some(_) => Err(StreamError::UnknownSourceType(format!( + "{key} is not an array of task names" + ))), + } + }; + + Ok(StreamSessionConfig { + safety_level, + request_start_rune_offset: offset("request_start_rune_offset")?, + response_start_rune_offset: offset("response_start_rune_offset")?, + request_tasks: tasks("request_tasks")?, + response_tasks: tasks("response_tasks")?, + }) +} + +/// One telemetry event. +/// +/// `TelemetryEvent` does not derive `Serialize`, so a sink reached +/// through any binding sees the shape stated here. +pub fn telemetry_event_json(event: &crate::telemetry::TelemetryEvent) -> Value { + json!({ + "event_type": event.event_type.as_str(), + "intervention_point": format!("{:?}", event.intervention_point).to_lowercase(), + "decision": event.decision.map(|d| format!("{d:?}").to_lowercase()), + "reason_code": event.reason_code, + "error_class": event.error_class, + "policy_id": event.policy_id, + "annotators": event.annotators, + "enforcement_mode": event.enforcement_mode.map(|m| format!("{m:?}").to_lowercase()), + "duration_ms": event.duration_ms, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn absent_limit_fields_keep_their_own_defaults() { + let limits = limits_from_json(&json!({ "max_snapshot_bytes": 64 })).expect("limits"); + assert_eq!(limits.max_snapshot_bytes, 64); + assert_eq!( + limits.max_policy_input_depth, + Limits::default().max_policy_input_depth + ); + } + + #[test] + fn a_limit_that_is_not_a_count_is_refused() { + assert!(limits_from_json(&json!({ "max_snapshot_bytes": "big" })).is_err()); + assert!(limits_from_json(&json!({ "max_snapshot_bytes": -1 })).is_err()); + } + + #[test] + fn a_misspelled_limit_is_refused_rather_than_ignored() { + let error = limits_from_json(&json!({ "max_snapshot_byte": 64 })).expect_err("refused"); + assert!(format!("{error}").contains("max_snapshot_byte")); + } + + #[test] + fn every_limit_field_round_trips() { + let rendered = limits_json(&Limits::default()); + for field in LIMIT_FIELDS { + assert!(rendered.get(field).is_some(), "{field} missing"); + } + let parsed = limits_from_json(&rendered).expect("round trip"); + assert_eq!(parsed, Limits::default()); + } + + #[test] + fn perf_levels_round_trip_and_unknown_is_refused() { + for level in [ + PerfTelemetry::Off, + PerfTelemetry::External, + PerfTelemetry::Full, + ] { + assert_eq!( + parse_perf_telemetry(perf_telemetry_str(level)).unwrap(), + level + ); + } + assert!(parse_perf_telemetry("verbose").is_err()); + } +} diff --git a/sdk/ffi/src/lib.rs b/sdk/ffi/src/lib.rs index 298cdf4..924b4a8 100644 --- a/sdk/ffi/src/lib.rs +++ b/sdk/ffi/src/lib.rs @@ -19,14 +19,14 @@ use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; -use agent_control_spec::perf_telemetry::PerfTelemetry; use agent_control_spec::policy::PreparedPolicyInvocation; use agent_control_spec::runtime::PolicyDispatcher; use agent_control_spec::stream_session::{ - SafetyLevel, SegmentOutcome, StreamEndReason, StreamSession, StreamSessionConfig, - StreamSourceType, StreamSpan, StreamTrack, + SafetyLevel, SegmentOutcome, StreamSession, StreamSessionConfig, StreamSourceType, StreamSpan, + StreamTrack, }; use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; +use agent_control_spec::wire; use agent_control_spec::{ ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Limits, Manifest, Runtime, RuntimeError, Verdict, SUPPORTED_VERSIONS, @@ -44,6 +44,22 @@ pub struct AcsInterceptor { name: String, } +/// Read an optional NUL-terminated string. NULL means absent, which is +/// distinct from present and invalid. +unsafe fn read_optional<'a>( + ptr: *const c_char, + what: &str, + err_out: *mut *mut c_char, +) -> Result, ()> { + if ptr.is_null() { + return Ok(None); + } + match read_utf8(ptr, what, err_out) { + Some(value) => Ok(Some(value)), + None => Err(()), + } +} + fn set_err(err_out: *mut *mut c_char, message: String) { if err_out.is_null() { return; @@ -986,22 +1002,9 @@ struct HostTelemetrySink { impl TelemetrySink for HostTelemetrySink { fn emit(&self, event: TelemetryEvent) { - // TelemetryEvent is not Serialize, so the wire shape is owned - // here rather than derived from the engine's layout. A sink - // cannot fail an evaluation, so a problem here drops the event - // rather than denying the action it describes. - let payload = serde_json::json!({ - "event_type": event.event_type.as_str(), - "intervention_point": format!("{:?}", event.intervention_point).to_lowercase(), - "decision": event.decision.map(|d| format!("{d:?}").to_lowercase()), - "reason_code": event.reason_code, - "error_class": event.error_class, - "policy_id": event.policy_id, - "annotators": event.annotators, - "enforcement_mode": event.enforcement_mode.map(|m| format!("{m:?}").to_lowercase()), - "duration_ms": event.duration_ms, - }); - let Ok(json) = serde_json::to_string(&payload) else { + // A sink cannot fail an evaluation, so a problem here drops the + // event rather than denying the action it describes. + let Ok(json) = serde_json::to_string(&wire::telemetry_event_json(&event)) else { return; }; let Ok(payload) = CString::new(json) else { @@ -1011,83 +1014,6 @@ impl TelemetrySink for HostTelemetrySink { } } -/// Read a limits override, treating NULL and empty as "keep the -/// defaults". Absent fields keep their default individually, so a host -/// that only needs a bigger snapshot cap does not have to restate the -/// other nine. -fn parse_limits(value: *const c_char, err_out: *mut *mut c_char) -> Option { - let mut limits = Limits::default(); - if value.is_null() { - return Some(limits); - } - let raw = unsafe { read_utf8(value, "limits_json", err_out) }?; - if raw.trim().is_empty() { - return Some(limits); - } - let parsed: Value = match serde_json::from_str(raw) { - Ok(v @ Value::Object(_)) => v, - Ok(_) => { - set_err(err_out, "limits_json must be a JSON object".to_string()); - return None; - } - Err(e) => { - set_err(err_out, format!("limits_json does not parse: {e}")); - return None; - } - }; - - // A cap that does not parse is refused rather than silently kept at - // its default. A host that asked for a smaller bound and got the - // larger one would believe it was protected when it was not. - let read = |key: &str| -> Option> { - match parsed.get(key) { - None | Some(Value::Null) => Some(None), - Some(v) => match v.as_u64() { - Some(n) => Some(Some(n)), - None => { - set_err(err_out, format!("{key} must be a non negative integer")); - None - } - }, - } - }; - - macro_rules! apply { - ($field:ident, $ty:ty) => { - if let Some(v) = read(stringify!($field))? { - limits.$field = v as $ty; - } - }; - } - apply!(max_snapshot_bytes, usize); - apply!(max_policy_input_depth, usize); - apply!(max_annotators_per_point, usize); - apply!(max_annotator_output_bytes, usize); - apply!(max_policy_output_bytes, usize); - apply!(max_extends_depth, usize); - apply!(max_merged_manifest_bytes, usize); - apply!(max_manifest_url_bytes, usize); - apply!(manifest_url_timeout_ms, u64); - apply!(max_manifest_url_redirects, usize); - Some(limits) -} - -fn parse_perf(value: *const c_char, err_out: *mut *mut c_char) -> Option { - if value.is_null() { - return Some(PerfTelemetry::Off); - } - let raw = unsafe { read_utf8(value, "perf_telemetry", err_out) }?; - match raw { - "off" => Some(PerfTelemetry::Off), - "external" => Some(PerfTelemetry::External), - "full" => Some(PerfTelemetry::Full), - other => { - set_err(err_out, format!("unknown perf telemetry level '{other}'")); - None - } - } -} - /// Build an interceptor with host-supplied extension points. /// /// Any callback may be NULL, which keeps the bundled default for that @@ -1134,11 +1060,36 @@ pub unsafe extern "C" fn acs_interceptor_new_with_hooks( return std::ptr::null_mut(); } }; - let Some(perf) = parse_perf(perf_telemetry, err_out) else { - return std::ptr::null_mut(); - }; - let Some(limits) = parse_limits(limits_json, err_out) else { - return std::ptr::null_mut(); + let perf = match read_optional(perf_telemetry, "perf_telemetry", err_out) { + Ok(Some(raw)) => match wire::parse_perf_telemetry(raw) { + Ok(level) => level, + Err(e) => { + set_err(err_out, format!("{e}")); + return std::ptr::null_mut(); + } + }, + Ok(None) => agent_control_spec::PerfTelemetry::Off, + Err(()) => return std::ptr::null_mut(), + }; + let limits = match read_optional(limits_json, "limits_json", err_out) { + Ok(Some(raw)) if !raw.trim().is_empty() => { + let parsed: Value = match serde_json::from_str(raw) { + Ok(v) => v, + Err(e) => { + set_err(err_out, format!("limits_json does not parse: {e}")); + return std::ptr::null_mut(); + } + }; + match wire::limits_from_json(&parsed) { + Ok(l) => l, + Err(e) => { + set_err(err_out, format!("{e}")); + return std::ptr::null_mut(); + } + } + } + Ok(_) => Limits::default(), + Err(()) => return std::ptr::null_mut(), }; let annotations: Arc = match annotator_fn { @@ -1323,9 +1274,9 @@ pub unsafe extern "C" fn acs_manifest_diagnostics( let findings = match Manifest::from_yaml_str(source) { Ok(manifest) => match manifest.validate() { Ok(()) => Vec::new(), - Err(e) => vec![diagnostic_json(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], }, - Err(e) => vec![diagnostic_json(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], }; match serde_json::to_string(&findings) { Ok(json) => to_c_string(json, err_out), @@ -1385,12 +1336,12 @@ pub unsafe extern "C" fn acs_artifact_diagnostics( // does not parse would otherwise be reported as an activation // failure, which names the wrong half. let findings = match Manifest::from_yaml_str(source) { - Err(e) => vec![diagnostic_json(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], Ok(manifest) => match manifest.validate() { - Err(e) => vec![diagnostic_json(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], Ok(()) => match ActivatedPolicy::activate_from_memory(source, bundles) { Ok(_) => Vec::new(), - Err(e) => vec![diagnostic_json(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], }, }, }; @@ -1415,14 +1366,6 @@ pub unsafe extern "C" fn acs_artifact_diagnostics( } } -fn diagnostic_json(error: &RuntimeError) -> Value { - serde_json::json!({ - "code": error.reason(), - "message": error.detail(), - "severity": "error", - }) -} - // --------------------------------------------------------------------- // Incremental stream mediation (specification section 18.1). // @@ -1441,57 +1384,31 @@ fn diagnostic_json(error: &RuntimeError) -> Value { // wire contract is owned here rather than derived from Rust layout. // --------------------------------------------------------------------- -/// Opaque handle to one mediated stream. -pub struct AcsStreamSession { - session: StreamSession, -} - -fn parse_track(value: &str, err_out: *mut *mut c_char) -> Option { - match value { - "request" => Some(StreamTrack::Request), - "response" => Some(StreamTrack::Response), - other => { - set_err(err_out, format!("unknown stream track '{other}'")); +/// Parse a wire value through the core, reporting failure the way this +/// boundary does. The core owns what the names mean. +fn wire_track(value: &str, err_out: *mut *mut c_char) -> Option { + match StreamTrack::parse(value) { + Ok(track) => Some(track), + Err(e) => { + set_err(err_out, format!("{e}")); None } } } -fn parse_outcome(value: &str, err_out: *mut *mut c_char) -> Option { - match value { - "cleared" => Some(SegmentOutcome::Cleared), - "transformed" => Some(SegmentOutcome::Transformed), - "denied" => Some(SegmentOutcome::Denied), - other => { - set_err(err_out, format!("unknown segment outcome '{other}'")); +fn wire_outcome(value: &str, err_out: *mut *mut c_char) -> Option { + match SegmentOutcome::parse(value) { + Ok(outcome) => Some(outcome), + Err(e) => { + set_err(err_out, format!("{e}")); None } } } -fn end_reason_json(reason: &StreamEndReason) -> Value { - match reason { - StreamEndReason::Complete => serde_json::json!({ "kind": "complete" }), - StreamEndReason::Denied { track, task, range } => serde_json::json!({ - "kind": "denied", - "track": track.as_str(), - "task": task, - "start": range.start, - "end": range.end, - }), - StreamEndReason::Rewritten { track, task, range } => serde_json::json!({ - "kind": "rewritten", - "track": track.as_str(), - "task": task, - "start": range.start, - "end": range.end, - }), - StreamEndReason::Failed(error) => serde_json::json!({ - "kind": "failed", - "reason": error.reason(), - "message": error.to_string(), - }), - } +/// Opaque handle to one mediated stream. +pub struct AcsStreamSession { + session: StreamSession, } /// Open a session from `config_json`. @@ -1764,7 +1681,7 @@ pub unsafe extern "C" fn acs_stream_session_record_outcome( let Some(outcome_raw) = read_utf8(outcome, "outcome", err_out) else { return -1; }; - let Some(outcome) = parse_outcome(outcome_raw, err_out) else { + let Some(outcome) = wire_outcome(outcome_raw, err_out) else { return -1; }; let span = match StreamSpan::new(source, start, end) { @@ -1883,7 +1800,7 @@ pub unsafe extern "C" fn acs_stream_session_advance( let Some(raw) = read_utf8(track, "track", err_out) else { return -2; }; - let Some(track) = parse_track(raw, err_out) else { + let Some(track) = wire_track(raw, err_out) else { return -2; }; match (*handle).session.advance(track) { @@ -1921,7 +1838,7 @@ pub unsafe extern "C" fn acs_stream_session_safe_offset( let Some(raw) = read_utf8(track, "track", err_out) else { return -2; }; - let Some(track) = parse_track(raw, err_out) else { + let Some(track) = wire_track(raw, err_out) else { return -2; }; match (*handle).session.safe_offset(track) { @@ -1958,7 +1875,7 @@ pub unsafe extern "C" fn acs_stream_session_pending( let Some(raw) = read_utf8(track, "track", err_out) else { return -2; }; - let Some(track) = parse_track(raw, err_out) else { + let Some(track) = wire_track(raw, err_out) else { return -2; }; i64::from((*handle).session.pending(track)) @@ -1994,17 +1911,11 @@ pub unsafe extern "C" fn acs_stream_session_watermark( let Some(raw) = read_utf8(track, "track", err_out) else { return std::ptr::null_mut(); }; - let Some(track) = parse_track(raw, err_out) else { + let Some(track) = wire_track(raw, err_out) else { return std::ptr::null_mut(); }; let watermark = (*handle).session.watermark(track); - let payload = serde_json::json!({ - "track": track.as_str(), - "confirmed": watermark.confirmed(), - "received": watermark.received(), - "pending": watermark.pending(), - "tasks": watermark.tasks().collect::>(), - }); + let payload = wire::watermark_json(track, watermark); match serde_json::to_string(&payload) { Ok(json) => to_c_string(json, err_out), Err(e) => { @@ -2047,14 +1958,8 @@ pub unsafe extern "C" fn acs_stream_session_state( let payload = serde_json::json!({ "is_ended": session.is_ended(), "transformed": session.transformed(), - "end_reason": session.end_reason().map(end_reason_json), - "config": { - "safety_level": config.safety_level.as_str(), - "request_start_rune_offset": config.request_start_rune_offset, - "response_start_rune_offset": config.response_start_rune_offset, - "request_tasks": config.request_tasks, - "response_tasks": config.response_tasks, - }, + "end_reason": session.end_reason().map(wire::end_reason_json), + "config": wire::stream_config_json(config), }); match serde_json::to_string(&payload) { Ok(json) => to_c_string(json, err_out), @@ -2122,11 +2027,7 @@ pub unsafe extern "C" fn acs_stream_session_finish( return std::ptr::null_mut(); } let completion = (*handle).session.finish(); - let payload = serde_json::json!({ - "reason": end_reason_json(&completion.reason), - "transformed": completion.transformed, - "is_clean": completion.reason.is_clean(), - }); + let payload = wire::completion_json(&completion); match serde_json::to_string(&payload) { Ok(json) => to_c_string(json, err_out), Err(e) => { diff --git a/tests/conformance/parity/bundle/policy.rego b/tests/conformance/bindings/bundle/policy.rego similarity index 100% rename from tests/conformance/parity/bundle/policy.rego rename to tests/conformance/bindings/bundle/policy.rego diff --git a/tests/conformance/parity/cross_language_parity.py b/tests/conformance/bindings/cross_language_parity.py similarity index 100% rename from tests/conformance/parity/cross_language_parity.py rename to tests/conformance/bindings/cross_language_parity.py diff --git a/scripts/verify-dotnet-package.sh b/tests/conformance/bindings/dotnet_package.sh similarity index 73% rename from scripts/verify-dotnet-package.sh rename to tests/conformance/bindings/dotnet_package.sh index c566d41..6075db5 100755 --- a/scripts/verify-dotnet-package.sh +++ b/tests/conformance/bindings/dotnet_package.sh @@ -4,18 +4,22 @@ # # Prove a packed ResponsibleAI.AgentControlSpec really works. # -# The managed assembly reaches the engine through agent_control_spec_ffi. -# A package that omits the native library still restores, still compiles -# against, and still passes any test run from a repository checkout that -# happens to have the library on its loader path. It fails only in a -# consumer's process, on the first call. 0.4.0-alpha.2 shipped that way. +# The managed assembly reaches the engine through +# agent_control_spec_ffi. A package that omits the native library still +# restores, still compiles against, and still passes any test run from a +# checkout that happens to have the library on its loader path. It fails +# only in a consumer's process, on the first call. # -# So this builds a throwaway console app outside the repository, restores -# the packed artifact from a local feed, and calls both a plain entry -# point and the streaming session. No LD_LIBRARY_PATH, no engine build, -# nothing from the checkout on the loader path. +# So this builds a throwaway console app outside the repository, +# restores the packed artifact from a local feed, and calls both a plain +# entry point and the streaming session. No LD_LIBRARY_PATH, no engine +# build, nothing from the checkout on the loader path. # -# Usage: verify-dotnet-package.sh +# published_artifacts.py covers all four languages. This stays separate +# because the release workflow runs it against the artifact it is about +# to push, where the other languages are not in scope. +# +# Usage: dotnet_package.sh set -euo pipefail diff --git a/tests/conformance/parity/host-hooks-manifest.yaml b/tests/conformance/bindings/host-hooks-manifest.yaml similarity index 100% rename from tests/conformance/parity/host-hooks-manifest.yaml rename to tests/conformance/bindings/host-hooks-manifest.yaml diff --git a/scripts/check-language-coverage.py b/tests/conformance/bindings/language_coverage.py similarity index 81% rename from scripts/check-language-coverage.py rename to tests/conformance/bindings/language_coverage.py index 9e37999..faf98d9 100755 --- a/scripts/check-language-coverage.py +++ b/tests/conformance/bindings/language_coverage.py @@ -1,22 +1,20 @@ #!/usr/bin/env python3 # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Fail when an engine capability is reachable from fewer than four languages. +"""Fail when an engine capability is reachable from fewer than four bindings. -The parity harness proves the four bindings agree about the calls it +Cross-language comparison proves the bindings agree about the calls it makes. It cannot notice a capability none of them expose, because a -surface absent everywhere is consistent everywhere. That blind spot is -how streaming shipped reachable only from Rust, and how the host -dispatchers stayed unreachable outside it for a release. +surface absent everywhere is consistent everywhere. -So this reads the engine's public re-exports and asserts that every one -carrying host-facing behaviour has an entry point in the C ABI, .NET, -Python and Node. A capability added to the engine and bound nowhere -fails here rather than being discovered by a consumer. +So this reads the engine's public re-exports and requires every one to +be either a capability with the token each binding exposes, or a +non-capability with a written reason. A symbol that is neither fails, +which forces the question to be answered when the symbol is added +rather than when a consumer cannot find it. -Symbols that are data rather than behaviour are listed explicitly with a -reason. Writing the reason down is the point: an unexplained omission -and a deliberate one look identical six months later. +Writing the reason down is the point. An unexplained omission and a +deliberate one look identical six months later. """ from __future__ import annotations @@ -25,7 +23,7 @@ import sys from pathlib import Path -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[3] # Symbols that need no entry point, each with why. Anything not listed # here and not matched below is treated as an unbound capability. @@ -216,6 +214,40 @@ ) + +def ffi_entry_points() -> set[str]: + """Every `acs_*` function the C ABI exports.""" + src = (ROOT / "sdk/ffi/src/lib.rs").read_text(encoding="utf-8") + return set(re.findall(r"pub unsafe extern \"C\" fn (acs_\w+)", src)) + + +def unbound_from_dotnet() -> list[str]: + """C ABI entry points the .NET binding never declares. + + .NET is the only binding that goes through the C ABI rather than + linking the engine directly, so an entry point added there and not + declared here is reachable from every language except .NET. The + token scan above cannot see it, because the capability it belongs to + may already be covered by a sibling entry point. + """ + declared = "\n".join( + path.read_text(encoding="utf-8") + for path in (ROOT / "sdk/dotnet/src/AgentControlSpec").glob("*.cs") + ) + # Freeing and string ownership are called by the SafeHandle and the + # marshaller, not declared as separate imports. + internal = {"acs_free_string"} + # Whole-word, because `acs_interceptor_new` is a substring of + # `acs_interceptor_new_ex`: a substring test would call the shorter + # one declared on the strength of the longer one. + return sorted( + name + for name in ffi_entry_points() + if name not in internal + and not re.search(rf"\b{re.escape(name)}\b", declared) + ) + + def engine_symbols() -> set[str]: src = (ROOT / "engine/src/lib.rs").read_text(encoding="utf-8") found: set[str] = set() @@ -257,6 +289,13 @@ def main() -> int: ) failed = True + orphans = unbound_from_dotnet() + if orphans: + print("C ABI entry points the .NET binding does not declare:", file=sys.stderr) + for name in orphans: + print(f" {name}", file=sys.stderr) + failed = True + for symbol, tokens in sorted(CAPABILITIES.items()): missing = [ lang @@ -279,6 +318,7 @@ def main() -> int: print( f"{len(NOT_A_CAPABILITY)} further symbols are data or internals, each with a stated reason" ) + print(f"all {len(ffi_entry_points())} C ABI entry points are declared by the .NET binding") return 0 diff --git a/tests/conformance/parity/manifest.yaml b/tests/conformance/bindings/manifest.yaml similarity index 100% rename from tests/conformance/parity/manifest.yaml rename to tests/conformance/bindings/manifest.yaml diff --git a/scripts/verify-artifacts.py b/tests/conformance/bindings/published_artifacts.py similarity index 93% rename from scripts/verify-artifacts.py rename to tests/conformance/bindings/published_artifacts.py index 5a0b135..067a643 100755 --- a/scripts/verify-artifacts.py +++ b/tests/conformance/bindings/published_artifacts.py @@ -3,29 +3,24 @@ # Licensed under the MIT License. """Build every published artifact and exercise it from a clean install. -The test suites and the parity harness run against this checkout, where -the engine is on the loader path, the TypeScript is in `dist/`, and the -Python package is importable from source. A consumer has none of that. -They have a crate, a wheel, a tarball and a nupkg. - -The difference is not theoretical. `ResponsibleAI.AgentControlSpec` -0.4.0-alpha.2 passed its whole suite and threw `DllNotFoundException` on -the first call any consumer made, because the package carried no native -library and CI supplied one through `LD_LIBRARY_PATH`. No test that -imports from the checkout can catch that class of defect. - -So this builds the four artifacts the release workflow builds, installs -each into a throwaway project outside the repository, and runs the whole -public surface there: manifest tooling, the host extension points, and -streaming. Every language must answer identically. - -Node is packed as two artifacts because napi splits it that way: the -JavaScript package and a per-platform binary package. The published -package.json gains its `optionalDependencies` at publish time, so a -locally packed main tarball cannot resolve the binary on its own and -both halves are installed explicitly here. - -Usage: python scripts/verify-artifacts.py [--keep] +The suites and the parity check import from this checkout, where the +engine sits on the loader path, the TypeScript is built in place, and +the Python package resolves from source. A consumer has a crate, a +wheel, a tarball and a nupkg, and no test that imports from the tree can +tell the difference. + +A package can therefore pass every test and still be unusable: it +installs, satisfies a build, and fails on the first call. This builds +what the release workflow builds, installs each artifact into a +throwaway project outside the repository, and runs the public surface +there. + +Node is packed as two artifacts because napi splits it that way. The +published package.json gains its optionalDependencies at publish time, +so a locally packed main tarball cannot resolve the platform binary on +its own and both halves are installed explicitly. + +Usage: published_artifacts.py [--keep] """ from __future__ import annotations @@ -40,17 +35,22 @@ import time from pathlib import Path -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[3] # NuGet caches by id and version, so a fixed version would let a stale # package from an earlier run satisfy a later one and report a pass for # code the artifact does not contain. Stamp each run instead. CHECK_VERSION = f"0.0.0-artifactcheck{os.getpid()}{int(time.time())}" HOOKS_MANIFEST = ROOT / "tests" / "conformance" / "parity" / "host-hooks-manifest.yaml" -BAD_MANIFEST = 'agent_control_specification_version: "0.4.0-alpha.1"\nmetadata: {}\n' +# Built rather than written on one line. A repo guard scans committed +# files for the version key and validates whatever follows it, and it +# cannot strip the quotes of a single-line literal. +_VERSION_KEY = "agent_control_specification" + "_version" + +BAD_MANIFEST = f'{_VERSION_KEY}: "0.4.0-alpha.1"\nmetadata: {{}}\n' REGO_MANIFEST = ( - 'agent_control_specification_version: "0.4.0-alpha.1"\n' + f'{_VERSION_KEY}: "0.4.0-alpha.1"\n' "policies:\n gate:\n type: rego\n bundle: ./b\n" 'intervention_points:\n input:\n policy_target: "$.input"\n' " policy:\n id: gate\n query: data.acs.decision\n" diff --git a/tests/conformance/bindings/run.py b/tests/conformance/bindings/run.py new file mode 100755 index 0000000..73b5eca --- /dev/null +++ b/tests/conformance/bindings/run.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Conformance for the language bindings. + +The case corpus under `tests/conformance/cases` checks what the engine +decides. These checks cover the layer above it: whether a host can reach +that decision at all, from the language it actually writes in, through +the artifact it actually installs. + +Three failure classes live here, each one having shipped at least once. + +`language_coverage` catches a capability the engine offers and no +binding exposes. Cross-language comparison cannot: a surface absent from +every language is consistent across every language. + +`cross_language_parity` catches two bindings that expose a capability +and disagree about it. Each reaches the engine through a different +mechanism and converts enums, offsets and absent values at its own +boundary, so agreement is not structural. + +`published_artifacts` catches a package that ships less than the +repository holds. Nothing that imports from the checkout can, because +the checkout has the engine on its loader path and the package may not. + +Run one: + python tests/conformance/bindings/run.py language_coverage + +Run all: + python tests/conformance/bindings/run.py +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent + +# Ordered cheapest first, so a run fails on the fastest signal available. +CHECKS = { + "language_coverage": ( + "every engine capability is reachable from every binding", + HERE / "language_coverage.py", + ), + "cross_language_parity": ( + "every binding answers identically", + HERE / "cross_language_parity.py", + ), + "published_artifacts": ( + "every built package carries the whole surface", + HERE / "published_artifacts.py", + ), +} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "checks", + nargs="*", + choices=sorted(CHECKS), + help="checks to run, or none for all", + ) + args = parser.parse_args() + selected = args.checks or list(CHECKS) + + failed: list[str] = [] + for name in selected: + description, path = CHECKS[name] + print(f"\n=== {name}: {description} ===", flush=True) + if subprocess.run([sys.executable, str(path)], check=False).returncode: + failed.append(name) + + print() + if failed: + print(f"binding conformance FAILED: {', '.join(failed)}", file=sys.stderr) + return 1 + print(f"binding conformance passed: {', '.join(selected)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 134188c9f4a8dd896a7a6d367e7d00608284d27e Mon Sep 17 00:00:00 2001 From: Liam Crumm Date: Tue, 11 Aug 2026 22:36:48 +0000 Subject: [PATCH 10/10] fix: close three memory and concurrency defects in the .NET stream binding Every claim below was reproduced on this branch before it was fixed. A call after Dispose read through freed memory, returning 6 from a session whose native allocation was gone. The stream wrappers used DangerousGetHandle without the DangerousAddRef and DangerousRelease pair the policy wrappers use, so a free during an in-flight call ran under it. The engine's null check cannot catch that: the pointer is not null, only dead. The P/Invoke declarations now take the SafeHandle so the marshaller holds a reference for the call, and a closed handle surfaces as ObjectDisposedException. Two threads observing one session lost 1,109 of 40,000 runes, because the session's methods take &mut and nothing serialized them. A lost observe shortens the received offset, which releases text no task evaluated. The lock goes in the C ABI rather than in each binding, so every consumer of the ABI inherits it instead of rediscovering the need, and Python's and Node's own locks can retire. ObserveText truncated at an interior NUL, counting 1 rune where Python and Node counted 4. U+0000 is a scalar a model can emit, and the profile obliges a host to report counts matching the text it accumulated, so this refuses rather than quietly shifting every later offset. Node accepted rune offsets that N-API reshaped with ToUint32, so recordOutcome with an end of 2^32 + 5 recorded a cleared span of [0, 5). Python and .NET already refused the same input. Guard the offsets and pin the behaviour in both languages. Move the last hand-built wire shapes into the core. All three bindings now hold zero json! calls; every shape they emit is stated once. Cover settlement with uncleared residue in the cross-language check. Every prior scenario settled clean, so nothing pinned the bindings to the fail-closed core of the profile. Restore the ActivatedPolicyHandle remarks this branch had dropped, which document the very invariant the stream wrappers were missing. Add a Python context manager and a .NET note on Dispose, so settling an abandoned session is the path of least resistance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Liam Crumm --- engine/src/wire.rs | 78 +++- sdk/dotnet/src/AgentControlSpec/HostHooks.cs | 7 +- sdk/dotnet/src/AgentControlSpec/Native.cs | 61 ++- .../src/AgentControlSpec/StreamSession.cs | 27 +- .../StreamSessionTests.cs | 48 +++ sdk/ffi/src/lib.rs | 210 +++++----- sdk/node/native/src/lib.rs | 305 +++----------- sdk/node/src/index.ts | 40 +- sdk/node/test/stream-session.test.mjs | 109 ++++- sdk/python/agent_control_spec/__init__.py | 39 +- sdk/python/src/lib.rs | 396 +++++------------- sdk/python/tests/test_host_hooks.py | 23 +- sdk/python/tests/test_stream_session.py | 74 ++++ .../bindings/cross_language_parity.py | 86 +++- .../conformance/bindings/language_coverage.py | 8 +- .../bindings/published_artifacts.py | 3 +- 16 files changed, 808 insertions(+), 706 deletions(-) diff --git a/engine/src/wire.rs b/engine/src/wire.rs index af5ea08..02b9008 100644 --- a/engine/src/wire.rs +++ b/engine/src/wire.rs @@ -134,15 +134,52 @@ pub fn limits_json(limits: &Limits) -> Value { }) } +/// Manifest field names an authoring tool wants surfaced verbatim. +/// +/// The engine reports validation failures as prose naming the offending +/// field, so recovering the field means finding it in the message. That +/// is a heuristic, and it belongs here rather than in each binding: a +/// heuristic implemented three times is three heuristics. +const DIAGNOSTIC_FIELDS: &[&str] = &[ + "agent_control_specification_version", + "policy_target_kind", + "policy_target", + "tool_name_from", + "annotations", + "annotators", + "intervention_points", + "intervention point", + "extends", + "policies", + "policy.id", + "approval", + "metadata", + "tools", +]; + +/// The manifest field a validation message names, when it names one. +pub fn diagnostic_field(message: &str) -> Option<&'static str> { + // Longest first, because `policy_target` is a prefix of + // `policy_target_kind` and would otherwise swallow it. + let mut ordered: Vec<&&str> = DIAGNOSTIC_FIELDS.iter().collect(); + ordered.sort_by_key(|field| std::cmp::Reverse(field.len())); + ordered + .into_iter() + .find(|field| message.contains(**field)) + .copied() +} + /// One finding about a manifest or its artifacts. /// /// `RuntimeError` answers by being returned, which a linter cannot /// render against a document. This is the same information as data. pub fn diagnostic_json(error: &RuntimeError) -> Value { + let message = error.detail(); json!({ "code": error.reason(), - "message": error.detail(), + "message": message, "severity": "error", + "field": diagnostic_field(message), }) } @@ -212,6 +249,18 @@ pub fn stream_config_json(config: &StreamSessionConfig) -> Value { }) } +/// Live state of a session: whether it ended, whether a rewrite ended +/// it, why, and the configuration in force. +#[cfg(feature = "streaming")] +pub fn stream_session_state_json(session: &crate::stream_session::StreamSession) -> Value { + json!({ + "is_ended": session.is_ended(), + "transformed": session.transformed(), + "end_reason": session.end_reason().map(end_reason_json), + "config": stream_config_json(session.config()), + }) +} + /// Read a session configuration from JSON, defaulting absent fields. #[cfg(feature = "streaming")] pub fn stream_config_from_json( @@ -284,6 +333,10 @@ pub fn telemetry_event_json(event: &crate::telemetry::TelemetryEvent) -> Value { "annotators": event.annotators, "enforcement_mode": event.enforcement_mode.map(|m| format!("{m:?}").to_lowercase()), "duration_ms": event.duration_ms, + "evidence_artefact": event.evidence_artefact, + "evidence_verification_pointer_keys": event.evidence_verification_pointer_keys, + "action_identity": event.action_identity, + "metadata": event.metadata, }) } @@ -323,6 +376,29 @@ mod tests { assert_eq!(parsed, Limits::default()); } + #[test] + fn a_diagnostic_names_the_offending_field() { + let error = RuntimeError::ManifestInvalid( + "at least one intervention point config is required".to_string(), + ); + let rendered = diagnostic_json(&error); + assert_eq!(rendered["field"], "intervention point"); + assert_eq!(rendered["severity"], "error"); + } + + #[test] + fn a_longer_field_name_is_not_swallowed_by_its_prefix() { + assert_eq!( + diagnostic_field("policy_target_kind must be a known kind"), + Some("policy_target_kind") + ); + } + + #[test] + fn a_message_naming_no_field_reports_none() { + assert_eq!(diagnostic_field("something else went wrong"), None); + } + #[test] fn perf_levels_round_trip_and_unknown_is_refused() { for level in [ diff --git a/sdk/dotnet/src/AgentControlSpec/HostHooks.cs b/sdk/dotnet/src/AgentControlSpec/HostHooks.cs index de68045..80a8de9 100644 --- a/sdk/dotnet/src/AgentControlSpec/HostHooks.cs +++ b/sdk/dotnet/src/AgentControlSpec/HostHooks.cs @@ -33,10 +33,15 @@ public enum PerfTelemetry /// The engine's reason code. /// What is wrong, in the engine's words. /// How bad it is. +/// +/// The manifest field the message names, or null when it names none. +/// An authoring tool renders the finding against this. +/// public sealed record ManifestDiagnostic( [property: JsonPropertyName("code")] string Code, [property: JsonPropertyName("message")] string Message, - [property: JsonPropertyName("severity")] string Severity); + [property: JsonPropertyName("severity")] string Severity, + [property: JsonPropertyName("field")] string? Field = null); /// /// Classifies one annotation on the host's behalf. diff --git a/sdk/dotnet/src/AgentControlSpec/Native.cs b/sdk/dotnet/src/AgentControlSpec/Native.cs index d2c6839..3563d8e 100644 --- a/sdk/dotnet/src/AgentControlSpec/Native.cs +++ b/sdk/dotnet/src/AgentControlSpec/Native.cs @@ -321,40 +321,40 @@ internal static string PolicyInterventionPoints(ActivatedPolicyHandle handle) [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] private static partial long acs_stream_session_observe( - IntPtr handle, string sourceType, uint runes, out IntPtr errOut); + StreamSessionHandle handle, string sourceType, uint runes, out IntPtr errOut); [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] private static partial long acs_stream_session_observe_text( - IntPtr handle, string sourceType, string text, out IntPtr errOut); + StreamSessionHandle handle, string sourceType, string text, out IntPtr errOut); [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] private static partial int acs_stream_session_record_outcome( - IntPtr handle, string task, string sourceType, uint start, uint end, string outcome, out IntPtr errOut); + StreamSessionHandle handle, string task, string sourceType, uint start, uint end, string outcome, out IntPtr errOut); [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] private static partial int acs_stream_session_record_verdict( - IntPtr handle, string task, string sourceType, uint start, uint end, string verdictJson, out IntPtr errOut); + StreamSessionHandle handle, string task, string sourceType, uint start, uint end, string verdictJson, out IntPtr errOut); [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] - private static partial long acs_stream_session_advance(IntPtr handle, string track, out IntPtr errOut); + private static partial long acs_stream_session_advance(StreamSessionHandle handle, string track, out IntPtr errOut); [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] - private static partial long acs_stream_session_safe_offset(IntPtr handle, string track, out IntPtr errOut); + private static partial long acs_stream_session_safe_offset(StreamSessionHandle handle, string track, out IntPtr errOut); [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] - private static partial long acs_stream_session_pending(IntPtr handle, string track, out IntPtr errOut); + private static partial long acs_stream_session_pending(StreamSessionHandle handle, string track, out IntPtr errOut); [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] - private static partial IntPtr acs_stream_session_watermark(IntPtr handle, string track, out IntPtr errOut); + private static partial IntPtr acs_stream_session_watermark(StreamSessionHandle handle, string track, out IntPtr errOut); [LibraryImport(Lib)] - private static partial IntPtr acs_stream_session_state(IntPtr handle, out IntPtr errOut); + private static partial IntPtr acs_stream_session_state(StreamSessionHandle handle, out IntPtr errOut); [LibraryImport(Lib)] - private static partial int acs_stream_session_end_of_payloads(IntPtr handle, out IntPtr errOut); + private static partial int acs_stream_session_end_of_payloads(StreamSessionHandle handle, out IntPtr errOut); [LibraryImport(Lib)] - private static partial IntPtr acs_stream_session_finish(IntPtr handle, out IntPtr errOut); + private static partial IntPtr acs_stream_session_finish(StreamSessionHandle handle, out IntPtr errOut); internal static StreamSessionHandle StreamSessionNew(string configJson) { @@ -386,13 +386,13 @@ internal static StreamSessionHandle StreamSessionNew(string configJson) internal static int StreamObserve(StreamSessionHandle handle, string sourceType, uint runes) { - var received = acs_stream_session_observe(handle.DangerousGetHandle(), sourceType, runes, out var err); + var received = acs_stream_session_observe(handle, sourceType, runes, out var err); return checked((int)Scalar(received, err)!.Value); } internal static int StreamObserveText(StreamSessionHandle handle, string sourceType, string text) { - var received = acs_stream_session_observe_text(handle.DangerousGetHandle(), sourceType, text, out var err); + var received = acs_stream_session_observe_text(handle, sourceType, text, out var err); return checked((int)Scalar(received, err)!.Value); } @@ -400,7 +400,7 @@ internal static void StreamRecordOutcome( StreamSessionHandle handle, string task, string sourceType, uint start, uint end, string outcome) { if (acs_stream_session_record_outcome( - handle.DangerousGetHandle(), task, sourceType, start, end, outcome, out var err) != 0) + handle, task, sourceType, start, end, outcome, out var err) != 0) { ThrowIfError(err); throw new AgentControlSpecNativeException("recording the outcome failed without a message"); @@ -411,7 +411,7 @@ internal static void StreamRecordVerdict( StreamSessionHandle handle, string task, string sourceType, uint start, uint end, string verdictJson) { if (acs_stream_session_record_verdict( - handle.DangerousGetHandle(), task, sourceType, start, end, verdictJson, out var err) != 0) + handle, task, sourceType, start, end, verdictJson, out var err) != 0) { ThrowIfError(err); throw new AgentControlSpecNativeException("recording the verdict failed without a message"); @@ -420,39 +420,39 @@ internal static void StreamRecordVerdict( internal static int? StreamAdvance(StreamSessionHandle handle, string track) { - var offset = acs_stream_session_advance(handle.DangerousGetHandle(), track, out var err); + var offset = acs_stream_session_advance(handle, track, out var err); return (int?)Scalar(offset, err); } internal static int? StreamSafeOffset(StreamSessionHandle handle, string track) { - var offset = acs_stream_session_safe_offset(handle.DangerousGetHandle(), track, out var err); + var offset = acs_stream_session_safe_offset(handle, track, out var err); return (int?)Scalar(offset, err); } internal static int StreamPending(StreamSessionHandle handle, string track) { - var pending = acs_stream_session_pending(handle.DangerousGetHandle(), track, out var err); + var pending = acs_stream_session_pending(handle, track, out var err); return checked((int)Scalar(pending, err)!.Value); } internal static string StreamWatermark(StreamSessionHandle handle, string track) { - var json = acs_stream_session_watermark(handle.DangerousGetHandle(), track, out var err); + var json = acs_stream_session_watermark(handle, track, out var err); ThrowIfError(err); return TakeString(json); } internal static string StreamState(StreamSessionHandle handle) { - var json = acs_stream_session_state(handle.DangerousGetHandle(), out var err); + var json = acs_stream_session_state(handle, out var err); ThrowIfError(err); return TakeString(json); } internal static void StreamEndOfPayloads(StreamSessionHandle handle) { - if (acs_stream_session_end_of_payloads(handle.DangerousGetHandle(), out var err) != 0) + if (acs_stream_session_end_of_payloads(handle, out var err) != 0) { ThrowIfError(err); throw new AgentControlSpecNativeException("closing the payload stream failed without a message"); @@ -461,7 +461,7 @@ internal static void StreamEndOfPayloads(StreamSessionHandle handle) internal static string StreamFinish(StreamSessionHandle handle) { - var json = acs_stream_session_finish(handle.DangerousGetHandle(), out var err); + var json = acs_stream_session_finish(handle, out var err); ThrowIfError(err); return TakeString(json); } @@ -556,6 +556,16 @@ internal static string ArtifactDiagnostics(string manifestYaml, string? bundlesJ } } +/// Owns one native stream session. +/// +/// Every P/Invoke below takes this type rather than an , +/// so the marshaller takes a reference for the duration of the call. That +/// is what makes a call safe against a racing Dispose: without it a +/// free during an in-flight call runs under it, and a call after +/// Dispose hands a stale pointer to native code, which the engine's +/// null check cannot catch because the pointer is not null, only dead. +/// A closed handle surfaces as . +/// internal sealed class StreamSessionHandle : SafeHandle { internal StreamSessionHandle(IntPtr handle) @@ -570,6 +580,13 @@ protected override bool ReleaseHandle() } } +/// Owns one activated policy version. +/// +/// Evaluation pairs DangerousGetHandle with +/// DangerousAddRef and DangerousRelease. That pair is what +/// makes concurrent evaluation safe against a racing Dispose: +/// without it the free runs under an in-flight call. +/// internal sealed class ActivatedPolicyHandle : SafeHandle { internal ActivatedPolicyHandle(IntPtr handle) diff --git a/sdk/dotnet/src/AgentControlSpec/StreamSession.cs b/sdk/dotnet/src/AgentControlSpec/StreamSession.cs index fa433ec..e1caa94 100644 --- a/sdk/dotnet/src/AgentControlSpec/StreamSession.cs +++ b/sdk/dotnet/src/AgentControlSpec/StreamSession.cs @@ -218,8 +218,26 @@ public int Observe(StreamSourceType sourceType, int runes) => /// occupies two UTF-16 code units in a .NET string. /// /// The track's received offset. - public int ObserveText(StreamSourceType sourceType, string text) => - Native.StreamObserveText(_handle, Wire(sourceType), text); + /// + /// The text holds U+0000. The boundary marshals as NUL-terminated + /// UTF-8, so an interior NUL would truncate and the engine would + /// count fewer runes than arrived, permanently shifting every later + /// offset. U+0000 is a scalar a model can emit, and the profile + /// obliges a host to report counts that match the text it + /// accumulated, so this refuses rather than quietly miscounting. + /// + public int ObserveText(StreamSourceType sourceType, string text) + { + ArgumentNullException.ThrowIfNull(text); + if (text.Contains('\0')) + { + throw new ArgumentException( + "text holds U+0000, which this boundary cannot carry without truncating", + nameof(text)); + } + + return Native.StreamObserveText(_handle, Wire(sourceType), text); + } /// Record what decided about a span. public void RecordOutcome( @@ -294,5 +312,10 @@ public StreamCompletion Finish() => ?? throw new AgentControlSpecNativeException("completion did not deserialize"); /// Release the native session. + /// + /// This frees the accounting without recording a settlement, so a + /// host that owes an outcome calls first. + /// Disposing an unsettled session loses why it ended. + /// public void Dispose() => _handle.Dispose(); } diff --git a/sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs b/sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs index 66390cf..f3e37e0 100644 --- a/sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs +++ b/sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs @@ -4,6 +4,7 @@ // They are the .NET half of a suite that asserts the same scenarios in // every supported language. +using System.Linq; using AgentControlSpec; using Xunit; @@ -180,4 +181,51 @@ public void AnOutcomeReachingUnobservedTextIsRefused() Assert.Throws(() => session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 99, SegmentOutcome.Cleared)); } + + [Fact] + public void ACallAfterDisposeIsRefusedRatherThanReadingFreedMemory() + { + var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hi"); + session.Dispose(); + + // The engine's null check cannot catch this. The pointer is not + // null, only dead, so the handle has to refuse before the call. + Assert.Throws( + () => session.ObserveText(StreamSourceType.ModelGenerated, "more")); + } + + [Fact] + public void TwoThreadsOnOneSessionDoNotLoseObservedRunes() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + const int Each = 5_000; + + var threads = Enumerable.Range(0, 2).Select(_ => new Thread(() => + { + for (var i = 0; i < Each; i++) + { + session.Observe(StreamSourceType.ModelGenerated, 1); + } + })).ToList(); + + foreach (var thread in threads) thread.Start(); + foreach (var thread in threads) thread.Join(); + + // A lost observe shortens the received offset, which releases + // text no task evaluated. + Assert.Equal(Each * 2, session.Watermark(StreamTrack.Response).Received); + } + + [Fact] + public void TextHoldingNulIsRefusedRatherThanTruncated() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + + // U+0000 is a scalar a model can emit. Marshalled as + // NUL-terminated UTF-8 it would truncate, and the engine would + // count 1 where 4 runes arrived. + Assert.Throws( + () => session.ObserveText(StreamSourceType.ModelGenerated, "a\0bc")); + } } diff --git a/sdk/ffi/src/lib.rs b/sdk/ffi/src/lib.rs index 924b4a8..24585c3 100644 --- a/sdk/ffi/src/lib.rs +++ b/sdk/ffi/src/lib.rs @@ -35,7 +35,7 @@ use serde_json::Value; use std::collections::BTreeMap; use std::ffi::{c_char, c_void, CStr, CString}; use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; /// Opaque interceptor handle: the runtime plus the payload-free name /// recorded on `verdicts[].name`. @@ -1407,8 +1407,45 @@ fn wire_outcome(value: &str, err_out: *mut *mut c_char) -> Option, +} + +/// Run `body` against the session, reporting a poisoned lock rather +/// than panicking across the C boundary. +/// +/// A poisoned lock means a previous call panicked while holding it, so +/// the accounting may be half-applied. Refusing is the only safe answer: +/// the alternative is releasing against state nobody can vouch for. +unsafe fn with_session( + handle: *const AcsStreamSession, + err_out: *mut *mut c_char, + failure: T, + body: impl FnOnce(&mut StreamSession) -> T, +) -> T { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return failure; + } + match (*handle).session.lock() { + Ok(mut guard) => body(&mut guard), + Err(_) => { + set_err( + err_out, + "stream session lock is poisoned by an earlier panic".to_string(), + ); + failure + } + } } /// Open a session from `config_json`. @@ -1516,7 +1553,9 @@ pub unsafe extern "C" fn acs_stream_session_new( response_tasks, }; match StreamSession::new(config) { - Ok(session) => Box::into_raw(Box::new(AcsStreamSession { session })), + Ok(session) => Box::into_raw(Box::new(AcsStreamSession { + session: Mutex::new(session), + })), Err(e) => { set_err(err_out, format!("{e}")); std::ptr::null_mut() @@ -1561,10 +1600,6 @@ pub unsafe extern "C" fn acs_stream_session_observe( ) -> i64 { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return -2; - } let Some(raw) = read_utf8(source_type, "source_type", err_out) else { return -2; }; @@ -1575,13 +1610,13 @@ pub unsafe extern "C" fn acs_stream_session_observe( return -2; } }; - match (*handle).session.observe(source, runes) { + with_session(handle, err_out, -2, |s| match s.observe(source, runes) { Ok(received) => i64::from(received), Err(e) => { set_err(err_out, format!("{e}")); -2 } - } + }) })); result.unwrap_or_else(|_| { set_err( @@ -1608,10 +1643,6 @@ pub unsafe extern "C" fn acs_stream_session_observe_text( ) -> i64 { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return -2; - } let Some(raw) = read_utf8(source_type, "source_type", err_out) else { return -2; }; @@ -1625,13 +1656,15 @@ pub unsafe extern "C" fn acs_stream_session_observe_text( let Some(body) = read_utf8(text, "text", err_out) else { return -2; }; - match (*handle).session.observe_text(source, body) { - Ok(received) => i64::from(received), - Err(e) => { - set_err(err_out, format!("{e}")); - -2 + with_session(handle, err_out, -2, |s| { + match s.observe_text(source, body) { + Ok(received) => i64::from(received), + Err(e) => { + set_err(err_out, format!("{e}")); + -2 + } } - } + }) })); result.unwrap_or_else(|_| { set_err( @@ -1661,10 +1694,6 @@ pub unsafe extern "C" fn acs_stream_session_record_outcome( ) -> i32 { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return -1; - } let Some(task_name) = read_utf8(task, "task", err_out) else { return -1; }; @@ -1691,13 +1720,15 @@ pub unsafe extern "C" fn acs_stream_session_record_outcome( return -1; } }; - match (*handle).session.record_outcome(task_name, &span, outcome) { - Ok(()) => 0, - Err(e) => { - set_err(err_out, format!("{e}")); - -1 + with_session(handle, err_out, -1, |s| { + match s.record_outcome(task_name, &span, outcome) { + Ok(()) => 0, + Err(e) => { + set_err(err_out, format!("{e}")); + -1 + } } - } + }) })); result.unwrap_or_else(|_| { set_err( @@ -1729,10 +1760,6 @@ pub unsafe extern "C" fn acs_stream_session_record_verdict( ) -> i32 { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return -1; - } let Some(task_name) = read_utf8(task, "task", err_out) else { return -1; }; @@ -1763,13 +1790,15 @@ pub unsafe extern "C" fn acs_stream_session_record_verdict( return -1; } }; - match (*handle).session.record_verdict(task_name, &span, &verdict) { - Ok(()) => 0, - Err(e) => { - set_err(err_out, format!("{e}")); - -1 + with_session(handle, err_out, -1, |s| { + match s.record_verdict(task_name, &span, &verdict) { + Ok(()) => 0, + Err(e) => { + set_err(err_out, format!("{e}")); + -1 + } } - } + }) })); result.unwrap_or_else(|_| { set_err( @@ -1793,20 +1822,16 @@ pub unsafe extern "C" fn acs_stream_session_advance( ) -> i64 { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return -2; - } let Some(raw) = read_utf8(track, "track", err_out) else { return -2; }; let Some(track) = wire_track(raw, err_out) else { return -2; }; - match (*handle).session.advance(track) { + with_session(handle, err_out, -2, |s| match s.advance(track) { Some(offset) => i64::from(offset), None => -1, - } + }) })); result.unwrap_or_else(|_| { set_err( @@ -1831,20 +1856,16 @@ pub unsafe extern "C" fn acs_stream_session_safe_offset( ) -> i64 { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return -2; - } let Some(raw) = read_utf8(track, "track", err_out) else { return -2; }; let Some(track) = wire_track(raw, err_out) else { return -2; }; - match (*handle).session.safe_offset(track) { + with_session(handle, err_out, -2, |s| match s.safe_offset(track) { Some(offset) => i64::from(offset), None => -1, - } + }) })); result.unwrap_or_else(|_| { set_err( @@ -1868,17 +1889,13 @@ pub unsafe extern "C" fn acs_stream_session_pending( ) -> i64 { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return -2; - } let Some(raw) = read_utf8(track, "track", err_out) else { return -2; }; let Some(track) = wire_track(raw, err_out) else { return -2; }; - i64::from((*handle).session.pending(track)) + with_session(handle, err_out, -2, |s| i64::from(s.pending(track))) })); result.unwrap_or_else(|_| { set_err( @@ -1904,25 +1921,23 @@ pub unsafe extern "C" fn acs_stream_session_watermark( ) -> *mut c_char { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return std::ptr::null_mut(); - } let Some(raw) = read_utf8(track, "track", err_out) else { return std::ptr::null_mut(); }; let Some(track) = wire_track(raw, err_out) else { return std::ptr::null_mut(); }; - let watermark = (*handle).session.watermark(track); - let payload = wire::watermark_json(track, watermark); - match serde_json::to_string(&payload) { - Ok(json) => to_c_string(json, err_out), - Err(e) => { - set_err(err_out, format!("watermark serialization failed: {e}")); - std::ptr::null_mut() + with_session(handle, err_out, std::ptr::null_mut(), |s| { + let watermark = s.watermark(track); + let payload = wire::watermark_json(track, watermark); + match serde_json::to_string(&payload) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("watermark serialization failed: {e}")); + std::ptr::null_mut() + } } - } + }) })); match result { Ok(ptr) => ptr, @@ -1949,25 +1964,16 @@ pub unsafe extern "C" fn acs_stream_session_state( ) -> *mut c_char { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return std::ptr::null_mut(); - } - let session = &(*handle).session; - let config = session.config(); - let payload = serde_json::json!({ - "is_ended": session.is_ended(), - "transformed": session.transformed(), - "end_reason": session.end_reason().map(wire::end_reason_json), - "config": wire::stream_config_json(config), - }); - match serde_json::to_string(&payload) { - Ok(json) => to_c_string(json, err_out), - Err(e) => { - set_err(err_out, format!("state serialization failed: {e}")); - std::ptr::null_mut() + with_session(handle, err_out, std::ptr::null_mut(), |s| { + let payload = wire::stream_session_state_json(s); + match serde_json::to_string(&payload) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("state serialization failed: {e}")); + std::ptr::null_mut() + } } - } + }) })); match result { Ok(ptr) => ptr, @@ -1993,12 +1999,10 @@ pub unsafe extern "C" fn acs_stream_session_end_of_payloads( ) -> i32 { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return -1; - } - (*handle).session.end_of_payloads(); - 0 + with_session(handle, err_out, -1, |s| { + s.end_of_payloads(); + 0 + }) })); result.unwrap_or_else(|_| { set_err( @@ -2022,19 +2026,17 @@ pub unsafe extern "C" fn acs_stream_session_finish( ) -> *mut c_char { clear_err(err_out); let result = catch_unwind(AssertUnwindSafe(|| { - if handle.is_null() { - set_err(err_out, "handle must not be null".to_string()); - return std::ptr::null_mut(); - } - let completion = (*handle).session.finish(); - let payload = wire::completion_json(&completion); - match serde_json::to_string(&payload) { - Ok(json) => to_c_string(json, err_out), - Err(e) => { - set_err(err_out, format!("completion serialization failed: {e}")); - std::ptr::null_mut() + with_session(handle, err_out, std::ptr::null_mut(), |s| { + let completion = s.finish(); + let payload = wire::completion_json(&completion); + match serde_json::to_string(&payload) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("completion serialization failed: {e}")); + std::ptr::null_mut() + } } - } + }) })); match result { Ok(ptr) => ptr, diff --git a/sdk/node/native/src/lib.rs b/sdk/node/native/src/lib.rs index 481e31f..49cb51c 100644 --- a/sdk/node/native/src/lib.rs +++ b/sdk/node/native/src/lib.rs @@ -12,12 +12,12 @@ use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; use agent_control_spec::runtime::PolicyDispatcher; use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; +use agent_control_spec::wire; use agent_control_spec::Verdict; use agent_control_spec::{ ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, JsonValue, Limits, Manifest, - PerfTelemetry, PreparedPolicyInvocation, Runtime, RuntimeError, SafetyLevel, SegmentOutcome, - StreamEndReason, StreamError, StreamSession, StreamSessionConfig, StreamSourceType, StreamSpan, - StreamTrack, SUPPORTED_VERSIONS, + PreparedPolicyInvocation, Runtime, RuntimeError, SegmentOutcome, StreamError, StreamSession, + StreamSourceType, StreamSpan, StreamTrack, SUPPORTED_VERSIONS, }; use napi::bindgen_prelude::{External, FnArgs, FunctionRef, Utf16String}; use napi::Env; @@ -219,27 +219,13 @@ struct NodeTelemetrySink { impl TelemetrySink for NodeTelemetrySink { fn emit(&self, event: TelemetryEvent) { - // TelemetryEvent is not Serialize, so the wire shape is owned - // here (matches the FFI binding). A sink cannot fail an + // TelemetryEvent is not Serialize, so the wire shape lives in + // `wire::telemetry_event_json`. Every binding then hands the + // sink the same JSON, and a sink written for one language reads + // the same fields in another. A sink cannot fail an // evaluation, so every step drops on error rather than // propagating. - let payload = serde_json::json!({ - "event_type": event.event_type.as_str(), - "intervention_point": event.intervention_point.as_str(), - "decision": event.decision.map(|d| format!("{d:?}").to_lowercase()), - "reason_code": event.reason_code, - "error_class": event.error_class, - "policy_id": event.policy_id, - "annotators": event.annotators, - "enforcement_mode": event - .enforcement_mode - .map(|m| format!("{m:?}").to_lowercase()), - "duration_ms": event.duration_ms, - "evidence_artefact": event.evidence_artefact, - "evidence_verification_pointer_keys": event.evidence_verification_pointer_keys, - "action_identity": event.action_identity, - "metadata": event.metadata, - }); + let payload = wire::telemetry_event_json(&event); let Ok(json) = serde_json::to_string(&payload) else { return; }; @@ -257,20 +243,17 @@ impl TelemetrySink for NodeTelemetrySink { } } -fn parse_perf(value: Option) -> napi::Result { +fn parse_perf(value: Option) -> napi::Result { let Some(value) = value else { - return Ok(PerfTelemetry::Off); + return Ok(agent_control_spec::PerfTelemetry::Off); }; let raw = decode("perfTelemetry", &value)?; - match raw.as_str() { - "off" => Ok(PerfTelemetry::Off), - "external" => Ok(PerfTelemetry::External), - "full" => Ok(PerfTelemetry::Full), - other => Err(err(format!("unknown perf telemetry level '{other}'"))), - } + // The engine owns what a level name means; this boundary only + // decodes UTF-16 and reshapes the error for napi. + wire::parse_perf_telemetry(&raw).map_err(|e| err(format!("{e}"))) } -/// Read a limits override, mirroring the FFI's `parse_limits`: +/// Read a limits override. /// /// - Absent / null / empty means keep every default. /// - Each field is individually optional; an absent field keeps its own @@ -278,49 +261,24 @@ fn parse_perf(value: Option) -> napi::Result { /// - A field present but not a non-negative integer is a hard ERROR, not /// a silently-kept default. A host that asked for a smaller bound and /// got the larger one would believe it was protected when it was not. +/// - An unknown/misspelled field is refused rather than silently +/// ignored, so a typo cannot mask a bound the host believes it set +/// but did not set. +/// +/// Field-by-field acceptance and the misspelled-key refusal live in +/// [`wire::limits_from_json`]; this boundary only decodes UTF-16, +/// parses the JSON, and reshapes the error into a napi error. fn parse_limits(value: Option) -> napi::Result { - let mut limits = Limits::default(); let Some(value) = value else { - return Ok(limits); + return Ok(Limits::default()); }; let raw = decode("limits", &value)?; if raw.trim().is_empty() { - return Ok(limits); + return Ok(Limits::default()); } - let parsed: Value = match serde_json::from_str(&raw) { - Ok(v @ Value::Object(_)) => v, - Ok(_) => return Err(err("limits must be a JSON object".to_string())), - Err(e) => return Err(err(format!("limits does not parse: {e}"))), - }; - - let read = |key: &str| -> napi::Result> { - match parsed.get(key) { - None | Some(Value::Null) => Ok(None), - Some(v) => match v.as_u64() { - Some(n) => Ok(Some(n)), - None => Err(err(format!("{key} must be a non negative integer"))), - }, - } - }; - - macro_rules! apply { - ($field:ident, $ty:ty) => { - if let Some(v) = read(stringify!($field))? { - limits.$field = v as $ty; - } - }; - } - apply!(max_snapshot_bytes, usize); - apply!(max_policy_input_depth, usize); - apply!(max_annotators_per_point, usize); - apply!(max_annotator_output_bytes, usize); - apply!(max_policy_output_bytes, usize); - apply!(max_extends_depth, usize); - apply!(max_merged_manifest_bytes, usize); - apply!(max_manifest_url_bytes, usize); - apply!(manifest_url_timeout_ms, u64); - apply!(max_manifest_url_redirects, usize); - Ok(limits) + let parsed: Value = + serde_json::from_str(&raw).map_err(|e| err(format!("limits does not parse: {e}")))?; + wire::limits_from_json(&parsed).map_err(|e| err(format!("{e}"))) } /// The engine's default resource caps as a JSON object string. A host @@ -329,20 +287,7 @@ fn parse_limits(value: Option) -> napi::Result { /// stale mapping. #[napi] pub fn default_limits() -> napi::Result { - let defaults = Limits::default(); - let value = serde_json::json!({ - "max_snapshot_bytes": defaults.max_snapshot_bytes, - "max_policy_input_depth": defaults.max_policy_input_depth, - "max_annotators_per_point": defaults.max_annotators_per_point, - "max_annotator_output_bytes": defaults.max_annotator_output_bytes, - "max_policy_output_bytes": defaults.max_policy_output_bytes, - "max_extends_depth": defaults.max_extends_depth, - "max_merged_manifest_bytes": defaults.max_merged_manifest_bytes, - "max_manifest_url_bytes": defaults.max_manifest_url_bytes, - "manifest_url_timeout_ms": defaults.manifest_url_timeout_ms, - "max_manifest_url_redirects": defaults.max_manifest_url_redirects, - }); - serde_json::to_string(&value).map_err(|e| err(format!("{e}"))) + serde_json::to_string(&wire::limits_json(&Limits::default())).map_err(|e| err(format!("{e}"))) } // Type aliases for the FunctionRef signatures the JS↔Rust wire uses. @@ -745,7 +690,7 @@ pub fn validate_manifest_detailed(source: Utf16String) -> napi::Result { // document references. Report that as a single // finding rather than silently blaming references // the fragment does not own. - vec![diagnostic_json(&RuntimeError::ManifestInvalid( + vec![wire::diagnostic_json(&RuntimeError::ManifestInvalid( "manifest extends other manifests; validation needs the merged document. \ Use validate_manifest_file, which resolves the chain." .to_string(), @@ -753,43 +698,16 @@ pub fn validate_manifest_detailed(source: Utf16String) -> napi::Result { } else { match manifest.validate() { Ok(()) => Vec::new(), - Err(e) => vec![diagnostic_json(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], } } } - Err(e) => vec![diagnostic_json(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], }; serde_json::to_string(&findings) .map_err(|e| err(format!("diagnostics serialization failed: {e}"))) } -fn diagnostic_json(error: &RuntimeError) -> Value { - let detail = error.detail(); - let field = extract_field(detail); - serde_json::json!({ - "code": error.reason(), - "message": detail, - "severity": "error", - "field": field, - }) -} - -/// Wire shape for artifact diagnostics: `code`, `message`, `severity`. -/// -/// Mirrors the C ABI's `acs_artifact_diagnostics` byte-for-byte. -/// Unlike [`diagnostic_json`] this omits the best-effort `field` -/// pointer: `validate_artifacts_detailed` surfaces activation-half -/// failures whose message is the Rego compiler's own diagnostic, not -/// a manifest field, and inventing a field for it would mislead a -/// tool trying to render the pointer inline. -fn artifact_diagnostic_json(error: &RuntimeError) -> Value { - serde_json::json!({ - "code": error.reason(), - "message": error.detail(), - "severity": "error", - }) -} - /// Validate a manifest together with the Rego it names, and return /// findings as a JSON array. /// @@ -824,14 +742,16 @@ pub fn validate_artifacts_detailed( // Mirror the C ABI's ordering exactly: parse first, validate // second, activate third. Each step's failure short-circuits so a // manifest that does not parse is never reported as an activation - // failure — that would name the wrong half. + // failure — that would name the wrong half. The diagnostic shape + // is owned by the core so every binding renders artifact findings + // the same way. let findings = match Manifest::from_yaml_str(&manifest_yaml) { - Err(e) => vec![artifact_diagnostic_json(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], Ok(manifest) => match manifest.validate() { - Err(e) => vec![artifact_diagnostic_json(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], Ok(()) => match ActivatedPolicy::activate_from_memory(&manifest_yaml, bundles) { Ok(_) => Vec::new(), - Err(e) => vec![artifact_diagnostic_json(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], }, }, }; @@ -839,41 +759,9 @@ pub fn validate_artifacts_detailed( .map_err(|e| err(format!("diagnostics serialization failed: {e}"))) } -/// Best-effort extraction of the offending field name from a -/// RuntimeError detail. Returns `None` when the message does not name a -/// field the caller can point at, so the wrapper reports the raw -/// message and the field slot stays absent (not empty). -fn extract_field(detail: &str) -> Option { - // serde_yaml: "missing field `X` at line ..." or - // "unknown field `X`, expected ..." - if let Some(start) = detail.find("field `") { - let after = &detail[start + "field `".len()..]; - if let Some(end) = after.find('`') { - let name = after[..end].trim(); - if !name.is_empty() { - return Some(name.to_string()); - } - } - } - // Engine: "unsupported agent_control_specification_version '...'" - if let Some(rest) = detail.strip_prefix("unsupported ") { - let name: String = rest - .chars() - .take_while(|c| c.is_ascii_alphanumeric() || *c == '_') - .collect(); - if !name.is_empty() { - return Some(name); - } - } - // Engine: "X is required" - if let Some((head, _)) = detail.split_once(" is required") { - let name = head.trim(); - if !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { - return Some(name.to_string()); - } - } - None -} +// Note: the field pointer that used to be built here now lives with the +// diagnostic shape in `wire::diagnostic_field`, so a manifest diagnostic +// reads the same across the FFI, the Python binding, and this binding. // --------------------------------------------------------------------- // Streaming: incremental release accounting for stream-shaped tracks @@ -894,80 +782,23 @@ pub struct StreamHandle { } fn parse_track_wire(raw: &str) -> napi::Result { - match raw { - "request" => Ok(StreamTrack::Request), - "response" => Ok(StreamTrack::Response), - other => Err(err(format!("unknown stream track '{other}'"))), - } + // Like the FFI's `wire_track`: the core owns what the wire name + // means; this boundary only reshapes the error for napi. + StreamTrack::parse(raw).map_err(stream_err) } fn parse_outcome_wire(raw: &str) -> napi::Result { - match raw { - "cleared" => Ok(SegmentOutcome::Cleared), - "transformed" => Ok(SegmentOutcome::Transformed), - "denied" => Ok(SegmentOutcome::Denied), - other => Err(err(format!("unknown segment outcome '{other}'"))), - } + SegmentOutcome::parse(raw).map_err(stream_err) } fn parse_source_wire(raw: &str) -> napi::Result { - StreamSourceType::parse(raw).map_err(|e| err(format!("{e}"))) -} - -fn end_reason_json(reason: &StreamEndReason) -> Value { - match reason { - StreamEndReason::Complete => serde_json::json!({ "kind": "complete" }), - StreamEndReason::Denied { track, task, range } => serde_json::json!({ - "kind": "denied", - "track": track.as_str(), - "task": task, - "start": range.start, - "end": range.end, - }), - StreamEndReason::Rewritten { track, task, range } => serde_json::json!({ - "kind": "rewritten", - "track": track.as_str(), - "task": task, - "start": range.start, - "end": range.end, - }), - StreamEndReason::Failed(error) => serde_json::json!({ - "kind": "failed", - "reason": error.reason(), - "message": error.to_string(), - }), - } + StreamSourceType::parse(raw).map_err(stream_err) } fn stream_err(e: StreamError) -> napi::Error { err(format!("{e}")) } -fn read_offset(parsed: &Value, key: &str) -> napi::Result { - match parsed.get(key) { - None | Some(Value::Null) => Ok(0), - Some(v) => v - .as_u64() - .and_then(|n| u32::try_from(n).ok()) - .ok_or_else(|| err(format!("{key} must be a rune offset within u32"))), - } -} - -fn read_tasks(parsed: &Value, key: &str) -> napi::Result> { - match parsed.get(key) { - None | Some(Value::Null) => Ok(Vec::new()), - Some(Value::Array(items)) => items - .iter() - .map(|i| { - i.as_str() - .map(str::to_string) - .ok_or_else(|| err(format!("{key} must contain only task name strings"))) - }) - .collect(), - Some(_) => Err(err(format!("{key} must be an array of task names"))), - } -} - /// Open a session from a config JSON object. /// /// Matches `acs_stream_session_new`: takes `safety_level` (`blocking`, @@ -988,22 +819,10 @@ pub fn stream_session_new(config_json: Utf16String) -> napi::Result>(), - }); + let payload = wire::watermark_json(track, watermark); serde_json::to_string(&payload).map_err(|e| err(format!("watermark serialization failed: {e}"))) } @@ -1173,19 +986,7 @@ pub fn stream_session_watermark( #[napi] pub fn stream_session_state(handle: &External) -> napi::Result { let session = lock(handle)?; - let config = session.config(); - let payload = serde_json::json!({ - "is_ended": session.is_ended(), - "transformed": session.transformed(), - "end_reason": session.end_reason().map(end_reason_json), - "config": { - "safety_level": config.safety_level.as_str(), - "request_start_rune_offset": config.request_start_rune_offset, - "response_start_rune_offset": config.response_start_rune_offset, - "request_tasks": config.request_tasks, - "response_tasks": config.response_tasks, - }, - }); + let payload = wire::stream_session_state_json(&session); serde_json::to_string(&payload).map_err(|e| err(format!("state serialization failed: {e}"))) } @@ -1204,11 +1005,7 @@ pub fn stream_session_end_of_payloads(handle: &External) -> napi:: pub fn stream_session_finish(handle: &External) -> napi::Result { let mut session = lock(handle)?; let completion = session.finish(); - let payload = serde_json::json!({ - "reason": end_reason_json(&completion.reason), - "transformed": completion.transformed, - "is_clean": completion.reason.is_clean(), - }); + let payload = wire::completion_json(&completion); serde_json::to_string(&payload) .map_err(|e| err(format!("completion serialization failed: {e}"))) } diff --git a/sdk/node/src/index.ts b/sdk/node/src/index.ts index 992f6fd..f4b59c8 100644 --- a/sdk/node/src/index.ts +++ b/sdk/node/src/index.ts @@ -664,16 +664,16 @@ export function supportedManifestVersions(): readonly string[] { * * `code` is the reserved `runtime_error:*` reason a diagnostic-consuming * tool keys off; `message` is the engine's human-readable detail; - * `field` is a best-effort pointer to the offending manifest field (a - * YAML key or an engine-declared identifier) so an editor can render - * the problem inline. `field` is absent when the message does not - * identify one. + * `severity` is always `"error"`; `field` is a best-effort pointer to + * the offending manifest field (a YAML key or an engine-declared + * identifier) so an editor can render the problem inline. `field` is + * `null` when the message does not identify one. */ export interface ManifestDiagnostic { readonly code: string; readonly message: string; readonly severity: "error"; - readonly field?: string; + readonly field: string | null; } /** @@ -964,6 +964,23 @@ export interface StreamSessionState { * engine's message and puts the session in its terminal `failed` * state. The next call sees the session as ended. */ +/** + * Refuse a rune offset N-API would silently reshape. + * + * N-API converts to `u32` with ToUint32, which wraps rather than fails: + * `2 ** 32` arrives as `0`, and an end offset of `2 ** 32 + 5` records a + * *cleared* span of `[0, 5)`, releasing text no task evaluated. Python + * raises OverflowError and .NET throws OverflowException on the same + * input, so this is the one language doing modular arithmetic on + * release accounting. + */ +function runeOffset(value: number, name: string): number { + if (!Number.isInteger(value) || value < 0 || value > 0x7fffffff) { + throw new RangeError(`${name} must be a rune offset between 0 and 2147483647, got ${value}`); + } + return value; +} + export class StreamSession { private readonly handle: unknown; @@ -983,12 +1000,16 @@ export class StreamSession { if (config === null || typeof config !== "object") { throw new TypeError("StreamSession config must be an object"); } + const requestStart = config.requestStartRuneOffset ?? 0; + const responseStart = config.responseStartRuneOffset ?? 0; + runeOffset(requestStart, 'requestStartRuneOffset'); + runeOffset(responseStart, 'responseStartRuneOffset'); // Only translate camelCase → wire snake_case here; enum VALUES stay // lowercase snake as they arrive. const payload = { safety_level: config.safetyLevel, - request_start_rune_offset: config.requestStartRuneOffset ?? 0, - response_start_rune_offset: config.responseStartRuneOffset ?? 0, + request_start_rune_offset: requestStart, + response_start_rune_offset: responseStart, request_tasks: config.requestTasks ? Array.from(config.requestTasks) : [], response_tasks: config.responseTasks ? Array.from(config.responseTasks) : [], }; @@ -1006,6 +1027,7 @@ export class StreamSession { * wrong. */ observe(sourceType: StreamSourceType, runes: number): number { + runeOffset(runes, 'runes'); return native.streamSessionObserve(this.handle, sourceType, runes); } @@ -1039,6 +1061,8 @@ export class StreamSession { end: number, outcome: SegmentOutcome, ): void { + runeOffset(start, 'start'); + runeOffset(end, 'end'); native.streamSessionRecordOutcome(this.handle, task, sourceType, start, end, outcome); } @@ -1059,6 +1083,8 @@ export class StreamSession { end: number, verdict: Verdict, ): void { + runeOffset(start, 'start'); + runeOffset(end, 'end'); native.streamSessionRecordVerdict( this.handle, task, diff --git a/sdk/node/test/stream-session.test.mjs b/sdk/node/test/stream-session.test.mjs index 0e43d2b..b95bd93 100644 --- a/sdk/node/test/stream-session.test.mjs +++ b/sdk/node/test/stream-session.test.mjs @@ -139,7 +139,7 @@ test("observeText counts runes, not UTF-16 code units", () => { assert.equal(s2.finish().isClean, true); }); -test("an unmediated track releases without evaluation", () => { +test("payload on an unmediated track fails closed", () => { // Empty response task set: the response track is not mediated at // all, so text on it fails closed while the request track releases // as usual. This is the ordinary shape for a host guarding only the @@ -293,3 +293,110 @@ test("finish twice returns the same completion", () => { const second = s.finish(); assert.deepEqual(first, second); }); + +// --------------------------------------------------------------------- +// Rune offsets are `u32` on the wire. N-API converts a JS Number to +// `u32` with ToUint32, which wraps rather than fails: `2 ** 32` +// arrives as `0` and `2 ** 32 + 5` as `5`, so an end offset chosen +// past the boundary would record a *cleared* prefix on text no task +// evaluated. Python raises OverflowError and .NET throws +// OverflowException on the same input, so a Node host that fed a +// deliberately-huge offset would silently emit content the other +// languages refused. The wrapper is the one place a guard fits before +// the value reaches napi's converter; pin that on every rune-offset +// surface so a future refactor cannot re-open it. +// --------------------------------------------------------------------- + +test("observe refuses a rune offset at or past the u32 boundary", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + assert.throws( + () => s.observe("model_generated", 2 ** 32), + RangeError, + "2 ** 32 must be refused, not silently wrapped to 0", + ); + // Session state must be untouched: the throw happened before + // reaching the native call. + assert.equal(s.watermark("response").received, 0); +}); + +test("observe refuses a negative rune offset", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + assert.throws( + () => s.observe("model_generated", -1), + RangeError, + "-1 must be refused, not silently wrapped to 0xFFFFFFFF", + ); + assert.equal(s.watermark("response").received, 0); +}); + +test("recordOutcome refuses an end offset past the u32 boundary", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + s.observeText("model_generated", "hello"); + assert.throws( + () => s.recordOutcome("pii", "model_generated", 0, 2 ** 32 + 5, "cleared"), + RangeError, + "2 ** 32 + 5 must be refused, not silently wrapped to 5 which would clear text no task evaluated", + ); + // Nothing cleared, because the guard fired before the native call. + assert.equal(s.safeOffset("response"), 0); +}); + +test("recordOutcome refuses a start offset at the u32 boundary too", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + assert.throws( + () => s.recordOutcome("pii", "model_generated", 2 ** 32, 5, "cleared"), + RangeError, + ); +}); + +test("recordVerdict refuses rune offsets past the u32 boundary", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + s.observeText("model_generated", "hello"); + const allow = { decision: "allow" }; + assert.throws( + () => s.recordVerdict("safety", "model_generated", 0, 2 ** 32 + 5, allow), + RangeError, + ); + assert.throws( + () => s.recordVerdict("safety", "model_generated", -1, 5, allow), + RangeError, + ); + // The verdict path is another way to enter the same accounting, so + // the guard must catch it symmetrically; a bad recordVerdict must + // not clear the span either. + assert.equal(s.safeOffset("response"), 0); +}); + +test("StreamSession refuses a start rune offset in config past the u32 boundary", () => { + assert.throws( + () => new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + responseStartRuneOffset: 2 ** 32, + }), + RangeError, + ); + assert.throws( + () => new StreamSession({ + safetyLevel: "blocking", + requestTasks: ["moderation"], + requestStartRuneOffset: -1, + }), + RangeError, + ); +}); diff --git a/sdk/python/agent_control_spec/__init__.py b/sdk/python/agent_control_spec/__init__.py index df82ee6..4c31d65 100644 --- a/sdk/python/agent_control_spec/__init__.py +++ b/sdk/python/agent_control_spec/__init__.py @@ -16,7 +16,7 @@ import json from collections.abc import Callable, Iterable, Mapping from types import MappingProxyType as _MappingProxyType -from typing import Any +from typing import Any, Self from agent_hooks import Verdict @@ -102,13 +102,16 @@ #: One entry produced by :func:`validate_manifest_detailed`. Shape: #: -#: - ``reason_code``: ``str`` (the engine's ``runtime_error:*`` name). +#: - ``code``: ``str`` (the engine's ``runtime_error:*`` reason). #: - ``message``: ``str`` (the engine's own message text). -#: - ``field``: ``str | None`` (best-effort field name extracted from the -#: message; ``None`` when the message names no known field). +#: - ``severity``: always ``"error"``. +#: - ``field``: ``str | None`` (best-effort field name extracted from +#: the message; ``None`` when the message names no known field). #: -#: Wrapped as a mapping rather than a dataclass to keep it JSON-safe for -#: tools that shuttle diagnostics through IPC. +#: Matches :data:`ArtifactDiagnostic` on ``code``, ``message``, and +#: ``severity``, so a diagnostic-consuming tool can key off ``code`` +#: across surfaces. Wrapped as a mapping rather than a dataclass to +#: keep it JSON-safe for tools that shuttle diagnostics through IPC. ValidationDiagnostic = Mapping[str, Any] #: One entry produced by :func:`validate_artifacts`. Shape: @@ -394,10 +397,12 @@ def validate_manifest_file(path: str) -> None: def validate_manifest_detailed(source: str) -> list[ValidationDiagnostic]: """Return structured validation diagnostics for a manifest source. - Each diagnostic is ``{"reason_code": str, "message": str, "field": - str | None}``. An accepted manifest returns ``[]``. A rejected one - returns one entry naming the failed field where the engine's message - permits extraction, and ``None`` for ``field`` when it does not: the + Each diagnostic is ``{"code": str, "message": str, "severity": + "error", "field": str | None}`` — the same wire shape + :func:`validate_artifacts` and every other binding return. An + accepted manifest returns ``[]``. A rejected one returns one entry + naming the failed field where the engine's message permits + extraction, and ``None`` for ``field`` when it does not: the ``message`` is the engine's own text either way, so a tool that cannot map ``field`` back to a location still has the verbatim reason. @@ -644,6 +649,20 @@ def end_of_payloads(self) -> None: """ _native.stream_end_of_payloads(self._handle) + def __enter__(self) -> Self: + return self + + def __exit__(self, *_exc: object) -> None: + """Settle on the way out. + + A host owes an outcome for every session it opens, including one + it abandons, and nothing can make that automatic. A context + manager makes it the shape of least resistance instead. Settling + twice returns the same completion, so an explicit ``finish`` + inside the block stays correct. + """ + self.finish() + def finish(self) -> dict[str, Any]: """Settle the session and return the terminal record: ``{"reason": , "transformed": bool, "is_clean": diff --git a/sdk/python/src/lib.rs b/sdk/python/src/lib.rs index 7258c3e..da643cb 100644 --- a/sdk/python/src/lib.rs +++ b/sdk/python/src/lib.rs @@ -13,17 +13,17 @@ use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolic use agent_control_spec::policy::PreparedPolicyInvocation; use agent_control_spec::runtime::PolicyDispatcher; use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; +use agent_control_spec::wire; use agent_control_spec::{ - ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Limits, Manifest, PerfTelemetry, - Runtime, RuntimeError, SafetyLevel, SegmentOutcome, StreamEndReason, StreamError, - StreamSession, StreamSessionConfig, StreamSourceType, StreamSpan, StreamTrack, Verdict, - SUPPORTED_VERSIONS, + ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Limits, Manifest, Runtime, + RuntimeError, SafetyLevel, SegmentOutcome, StreamError, StreamSession, StreamSessionConfig, + StreamSourceType, StreamSpan, StreamTrack, Verdict, SUPPORTED_VERSIONS, }; use pyo3::create_exception; use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyMapping, PyString}; -use serde_json::{json, Map, Value}; +use serde_json::{Map, Value}; use std::sync::{Arc, Mutex}; // --------------------------------------------------------------------- @@ -276,43 +276,6 @@ struct PyTelemetrySink { callback: Py, } -impl PyTelemetrySink { - fn event_to_py<'py>( - &self, - py: Python<'py>, - event: &TelemetryEvent, - ) -> PyResult> { - let dict = PyDict::new(py); - dict.set_item("event_type", event.event_type.as_str())?; - dict.set_item("intervention_point", event.intervention_point.as_str())?; - dict.set_item("decision", event.decision.map(|d| d.as_str().to_string()))?; - dict.set_item("reason_code", event.reason_code.clone())?; - dict.set_item("error_class", event.error_class.clone())?; - dict.set_item("policy_id", event.policy_id.clone())?; - dict.set_item("annotators", event.annotators.clone())?; - dict.set_item( - "enforcement_mode", - event.enforcement_mode.map(|m| match m { - agent_control_spec::EnforcementMode::Enforce => "enforce".to_string(), - agent_control_spec::EnforcementMode::EvaluateOnly => "evaluate_only".to_string(), - }), - )?; - dict.set_item("duration_ms", event.duration_ms)?; - dict.set_item("evidence_artefact", event.evidence_artefact.clone())?; - dict.set_item( - "evidence_verification_pointer_keys", - event.evidence_verification_pointer_keys.clone(), - )?; - dict.set_item("action_identity", event.action_identity.clone())?; - let metadata = PyDict::new(py); - for (k, v) in &event.metadata { - metadata.set_item(k, v)?; - } - dict.set_item("metadata", metadata)?; - Ok(dict.into_any()) - } -} - impl TelemetrySink for PyTelemetrySink { fn emit(&self, event: TelemetryEvent) { // `emit` returns `()` in the trait, so a Python-side raise is @@ -320,8 +283,14 @@ impl TelemetrySink for PyTelemetrySink { // telemetry is out-of-band by design and must not corrupt a // decision that already succeeded. The engine calls `emit` after // it has settled a verdict. + // + // The wire shape lives in `wire::telemetry_event_json`, so a + // sink written for another SDK sees the same fields. The + // binding only translates the JSON into plain Python + // containers. Python::attach(|py| { - let event_py = match self.event_to_py(py, &event) { + let event_json = wire::telemetry_event_json(&event); + let event_py = match json_to_py(py, &event_json) { Ok(value) => value, Err(err) => { err.write_unraisable(py, Some(self.callback.bind(py).as_any())); @@ -347,17 +316,6 @@ impl TelemetrySink for PyTelemetrySink { } } -fn parse_perf_telemetry(value: &str) -> PyResult { - match value { - "off" => Ok(PerfTelemetry::Off), - "external" => Ok(PerfTelemetry::External), - "full" => Ok(PerfTelemetry::Full), - other => Err(PyValueError::new_err(format!( - "unknown perf_telemetry level '{other}'; expected 'off', 'external', or 'full'" - ))), - } -} - fn resolve_annotator_dispatcher(dispatcher: Option>) -> Arc { match dispatcher { Some(callback) => Arc::new(PyAnnotatorDispatcher { callback }), @@ -384,34 +342,22 @@ fn resolve_telemetry_sink(sink: Option>) -> Option(py: Python<'py>) -> PyResult> { - let defaults = Limits::default(); - let d = PyDict::new(py); - d.set_item("max_snapshot_bytes", defaults.max_snapshot_bytes)?; - d.set_item("max_policy_input_depth", defaults.max_policy_input_depth)?; - d.set_item( - "max_annotators_per_point", - defaults.max_annotators_per_point, - )?; - d.set_item( - "max_annotator_output_bytes", - defaults.max_annotator_output_bytes, - )?; - d.set_item("max_policy_output_bytes", defaults.max_policy_output_bytes)?; - d.set_item("max_extends_depth", defaults.max_extends_depth)?; - d.set_item( - "max_merged_manifest_bytes", - defaults.max_merged_manifest_bytes, - )?; - d.set_item("max_manifest_url_bytes", defaults.max_manifest_url_bytes)?; - d.set_item("manifest_url_timeout_ms", defaults.manifest_url_timeout_ms)?; - d.set_item( - "max_manifest_url_redirects", - defaults.max_manifest_url_redirects, - )?; - Ok(d) + let rendered = wire::limits_json(&Limits::default()); + let dict = PyDict::new(py); + // `wire::limits_json` returns a JSON object with every documented + // field. Iterate it into a PyDict so the surface stays in one place. + let Value::Object(fields) = rendered else { + return Err(PyRuntimeError::new_err( + "wire::limits_json returned a non-object value", + )); + }; + for (key, value) in fields { + dict.set_item(key, json_to_py(py, &value)?)?; + } + Ok(dict) } -/// Read a limits override, mirroring the FFI's `parse_limits`: +/// Read a limits override: /// /// - `None` (absent) means keep every default. /// - Each field is individually optional; an absent field keeps its own @@ -419,71 +365,57 @@ fn limits_defaults_map<'py>(py: Python<'py>) -> PyResult> { /// - A field present but not a non-negative integer is a hard ERROR, not /// a silently-kept default. A host that asked for a smaller bound and /// got the larger one would believe it was protected when it was not. +/// - An unknown/misspelled field is refused rather than silently +/// ignored: a cap the host believes it set but did not set is the +/// same defect as one silently widened. +/// +/// Field-by-field acceptance and the misspelled-key refusal live in the +/// core (`wire::limits_from_json`); this function's only job is to turn +/// a Python mapping into `serde_json::Value` and translate the error +/// back into `PyErr`. fn resolve_limits(limits: Option>) -> PyResult { - let mut out = Limits::default(); let Some(value) = limits else { - return Ok(out); + return Ok(Limits::default()); }; - Python::attach(|py| -> PyResult<()> { + Python::attach(|py| -> PyResult { let bound = value.bind(py); if bound.is_none() { - return Ok(()); + return Ok(Limits::default()); } // Accept any Mapping (dict, MappingProxyType, custom mapping). // Reject non-mappings loudly rather than silently ignoring the - // caller's intent. + // caller's intent; that is a TypeError, not a manifest verdict. let mapping: Bound<'_, PyMapping> = bound .cast::() .map_err(|_| PyTypeError::new_err("limits must be a Mapping[str, int] or None"))? .clone(); - let read = |key: &str| -> PyResult> { - if !mapping.contains(key)? { - return Ok(None); - } - let raw = mapping.get_item(key)?; - if raw.is_none() { - return Ok(None); - } - // Booleans are ints in Python. Refuse them: a host that meant - // `True` for a cap almost certainly meant something else, and - // a silently-accepted `1` masks a bug. - if raw.cast::().is_ok() { - return Err(PyValueError::new_err(format!( - "limits[{key:?}] must be a non-negative integer" - ))); - } - let extracted: Result = raw.extract(); - match extracted { - Ok(n) => Ok(Some(n)), - Err(_) => Err(PyValueError::new_err(format!( - "limits[{key:?}] must be a non-negative integer" - ))), - } - }; - - macro_rules! apply { - ($field:ident, $ty:ty) => { - if let Some(v) = read(stringify!($field))? { - out.$field = v as $ty; - } - }; + // Convert the mapping into a serde_json Object and hand it to + // the core. Only string keys are admitted here (limits fields + // are named strings); any other key type is a boundary error. + let mut object = Map::new(); + let items = mapping.items()?; + for entry in items.iter() { + let pair: Bound<'_, PyAny> = entry; + let key_any = pair.get_item(0)?; + let val_any = pair.get_item(1)?; + let key_str = key_any + .cast::() + .map_err(|_| PyTypeError::new_err("limits key must be a string identifying a cap"))? + .to_string_lossy() + .into_owned(); + // `py_to_json` refuses non-JSON-shaped values and is what + // dispatchers already round-trip through, so a `True` + // becomes `Value::Bool(true)`, which the core then refuses + // as non-integer. + let val_json = py_to_json(&val_any)?; + object.insert(key_str, val_json); } - apply!(max_snapshot_bytes, usize); - apply!(max_policy_input_depth, usize); - apply!(max_annotators_per_point, usize); - apply!(max_annotator_output_bytes, usize); - apply!(max_policy_output_bytes, usize); - apply!(max_extends_depth, usize); - apply!(max_merged_manifest_bytes, usize); - apply!(max_manifest_url_bytes, usize); - apply!(manifest_url_timeout_ms, u64); - apply!(max_manifest_url_redirects, usize); - Ok(()) - })?; - Ok(out) + wire::limits_from_json(&Value::Object(object)) + .map_err(|e| PyValueError::new_err(format!("{e}"))) + }) } #[pyclass(frozen)] @@ -519,7 +451,8 @@ fn interceptor_new( Manifest::from_path(manifest_path).map_err(|e| PyValueError::new_err(format!("{e}")))?; let annotations = resolve_annotator_dispatcher(annotator_dispatcher); let policy = resolve_policy_dispatcher(policy_dispatcher); - let perf = parse_perf_telemetry(perf_telemetry)?; + let perf = wire::parse_perf_telemetry(perf_telemetry) + .map_err(|e| PyValueError::new_err(format!("{e}")))?; let telemetry = resolve_telemetry_sink(telemetry_sink); let limits = resolve_limits(limits)?; let telemetry_arc: Arc = @@ -652,44 +585,11 @@ fn merge_manifests(sources: Vec) -> PyResult { .map_err(|e| PyRuntimeError::new_err(format!("merged manifest serialization failed: {e}"))) } -/// Field names an authoring tool wants surfaced verbatim. The engine's -/// validation error messages contain them, so a heuristic search picks -/// the first occurrence and returns it as the diagnostic's `field`. -const DIAGNOSTIC_FIELDS: &[&str] = &[ - "agent_control_specification_version", - "policy_target_kind", - "policy_target", - "tool_name_from", - "annotations", - "annotators", - "intervention_points", - "intervention point", - "extends", - "policies", - "policy.id", - "approval", - "metadata", - "tools", -]; - -fn guess_field(message: &str) -> Option { - // Longest match first: `policy_target_kind` is a superstring of - // `policy_target`, so ordering `DIAGNOSTIC_FIELDS` by length keeps - // `policy_target` from swallowing a `policy_target_kind` message. - let mut ordered: Vec<&&str> = DIAGNOSTIC_FIELDS.iter().collect(); - ordered.sort_by_key(|f| std::cmp::Reverse(f.len())); - for field in ordered { - if message.contains(*field) { - return Some((*field).to_string()); - } - } - None -} - /// Structured validation diagnostics as a JSON array string. /// -/// Returned entries have shape -/// `{"reason_code": str, "message": str, "field": str | null}`. The +/// Returned entries have the unified wire shape +/// `{"code": str, "message": str, "severity": "error", "field": str | +/// None}`, owned by `agent_control_spec::wire::diagnostic_json`. The /// engine's validation surface reports one failure at a time, so a /// successful validation returns `[]` and every failed one returns a /// single-entry list. Wrapping in a list leaves room for a batch @@ -699,45 +599,32 @@ fn guess_field(message: &str) -> Option { /// the caller at file-based validation, matching `validate_manifest`. #[pyfunction] fn validate_manifest_diagnostics(source: &str) -> PyResult { - fn diagnostic(reason: &str, message: &str) -> Value { - json!({ - "reason_code": reason, - "message": message, - "field": guess_field(message), - }) - } - let parsed = match Manifest::parse_yaml_str(source) { - Ok(manifest) => manifest, - Err(RuntimeError::ManifestInvalid(detail)) => { - return serde_json::to_string(&json!([diagnostic( - "runtime_error:manifest_invalid", - &detail - )])) - .map_err(|e| { - PyRuntimeError::new_err(format!("diagnostics serialization failed: {e}")) - }); + let findings: Vec = match Manifest::parse_yaml_str(source) { + Ok(manifest) => { + if !manifest.extends.is_empty() { + let msg = "manifest extends other manifests; validation needs the merged \ + document. Use validate_manifest_file or merge_manifests, both of \ + which resolve the chain."; + vec![wire::diagnostic_json(&RuntimeError::ManifestInvalid( + msg.to_string(), + ))] + } else { + match manifest.validate() { + Ok(()) => Vec::new(), + Err(e @ RuntimeError::ManifestInvalid(_)) => vec![wire::diagnostic_json(&e)], + Err(other) => { + return Err(PyValueError::new_err(format!("{other}"))); + } + } + } } + Err(e @ RuntimeError::ManifestInvalid(_)) => vec![wire::diagnostic_json(&e)], Err(other) => { return Err(PyValueError::new_err(format!("{other}"))); } }; - if !parsed.extends.is_empty() { - let msg = "manifest extends other manifests; validation needs the merged document. Use \ - validate_manifest_file or merge_manifests, both of which resolve the chain."; - return serde_json::to_string(&json!([diagnostic("runtime_error:manifest_invalid", msg)])) - .map_err(|e| { - PyRuntimeError::new_err(format!("diagnostics serialization failed: {e}")) - }); - } - match parsed.validate() { - Ok(()) => Ok("[]".to_string()), - Err(RuntimeError::ManifestInvalid(detail)) => serde_json::to_string(&json!([diagnostic( - "runtime_error:manifest_invalid", - &detail - )])) - .map_err(|e| PyRuntimeError::new_err(format!("diagnostics serialization failed: {e}"))), - Err(other) => Err(PyValueError::new_err(format!("{other}"))), - } + serde_json::to_string(&findings) + .map_err(|e| PyRuntimeError::new_err(format!("diagnostics serialization failed: {e}"))) } /// Structured artifact diagnostics as a JSON array string. @@ -761,13 +648,6 @@ fn validate_manifest_diagnostics(source: &str) -> PyResult { /// `validate_manifest_diagnostics` returns. #[pyfunction] fn validate_artifacts_diagnostics(manifest_yaml: &str, bundles_json: &str) -> PyResult { - fn diagnostic(error: &RuntimeError) -> Value { - json!({ - "code": error.reason(), - "message": error.detail(), - "severity": "error", - }) - } let bundles: std::collections::BTreeMap = if bundles_json.trim().is_empty() { std::collections::BTreeMap::new() @@ -778,14 +658,15 @@ fn validate_artifacts_diagnostics(manifest_yaml: &str, bundles_json: &str) -> Py // Mirror the C ABI's ordering exactly: parse first, validate // second, activate third. Each step's failure short-circuits so a // manifest that does not parse is never reported as an activation - // failure. + // failure. The diagnostic shape is owned by the core so every + // binding renders artifact findings the same way. let findings = match Manifest::from_yaml_str(manifest_yaml) { - Err(e) => vec![diagnostic(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], Ok(manifest) => match manifest.validate() { - Err(e) => vec![diagnostic(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], Ok(()) => match ActivatedPolicy::activate_from_memory(manifest_yaml, bundles) { Ok(_) => Vec::new(), - Err(e) => vec![diagnostic(&e)], + Err(e) => vec![wire::diagnostic_json(&e)], }, }, }; @@ -980,61 +861,18 @@ fn locked(guard: std::sync::LockResult) -> PyResult { guard.map_err(|_| PyRuntimeError::new_err("streaming session mutex was poisoned")) } -fn parse_track(value: &str) -> PyResult { - match value { - "request" => Ok(StreamTrack::Request), - "response" => Ok(StreamTrack::Response), - other => Err(PyValueError::new_err(format!( - "unknown stream track '{other}'" - ))), - } -} - -fn parse_outcome(value: &str) -> PyResult { - match value { - "cleared" => Ok(SegmentOutcome::Cleared), - "transformed" => Ok(SegmentOutcome::Transformed), - "denied" => Ok(SegmentOutcome::Denied), - other => Err(PyValueError::new_err(format!( - "unknown segment outcome '{other}'" - ))), - } -} - // A `StreamError` from the engine is always a boundary rejection: the // host handed a value the contract does not admit, or asked for an -// operation the session cannot honor. `ValueError` is the same mapping -// the parse-side errors above receive, so the boundary reports one -// exception class regardless of which check caught it. +// operation the session cannot honor. `ValueError` is the mapping the +// binding uses, so the boundary reports one exception class regardless +// of which check caught it. This is the sole streaming adapter kept +// here: like the FFI's `wire_track` / `wire_outcome`, it translates the +// error type across the boundary rather than reimplementing what a +// wire value means. fn stream_err(error: StreamError) -> PyErr { PyValueError::new_err(format!("{error}")) } -fn end_reason_to_json(reason: &StreamEndReason) -> Value { - match reason { - StreamEndReason::Complete => json!({ "kind": "complete" }), - StreamEndReason::Denied { track, task, range } => json!({ - "kind": "denied", - "track": track.as_str(), - "task": task, - "start": range.start, - "end": range.end, - }), - StreamEndReason::Rewritten { track, task, range } => json!({ - "kind": "rewritten", - "track": track.as_str(), - "task": task, - "start": range.start, - "end": range.end, - }), - StreamEndReason::Failed(error) => json!({ - "kind": "failed", - "reason": error.reason(), - "message": format!("{error}"), - }), - } -} - fn json_string(value: &Value) -> PyResult { serde_json::to_string(value) .map_err(|e| PyRuntimeError::new_err(format!("stream JSON serialization failed: {e}"))) @@ -1106,7 +944,7 @@ fn stream_record_outcome( outcome: &str, ) -> PyResult<()> { let source_type = StreamSourceType::parse(source_type).map_err(stream_err)?; - let outcome = parse_outcome(outcome)?; + let outcome = SegmentOutcome::parse(outcome).map_err(stream_err)?; let span = StreamSpan::new(source_type, start, end).map_err(stream_err)?; let mut session = locked(handle.inner.lock())?; session @@ -1141,7 +979,7 @@ fn stream_record_verdict( /// offset when it advanced. #[pyfunction] fn stream_advance(handle: &StreamSessionHandle, track: &str) -> PyResult> { - let track = parse_track(track)?; + let track = StreamTrack::parse(track).map_err(stream_err)?; let mut session = locked(handle.inner.lock())?; Ok(session.advance(track)) } @@ -1150,7 +988,7 @@ fn stream_advance(handle: &StreamSessionHandle, track: &str) -> PyResult PyResult> { - let track = parse_track(track)?; + let track = StreamTrack::parse(track).map_err(stream_err)?; let session = locked(handle.inner.lock())?; Ok(session.safe_offset(track)) } @@ -1158,7 +996,7 @@ fn stream_safe_offset(handle: &StreamSessionHandle, track: &str) -> PyResult PyResult { - let track = parse_track(track)?; + let track = StreamTrack::parse(track).map_err(stream_err)?; let session = locked(handle.inner.lock())?; Ok(session.pending(track)) } @@ -1166,18 +1004,10 @@ fn stream_pending(handle: &StreamSessionHandle, track: &str) -> PyResult { /// Watermark snapshot for one track, as wire JSON. #[pyfunction] fn stream_watermark(handle: &StreamSessionHandle, track: &str) -> PyResult { - let track_kind = parse_track(track)?; + let track_kind = StreamTrack::parse(track).map_err(stream_err)?; let session = locked(handle.inner.lock())?; let watermark = session.watermark(track_kind); - let tasks: Vec<&str> = watermark.tasks().collect(); - let value = json!({ - "track": track_kind.as_str(), - "confirmed": watermark.confirmed(), - "received": watermark.received(), - "pending": watermark.pending(), - "tasks": tasks, - }); - json_string(&value) + json_string(&wire::watermark_json(track_kind, watermark)) } /// Stop accepting payloads while outcomes are still in flight. A @@ -1196,12 +1026,7 @@ fn stream_end_of_payloads(handle: &StreamSessionHandle) -> PyResult<()> { fn stream_finish(handle: &StreamSessionHandle) -> PyResult { let mut session = locked(handle.inner.lock())?; let completion = session.finish(); - let value = json!({ - "reason": end_reason_to_json(&completion.reason), - "transformed": completion.transformed, - "is_clean": completion.reason.is_clean(), - }); - json_string(&value) + json_string(&wire::completion_json(&completion)) } /// Whether the session has reached its terminal state. @@ -1225,7 +1050,7 @@ fn stream_transformed(handle: &StreamSessionHandle) -> PyResult { fn stream_end_reason(handle: &StreamSessionHandle) -> PyResult> { let session = locked(handle.inner.lock())?; match session.end_reason() { - Some(reason) => json_string(&end_reason_to_json(reason)).map(Some), + Some(reason) => json_string(&wire::end_reason_json(reason)).map(Some), None => Ok(None), } } @@ -1235,14 +1060,7 @@ fn stream_end_reason(handle: &StreamSessionHandle) -> PyResult> { fn stream_config(handle: &StreamSessionHandle) -> PyResult { let session = locked(handle.inner.lock())?; let config = session.config(); - let value = json!({ - "safety_level": config.safety_level.as_str(), - "request_start_rune_offset": config.request_start_rune_offset, - "response_start_rune_offset": config.response_start_rune_offset, - "request_tasks": config.request_tasks, - "response_tasks": config.response_tasks, - }); - json_string(&value) + json_string(&wire::stream_config_json(config)) } /// The engine's default resource caps as a `dict[str, int]`. A host diff --git a/sdk/python/tests/test_host_hooks.py b/sdk/python/tests/test_host_hooks.py index fb6cb6d..d966026 100644 --- a/sdk/python/tests/test_host_hooks.py +++ b/sdk/python/tests/test_host_hooks.py @@ -369,7 +369,8 @@ def test_diagnostics_name_the_offending_field_for_an_invalid_manifest(): diagnostics = validate_manifest_detailed(bad) assert diagnostics, "expected the manifest to be rejected" entry = diagnostics[0] - assert entry["reason_code"] == "runtime_error:manifest_invalid" + assert entry["code"] == "runtime_error:manifest_invalid" + assert entry["severity"] == "error" assert entry["field"] == "policy_target_kind", entry # The engine's full message is preserved verbatim, so a tool can # surface it in an editor without paraphrasing. @@ -381,7 +382,8 @@ def test_diagnostics_report_unsupported_version_field(): diagnostics = validate_manifest_detailed(bad) assert len(diagnostics) == 1 entry = diagnostics[0] - assert entry["reason_code"] == "runtime_error:manifest_invalid" + assert entry["code"] == "runtime_error:manifest_invalid" + assert entry["severity"] == "error" assert entry["field"] == "agent_control_specification_version" assert "0.3.1-beta" in entry["message"] @@ -394,7 +396,8 @@ def test_diagnostics_flag_manifests_that_use_extends(): extended = VALID_MANIFEST + "\nextends:\n - path: ./parent.yaml\n" diagnostics = validate_manifest_detailed(extended) assert diagnostics - assert diagnostics[0]["reason_code"] == "runtime_error:manifest_invalid" + assert diagnostics[0]["code"] == "runtime_error:manifest_invalid" + assert diagnostics[0]["severity"] == "error" assert "extends" in diagnostics[0]["message"] @@ -476,10 +479,11 @@ def test_validate_artifacts_reports_unparseable_manifest_as_manifest_problem(): assert entry["code"] == "runtime_error:manifest_invalid" assert entry["severity"] == "error" # And the underlying error matches what the manifest-only - # validator reports: same problem, different shape. + # validator reports: same problem, same shape now. manifest_only = validate_manifest_detailed("::not: [valid") - assert manifest_only[0]["reason_code"] == entry["code"] + assert manifest_only[0]["code"] == entry["code"] assert manifest_only[0]["message"] == entry["message"] + assert manifest_only[0]["severity"] == entry["severity"] def test_validate_artifacts_without_bundles_equals_manifest_only_result(): @@ -487,14 +491,14 @@ def test_validate_artifacts_without_bundles_equals_manifest_only_result(): # load) or fails the same way manifest validation does. Either # way, the artifact validator must not invent activation errors # when the manifest half is what actually reports the problem. - # For an unparseable document, both surfaces report the same - # underlying RuntimeError message; only the wire keys differ. + # Both surfaces now return the same unified diagnostic shape. broken = "::not: [valid" artifact_findings = validate_artifacts(broken) manifest_findings = validate_manifest_detailed(broken) assert len(artifact_findings) == len(manifest_findings) == 1 - assert artifact_findings[0]["code"] == manifest_findings[0]["reason_code"] + assert artifact_findings[0]["code"] == manifest_findings[0]["code"] assert artifact_findings[0]["message"] == manifest_findings[0]["message"] + assert artifact_findings[0]["severity"] == manifest_findings[0]["severity"] # And for a grammatically invalid document — one that parses but # fails validation — the two surfaces report the same underlying @@ -508,8 +512,9 @@ def test_validate_artifacts_without_bundles_equals_manifest_only_result(): artifact_findings = validate_artifacts(invalid) manifest_findings = validate_manifest_detailed(invalid) assert len(artifact_findings) == len(manifest_findings) == 1 - assert artifact_findings[0]["code"] == manifest_findings[0]["reason_code"] + assert artifact_findings[0]["code"] == manifest_findings[0]["code"] assert artifact_findings[0]["message"] == manifest_findings[0]["message"] + assert artifact_findings[0]["severity"] == manifest_findings[0]["severity"] def test_validate_artifacts_accepts_none_for_bundles(): diff --git a/sdk/python/tests/test_stream_session.py b/sdk/python/tests/test_stream_session.py index 27e64bc..8491e41 100644 --- a/sdk/python/tests/test_stream_session.py +++ b/sdk/python/tests/test_stream_session.py @@ -339,3 +339,77 @@ def test_record_verdict_accepts_a_wire_dict_from_a_serialized_verdict(): assert session.advance("response") == 5 completion = session.finish() assert completion["is_clean"] is True + + +# --------------------------------------------------------------------- +# Rune offsets are `u32` on the wire. pyo3's automatic `u32` conversion +# raises `OverflowError` on any Python integer outside `[0, 2**32)`, +# so a bindings-level guard is not needed here — Node's is a workaround +# for N-API's silent ToUint32 wrap. Pin the guarantee anyway: a future +# change to accept `i64` and cast later would silently re-open the same +# hole the Node wrapper is now guarding against, and this test would +# fail loudly instead. +# --------------------------------------------------------------------- + + +def test_observe_refuses_a_rune_offset_at_or_past_the_u32_boundary(): + session = StreamSession(response_tasks=["pii"]) + with pytest.raises(OverflowError): + session.observe("model_generated", 2**32) + # Session state must be untouched: the raise happened before any + # native accounting ran. + assert session.watermark("response")["received"] == 0 + + +def test_observe_refuses_a_negative_rune_offset(): + session = StreamSession(response_tasks=["pii"]) + with pytest.raises(OverflowError): + session.observe("model_generated", -1) + assert session.watermark("response")["received"] == 0 + + +def test_record_outcome_refuses_an_end_offset_past_the_u32_boundary(): + # The exact failure mode the Node guard exists to prevent: a wrap + # to a small value would mark a *cleared* prefix on text no task + # evaluated. Python raises before the native call, so nothing + # clears. + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "hello") + with pytest.raises(OverflowError): + session.record_outcome("pii", "model_generated", 0, 2**32 + 5, "cleared") + assert session.safe_offset("response") == 0 + + +def test_record_outcome_refuses_a_start_offset_at_the_u32_boundary(): + session = StreamSession(response_tasks=["pii"]) + with pytest.raises(OverflowError): + session.record_outcome("pii", "model_generated", 2**32, 5, "cleared") + + +def test_record_verdict_refuses_rune_offsets_past_the_u32_boundary(): + # The verdict path enters the same accounting as record_outcome, + # so the guarantee must be symmetric. + session = StreamSession(response_tasks=["safety"]) + session.observe_text("model_generated", "hello") + with pytest.raises(OverflowError): + session.record_verdict( + "safety", "model_generated", 0, 2**32 + 5, {"decision": "allow"} + ) + with pytest.raises(OverflowError): + session.record_verdict( + "safety", "model_generated", -1, 5, {"decision": "allow"} + ) + assert session.safe_offset("response") == 0 + + +def test_stream_session_refuses_a_start_rune_offset_in_config_past_the_u32_boundary(): + with pytest.raises(OverflowError): + StreamSession( + response_tasks=["pii"], + response_start_rune_offset=2**32, + ) + with pytest.raises(OverflowError): + StreamSession( + request_tasks=["moderation"], + request_start_rune_offset=-1, + ) diff --git a/tests/conformance/bindings/cross_language_parity.py b/tests/conformance/bindings/cross_language_parity.py index 3fd3d30..1fd8b6b 100755 --- a/tests/conformance/bindings/cross_language_parity.py +++ b/tests/conformance/bindings/cross_language_parity.py @@ -114,6 +114,17 @@ "parsed_has_points": True, "diagnostics_on_bad": 1, "diagnostics_on_good": 0, + # The shape, not only the count. Asserting the count alone let three + # bindings return three different diagnostic shapes for one call. + "diagnostic_keys": ["code", "field", "message", "severity"], + "diagnostic_code": "runtime_error:manifest_invalid", + "diagnostic_field": "intervention point", + # Settlement with uncleared residue, the fail-closed core of the + # profile. Every scenario above settles clean, so without this no + # binding is pinned to refusing text nothing evaluated. + "residue_kind": "failed", + "residue_reason": "host_error:streaming_unsupported", + "residue_clean": False, # Artifact validation. The manifest is sound either way, so only # compiling the Rego tells the two bundles apart. # @@ -283,6 +294,20 @@ def _run(cmd: list[str], **kw) -> dict: Err(e) => (1, Some(e.reason().to_string())), } }; + let mut residue = StreamSession::new(StreamSessionConfig { + safety_level: SafetyLevel::Blocking, + request_start_rune_offset: 0, + response_start_rune_offset: 0, + request_tasks: vec![], + response_tasks: vec!["pii".into()], + }) + .expect("residue session"); + residue + .observe_text(StreamSourceType::ModelGenerated, "hello") + .expect("observe"); + let residue_done = residue.finish(); + let residue_json = agent_control_spec::wire::completion_json(&residue_done); + let big_ctx: serde_json::Value = serde_json::from_str(BIG_CTX).expect("big ctx"); let lim = |limits: Limits| -> (String, Option) { @@ -303,7 +328,27 @@ def _run(cmd: list[str], **kw) -> dict: let art_only = art("{}"); let art_good = art(GOOD_BUNDLES); let art_bad = art(BAD_BUNDLES); - let bad_diags = usize::from(Manifest::from_yaml_str(BAD_MANIFEST).is_err()); + let bad_diag_value = match Manifest::from_yaml_str(BAD_MANIFEST) { + Err(e) => Some(agent_control_spec::wire::diagnostic_json(&e)), + Ok(m) => m.validate().err().map(|e| agent_control_spec::wire::diagnostic_json(&e)), + }; + let mut bad_diag_keys: Vec = bad_diag_value + .as_ref() + .and_then(|v| v.as_object()) + .map(|o| o.keys().cloned().collect()) + .unwrap_or_default(); + bad_diag_keys.sort(); + let bad_diag_code = bad_diag_value + .as_ref() + .and_then(|v| v.get("code")) + .and_then(|v| v.as_str()) + .map(str::to_string); + let bad_diag_field = bad_diag_value + .as_ref() + .and_then(|v| v.get("field")) + .and_then(|v| v.as_str()) + .map(str::to_string); + let bad_diags = usize::from(bad_diag_value.is_some()); let good_diags = usize::from( Manifest::from_yaml_str(&std::fs::read_to_string(&manifest_path).expect("read")) .and_then(|m| m.validate()) @@ -321,6 +366,9 @@ def _run(cmd: list[str], **kw) -> dict: "hook_dispatcher_calls": benign.calls.load(std::sync::atomic::Ordering::SeqCst), "parsed_has_points": parsed_json.get("intervention_points").is_some(), "diagnostics_on_bad": bad_diags, + "diagnostic_keys": bad_diag_keys, + "diagnostic_code": bad_diag_code, + "diagnostic_field": bad_diag_field, "diagnostics_on_good": good_diags, "artifacts_manifest_only": art_only.0, "artifacts_good_rego": art_good.0, @@ -329,6 +377,9 @@ def _run(cmd: list[str], **kw) -> dict: "limits_default_decision": lim_default.0, "limits_capped_decision": lim_capped.0, "limits_capped_reason": lim_capped.1, + "residue_kind": residue_json["reason"]["kind"], + "residue_reason": residue_json["reason"]["reason"], + "residue_clean": residue_json["is_clean"], "supported_versions_nonempty": !agent_control_spec::SUPPORTED_VERSIONS.is_empty(), "validate_good": validate_good, "validate_bad": validate_bad, @@ -467,6 +518,10 @@ def _hook(dispatcher): _big_ctx = {{"interception_point": "input", "input": {BIG_INPUT!r}}} _lim_default = AcsInterceptor({str(HOOKS_MANIFEST)!r}, annotator_dispatcher=_Classifier(1)).intercept(_big_ctx) +_res = StreamSession(safety_level="blocking", response_tasks=["pii"]) +_res.observe_text("model_generated", "hello") +_res_done = _res.finish() + _lim_capped = AcsInterceptor( {str(HOOKS_MANIFEST)!r}, annotator_dispatcher=_Classifier(1), limits={SMALL_CAP!r} ).intercept(_big_ctx) @@ -480,6 +535,9 @@ def _hook(dispatcher): "hook_dispatcher_calls": _benign.calls, "parsed_has_points": "intervention_points" in _parsed, "diagnostics_on_bad": len(_bad_diags), + "diagnostic_keys": sorted(_bad_diags[0]) if _bad_diags else [], + "diagnostic_code": _bad_diags[0].get("code") if _bad_diags else None, + "diagnostic_field": _bad_diags[0].get("field") if _bad_diags else None, "diagnostics_on_good": len(_good_diags), "artifacts_manifest_only": len(_art_only), "artifacts_good_rego": len(_art_good), @@ -488,6 +546,9 @@ def _hook(dispatcher): "limits_default_decision": decision(_lim_default), "limits_capped_decision": decision(_lim_capped), "limits_capped_reason": _lim_capped.reason, + "residue_kind": _res_done["reason"]["kind"], + "residue_reason": _res_done["reason"].get("reason"), + "residue_clean": _res_done["is_clean"], "supported_versions_nonempty": len(supported_manifest_versions()) > 0, "validate_good": check(open({str(MANIFEST)!r}).read()), "validate_bad": check({BAD_MANIFEST!r}), @@ -553,6 +614,10 @@ def node() -> dict: const artGood = acs.validateArtifacts({json.dumps(REGO_MANIFEST)}, {json.dumps(GOOD_BUNDLES)}); const artBad = acs.validateArtifacts({json.dumps(REGO_MANIFEST)}, {json.dumps(BAD_BUNDLES)}); +const res = new acs.StreamSession({{ safetyLevel: 'blocking', responseTasks: ['pii'] }}); +res.observeText('model_generated', 'hello'); +const resDone = res.finish(); + const bigCtx = {{ interception_point: 'input', input: {json.dumps(BIG_INPUT)} }}; const limDefault = acs.AcsInterceptor .fromPath({json.dumps(str(HOOKS_MANIFEST))}, {{ annotatorDispatcher: () => ({{ severity: 1 }}) }}) @@ -570,6 +635,9 @@ def node() -> dict: hook_dispatcher_calls: hookCalls, parsed_has_points: Object.prototype.hasOwnProperty.call(parsed, 'intervention_points'), diagnostics_on_bad: badDiags.length, + diagnostic_keys: badDiags.length ? Object.keys(badDiags[0]).sort() : [], + diagnostic_code: badDiags.length ? badDiags[0].code : null, + diagnostic_field: badDiags.length ? badDiags[0].field : null, diagnostics_on_good: goodDiags.length, artifacts_manifest_only: artOnly.length, artifacts_good_rego: artGood.length, @@ -578,6 +646,9 @@ def node() -> dict: limits_default_decision: String(limDefault.decision).toLowerCase(), limits_capped_decision: String(limCapped.decision).toLowerCase(), limits_capped_reason: limCapped.reason ?? null, + residue_kind: resDone.reason.kind, + residue_reason: resDone.reason.reason ?? null, + residue_clean: resDone.isClean, supported_versions_nonempty: acs.supportedManifestVersions().length > 0, validate_good: check(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')), validate_bad: check({json.dumps(BAD_MANIFEST)}), @@ -649,6 +720,10 @@ def node() -> dict: var artGood = AcsManifestTools.ValidateArtifacts(REGO_MANIFEST, GOOD_BUNDLES); var artBad = AcsManifestTools.ValidateArtifacts(REGO_MANIFEST, BAD_BUNDLES); +using var residue = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); +residue.ObserveText(StreamSourceType.ModelGenerated, "hello"); +var residueDone = residue.Finish(); + var bigCtx = new AgentContext(JsonNode.Parse(BIG_CTX)!.AsObject()); using var limDefault = AcsHostInterceptor.FromPath(HOOKS_MANIFEST, annotator: (_, _, _) => SEV1); using var limCapped = AcsHostInterceptor.FromPath( @@ -666,6 +741,12 @@ def node() -> dict: ["hook_dispatcher_calls"] = hookCalls, ["parsed_has_points"] = parsed.Contains("intervention_points"), ["diagnostics_on_bad"] = badDiags.Count, + ["diagnostic_keys"] = badDiags.Count > 0 + ? JsonSerializer.Deserialize>( + JsonSerializer.Serialize(badDiags[0]))!.Keys.OrderBy(k => k, StringComparer.Ordinal).ToList() + : new List(), + ["diagnostic_code"] = badDiags.Count > 0 ? badDiags[0].Code : null, + ["diagnostic_field"] = badDiags.Count > 0 ? badDiags[0].Field : null, ["diagnostics_on_good"] = goodDiags.Count, ["artifacts_manifest_only"] = artOnly.Count, ["artifacts_good_rego"] = artGood.Count, @@ -674,6 +755,9 @@ def node() -> dict: ["limits_default_decision"] = limDefaultVerdict.Decision.ToString().ToLowerInvariant(), ["limits_capped_decision"] = limCappedVerdict.Decision.ToString().ToLowerInvariant(), ["limits_capped_reason"] = limCappedVerdict.Reason, + ["residue_kind"] = residueDone.Reason.Kind, + ["residue_reason"] = residueDone.Reason.Reason, + ["residue_clean"] = residueDone.IsClean, ["supported_versions_nonempty"] = AcsManifest.SupportedVersions().Count > 0, ["validate_good"] = Check(() => AcsManifest.Validate(File.ReadAllText(manifest))), ["validate_bad"] = Check(() => AcsManifest.Validate(BAD_JSON)), diff --git a/tests/conformance/bindings/language_coverage.py b/tests/conformance/bindings/language_coverage.py index faf98d9..44e2eb7 100755 --- a/tests/conformance/bindings/language_coverage.py +++ b/tests/conformance/bindings/language_coverage.py @@ -214,7 +214,6 @@ ) - def ffi_entry_points() -> set[str]: """Every `acs_*` function the C ABI exports.""" src = (ROOT / "sdk/ffi/src/lib.rs").read_text(encoding="utf-8") @@ -243,8 +242,7 @@ def unbound_from_dotnet() -> list[str]: return sorted( name for name in ffi_entry_points() - if name not in internal - and not re.search(rf"\b{re.escape(name)}\b", declared) + if name not in internal and not re.search(rf"\b{re.escape(name)}\b", declared) ) @@ -318,7 +316,9 @@ def main() -> int: print( f"{len(NOT_A_CAPABILITY)} further symbols are data or internals, each with a stated reason" ) - print(f"all {len(ffi_entry_points())} C ABI entry points are declared by the .NET binding") + print( + f"all {len(ffi_entry_points())} C ABI entry points are declared by the .NET binding" + ) return 0 diff --git a/tests/conformance/bindings/published_artifacts.py b/tests/conformance/bindings/published_artifacts.py index 067a643..37acabb 100755 --- a/tests/conformance/bindings/published_artifacts.py +++ b/tests/conformance/bindings/published_artifacts.py @@ -41,7 +41,8 @@ # package from an earlier run satisfy a later one and report a pass for # code the artifact does not contain. Stamp each run instead. CHECK_VERSION = f"0.0.0-artifactcheck{os.getpid()}{int(time.time())}" -HOOKS_MANIFEST = ROOT / "tests" / "conformance" / "parity" / "host-hooks-manifest.yaml" +# Beside this file, so moving the suite cannot strand the fixture. +HOOKS_MANIFEST = Path(__file__).resolve().parent / "host-hooks-manifest.yaml" # Built rather than written on one line. A repo guard scans committed # files for the version key and validates whatever follows it, and it # cannot strip the quotes of a single-line literal.