diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f50519..2c93f69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,6 +85,12 @@ jobs: ./actionlint -color - name: Version surfaces agree run: python3 scripts/check-version-consistency.py + # The parity harness proves the bindings agree about the calls it + # makes. It cannot notice a capability none of them expose, because + # a surface absent everywhere is consistent everywhere. That blind + # spot is how streaming shipped reachable only from Rust. + - name: Every engine capability is reachable from every language + run: python3 tests/conformance/bindings/run.py language_coverage - name: cargo-audit over example lockfiles (pinned, checksum-verified) # Examples are deliberately outside the Dependabot update config, # and the coding_agent app's path dependency predates the engine @@ -151,3 +157,74 @@ jobs: working-directory: sdk/dotnet env: LD_LIBRARY_PATH: ${{ github.workspace }}/target/release + # The suite above runs with the engine on the loader path, which a + # consumer installing from NuGet does not have. Pack and run the + # artifact itself so a package that omits the native library fails + # here rather than in someone else's process. + - name: Verify the packed package works without a local engine build + run: | + set -euo pipefail + mkdir -p sdk/dotnet/native/runtimes/linux-x64/native + cp target/release/libagent_control_spec_ffi.so \ + sdk/dotnet/native/runtimes/linux-x64/native/ + dotnet pack sdk/dotnet/src/AgentControlSpec -c Release -o "$PWD/ci-feed" \ + --nologo -p:AcsNativeAssetsRequired=true -p:Version=0.0.0-ci + bash tests/conformance/bindings/dotnet_package.sh "$PWD/ci-feed" 0.0.0-ci + + # Streaming reaches each language through a different binding + # mechanism, so agreement between them is not structural. Run one + # scenario in all four and diff. + streaming-parity: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: "8.0.x" + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - run: cargo build --release --locked -p agent-control-spec-ffi + - name: Build the node binding + run: | + npm ci + npm run build + working-directory: sdk/node + - name: Build and install the python wheel + run: | + pip install maturin==1.8.7 + maturin build --release -m sdk/python/Cargo.toml -o dist + pip install dist/*.whl + - name: Compare every language across the whole surface + run: python tests/conformance/bindings/run.py cross_language_parity + env: + LD_LIBRARY_PATH: ${{ github.workspace }}/target/release + + # The suites and the parity harness run against this checkout, where the + # engine is on the loader path and the packages are importable from + # source. A consumer has a crate, a wheel, a tarball and a nupkg. A + # published .NET package once passed every test and threw + # DllNotFoundException on the first call a consumer made, so build the + # real artifacts and run the surface from a clean install of each. + artifacts: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: "8.0.x" + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - run: pip install maturin==1.8.7 + - run: npm ci + working-directory: sdk/node + - run: python tests/conformance/bindings/run.py published_artifacts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 969f87a..ce5e29f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -226,7 +226,43 @@ jobs: fi working-directory: sdk/node + # The managed assembly calls the engine through agent_control_spec_ffi, + # so a package without the native library installs cleanly and then + # throws DllNotFoundException on the first call. Build one library per + # supported RID and let the pack step assemble them. + dotnet-native: + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, rid: linux-x64, lib: libagent_control_spec_ffi.so } + - { os: ubuntu-24.04-arm, target: aarch64-unknown-linux-gnu, rid: linux-arm64, lib: libagent_control_spec_ffi.so } + - { os: macos-latest, target: x86_64-apple-darwin, rid: osx-x64, lib: libagent_control_spec_ffi.dylib } + - { os: macos-latest, target: aarch64-apple-darwin, rid: osx-arm64, lib: libagent_control_spec_ffi.dylib } + - { os: windows-latest, target: x86_64-pc-windows-msvc, rid: win-x64, lib: agent_control_spec_ffi.dll } + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c # master; stable toolchain + with: + toolchain: stable + targets: ${{ matrix.target }} + - name: Build the engine binding for ${{ matrix.rid }} + shell: bash + run: cargo build --release --locked -p agent-control-spec-ffi --target ${{ matrix.target }} + - name: Stage under its runtime identifier + shell: bash + run: | + mkdir -p "sdk/dotnet/native/runtimes/${{ matrix.rid }}/native" + cp "target/${{ matrix.target }}/release/${{ matrix.lib }}" \ + "sdk/dotnet/native/runtimes/${{ matrix.rid }}/native/${{ matrix.lib }}" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dotnet-native-${{ matrix.rid }} + path: sdk/dotnet/native/runtimes/${{ matrix.rid }}/native/${{ matrix.lib }} + dotnet: + needs: dotnet-native runs-on: ubuntu-latest environment: release permissions: @@ -237,11 +273,34 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: { dotnet-version: "8.0.x" } + - name: Collect the native libraries + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + pattern: dotnet-native-* + path: sdk/dotnet/native/staging + - name: Lay them out by runtime identifier + shell: bash + run: | + set -euo pipefail + for dir in sdk/dotnet/native/staging/dotnet-native-*; do + rid="${dir##*/dotnet-native-}" + mkdir -p "sdk/dotnet/native/runtimes/$rid/native" + cp "$dir"/* "sdk/dotnet/native/runtimes/$rid/native/" + done + find sdk/dotnet/native/runtimes -type f | sort - name: Pack run: | dotnet restore --nologo - dotnet pack src/AgentControlSpec -c Release -o dist --nologo + dotnet pack src/AgentControlSpec -c Release -o dist --nologo -p:AcsNativeAssetsRequired=true working-directory: sdk/dotnet + - name: Prove the packed artifact runs without a local engine build + shell: bash + run: | + set -euo pipefail + nupkg=$(find sdk/dotnet/dist -name 'ResponsibleAI.AgentControlSpec.*.nupkg' -print -quit) + version=$(basename "$nupkg" .nupkg) + version=${version#ResponsibleAI.AgentControlSpec.} + bash tests/conformance/bindings/dotnet_package.sh "$PWD/sdk/dotnet/dist" "$version" - name: SBOM uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0 with: diff --git a/.gitignore b/.gitignore index 32e9518..2effd7f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,12 @@ __pycache__/ dist/ bin/ obj/ + +# Per-RID engine binaries staged for the .NET package (built, never committed). +/sdk/dotnet/native/ + +# napi writes these next to the committed binding.js loader when the +# platform build runs in place. The loader the package ships is +# binding.js; these are byproducts. +/sdk/node/index.js +/sdk/node/index.d.ts diff --git a/README.md b/README.md index ddd6cef..d422ee7 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,10 @@ those are host obligations defined by agent-hooks. | Path | Contents | | --- | --- | | `engine/` | Rust evaluation core (`agent-control-spec` crate): manifest, dispatchers, annotators, policy-output normalization, the `AcsInterceptor` | +| `sdk/ffi/` | C ABI over the engine (`agent-control-spec-ffi`), which the .NET binding calls | | `sdk/python/` | Python binding: `agent_control_spec` package wrapping the engine as an `agent_hooks` interceptor | +| `sdk/node/` | Node binding: `@responsibleai/agent-control-spec` | +| `sdk/dotnet/` | .NET binding: `ResponsibleAI.AgentControlSpec` | | `spec/` | The ACS specification (policy plane) and schemas | | `policy/` | Cedar and Rego policy libraries | | `fixtures/` | Evaluation fixtures | diff --git a/engine/src/lib.rs b/engine/src/lib.rs index 1008be8..609af56 100644 --- a/engine/src/lib.rs +++ b/engine/src/lib.rs @@ -38,6 +38,7 @@ pub mod runtime; pub mod stream_session; pub mod telemetry; pub mod tool_projection; +pub mod wire; // The interception contract, re-exported for consumers that want a // single dependency. diff --git a/engine/src/stream_session.rs b/engine/src/stream_session.rs index b86eca8..aaed109 100644 --- a/engine/src/stream_session.rs +++ b/engine/src/stream_session.rs @@ -204,6 +204,19 @@ pub enum StreamTrack { } impl StreamTrack { + /// Parse a track's wire name. + /// + /// Every binding needs this and none should own it. Three copies of + /// what `"response"` means are three chances to disagree, which is + /// the drift this module exists to prevent. + pub fn parse(value: &str) -> Result { + match value { + "request" => Ok(Self::Request), + "response" => Ok(Self::Response), + _ => Err(StreamError::UnknownStreamTrack(value.to_string())), + } + } + pub fn as_str(self) -> &'static str { match self { Self::Request => "request", @@ -223,6 +236,8 @@ impl StreamTrack { pub enum StreamError { UnknownSafetyLevel(String), UnknownSourceType(String), + UnknownStreamTrack(String), + UnknownSegmentOutcome(String), /// Payload arrived on a track the session does not mediate. /// /// An empty task set means that track is not mediated, which is the @@ -302,6 +317,10 @@ impl fmt::Display for StreamError { write!(f, "unknown streaming safety level {value}") } Self::UnknownSourceType(value) => write!(f, "unknown stream source type {value}"), + Self::UnknownStreamTrack(value) => write!(f, "unknown stream track {value}"), + Self::UnknownSegmentOutcome(value) => { + write!(f, "unknown segment outcome {value}") + } Self::NoTasks(track) => write!( f, "payload arrived on the unmediated {} track", @@ -421,6 +440,26 @@ pub enum SegmentOutcome { Denied, } +impl SegmentOutcome { + /// Parse an outcome's wire name. + pub fn parse(value: &str) -> Result { + match value { + "cleared" => Ok(Self::Cleared), + "transformed" => Ok(Self::Transformed), + "denied" => Ok(Self::Denied), + _ => Err(StreamError::UnknownSegmentOutcome(value.to_string())), + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Cleared => "cleared", + Self::Transformed => "transformed", + Self::Denied => "denied", + } + } +} + /// Reason a session reached its terminal state. #[derive(Debug, Clone, PartialEq, Eq)] pub enum StreamEndReason { diff --git a/engine/src/wire.rs b/engine/src/wire.rs new file mode 100644 index 0000000..02b9008 --- /dev/null +++ b/engine/src/wire.rs @@ -0,0 +1,416 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +//! The JSON shapes every binding answers with. +//! +//! None of the streaming or telemetry types derives `Serialize`, so +//! something has to decide what a watermark looks like on the wire. +//! Before this module each binding decided separately, which put three +//! copies of that decision in three crates. Three copies of what +//! `"response"` means are three chances to disagree, and a disagreement +//! here is a host releasing text no task evaluated. +//! +//! So the contract lives with the engine that defines the behaviour. +//! The bindings translate calling conventions, not meaning, and the +//! cross-language conformance suite then checks the bindings rather +//! than re-litigating the contract in each one. + +use crate::error::RuntimeError; +use crate::limits::Limits; +use crate::perf_telemetry::PerfTelemetry; +use serde_json::{json, Map, Value}; + +#[cfg(feature = "streaming")] +use crate::stream_session::{ + StreamCompletion, StreamEndReason, StreamSessionConfig, StreamWatermark, +}; + +/// Parse a perf telemetry level's wire name. +pub fn parse_perf_telemetry(value: &str) -> Result { + match value { + "off" => Ok(PerfTelemetry::Off), + "external" => Ok(PerfTelemetry::External), + "full" => Ok(PerfTelemetry::Full), + other => Err(RuntimeError::ManifestInvalid(format!( + "unknown perf telemetry level '{other}'" + ))), + } +} + +/// A perf telemetry level's wire name. +pub fn perf_telemetry_str(level: PerfTelemetry) -> &'static str { + match level { + PerfTelemetry::Off => "off", + PerfTelemetry::External => "external", + PerfTelemetry::Full => "full", + } +} + +/// Apply a JSON object of resource cap overrides onto the defaults. +/// +/// Each field is individually optional, so a host raising one cap does +/// not restate the other nine. A field present but not a non-negative +/// integer is refused rather than silently kept at its default: a host +/// that asked for a smaller bound and got the larger one would believe +/// it was protected when it was not. +pub fn limits_from_json(value: &Value) -> Result { + let Value::Object(fields) = value else { + return Err(RuntimeError::ManifestInvalid( + "limits must be a JSON object".to_string(), + )); + }; + let mut limits = Limits::default(); + + let read = |key: &str| -> Result, RuntimeError> { + match fields.get(key) { + None | Some(Value::Null) => Ok(None), + Some(found) => found.as_u64().map(Some).ok_or_else(|| { + RuntimeError::ManifestInvalid(format!("{key} must be a non negative integer")) + }), + } + }; + + macro_rules! apply { + ($field:ident, $ty:ty) => { + if let Some(found) = read(stringify!($field))? { + limits.$field = found as $ty; + } + }; + } + apply!(max_snapshot_bytes, usize); + apply!(max_policy_input_depth, usize); + apply!(max_annotators_per_point, usize); + apply!(max_annotator_output_bytes, usize); + apply!(max_policy_output_bytes, usize); + apply!(max_extends_depth, usize); + apply!(max_merged_manifest_bytes, usize); + apply!(max_manifest_url_bytes, usize); + apply!(manifest_url_timeout_ms, u64); + apply!(max_manifest_url_redirects, usize); + + let unknown: Vec<&str> = fields + .keys() + .map(String::as_str) + .filter(|key| !LIMIT_FIELDS.contains(key)) + .collect(); + if !unknown.is_empty() { + // A misspelled cap that is quietly ignored is the same defect as + // one that is quietly widened: the host believes it set a bound + // it did not set. + return Err(RuntimeError::ManifestInvalid(format!( + "unknown limit field(s): {}", + unknown.join(", ") + ))); + } + Ok(limits) +} + +/// Every field [`limits_from_json`] accepts, in declaration order. +pub const LIMIT_FIELDS: [&str; 10] = [ + "max_snapshot_bytes", + "max_policy_input_depth", + "max_annotators_per_point", + "max_annotator_output_bytes", + "max_policy_output_bytes", + "max_extends_depth", + "max_merged_manifest_bytes", + "max_manifest_url_bytes", + "manifest_url_timeout_ms", + "max_manifest_url_redirects", +]; + +/// The resource caps in force, as JSON. +pub fn limits_json(limits: &Limits) -> Value { + json!({ + "max_snapshot_bytes": limits.max_snapshot_bytes, + "max_policy_input_depth": limits.max_policy_input_depth, + "max_annotators_per_point": limits.max_annotators_per_point, + "max_annotator_output_bytes": limits.max_annotator_output_bytes, + "max_policy_output_bytes": limits.max_policy_output_bytes, + "max_extends_depth": limits.max_extends_depth, + "max_merged_manifest_bytes": limits.max_merged_manifest_bytes, + "max_manifest_url_bytes": limits.max_manifest_url_bytes, + "manifest_url_timeout_ms": limits.manifest_url_timeout_ms, + "max_manifest_url_redirects": limits.max_manifest_url_redirects, + }) +} + +/// Manifest field names an authoring tool wants surfaced verbatim. +/// +/// The engine reports validation failures as prose naming the offending +/// field, so recovering the field means finding it in the message. That +/// is a heuristic, and it belongs here rather than in each binding: a +/// heuristic implemented three times is three heuristics. +const DIAGNOSTIC_FIELDS: &[&str] = &[ + "agent_control_specification_version", + "policy_target_kind", + "policy_target", + "tool_name_from", + "annotations", + "annotators", + "intervention_points", + "intervention point", + "extends", + "policies", + "policy.id", + "approval", + "metadata", + "tools", +]; + +/// The manifest field a validation message names, when it names one. +pub fn diagnostic_field(message: &str) -> Option<&'static str> { + // Longest first, because `policy_target` is a prefix of + // `policy_target_kind` and would otherwise swallow it. + let mut ordered: Vec<&&str> = DIAGNOSTIC_FIELDS.iter().collect(); + ordered.sort_by_key(|field| std::cmp::Reverse(field.len())); + ordered + .into_iter() + .find(|field| message.contains(**field)) + .copied() +} + +/// One finding about a manifest or its artifacts. +/// +/// `RuntimeError` answers by being returned, which a linter cannot +/// render against a document. This is the same information as data. +pub fn diagnostic_json(error: &RuntimeError) -> Value { + let message = error.detail(); + json!({ + "code": error.reason(), + "message": message, + "severity": "error", + "field": diagnostic_field(message), + }) +} + +/// A list of findings. Empty means sound. +pub fn diagnostics_json(errors: &[RuntimeError]) -> Value { + Value::Array(errors.iter().map(diagnostic_json).collect()) +} + +/// Why a session ended. +#[cfg(feature = "streaming")] +pub fn end_reason_json(reason: &StreamEndReason) -> Value { + match reason { + StreamEndReason::Complete => json!({ "kind": "complete" }), + StreamEndReason::Denied { track, task, range } => json!({ + "kind": "denied", + "track": track.as_str(), + "task": task, + "start": range.start, + "end": range.end, + }), + StreamEndReason::Rewritten { track, task, range } => json!({ + "kind": "rewritten", + "track": track.as_str(), + "task": task, + "start": range.start, + "end": range.end, + }), + StreamEndReason::Failed(error) => json!({ + "kind": "failed", + "reason": error.reason(), + "message": error.to_string(), + }), + } +} + +/// How far one track got and what still owes a decision. +#[cfg(feature = "streaming")] +pub fn watermark_json(track: crate::stream_session::StreamTrack, mark: &StreamWatermark) -> Value { + json!({ + "track": track.as_str(), + "confirmed": mark.confirmed(), + "received": mark.received(), + "pending": mark.pending(), + "tasks": mark.tasks().collect::>(), + }) +} + +/// Terminal settlement of a session. +#[cfg(feature = "streaming")] +pub fn completion_json(completion: &StreamCompletion) -> Value { + json!({ + "reason": end_reason_json(&completion.reason), + "transformed": completion.transformed, + "is_clean": completion.reason.is_clean(), + }) +} + +/// The offsets and task sets a session was opened with. +#[cfg(feature = "streaming")] +pub fn stream_config_json(config: &StreamSessionConfig) -> Value { + json!({ + "safety_level": config.safety_level.as_str(), + "request_start_rune_offset": config.request_start_rune_offset, + "response_start_rune_offset": config.response_start_rune_offset, + "request_tasks": config.request_tasks, + "response_tasks": config.response_tasks, + }) +} + +/// Live state of a session: whether it ended, whether a rewrite ended +/// it, why, and the configuration in force. +#[cfg(feature = "streaming")] +pub fn stream_session_state_json(session: &crate::stream_session::StreamSession) -> Value { + json!({ + "is_ended": session.is_ended(), + "transformed": session.transformed(), + "end_reason": session.end_reason().map(end_reason_json), + "config": stream_config_json(session.config()), + }) +} + +/// Read a session configuration from JSON, defaulting absent fields. +#[cfg(feature = "streaming")] +pub fn stream_config_from_json( + value: &Value, +) -> Result { + use crate::stream_session::{SafetyLevel, StreamError}; + + let empty = Map::new(); + let fields = value.as_object().unwrap_or(&empty); + + let safety_level = SafetyLevel::parse( + fields + .get("safety_level") + .and_then(Value::as_str) + .unwrap_or("blocking"), + )?; + + let offset = |key: &str| -> Result { + match fields.get(key) { + None | Some(Value::Null) => Ok(0), + Some(found) => found + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| { + StreamError::UnknownSourceType(format!("{key} is not a rune offset")) + }), + } + }; + + let tasks = |key: &str| -> Result, StreamError> { + match fields.get(key) { + None | Some(Value::Null) => Ok(Vec::new()), + Some(Value::Array(items)) => items + .iter() + .map(|item| { + item.as_str().map(str::to_string).ok_or_else(|| { + StreamError::UnknownSourceType(format!( + "{key} holds a non string task name" + )) + }) + }) + .collect(), + Some(_) => Err(StreamError::UnknownSourceType(format!( + "{key} is not an array of task names" + ))), + } + }; + + Ok(StreamSessionConfig { + safety_level, + request_start_rune_offset: offset("request_start_rune_offset")?, + response_start_rune_offset: offset("response_start_rune_offset")?, + request_tasks: tasks("request_tasks")?, + response_tasks: tasks("response_tasks")?, + }) +} + +/// One telemetry event. +/// +/// `TelemetryEvent` does not derive `Serialize`, so a sink reached +/// through any binding sees the shape stated here. +pub fn telemetry_event_json(event: &crate::telemetry::TelemetryEvent) -> Value { + json!({ + "event_type": event.event_type.as_str(), + "intervention_point": format!("{:?}", event.intervention_point).to_lowercase(), + "decision": event.decision.map(|d| format!("{d:?}").to_lowercase()), + "reason_code": event.reason_code, + "error_class": event.error_class, + "policy_id": event.policy_id, + "annotators": event.annotators, + "enforcement_mode": event.enforcement_mode.map(|m| format!("{m:?}").to_lowercase()), + "duration_ms": event.duration_ms, + "evidence_artefact": event.evidence_artefact, + "evidence_verification_pointer_keys": event.evidence_verification_pointer_keys, + "action_identity": event.action_identity, + "metadata": event.metadata, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn absent_limit_fields_keep_their_own_defaults() { + let limits = limits_from_json(&json!({ "max_snapshot_bytes": 64 })).expect("limits"); + assert_eq!(limits.max_snapshot_bytes, 64); + assert_eq!( + limits.max_policy_input_depth, + Limits::default().max_policy_input_depth + ); + } + + #[test] + fn a_limit_that_is_not_a_count_is_refused() { + assert!(limits_from_json(&json!({ "max_snapshot_bytes": "big" })).is_err()); + assert!(limits_from_json(&json!({ "max_snapshot_bytes": -1 })).is_err()); + } + + #[test] + fn a_misspelled_limit_is_refused_rather_than_ignored() { + let error = limits_from_json(&json!({ "max_snapshot_byte": 64 })).expect_err("refused"); + assert!(format!("{error}").contains("max_snapshot_byte")); + } + + #[test] + fn every_limit_field_round_trips() { + let rendered = limits_json(&Limits::default()); + for field in LIMIT_FIELDS { + assert!(rendered.get(field).is_some(), "{field} missing"); + } + let parsed = limits_from_json(&rendered).expect("round trip"); + assert_eq!(parsed, Limits::default()); + } + + #[test] + fn a_diagnostic_names_the_offending_field() { + let error = RuntimeError::ManifestInvalid( + "at least one intervention point config is required".to_string(), + ); + let rendered = diagnostic_json(&error); + assert_eq!(rendered["field"], "intervention point"); + assert_eq!(rendered["severity"], "error"); + } + + #[test] + fn a_longer_field_name_is_not_swallowed_by_its_prefix() { + assert_eq!( + diagnostic_field("policy_target_kind must be a known kind"), + Some("policy_target_kind") + ); + } + + #[test] + fn a_message_naming_no_field_reports_none() { + assert_eq!(diagnostic_field("something else went wrong"), None); + } + + #[test] + fn perf_levels_round_trip_and_unknown_is_refused() { + for level in [ + PerfTelemetry::Off, + PerfTelemetry::External, + PerfTelemetry::Full, + ] { + assert_eq!( + parse_perf_telemetry(perf_telemetry_str(level)).unwrap(), + level + ); + } + assert!(parse_perf_telemetry("verbose").is_err()); + } +} diff --git a/sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj b/sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj index c002b01..febdfea 100644 --- a/sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj +++ b/sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj @@ -10,8 +10,38 @@ Agent Control Specification: a policy decision runtime plugging into agent-hooks as an interceptor. .NET wrapper over the Rust engine (native library agent_control_spec_ffi resolved from the loader path). MIT https://github.com/responsibleai/agent-control-spec + + $(MSBuildThisFileDirectory)..\..\native\ + + + <_AcsNativeAsset Include="$(AcsNativeDir)runtimes\**\*" /> + + + + + + + + diff --git a/sdk/dotnet/src/AgentControlSpec/HostHooks.cs b/sdk/dotnet/src/AgentControlSpec/HostHooks.cs new file mode 100644 index 0000000..80a8de9 --- /dev/null +++ b/sdk/dotnet/src/AgentControlSpec/HostHooks.cs @@ -0,0 +1,290 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Host extension points and manifest tooling. +// +// The engine takes an annotator dispatcher, a policy dispatcher, a +// telemetry sink and a perf level. The zero-config constructors pick +// defaults for all four, which is right for a host that wants a policy +// decision and nothing else. A host that classifies through its own +// service, evaluates through its own engine, or records its own audit +// trail supplies them here. + +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using AgentHooks; + +namespace AgentControlSpec; + +/// How much timing detail the engine records. +public enum PerfTelemetry +{ + /// Record nothing. + Off, + + /// Record the time spent outside the engine. + External, + + /// Record every phase. + Full, +} + +/// One finding about a manifest. +/// The engine's reason code. +/// What is wrong, in the engine's words. +/// How bad it is. +/// +/// The manifest field the message names, or null when it names none. +/// An authoring tool renders the finding against this. +/// +public sealed record ManifestDiagnostic( + [property: JsonPropertyName("code")] string Code, + [property: JsonPropertyName("message")] string Message, + [property: JsonPropertyName("severity")] string Severity, + [property: JsonPropertyName("field")] string? Field = null); + +/// +/// Classifies one annotation on the host's behalf. +/// +/// The annotator the manifest bound. +/// The binding's configured fields. +/// The policy input built so far. +/// The annotation value as JSON. +/// +/// Throwing fails the evaluation closed. An annotation that could not be +/// produced must not read as an annotation that found nothing. +/// +public delegate string AnnotatorDispatcher( + string annotatorName, string invocationJson, string policyInputJson); + +/// Evaluates one prepared policy invocation on the host's behalf. +/// The prepared invocation, tagged by engine type. +/// The policy output as JSON. +public delegate string PolicyDispatcher(string invocationJson); + +/// Receives one telemetry event as JSON. +/// A sink cannot fail an evaluation, so it has no error channel. +public delegate void TelemetrySink(string eventJson); + +/// Manifest reading and validation for authoring and tooling. +public static class AcsManifestTools +{ + private static readonly JsonSerializerOptions Json = new() + { + PropertyNameCaseInsensitive = true, + }; + + /// + /// Parse manifest text and return it as JSON. Parsing is not + /// validation: this answers what the document says, which a tool + /// needs before the document is runnable. + /// + public static string Parse(string yaml) => Native.ManifestParse(yaml); + + /// + /// Compose a chain of manifest documents into one, outermost base + /// first. This is the overlay case: a base policy plus the deltas an + /// environment layers on it. + /// + public static string Merge(IEnumerable yamls) => + Native.ManifestMerge(JsonSerializer.Serialize(yamls.ToArray())); + + /// + /// Validate manifest text and return every finding. An empty list + /// means valid. + /// + /// + /// answers yes or no by throwing, + /// which a linter cannot render against a document. This returns the + /// findings instead. + /// + public static IReadOnlyList Diagnostics(string yaml) => + JsonSerializer.Deserialize>(Native.ManifestDiagnostics(yaml), Json) + ?? throw new AgentControlSpecNativeException("diagnostics did not deserialize"); + + /// + /// Validate a manifest together with the Rego it names, and return + /// every finding. An empty list means both halves are sound. + /// + /// The manifest source. + /// + /// Policy id to in-memory Rego bundle, the same shape + /// takes. Null means the + /// manifest names no Rego, and the answer then equals + /// . + /// + /// + /// answers only for the document. A manifest + /// can name a bundle, satisfy the grammar, and still fail at + /// activation because the Rego does not compile. Compilation happens + /// at activation, so this activates in memory and reports what that + /// surfaced, which moves the failure from a host's first agent action + /// to its CI. + /// + public static IReadOnlyList ValidateArtifacts( + string manifestYaml, string? bundles = null) => + JsonSerializer.Deserialize>( + Native.ArtifactDiagnostics(manifestYaml, bundles), Json) + ?? throw new AgentControlSpecNativeException("diagnostics did not deserialize"); +} + +/// +/// An interceptor wired to host-supplied extension points. +/// +/// Anything left null keeps the bundled default for that slot, so a host +/// overrides only what it needs. +/// +/// +/// +/// using var interceptor = AcsHostInterceptor.FromPath( +/// "manifest.yaml", +/// annotator: (name, invocation, input) => Classify(input)); +/// +/// +public sealed class AcsHostInterceptor : IInterceptor, IDisposable +{ + // The delegates are held so the GC cannot collect them while native + // code still holds their function pointers. Dropping this field is + // the classic way to turn a working callback into an intermittent + // crash under load. + private readonly List _pinned = []; + private readonly IntPtr _handle; + private bool _disposed; + + private AcsHostInterceptor(IntPtr handle) => _handle = handle; + + private delegate IntPtr NativeAnnotator( + IntPtr ctx, IntPtr name, IntPtr invocation, IntPtr input, out IntPtr errOut); + + private delegate IntPtr NativePolicy(IntPtr ctx, IntPtr invocation, out IntPtr errOut); + + private delegate void NativeTelemetry(IntPtr ctx, IntPtr eventJson); + + private delegate void NativeFree(IntPtr ctx, IntPtr value); + + private static string Read(IntPtr p) => Marshal.PtrToStringUTF8(p) ?? string.Empty; + + /// Build an interceptor with host-supplied extension points. + /// Manifest to load. + /// Host classifier, or null for the bundled annotators. + /// Host policy engine, or null for the bundled dispatchers. + /// Host telemetry sink, or null to record nothing. + /// How much timing detail to record. + /// + /// Resource caps as JSON, overriding the engine's defaults field by + /// field. Null keeps every default. A host feeding large payloads + /// raises max_snapshot_bytes; one hardening against a hostile + /// manifest lowers max_extends_depth or + /// manifest_url_timeout_ms. + /// + public static AcsHostInterceptor FromPath( + string manifestPath, + AnnotatorDispatcher? annotator = null, + PolicyDispatcher? policy = null, + TelemetrySink? telemetry = null, + PerfTelemetry perfTelemetry = PerfTelemetry.Off, + string? limits = null) + { + var pinned = new List(); + + // Freed by the native side through this callback, so the string a + // host callback returns never crosses allocators. + NativeFree free = (_, value) => Marshal.FreeCoTaskMem(value); + pinned.Add(free); + + IntPtr annotatorPtr = IntPtr.Zero; + if (annotator is not null) + { + NativeAnnotator shim = (IntPtr _, IntPtr name, IntPtr invocation, IntPtr input, out IntPtr err) => + { + err = IntPtr.Zero; + try + { + return Marshal.StringToCoTaskMemUTF8( + annotator(Read(name), Read(invocation), Read(input))); + } + catch (Exception e) + { + err = Marshal.StringToCoTaskMemUTF8(e.Message); + return IntPtr.Zero; + } + }; + pinned.Add(shim); + annotatorPtr = Marshal.GetFunctionPointerForDelegate(shim); + } + + IntPtr policyPtr = IntPtr.Zero; + if (policy is not null) + { + NativePolicy shim = (IntPtr _, IntPtr invocation, out IntPtr err) => + { + err = IntPtr.Zero; + try + { + return Marshal.StringToCoTaskMemUTF8(policy(Read(invocation))); + } + catch (Exception e) + { + err = Marshal.StringToCoTaskMemUTF8(e.Message); + return IntPtr.Zero; + } + }; + pinned.Add(shim); + policyPtr = Marshal.GetFunctionPointerForDelegate(shim); + } + + IntPtr telemetryPtr = IntPtr.Zero; + if (telemetry is not null) + { + NativeTelemetry shim = (IntPtr _, IntPtr payload) => + { + // A sink that throws must not fail the action it merely + // describes, and an exception here would cross the + // native boundary. + try + { + telemetry(Read(payload)); + } + catch + { + // Intentionally swallowed: see above. + } + }; + pinned.Add(shim); + telemetryPtr = Marshal.GetFunctionPointerForDelegate(shim); + } + + var handle = Native.InterceptorNewWithHooks( + manifestPath, + annotatorPtr, IntPtr.Zero, + policyPtr, IntPtr.Zero, + telemetryPtr, IntPtr.Zero, + Marshal.GetFunctionPointerForDelegate(free), + perfTelemetry.ToString().ToLowerInvariant(), + limits); + + var interceptor = new AcsHostInterceptor(handle); + interceptor._pinned.AddRange(pinned); + return interceptor; + } + + /// Evaluate one agent context. + public ValueTask InterceptAsync(AgentContext context, CancellationToken ct = default) + { + ObjectDisposedException.ThrowIf(_disposed, this); + var wire = Native.Intercept(_handle, context.Json.ToJsonString()); + var parsed = System.Text.Json.Nodes.JsonNode.Parse(wire) as System.Text.Json.Nodes.JsonObject + ?? throw new AgentControlSpecNativeException("engine returned a non-object verdict"); + return ValueTask.FromResult(Verdict.FromWire(parsed)); + } + + /// Release the native interceptor. + public void Dispose() + { + if (_disposed) + return; + _disposed = true; + Native.Free(_handle); + _pinned.Clear(); + } +} diff --git a/sdk/dotnet/src/AgentControlSpec/Native.cs b/sdk/dotnet/src/AgentControlSpec/Native.cs index 235a49f..3563d8e 100644 --- a/sdk/dotnet/src/AgentControlSpec/Native.cs +++ b/sdk/dotnet/src/AgentControlSpec/Native.cs @@ -312,15 +312,280 @@ internal static string PolicyInterventionPoints(ActivatedPolicyHandle handle) } internal static void PolicyFree(IntPtr handle) => acs_policy_free(handle); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_stream_session_new(string configJson, out IntPtr errOut); + + [LibraryImport(Lib)] + private static partial void acs_stream_session_free(IntPtr handle); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial long acs_stream_session_observe( + StreamSessionHandle handle, string sourceType, uint runes, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial long acs_stream_session_observe_text( + StreamSessionHandle handle, string sourceType, string text, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial int acs_stream_session_record_outcome( + StreamSessionHandle handle, string task, string sourceType, uint start, uint end, string outcome, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial int acs_stream_session_record_verdict( + StreamSessionHandle handle, string task, string sourceType, uint start, uint end, string verdictJson, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial long acs_stream_session_advance(StreamSessionHandle handle, string track, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial long acs_stream_session_safe_offset(StreamSessionHandle handle, string track, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial long acs_stream_session_pending(StreamSessionHandle handle, string track, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_stream_session_watermark(StreamSessionHandle handle, string track, out IntPtr errOut); + + [LibraryImport(Lib)] + private static partial IntPtr acs_stream_session_state(StreamSessionHandle handle, out IntPtr errOut); + + [LibraryImport(Lib)] + private static partial int acs_stream_session_end_of_payloads(StreamSessionHandle handle, out IntPtr errOut); + + [LibraryImport(Lib)] + private static partial IntPtr acs_stream_session_finish(StreamSessionHandle handle, out IntPtr errOut); + + internal static StreamSessionHandle StreamSessionNew(string configJson) + { + var handle = acs_stream_session_new(configJson, out var err); + if (handle == IntPtr.Zero) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("stream session creation returned no handle"); + } + + return new StreamSessionHandle(handle); + } + + internal static void StreamSessionFree(IntPtr handle) => acs_stream_session_free(handle); + + // A scalar query answers with the value, -1 for absent, or -2 for a + // boundary failure. Absent is a real answer (a settled session has no + // safe offset), so it becomes null rather than an exception. + private static long? Scalar(long value, IntPtr err) + { + if (value == -2) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("stream session call failed without a message"); + } + + return value == -1 ? null : value; + } + + internal static int StreamObserve(StreamSessionHandle handle, string sourceType, uint runes) + { + var received = acs_stream_session_observe(handle, sourceType, runes, out var err); + return checked((int)Scalar(received, err)!.Value); + } + + internal static int StreamObserveText(StreamSessionHandle handle, string sourceType, string text) + { + var received = acs_stream_session_observe_text(handle, sourceType, text, out var err); + return checked((int)Scalar(received, err)!.Value); + } + + internal static void StreamRecordOutcome( + StreamSessionHandle handle, string task, string sourceType, uint start, uint end, string outcome) + { + if (acs_stream_session_record_outcome( + handle, task, sourceType, start, end, outcome, out var err) != 0) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("recording the outcome failed without a message"); + } + } + + internal static void StreamRecordVerdict( + StreamSessionHandle handle, string task, string sourceType, uint start, uint end, string verdictJson) + { + if (acs_stream_session_record_verdict( + handle, task, sourceType, start, end, verdictJson, out var err) != 0) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("recording the verdict failed without a message"); + } + } + + internal static int? StreamAdvance(StreamSessionHandle handle, string track) + { + var offset = acs_stream_session_advance(handle, track, out var err); + return (int?)Scalar(offset, err); + } + + internal static int? StreamSafeOffset(StreamSessionHandle handle, string track) + { + var offset = acs_stream_session_safe_offset(handle, track, out var err); + return (int?)Scalar(offset, err); + } + + internal static int StreamPending(StreamSessionHandle handle, string track) + { + var pending = acs_stream_session_pending(handle, track, out var err); + return checked((int)Scalar(pending, err)!.Value); + } + + internal static string StreamWatermark(StreamSessionHandle handle, string track) + { + var json = acs_stream_session_watermark(handle, track, out var err); + ThrowIfError(err); + return TakeString(json); + } + + internal static string StreamState(StreamSessionHandle handle) + { + var json = acs_stream_session_state(handle, out var err); + ThrowIfError(err); + return TakeString(json); + } + + internal static void StreamEndOfPayloads(StreamSessionHandle handle) + { + if (acs_stream_session_end_of_payloads(handle, out var err) != 0) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("closing the payload stream failed without a message"); + } + } + + internal static string StreamFinish(StreamSessionHandle handle) + { + var json = acs_stream_session_finish(handle, out var err); + ThrowIfError(err); + return TakeString(json); + } + + [LibraryImport(Lib)] + private static partial IntPtr acs_interceptor_new_with_hooks( + ReadOnlySpan manifestPath, nuint manifestPathLen, + IntPtr annotatorFn, IntPtr annotatorCtx, + IntPtr policyFn, IntPtr policyCtx, + IntPtr telemetryFn, IntPtr telemetryCtx, + IntPtr hookFree, + IntPtr perfTelemetry, + IntPtr limitsJson, + out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_manifest_parse(string yaml, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_manifest_merge(string yamlsJson, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_manifest_diagnostics(string yaml, out IntPtr errOut); + + [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr acs_artifact_diagnostics( + string manifestYaml, string? bundlesJson, out IntPtr errOut); + + internal static IntPtr InterceptorNewWithHooks( + string manifestPath, + IntPtr annotatorFn, IntPtr annotatorCtx, + IntPtr policyFn, IntPtr policyCtx, + IntPtr telemetryFn, IntPtr telemetryCtx, + IntPtr hookFree, + string? perfTelemetry, + string? limitsJson) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(manifestPath); + var perf = perfTelemetry is null + ? IntPtr.Zero + : Marshal.StringToCoTaskMemUTF8(perfTelemetry); + var limits = limitsJson is null ? IntPtr.Zero : Marshal.StringToCoTaskMemUTF8(limitsJson); + try + { + var handle = acs_interceptor_new_with_hooks( + bytes, (nuint)bytes.Length, + annotatorFn, annotatorCtx, policyFn, policyCtx, + telemetryFn, telemetryCtx, hookFree, perf, limits, out var err); + if (handle == IntPtr.Zero) + { + ThrowIfError(err); + throw new AgentControlSpecNativeException("interceptor construction returned no handle"); + } + + return handle; + } + finally + { + if (perf != IntPtr.Zero) + Marshal.FreeCoTaskMem(perf); + if (limits != IntPtr.Zero) + Marshal.FreeCoTaskMem(limits); + } + } + + internal static string ManifestParse(string yaml) + { + var json = acs_manifest_parse(yaml, out var err); + ThrowIfError(err); + return TakeString(json); + } + + internal static string ManifestMerge(string yamlsJson) + { + var json = acs_manifest_merge(yamlsJson, out var err); + ThrowIfError(err); + return TakeString(json); + } + + internal static string ManifestDiagnostics(string yaml) + { + var json = acs_manifest_diagnostics(yaml, out var err); + ThrowIfError(err); + return TakeString(json); + } + + internal static string ArtifactDiagnostics(string manifestYaml, string? bundlesJson) + { + var json = acs_artifact_diagnostics(manifestYaml, bundlesJson, out var err); + ThrowIfError(err); + return TakeString(json); + } +} + +/// Owns one native stream session. +/// +/// Every P/Invoke below takes this type rather than an , +/// so the marshaller takes a reference for the duration of the call. That +/// is what makes a call safe against a racing Dispose: without it a +/// free during an in-flight call runs under it, and a call after +/// Dispose hands a stale pointer to native code, which the engine's +/// null check cannot catch because the pointer is not null, only dead. +/// A closed handle surfaces as . +/// +internal sealed class StreamSessionHandle : SafeHandle +{ + internal StreamSessionHandle(IntPtr handle) + : base(IntPtr.Zero, ownsHandle: true) => SetHandle(handle); + + public override bool IsInvalid => handle == IntPtr.Zero; + + protected override bool ReleaseHandle() + { + Native.StreamSessionFree(handle); + return true; + } } -/// Owns one activated policy version's native allocation. +/// Owns one activated policy version. /// -/// A rather than the bare pointer the -/// interceptor keeps, because an activated policy is shared across -/// threads by design: the ref count holds the pointer alive for the -/// duration of every in-flight evaluation, so disposing while another -/// thread evaluates frees after that call rather than under it. +/// Evaluation pairs DangerousGetHandle with +/// DangerousAddRef and DangerousRelease. That pair is what +/// makes concurrent evaluation safe against a racing Dispose: +/// without it the free runs under an in-flight call. /// internal sealed class ActivatedPolicyHandle : SafeHandle { diff --git a/sdk/dotnet/src/AgentControlSpec/StreamSession.cs b/sdk/dotnet/src/AgentControlSpec/StreamSession.cs new file mode 100644 index 0000000..e1caa94 --- /dev/null +++ b/sdk/dotnet/src/AgentControlSpec/StreamSession.cs @@ -0,0 +1,321 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Incremental stream mediation (specification section 18.1). +// +// This is a binding, not an implementation. Every decision about what +// may be released is made by the Rust engine through the C ABI, so a +// host on this SDK and a host on any other answer identically by +// construction rather than by agreement. + +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AgentControlSpec; + +/// How much a host withholds while a stream is mediated. +public enum SafetyLevel +{ + /// Withhold every rune until a task clears it. + Blocking, + + /// Withhold until the whole payload has been evaluated. + Complete, + + /// Emit on arrival and evaluate behind the stream. + Deferred, +} + +/// Which side of the exchange a payload belongs to. +public enum StreamSourceType +{ + /// Text authored by the caller. + UserRequest, + + /// Text generated by the model. + ModelGenerated, +} + +/// The two independently accounted halves of a stream. +public enum StreamTrack +{ + /// Runes flowing toward the model. + Request, + + /// Runes flowing back to the caller. + Response, +} + +/// What a task decided about one evaluated span. +public enum SegmentOutcome +{ + /// The span may be released unchanged. + Cleared, + + /// The host substitutes its own text. This ends the stream. + Transformed, + + /// The span is refused. This ends the stream. + Denied, +} + +/// How far one track got and what still owes a decision. +/// The track described. +/// Runes every task has cleared. +/// Runes observed. +/// Runes observed but not yet released. +/// The tasks that must clear a span before it releases. +public sealed record StreamWatermark( + [property: JsonPropertyName("track")] string Track, + [property: JsonPropertyName("confirmed")] int Confirmed, + [property: JsonPropertyName("received")] int Received, + [property: JsonPropertyName("pending")] int Pending, + [property: JsonPropertyName("tasks")] IReadOnlyList Tasks); + +/// Why a session ended. +/// One of complete, denied, rewritten or failed. +/// The track carrying the span, when a span ended it. +/// The task that decided, when a task ended it. +/// Start rune of the deciding span. +/// End rune of the deciding span. +/// The engine's reason code, when the session failed. +/// The engine's message, when the session failed. +public sealed record StreamEndReason( + [property: JsonPropertyName("kind")] string Kind, + [property: JsonPropertyName("track")] string? Track = null, + [property: JsonPropertyName("task")] string? Task = null, + [property: JsonPropertyName("start")] int? Start = null, + [property: JsonPropertyName("end")] int? End = null, + [property: JsonPropertyName("reason")] string? Reason = null, + [property: JsonPropertyName("message")] string? Message = null); + +/// Terminal settlement of a session. +/// Why the session ended. +/// Whether the emitted text is not verbatim model output. +/// Whether the stream ran to completion without a refusal. +public sealed record StreamCompletion( + [property: JsonPropertyName("reason")] StreamEndReason Reason, + [property: JsonPropertyName("transformed")] bool Transformed, + [property: JsonPropertyName("is_clean")] bool IsClean); + +/// The offsets and task sets a session was opened with. +/// The withholding level. +/// Where request accounting begins. +/// Where response accounting begins. +/// Tasks mediating the request track. +/// Tasks mediating the response track. +public sealed record StreamSessionConfig( + [property: JsonPropertyName("safety_level")] string SafetyLevel, + [property: JsonPropertyName("request_start_rune_offset")] int RequestStartRuneOffset, + [property: JsonPropertyName("response_start_rune_offset")] int ResponseStartRuneOffset, + [property: JsonPropertyName("request_tasks")] IReadOnlyList RequestTasks, + [property: JsonPropertyName("response_tasks")] IReadOnlyList ResponseTasks); + +/// Live state of a session. +/// Whether the session has settled. +/// Whether a rewrite ended it. +/// Why it ended, or null while it is live. +/// The configuration in force. +public sealed record StreamSessionState( + [property: JsonPropertyName("is_ended")] bool IsEnded, + [property: JsonPropertyName("transformed")] bool Transformed, + [property: JsonPropertyName("end_reason")] StreamEndReason? EndReason, + [property: JsonPropertyName("config")] StreamSessionConfig Config); + +/// +/// One mediated stream. +/// +/// A host reports arriving payloads, reports what each task decided +/// about each evaluated span, and reads back how far the stream is safe +/// to release. The session holds no text, so applying a rewrite and +/// emitting released runes stay the host's job. +/// +/// +/// +/// using var session = new StreamSession( +/// SafetyLevel.Blocking, responseTasks: ["pii"]); +/// session.ObserveText(StreamSourceType.ModelGenerated, "hello"); +/// session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); +/// int? release = session.SafeOffset(StreamTrack.Response); +/// +/// +public sealed class StreamSession : IDisposable +{ + private static readonly JsonSerializerOptions Json = new() + { + PropertyNameCaseInsensitive = true, + }; + + private readonly StreamSessionHandle _handle; + + /// Open a session. + /// How much to withhold while mediating. + /// Tasks mediating the request track. Empty leaves it unmediated. + /// Tasks mediating the response track. Empty leaves it unmediated. + /// Where request accounting begins, for a resumed stream. + /// Where response accounting begins, for a resumed stream. + /// + /// The engine refused the configuration, which includes leaving both + /// tracks unmediated, because a session that evaluates nothing + /// releases everything. + /// + public StreamSession( + SafetyLevel safetyLevel = SafetyLevel.Blocking, + IEnumerable? requestTasks = null, + IEnumerable? responseTasks = null, + int requestStartRuneOffset = 0, + int responseStartRuneOffset = 0) + { + var config = JsonSerializer.Serialize(new Dictionary + { + ["safety_level"] = Wire(safetyLevel), + ["request_start_rune_offset"] = requestStartRuneOffset, + ["response_start_rune_offset"] = responseStartRuneOffset, + ["request_tasks"] = requestTasks?.ToArray() ?? [], + ["response_tasks"] = responseTasks?.ToArray() ?? [], + }); + _handle = Native.StreamSessionNew(config); + } + + internal static string Wire(SafetyLevel level) => level switch + { + AgentControlSpec.SafetyLevel.Blocking => "blocking", + AgentControlSpec.SafetyLevel.Complete => "complete", + AgentControlSpec.SafetyLevel.Deferred => "deferred", + _ => throw new ArgumentOutOfRangeException(nameof(level), level, "unknown safety level"), + }; + + internal static string Wire(StreamSourceType source) => source switch + { + StreamSourceType.UserRequest => "user_request", + StreamSourceType.ModelGenerated => "model_generated", + _ => throw new ArgumentOutOfRangeException(nameof(source), source, "unknown stream source type"), + }; + + internal static string Wire(StreamTrack track) => track switch + { + StreamTrack.Request => "request", + StreamTrack.Response => "response", + _ => throw new ArgumentOutOfRangeException(nameof(track), track, "unknown stream track"), + }; + + internal static string Wire(SegmentOutcome outcome) => outcome switch + { + SegmentOutcome.Cleared => "cleared", + SegmentOutcome.Transformed => "transformed", + SegmentOutcome.Denied => "denied", + _ => throw new ArgumentOutOfRangeException(nameof(outcome), outcome, "unknown segment outcome"), + }; + + /// Record that more runes arrived. + /// The track's received offset. + public int Observe(StreamSourceType sourceType, int runes) => + Native.StreamObserve(_handle, Wire(sourceType), checked((uint)runes)); + + /// + /// Record an arriving payload by its text, counting runes the way the + /// engine does so a host never counts them itself. A rune is a Unicode + /// scalar, so an astral-plane character counts once even though it + /// occupies two UTF-16 code units in a .NET string. + /// + /// The track's received offset. + /// + /// The text holds U+0000. The boundary marshals as NUL-terminated + /// UTF-8, so an interior NUL would truncate and the engine would + /// count fewer runes than arrived, permanently shifting every later + /// offset. U+0000 is a scalar a model can emit, and the profile + /// obliges a host to report counts that match the text it + /// accumulated, so this refuses rather than quietly miscounting. + /// + public int ObserveText(StreamSourceType sourceType, string text) + { + ArgumentNullException.ThrowIfNull(text); + if (text.Contains('\0')) + { + throw new ArgumentException( + "text holds U+0000, which this boundary cannot carry without truncating", + nameof(text)); + } + + return Native.StreamObserveText(_handle, Wire(sourceType), text); + } + + /// Record what decided about a span. + public void RecordOutcome( + string task, StreamSourceType sourceType, int start, int end, SegmentOutcome outcome) => + Native.StreamRecordOutcome( + _handle, task, Wire(sourceType), checked((uint)start), checked((uint)end), Wire(outcome)); + + /// + /// Record a verdict against a span, mapping its decision onto an + /// outcome. Takes a verdict exactly as + /// returns one, so a decision feeds straight back with nothing to + /// translate. + /// + public void RecordVerdict( + string task, StreamSourceType sourceType, int start, int end, string verdictJson) => + Native.StreamRecordVerdict( + _handle, task, Wire(sourceType), checked((uint)start), checked((uint)end), verdictJson); + + /// + /// Recompute a track's watermark against the outcomes recorded so far. + /// This is the call that moves the releasable offset; recording an + /// outcome on its own does not. + /// + /// The offset it advanced to, or null when it did not advance. + public int? Advance(StreamTrack track) => Native.StreamAdvance(_handle, Wire(track)); + + /// + /// The offset safe to release, as of the last . + /// + /// + /// The releasable offset, or null once the session has ended. Null + /// means release nothing further; it is not an error, and a settled + /// session reports it rather than a stale number that would release + /// text no task cleared. + /// + public int? SafeOffset(StreamTrack track) => Native.StreamSafeOffset(_handle, Wire(track)); + + /// Runes observed on a track but not yet released. + public int Pending(StreamTrack track) => Native.StreamPending(_handle, Wire(track)); + + /// + /// A track's watermark. The confirmed offset stays readable after + /// settlement, so an audit record can still say how far the stream got. + /// + public StreamWatermark Watermark(StreamTrack track) => + JsonSerializer.Deserialize(Native.StreamWatermark(_handle, Wire(track)), Json) + ?? throw new AgentControlSpecNativeException("watermark did not deserialize"); + + /// Live session state. + public StreamSessionState State => + JsonSerializer.Deserialize(Native.StreamState(_handle), Json) + ?? throw new AgentControlSpecNativeException("state did not deserialize"); + + /// Whether the session has settled. + public bool IsEnded => State.IsEnded; + + /// Whether a rewrite ended the session. + public bool Transformed => State.Transformed; + + /// Why the session ended, or null while it is live. + public StreamEndReason? EndReason => State.EndReason; + + /// The configuration in force. + public StreamSessionConfig Config => State.Config; + + /// Declare that no further payload will arrive. + public void EndOfPayloads() => Native.StreamEndOfPayloads(_handle); + + /// Settle the session. Settling twice returns the same completion. + public StreamCompletion Finish() => + JsonSerializer.Deserialize(Native.StreamFinish(_handle), Json) + ?? throw new AgentControlSpecNativeException("completion did not deserialize"); + + /// Release the native session. + /// + /// This frees the accounting without recording a settlement, so a + /// host that owes an outcome calls first. + /// Disposing an unsettled session loses why it ended. + /// + public void Dispose() => _handle.Dispose(); +} diff --git a/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs b/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs new file mode 100644 index 0000000..bc36380 --- /dev/null +++ b/sdk/dotnet/tests/AgentControlSpec.Tests/HostHooksTests.cs @@ -0,0 +1,315 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Host extension points: the engine's own dispatchers, telemetry sink +// and perf level, reachable from .NET. +// +// The scenario is the one that blocked a real consumer on 0.3: a host +// classifier reached over HTTP, bound as an annotator, whose answer +// decides the verdict. Before these entry points there was no way to +// supply one from any language but Rust. + +using System.Text.Json.Nodes; +using AgentControlSpec; +using AgentHooks; +using Xunit; + +namespace AgentControlSpec.Tests; + +public sealed class HostHooksTests : IDisposable +{ + private readonly string _dir = Directory.CreateTempSubdirectory("acs-hooks").FullName; + + private string WriteFixture() + { + var bundle = Path.Combine(_dir, "bundle"); + Directory.CreateDirectory(bundle); + File.WriteAllText(Path.Combine(bundle, "policy.rego"), """ + package acs + + decision := {"decision": "deny", "reason": "unsafe_content"} if { + input.annotations.content_safety.severity >= 4 + } else := {"decision": "allow"} + """); + + var manifest = Path.Combine(_dir, "manifest.yaml"); + File.WriteAllText(manifest, """ + agent_control_specification_version: "0.4.0-alpha.1" + metadata: + name: host-hooks-test + annotators: + content_safety: + type: classifier + policies: + gate: + type: rego + bundle: ./bundle + intervention_points: + input: + policy_target: "$snap.input" + annotations: + content_safety: + from: "$target" + policy: + id: gate + query: data.acs.decision + """); + return manifest; + } + + private static AgentContext Input(string text) => + new((JsonNode.Parse($$"""{"interception_point":"input","input":{{System.Text.Json.JsonSerializer.Serialize(text)}}}""")!).AsObject()); + + [Fact] + public async Task AHostClassifierDecidesTheVerdict() + { + var manifest = WriteFixture(); + var calls = 0; + + using var benign = AcsHostInterceptor.FromPath( + manifest, + annotator: (name, _, _) => + { + calls++; + Assert.Equal("content_safety", name); + return """{"severity":1}"""; + }); + + var allowed = await benign.InterceptAsync(Input("hello")); + Assert.Equal(Decision.Allow, allowed.Decision); + Assert.Equal(1, calls); + + using var harmful = AcsHostInterceptor.FromPath( + manifest, annotator: (_, _, _) => """{"severity":7}"""); + + var denied = await harmful.InterceptAsync(Input("hello")); + Assert.Equal(Decision.Deny, denied.Decision); + Assert.Equal("unsafe_content", denied.Reason); + } + + [Fact] + public async Task AClassifierThatFailsDeniesRatherThanFindingNothing() + { + var manifest = WriteFixture(); + + using var broken = AcsHostInterceptor.FromPath( + manifest, + annotator: (_, _, _) => throw new InvalidOperationException("classifier unreachable")); + + var verdict = await broken.InterceptAsync(Input("hello")); + + // The point of the test: an unreachable classifier must not read + // as a classifier that found nothing. + Assert.Equal(Decision.Deny, verdict.Decision); + Assert.Equal("runtime_error:annotation_failed", verdict.Reason); + } + + [Fact] + public async Task ATelemetrySinkSeesTheEvaluation() + { + var manifest = WriteFixture(); + var events = new List(); + + using var interceptor = AcsHostInterceptor.FromPath( + manifest, + annotator: (_, _, _) => """{"severity":1}""", + telemetry: events.Add, + perfTelemetry: PerfTelemetry.Full); + + await interceptor.InterceptAsync(Input("hello")); + + Assert.NotEmpty(events); + Assert.Contains(events, e => e.Contains("intervention_point")); + } + + [Fact] + public async Task ASinkThatThrowsDoesNotFailTheAction() + { + var manifest = WriteFixture(); + + using var interceptor = AcsHostInterceptor.FromPath( + manifest, + annotator: (_, _, _) => """{"severity":1}""", + telemetry: _ => throw new InvalidOperationException("sink is down")); + + // A sink records what happened. It does not get a vote on it. + var verdict = await interceptor.InterceptAsync(Input("hello")); + Assert.Equal(Decision.Allow, verdict.Decision); + } + + [Fact] + public void ParseReadsAManifestWithoutRunningIt() + { + var json = AcsManifestTools.Parse(""" + agent_control_specification_version: "0.4.0-alpha.1" + policies: + p: + type: test + intervention_points: + input: + policy_target: "$.input" + policy: + id: p + """); + + Assert.Contains("intervention_points", json); + } + + [Fact] + public void ParseRejectsTextThatIsNotAManifest() + { + Assert.Throws( + () => AcsManifestTools.Parse("this: [is not")); + } + + [Fact] + public void DiagnosticsNameTheProblemRatherThanThrowing() + { + var findings = AcsManifestTools.Diagnostics(""" + agent_control_specification_version: "0.4.0-alpha.1" + metadata: {} + """); + + var finding = Assert.Single(findings); + Assert.Equal("error", finding.Severity); + Assert.Contains("intervention point", finding.Message); + Assert.StartsWith("runtime_error:", finding.Code); + } + + [Fact] + public void DiagnosticsAreEmptyForAValidManifest() + { + Assert.Empty(AcsManifestTools.Diagnostics(""" + agent_control_specification_version: "0.4.0-alpha.1" + policies: + p: + type: test + intervention_points: + input: + policy_target: "$.input" + policy: + id: p + """)); + } + + [Fact] + public void MergeComposesABaseWithAnOverlay() + { + var merged = AcsManifestTools.Merge([ + """ + agent_control_specification_version: "0.4.0-alpha.1" + policies: + p: + type: test + intervention_points: + input: + policy_target: "$.input" + policy: + id: p + """, + """ + agent_control_specification_version: "0.4.0-alpha.1" + metadata: + name: overlay-applied + """, + ]); + + Assert.Contains("overlay-applied", merged); + } + + + private const string RegoManifest = """ + agent_control_specification_version: "0.4.0-alpha.1" + policies: + gate: + type: rego + bundle: ./b + intervention_points: + input: + policy_target: "$.input" + policy: + id: gate + query: data.acs.decision + """; + + [Fact] + public void ArtifactValidationClearsAManifestWhoseRegoCompiles() + { + var bundles = """ + {"gate":{"modules":{"p.rego":"package acs\ndecision := {\"decision\":\"allow\"}\n"}}} + """; + + Assert.Empty(AcsManifestTools.ValidateArtifacts(RegoManifest, bundles)); + } + + [Fact] + public void ArtifactValidationCatchesRegoTheManifestCheckCannot() + { + var broken = """ + {"gate":{"modules":{"p.rego":"package acs\nthis is not rego at all ***\n"}}} + """; + + // The manifest itself is sound, so the document check passes it. + Assert.Empty(AcsManifestTools.Diagnostics(RegoManifest)); + + // The Rego is not, and only activation finds that out. + var finding = Assert.Single(AcsManifestTools.ValidateArtifacts(RegoManifest, broken)); + Assert.StartsWith("runtime_error:", finding.Code); + Assert.Contains("p.rego", finding.Message); + } + + [Fact] + public void AManifestThatDoesNotParseIsReportedAsAManifestProblem() + { + var finding = Assert.Single(AcsManifestTools.ValidateArtifacts("this: [is not", null)); + + // Naming this an activation failure would blame the wrong half. + Assert.Contains("manifest", finding.Code); + } + + [Fact] + public void WithNoBundlesArtifactValidationAgreesWithTheManifestCheck() + { + const string Bad = """ + agent_control_specification_version: "0.4.0-alpha.1" + metadata: {} + """; + + Assert.Equal( + AcsManifestTools.Diagnostics(Bad).Count, + AcsManifestTools.ValidateArtifacts(Bad, null).Count); + } + + + [Fact] + public async Task ALoweredSnapshotCapIsEnforced() + { + var manifest = WriteFixture(); + var big = new string('x', 4096); + + using var permissive = AcsHostInterceptor.FromPath( + manifest, annotator: (_, _, _) => """{"severity":1}"""); + Assert.Equal(Decision.Allow, (await permissive.InterceptAsync(Input(big))).Decision); + + // The same context against a cap smaller than it. A host that + // asked for a smaller bound and kept the larger one would believe + // it was protected when it was not. + using var capped = AcsHostInterceptor.FromPath( + manifest, + annotator: (_, _, _) => """{"severity":1}""", + limits: """{"max_snapshot_bytes": 64}"""); + + var verdict = await capped.InterceptAsync(Input(big)); + Assert.Equal(Decision.Deny, verdict.Decision); + Assert.StartsWith("runtime_error:", verdict.Reason); + } + + [Fact] + public void ALimitThatIsNotANumberIsRefused() + { + var manifest = WriteFixture(); + Assert.Throws(() => + AcsHostInterceptor.FromPath(manifest, limits: """{"max_snapshot_bytes": "big"}""")); + } + + public void Dispose() => Directory.Delete(_dir, recursive: true); +} diff --git a/sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs b/sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs new file mode 100644 index 0000000..f3e37e0 --- /dev/null +++ b/sdk/dotnet/tests/AgentControlSpec.Tests/StreamSessionTests.cs @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// Streaming reaches .NET through the C ABI, so these run the real engine. +// They are the .NET half of a suite that asserts the same scenarios in +// every supported language. + +using System.Linq; +using AgentControlSpec; +using Xunit; + +namespace AgentControlSpec.Tests; + +public sealed class StreamSessionTests +{ + [Fact] + public void AClearedSpanReleasesUpToItsEnd() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + + Assert.Equal(5, session.ObserveText(StreamSourceType.ModelGenerated, "hello")); + Assert.Equal(0, session.SafeOffset(StreamTrack.Response)); + + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); + + Assert.Equal(5, session.Advance(StreamTrack.Response)); + Assert.Equal(5, session.SafeOffset(StreamTrack.Response)); + Assert.Equal(0, session.Pending(StreamTrack.Response)); + + var completion = session.Finish(); + Assert.Equal("complete", completion.Reason.Kind); + Assert.True(completion.IsClean); + Assert.False(completion.Transformed); + } + + [Fact] + public void ARefusalEndsTheSessionAndStillReportsHowFarItGot() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hello world"); + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); + session.Advance(StreamTrack.Response); + Assert.Equal(5, session.SafeOffset(StreamTrack.Response)); + + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 5, 11, SegmentOutcome.Denied); + + Assert.True(session.IsEnded); + Assert.Null(session.SafeOffset(StreamTrack.Response)); + + // The audit path: the offset the stream reached survives settlement. + Assert.Equal(5, session.Watermark(StreamTrack.Response).Confirmed); + + var reason = session.EndReason; + Assert.NotNull(reason); + Assert.Equal("denied", reason!.Kind); + Assert.Equal("pii", reason.Task); + Assert.Equal("response", reason.Track); + Assert.False(session.Finish().IsClean); + } + + [Fact] + public void EveryTaskMustClearASpanBeforeItReleases() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii", "harm"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hello"); + + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); + Assert.Null(session.Advance(StreamTrack.Response)); + Assert.Equal(0, session.SafeOffset(StreamTrack.Response)); + + session.RecordOutcome("harm", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); + Assert.Equal(5, session.Advance(StreamTrack.Response)); + Assert.Equal(5, session.SafeOffset(StreamTrack.Response)); + } + + [Fact] + public void ObserveTextCountsRunesNotUtf16CodeUnits() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + + // One astral-plane scalar. .NET stores it as two UTF-16 code units, + // so a host counting string.Length would release twice what was + // evaluated. + const string Emoji = "\U0001F600"; + Assert.Equal(2, Emoji.Length); + Assert.Equal(1, session.ObserveText(StreamSourceType.ModelGenerated, Emoji)); + } + + [Fact] + public void APayloadOnAnUnmediatedTrackIsRefused() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + + // No task mediates the request track, so nothing would ever clear + // text sent there. The engine refuses the payload rather than + // releasing it unevaluated. + var error = Assert.Throws(() => + session.ObserveText(StreamSourceType.UserRequest, "hi")); + Assert.Contains("unmediated", error.Message); + Assert.Empty(session.Watermark(StreamTrack.Request).Tasks); + } + + [Fact] + public void TheTwoTracksAccountIndependently() + { + using var session = new StreamSession( + SafetyLevel.Blocking, requestTasks: ["pii"], responseTasks: ["pii"]); + + session.ObserveText(StreamSourceType.UserRequest, "abc"); + session.ObserveText(StreamSourceType.ModelGenerated, "defghi"); + + session.RecordOutcome("pii", StreamSourceType.UserRequest, 0, 3, SegmentOutcome.Cleared); + session.Advance(StreamTrack.Request); + + Assert.Equal(3, session.SafeOffset(StreamTrack.Request)); + Assert.Equal(0, session.SafeOffset(StreamTrack.Response)); + Assert.Equal(6, session.Watermark(StreamTrack.Response).Received); + } + + [Fact] + public void AResumedStreamStartsFromItsRecordedOffsets() + { + using var session = new StreamSession( + SafetyLevel.Blocking, responseTasks: ["pii"], responseStartRuneOffset: 10); + + Assert.Equal(10, session.Config.ResponseStartRuneOffset); + session.ObserveText(StreamSourceType.ModelGenerated, "abc"); + Assert.Equal(13, session.Watermark(StreamTrack.Response).Received); + } + + [Fact] + public void ARewriteIsTerminalAndReportsItself() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hello"); + + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Transformed); + + Assert.True(session.IsEnded); + Assert.True(session.Transformed); + Assert.Equal("rewritten", session.EndReason!.Kind); + Assert.Null(session.SafeOffset(StreamTrack.Response)); + } + + [Fact] + public void AVerdictFeedsBackWithoutTranslation() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hello"); + + // Shaped as ActivatedPolicy.Evaluate returns one. + session.RecordVerdict( + "pii", StreamSourceType.ModelGenerated, 0, 5, + """{"decision":"allow","reasons":[]}"""); + + Assert.Equal(5, session.Advance(StreamTrack.Response)); + Assert.Equal(5, session.SafeOffset(StreamTrack.Response)); + } + + [Fact] + public void AnUnknownTaskIsRefusedRatherThanIgnored() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hello"); + + Assert.Throws(() => + session.RecordOutcome("nope", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared)); + } + + [Fact] + public void ASessionThatEvaluatesNothingIsRefused() + { + Assert.Throws(() => new StreamSession(SafetyLevel.Blocking)); + } + + [Fact] + public void AnOutcomeReachingUnobservedTextIsRefused() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hi"); + + Assert.Throws(() => + session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 99, SegmentOutcome.Cleared)); + } + + [Fact] + public void ACallAfterDisposeIsRefusedRatherThanReadingFreedMemory() + { + var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + session.ObserveText(StreamSourceType.ModelGenerated, "hi"); + session.Dispose(); + + // The engine's null check cannot catch this. The pointer is not + // null, only dead, so the handle has to refuse before the call. + Assert.Throws( + () => session.ObserveText(StreamSourceType.ModelGenerated, "more")); + } + + [Fact] + public void TwoThreadsOnOneSessionDoNotLoseObservedRunes() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + const int Each = 5_000; + + var threads = Enumerable.Range(0, 2).Select(_ => new Thread(() => + { + for (var i = 0; i < Each; i++) + { + session.Observe(StreamSourceType.ModelGenerated, 1); + } + })).ToList(); + + foreach (var thread in threads) thread.Start(); + foreach (var thread in threads) thread.Join(); + + // A lost observe shortens the received offset, which releases + // text no task evaluated. + Assert.Equal(Each * 2, session.Watermark(StreamTrack.Response).Received); + } + + [Fact] + public void TextHoldingNulIsRefusedRatherThanTruncated() + { + using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); + + // U+0000 is a scalar a model can emit. Marshalled as + // NUL-terminated UTF-8 it would truncate, and the engine would + // count 1 where 4 runes arrived. + Assert.Throws( + () => session.ObserveText(StreamSourceType.ModelGenerated, "a\0bc")); + } +} diff --git a/sdk/ffi/Cargo.toml b/sdk/ffi/Cargo.toml index 4e6f2b3..4a7d24d 100644 --- a/sdk/ffi/Cargo.toml +++ b/sdk/ffi/Cargo.toml @@ -11,5 +11,5 @@ name = "agent_control_spec_ffi" crate-type = ["cdylib", "rlib"] [dependencies] -agent-control-spec = { path = "../../engine", features = ["default-dispatchers"] } +agent-control-spec = { path = "../../engine", features = ["default-dispatchers", "streaming"] } serde_json = "1" diff --git a/sdk/ffi/src/lib.rs b/sdk/ffi/src/lib.rs index 38c0291..24585c3 100644 --- a/sdk/ffi/src/lib.rs +++ b/sdk/ffi/src/lib.rs @@ -17,16 +17,25 @@ // every schema-valid context. Errors on that path are boundary // problems only (bad UTF-8, non-object context, poisoned handle). +use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; +use agent_control_spec::policy::PreparedPolicyInvocation; +use agent_control_spec::runtime::PolicyDispatcher; +use agent_control_spec::stream_session::{ + SafetyLevel, SegmentOutcome, StreamSession, StreamSessionConfig, StreamSourceType, StreamSpan, + StreamTrack, +}; +use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; +use agent_control_spec::wire; use agent_control_spec::{ - ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, Runtime, RuntimeError, - SUPPORTED_VERSIONS, + ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Limits, Manifest, Runtime, + RuntimeError, Verdict, SUPPORTED_VERSIONS, }; use serde_json::Value; use std::collections::BTreeMap; -use std::ffi::{c_char, CStr, CString}; +use std::ffi::{c_char, c_void, CStr, CString}; use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; /// Opaque interceptor handle: the runtime plus the payload-free name /// recorded on `verdicts[].name`. @@ -35,6 +44,22 @@ pub struct AcsInterceptor { name: String, } +/// Read an optional NUL-terminated string. NULL means absent, which is +/// distinct from present and invalid. +unsafe fn read_optional<'a>( + ptr: *const c_char, + what: &str, + err_out: *mut *mut c_char, +) -> Result, ()> { + if ptr.is_null() { + return Ok(None); + } + match read_utf8(ptr, what, err_out) { + Some(value) => Ok(Some(value)), + None => Err(()), + } +} + fn set_err(err_out: *mut *mut c_char, message: String) { if err_out.is_null() { return; @@ -806,6 +831,1225 @@ pub unsafe extern "C" fn acs_policy_free(handle: *mut AcsActivatedPolicy) { drop(Box::from_raw(handle)); } +// --------------------------------------------------------------------- +// Host extension points and manifest tooling. +// +// The engine takes an annotator dispatcher, a policy dispatcher, a +// telemetry sink and a perf level. The zero-config constructors above +// pick defaults for all four, which is right for a host that wants a +// policy decision and nothing else. A host that classifies through its +// own service, evaluates through its own engine, or records its own +// audit trail needs to supply them, and before these entry points +// existed there was no way in from any language but Rust. +// +// Callbacks cross the boundary as JSON and answer with JSON. A callback +// returns NULL and sets its own error string to fail, and the engine +// turns that into a fail-closed deny rather than treating it as an +// absent annotation: a classifier that could not be reached must not +// read as "found nothing". +// +// Ownership: a string the host returns is freed by the host, through +// the `free` callback registered alongside. The engine copies what it +// needs first. It never calls `acs_free_string` on host memory. +// --------------------------------------------------------------------- + +/// Free a string a host callback returned. +pub type AcsHookFree = unsafe extern "C" fn(ctx: *mut c_void, value: *mut c_char); + +/// Classify one annotation. Returns the annotation value as JSON, or +/// NULL with `*err_out` set. +pub type AcsAnnotatorFn = unsafe extern "C" fn( + ctx: *mut c_void, + annotator_name: *const c_char, + invocation_json: *const c_char, + policy_input_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char; + +/// Evaluate one prepared policy invocation. Returns the policy output as +/// JSON, or NULL with `*err_out` set. +pub type AcsPolicyFn = unsafe extern "C" fn( + ctx: *mut c_void, + invocation_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char; + +/// Receive one telemetry event as JSON. A sink cannot fail the +/// evaluation, so it has no error channel. +pub type AcsTelemetryFn = unsafe extern "C" fn(ctx: *mut c_void, event_json: *const c_char); + +// A host context is an opaque pointer the engine only hands back. The +// engine calls dispatchers from whatever thread is evaluating, so the +// host is responsible for its context being safe to use from more than +// one. Stated here because the compiler cannot check it. +struct HostCtx { + ctx: *mut c_void, + free: Option, +} + +unsafe impl Send for HostCtx {} +unsafe impl Sync for HostCtx {} + +impl HostCtx { + /// Copy a string the host returned, then hand the original back to + /// the host's own allocator. + unsafe fn take(&self, raw: *mut c_char) -> Option { + if raw.is_null() { + return None; + } + let copied = CStr::from_ptr(raw).to_str().ok().map(str::to_string); + if let Some(free) = self.free { + free(self.ctx, raw); + } + copied + } +} + +fn host_error(what: &str, err_out: *mut *mut c_char) -> RuntimeError { + let detail = if err_out.is_null() { + None + } else { + let raw = unsafe { *err_out }; + if raw.is_null() { + None + } else { + let message = unsafe { CStr::from_ptr(raw) } + .to_string_lossy() + .into_owned(); + unsafe { acs_free_string(raw) }; + Some(message) + } + }; + RuntimeError::PolicyInvocationFailed(match detail { + Some(message) => format!("host {what} failed: {message}"), + None => format!("host {what} failed without a message"), + }) +} + +struct HostAnnotatorDispatcher { + host: HostCtx, + call: AcsAnnotatorFn, +} + +impl AnnotatorDispatcher for HostAnnotatorDispatcher { + fn dispatch( + &self, + annotator_name: &str, + annotator: &AnnotatorInvocation, + preliminary_policy_input: &Value, + ) -> Result { + let name = CString::new(annotator_name).map_err(|_| { + RuntimeError::PolicyInvocationFailed("annotator name held a NUL".into()) + })?; + let invocation = CString::new(serde_json::to_string(annotator).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("annotator invocation: {e}")) + })?) + .map_err(|_| { + RuntimeError::PolicyInvocationFailed("annotator invocation held a NUL".into()) + })?; + let input = CString::new( + serde_json::to_string(preliminary_policy_input) + .map_err(|e| RuntimeError::PolicyInvocationFailed(format!("policy input: {e}")))?, + ) + .map_err(|_| RuntimeError::PolicyInvocationFailed("policy input held a NUL".into()))?; + + let mut err: *mut c_char = std::ptr::null_mut(); + let raw = unsafe { + (self.call)( + self.host.ctx, + name.as_ptr(), + invocation.as_ptr(), + input.as_ptr(), + &mut err, + ) + }; + let Some(json) = (unsafe { self.host.take(raw) }) else { + return Err(host_error("annotator", &mut err)); + }; + serde_json::from_str(&json).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("host annotator returned non JSON: {e}")) + }) + } +} + +struct HostPolicyDispatcher { + host: HostCtx, + call: AcsPolicyFn, +} + +impl PolicyDispatcher for HostPolicyDispatcher { + fn evaluate(&self, invocation: &PreparedPolicyInvocation) -> Result { + let payload = CString::new(serde_json::to_string(invocation).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("policy invocation: {e}")) + })?) + .map_err(|_| RuntimeError::PolicyInvocationFailed("policy invocation held a NUL".into()))?; + + let mut err: *mut c_char = std::ptr::null_mut(); + let raw = unsafe { (self.call)(self.host.ctx, payload.as_ptr(), &mut err) }; + let Some(json) = (unsafe { self.host.take(raw) }) else { + return Err(host_error("policy dispatcher", &mut err)); + }; + serde_json::from_str(&json).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("host policy returned non JSON: {e}")) + }) + } +} + +struct HostTelemetrySink { + host: HostCtx, + call: AcsTelemetryFn, +} + +impl TelemetrySink for HostTelemetrySink { + fn emit(&self, event: TelemetryEvent) { + // A sink cannot fail an evaluation, so a problem here drops the + // event rather than denying the action it describes. + let Ok(json) = serde_json::to_string(&wire::telemetry_event_json(&event)) else { + return; + }; + let Ok(payload) = CString::new(json) else { + return; + }; + unsafe { (self.call)(self.host.ctx, payload.as_ptr()) }; + } +} + +/// Build an interceptor with host-supplied extension points. +/// +/// Any callback may be NULL, which keeps the bundled default for that +/// slot, so a host overrides only what it needs. `perf_telemetry` is +/// `off`, `external` or `full`, and NULL means `off`. +/// +/// `annotator_ctx`, `policy_ctx` and `telemetry_ctx` are opaque to the +/// engine. Dispatch happens on whichever thread evaluates, so a context +/// shared across threads must be safe to use from all of them. +/// +/// Returns NULL and sets `*err_out` on failure. Free with +/// `acs_interceptor_free`. +/// +/// # Safety +/// `manifest_path` must point to `manifest_path_len` readable bytes. +/// Every non-null callback must remain valid, and every context must +/// stay alive, until the handle is freed. +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub unsafe extern "C" fn acs_interceptor_new_with_hooks( + manifest_path: *const u8, + manifest_path_len: usize, + annotator_fn: Option, + annotator_ctx: *mut c_void, + policy_fn: Option, + policy_ctx: *mut c_void, + telemetry_fn: Option, + telemetry_ctx: *mut c_void, + hook_free: Option, + perf_telemetry: *const c_char, + limits_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut AcsInterceptor { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let path = match read_path(manifest_path, manifest_path_len, err_out) { + Some(p) => p, + None => return std::ptr::null_mut(), + }; + let manifest = match Manifest::from_path(path) { + Ok(m) => m, + Err(e) => { + set_err(err_out, format!("{e}")); + return std::ptr::null_mut(); + } + }; + let perf = match read_optional(perf_telemetry, "perf_telemetry", err_out) { + Ok(Some(raw)) => match wire::parse_perf_telemetry(raw) { + Ok(level) => level, + Err(e) => { + set_err(err_out, format!("{e}")); + return std::ptr::null_mut(); + } + }, + Ok(None) => agent_control_spec::PerfTelemetry::Off, + Err(()) => return std::ptr::null_mut(), + }; + let limits = match read_optional(limits_json, "limits_json", err_out) { + Ok(Some(raw)) if !raw.trim().is_empty() => { + let parsed: Value = match serde_json::from_str(raw) { + Ok(v) => v, + Err(e) => { + set_err(err_out, format!("limits_json does not parse: {e}")); + return std::ptr::null_mut(); + } + }; + match wire::limits_from_json(&parsed) { + Ok(l) => l, + Err(e) => { + set_err(err_out, format!("{e}")); + return std::ptr::null_mut(); + } + } + } + Ok(_) => Limits::default(), + Err(()) => return std::ptr::null_mut(), + }; + + let annotations: Arc = match annotator_fn { + Some(call) => Arc::new(HostAnnotatorDispatcher { + host: HostCtx { + ctx: annotator_ctx, + free: hook_free, + }, + call, + }), + None => default_annotator_dispatcher(), + }; + let policy: Arc = match policy_fn { + Some(call) => Arc::new(HostPolicyDispatcher { + host: HostCtx { + ctx: policy_ctx, + free: hook_free, + }, + call, + }), + None => Arc::new(BindingPolicyDispatcher::new()), + }; + let telemetry: Arc = match telemetry_fn { + Some(call) => Arc::new(HostTelemetrySink { + host: HostCtx { + ctx: telemetry_ctx, + free: hook_free, + }, + call, + }), + None => Arc::new(NoopTelemetrySink), + }; + + match Runtime::with_telemetry_perf_and_limits( + manifest, + annotations, + policy, + telemetry, + perf, + limits, + ) { + Ok(runtime) => Box::into_raw(Box::new(AcsInterceptor { + runtime, + name: "acs".to_string(), + })), + Err(e) => { + set_err(err_out, format!("{e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_interceptor_new_with_hooks".to_string(), + ); + std::ptr::null_mut() + } + } +} + +/// Parse manifest text and return it as JSON. +/// +/// Parsing is not validation: this answers what the document says, which +/// an authoring or migration tool needs before the document is +/// runnable. Freed with `acs_free_string`. +/// +/// # Safety +/// `yaml` must be a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_manifest_parse( + yaml: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(source) = read_utf8(yaml, "yaml", err_out) else { + return std::ptr::null_mut(); + }; + match Manifest::from_yaml_str(source) { + Ok(manifest) => match serde_json::to_string(&manifest) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("manifest serialization failed: {e}")); + std::ptr::null_mut() + } + }, + Err(e) => { + set_err(err_out, format!("{e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err(err_out, "internal panic in acs_manifest_parse".to_string()); + std::ptr::null_mut() + } + } +} + +/// Compose a chain of manifest documents into one and return it as JSON. +/// +/// `yamls_json` is a JSON array of manifest sources, outermost base +/// first. This is the overlay case: a base policy plus the deltas an +/// environment layers on it, resolved the same way the engine resolves +/// `extends`. Freed with `acs_free_string`. +/// +/// # Safety +/// `yamls_json` must be a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_manifest_merge( + yamls_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(raw) = read_utf8(yamls_json, "yamls_json", err_out) else { + return std::ptr::null_mut(); + }; + let sources: Vec = match serde_json::from_str(raw) { + Ok(v) => v, + Err(e) => { + set_err( + err_out, + format!("yamls_json must be a JSON array of manifest sources: {e}"), + ); + return std::ptr::null_mut(); + } + }; + if sources.is_empty() { + set_err(err_out, "yamls_json must name at least one source".into()); + return std::ptr::null_mut(); + } + let borrowed: Vec<&str> = sources.iter().map(String::as_str).collect(); + match Manifest::from_yaml_chain(&borrowed) { + Ok(manifest) => match serde_json::to_string(&manifest) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("manifest serialization failed: {e}")); + std::ptr::null_mut() + } + }, + Err(e) => { + set_err(err_out, format!("{e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err(err_out, "internal panic in acs_manifest_merge".to_string()); + std::ptr::null_mut() + } + } +} + +/// Validate manifest text and return the findings as a JSON array. +/// +/// An empty array means valid. Each entry carries `code`, `message` and +/// `severity`. This is the shape an authoring tool or a CI linter needs: +/// `acs_validate_manifest` answers yes or no through an error string, +/// which cannot be rendered against a document. Freed with +/// `acs_free_string`. +/// +/// # Safety +/// `yaml` must be a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_manifest_diagnostics( + yaml: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(source) = read_utf8(yaml, "yaml", err_out) else { + return std::ptr::null_mut(); + }; + let findings = match Manifest::from_yaml_str(source) { + Ok(manifest) => match manifest.validate() { + Ok(()) => Vec::new(), + Err(e) => vec![wire::diagnostic_json(&e)], + }, + Err(e) => vec![wire::diagnostic_json(&e)], + }; + match serde_json::to_string(&findings) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("diagnostics serialization failed: {e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_manifest_diagnostics".to_string(), + ); + std::ptr::null_mut() + } + } +} + +/// Validate a manifest together with the Rego it names, and return the +/// findings as a JSON array. +/// +/// An empty array means both halves are sound. `acs_manifest_diagnostics` +/// answers only for the document: a manifest can name a bundle, satisfy +/// the grammar, and still fail at activation because the Rego does not +/// compile. Compilation happens at activation, so this activates against +/// the supplied bundles in memory and reports what that surfaced, which +/// moves the failure from a host's first agent action to its CI. +/// +/// `bundles_json` maps policy id to an in-memory bundle, the same shape +/// `acs_policy_activate_from_memory` takes. NULL or empty means the +/// manifest names no Rego, in which case this answers exactly as +/// `acs_manifest_diagnostics` does. Freed with `acs_free_string`. +/// +/// # Safety +/// `manifest_yaml` must be a valid NUL-terminated string. `bundles_json` +/// must be NULL or a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_artifact_diagnostics( + manifest_yaml: *const c_char, + bundles_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(source) = read_utf8(manifest_yaml, "manifest_yaml", err_out) else { + return std::ptr::null_mut(); + }; + let bundles = match read_in_memory_bundles(bundles_json, err_out) { + Some(bundles) => bundles, + None => return std::ptr::null_mut(), + }; + + // The manifest is checked first and on its own. A document that + // does not parse would otherwise be reported as an activation + // failure, which names the wrong half. + let findings = match Manifest::from_yaml_str(source) { + Err(e) => vec![wire::diagnostic_json(&e)], + Ok(manifest) => match manifest.validate() { + Err(e) => vec![wire::diagnostic_json(&e)], + Ok(()) => match ActivatedPolicy::activate_from_memory(source, bundles) { + Ok(_) => Vec::new(), + Err(e) => vec![wire::diagnostic_json(&e)], + }, + }, + }; + + match serde_json::to_string(&findings) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("diagnostics serialization failed: {e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_artifact_diagnostics".to_string(), + ); + std::ptr::null_mut() + } + } +} + +// --------------------------------------------------------------------- +// Incremental stream mediation (specification section 18.1). +// +// A `StreamSession` is stateful, so it follows the handle shape used by +// `AcsActivatedPolicy`: create once, drive it as payloads arrive, free +// exactly once. The runtime underneath stays stateless; the session only +// records what each ordinary evaluation cleared. +// +// Scalar queries return `i64` so an absent value needs no allocation: +// `>= 0` is the value, `-1` is absent (a released offset the caller must +// treat as "release nothing"), and `-2` means the call failed and +// `*err_out` carries why. Absent and failed are distinct because a +// settled session legitimately has no safe offset, which is not an error. +// +// Structured queries return JSON, freed with `acs_free_string`, so the +// wire contract is owned here rather than derived from Rust layout. +// --------------------------------------------------------------------- + +/// Parse a wire value through the core, reporting failure the way this +/// boundary does. The core owns what the names mean. +fn wire_track(value: &str, err_out: *mut *mut c_char) -> Option { + match StreamTrack::parse(value) { + Ok(track) => Some(track), + Err(e) => { + set_err(err_out, format!("{e}")); + None + } + } +} + +fn wire_outcome(value: &str, err_out: *mut *mut c_char) -> Option { + match SegmentOutcome::parse(value) { + Ok(outcome) => Some(outcome), + Err(e) => { + set_err(err_out, format!("{e}")); + None + } + } +} + +/// Opaque handle to one mediated stream. +/// +/// The session is behind a lock because its methods take `&mut self`, +/// unlike `AcsActivatedPolicy`, whose evaluation takes `&self` and is +/// `Send + Sync`. Without one, two host threads driving one stream race +/// in the engine, and a lost `observe` silently shortens the received +/// offset, which releases text no task evaluated. The lock costs +/// nothing next to the JSON on either side of this boundary, and it +/// means every C consumer gets the guarantee rather than each binding +/// having to rediscover it. +pub struct AcsStreamSession { + session: Mutex, +} + +/// Run `body` against the session, reporting a poisoned lock rather +/// than panicking across the C boundary. +/// +/// A poisoned lock means a previous call panicked while holding it, so +/// the accounting may be half-applied. Refusing is the only safe answer: +/// the alternative is releasing against state nobody can vouch for. +unsafe fn with_session( + handle: *const AcsStreamSession, + err_out: *mut *mut c_char, + failure: T, + body: impl FnOnce(&mut StreamSession) -> T, +) -> T { + if handle.is_null() { + set_err(err_out, "handle must not be null".to_string()); + return failure; + } + match (*handle).session.lock() { + Ok(mut guard) => body(&mut guard), + Err(_) => { + set_err( + err_out, + "stream session lock is poisoned by an earlier panic".to_string(), + ); + failure + } + } +} + +/// Open a session from `config_json`. +/// +/// The object takes `safety_level` (`blocking`, `complete` or +/// `deferred`), the per track start offsets `request_start_rune_offset` +/// and `response_start_rune_offset`, and the task name arrays +/// `request_tasks` and `response_tasks`. An absent field takes its +/// default; an empty task array means that track is unmediated. +/// +/// Returns NULL and sets `*err_out` on failure. Free with +/// `acs_stream_session_free`. +/// +/// # Safety +/// `config_json` must be a valid NUL-terminated string. `err_out` must +/// be null or point to a writable pointer. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_new( + config_json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut AcsStreamSession { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(raw) = read_utf8(config_json, "config_json", err_out) else { + return std::ptr::null_mut(); + }; + let parsed: Value = match serde_json::from_str(raw) { + Ok(v @ Value::Object(_)) => v, + Ok(_) => { + set_err(err_out, "config_json must be a JSON object".to_string()); + return std::ptr::null_mut(); + } + Err(e) => { + set_err(err_out, format!("config_json does not parse: {e}")); + return std::ptr::null_mut(); + } + }; + let level_raw = parsed + .get("safety_level") + .and_then(Value::as_str) + .unwrap_or("blocking"); + let safety_level = match SafetyLevel::parse(level_raw) { + Ok(l) => l, + Err(e) => { + set_err(err_out, format!("{e}")); + return std::ptr::null_mut(); + } + }; + let offset = |key: &str| -> Result { + match parsed.get(key) { + None | Some(Value::Null) => Ok(0), + Some(v) => v + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| format!("{key} must be a rune offset within u32")), + } + }; + let request_start_rune_offset = match offset("request_start_rune_offset") { + Ok(v) => v, + Err(e) => { + set_err(err_out, e); + return std::ptr::null_mut(); + } + }; + let response_start_rune_offset = match offset("response_start_rune_offset") { + Ok(v) => v, + Err(e) => { + set_err(err_out, e); + return std::ptr::null_mut(); + } + }; + let tasks = |key: &str| -> Result, String> { + match parsed.get(key) { + None | Some(Value::Null) => Ok(Vec::new()), + Some(Value::Array(items)) => items + .iter() + .map(|i| { + i.as_str() + .map(str::to_string) + .ok_or_else(|| format!("{key} must contain only task name strings")) + }) + .collect(), + Some(_) => Err(format!("{key} must be an array of task names")), + } + }; + let request_tasks = match tasks("request_tasks") { + Ok(v) => v, + Err(e) => { + set_err(err_out, e); + return std::ptr::null_mut(); + } + }; + let response_tasks = match tasks("response_tasks") { + Ok(v) => v, + Err(e) => { + set_err(err_out, e); + return std::ptr::null_mut(); + } + }; + let config = StreamSessionConfig { + safety_level, + request_start_rune_offset, + response_start_rune_offset, + request_tasks, + response_tasks, + }; + match StreamSession::new(config) { + Ok(session) => Box::into_raw(Box::new(AcsStreamSession { + session: Mutex::new(session), + })), + Err(e) => { + set_err(err_out, format!("{e}")); + std::ptr::null_mut() + } + } + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_stream_session_new".to_string(), + ); + std::ptr::null_mut() + } + } +} + +/// Free a session handle. Freeing NULL is a no-op. +/// +/// # Safety +/// `handle` must come from `acs_stream_session_new` and be freed once. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_free(handle: *mut AcsStreamSession) { + if handle.is_null() { + return; + } + let _ = catch_unwind(AssertUnwindSafe(|| drop(Box::from_raw(handle)))); +} + +/// Record that `runes` more runes of `source_type` arrived. Returns the +/// track's received offset, or -2 on failure. +/// +/// # Safety +/// `handle` must be live; `source_type` a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_observe( + handle: *mut AcsStreamSession, + source_type: *const c_char, + runes: u32, + err_out: *mut *mut c_char, +) -> i64 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(raw) = read_utf8(source_type, "source_type", err_out) else { + return -2; + }; + let source = match StreamSourceType::parse(raw) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -2; + } + }; + with_session(handle, err_out, -2, |s| match s.observe(source, runes) { + Ok(received) => i64::from(received), + Err(e) => { + set_err(err_out, format!("{e}")); + -2 + } + }) + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_observe".to_string(), + ); + -2 + }) +} + +/// Record an arriving payload by its text, counting runes the way the +/// engine does so a host never has to count them itself. Returns the +/// track's received offset, or -2 on failure. +/// +/// # Safety +/// `handle` must be live; `source_type` and `text` valid NUL-terminated +/// strings. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_observe_text( + handle: *mut AcsStreamSession, + source_type: *const c_char, + text: *const c_char, + err_out: *mut *mut c_char, +) -> i64 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(raw) = read_utf8(source_type, "source_type", err_out) else { + return -2; + }; + let source = match StreamSourceType::parse(raw) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -2; + } + }; + let Some(body) = read_utf8(text, "text", err_out) else { + return -2; + }; + with_session(handle, err_out, -2, |s| { + match s.observe_text(source, body) { + Ok(received) => i64::from(received), + Err(e) => { + set_err(err_out, format!("{e}")); + -2 + } + } + }) + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_observe_text".to_string(), + ); + -2 + }) +} + +/// Record what `task` decided about the span `[start, end)` of +/// `source_type`. `outcome` is `cleared`, `transformed` or `denied`. +/// Returns 0, or -1 on failure. +/// +/// # Safety +/// `handle` must be live; `task`, `source_type` and `outcome` valid +/// NUL-terminated strings. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_record_outcome( + handle: *mut AcsStreamSession, + task: *const c_char, + source_type: *const c_char, + start: u32, + end: u32, + outcome: *const c_char, + err_out: *mut *mut c_char, +) -> i32 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(task_name) = read_utf8(task, "task", err_out) else { + return -1; + }; + let Some(source_raw) = read_utf8(source_type, "source_type", err_out) else { + return -1; + }; + let source = match StreamSourceType::parse(source_raw) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -1; + } + }; + let Some(outcome_raw) = read_utf8(outcome, "outcome", err_out) else { + return -1; + }; + let Some(outcome) = wire_outcome(outcome_raw, err_out) else { + return -1; + }; + let span = match StreamSpan::new(source, start, end) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -1; + } + }; + with_session(handle, err_out, -1, |s| { + match s.record_outcome(task_name, &span, outcome) { + Ok(()) => 0, + Err(e) => { + set_err(err_out, format!("{e}")); + -1 + } + } + }) + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_record_outcome".to_string(), + ); + -1 + }) +} + +/// Record an Agent Control Specification verdict against the span +/// `[start, end)` of `source_type`, mapping its decision onto an +/// outcome. `verdict_json` is a verdict as `acs_policy_evaluate` +/// returns one, so a host feeds a decision straight back without +/// translating it. Returns 0, or -1 on failure. +/// +/// # Safety +/// `handle` must be live; `task`, `source_type` and `verdict_json` +/// valid NUL-terminated strings. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_record_verdict( + handle: *mut AcsStreamSession, + task: *const c_char, + source_type: *const c_char, + start: u32, + end: u32, + verdict_json: *const c_char, + err_out: *mut *mut c_char, +) -> i32 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(task_name) = read_utf8(task, "task", err_out) else { + return -1; + }; + let Some(source_raw) = read_utf8(source_type, "source_type", err_out) else { + return -1; + }; + let source = match StreamSourceType::parse(source_raw) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -1; + } + }; + let Some(raw) = read_utf8(verdict_json, "verdict_json", err_out) else { + return -1; + }; + let verdict: Verdict = match serde_json::from_str(raw) { + Ok(v) => v, + Err(e) => { + set_err(err_out, format!("verdict_json does not parse: {e}")); + return -1; + } + }; + let span = match StreamSpan::new(source, start, end) { + Ok(s) => s, + Err(e) => { + set_err(err_out, format!("{e}")); + return -1; + } + }; + with_session(handle, err_out, -1, |s| { + match s.record_verdict(task_name, &span, &verdict) { + Ok(()) => 0, + Err(e) => { + set_err(err_out, format!("{e}")); + -1 + } + } + }) + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_record_verdict".to_string(), + ); + -1 + }) +} + +/// Recompute `track`'s watermark and return the offset it advanced to, +/// -1 when it did not advance or the session has ended, -2 on failure. +/// +/// # Safety +/// `handle` must be live; `track` a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_advance( + handle: *mut AcsStreamSession, + track: *const c_char, + err_out: *mut *mut c_char, +) -> i64 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(raw) = read_utf8(track, "track", err_out) else { + return -2; + }; + let Some(track) = wire_track(raw, err_out) else { + return -2; + }; + with_session(handle, err_out, -2, |s| match s.advance(track) { + Some(offset) => i64::from(offset), + None => -1, + }) + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_advance".to_string(), + ); + -2 + }) +} + +/// The offset of `track` safe to release, -1 once the session has ended, +/// -2 on failure. A settled session has no safe offset, which is not an +/// error: it means release nothing further. +/// +/// # Safety +/// `handle` must be live; `track` a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_safe_offset( + handle: *const AcsStreamSession, + track: *const c_char, + err_out: *mut *mut c_char, +) -> i64 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(raw) = read_utf8(track, "track", err_out) else { + return -2; + }; + let Some(track) = wire_track(raw, err_out) else { + return -2; + }; + with_session(handle, err_out, -2, |s| match s.safe_offset(track) { + Some(offset) => i64::from(offset), + None => -1, + }) + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_safe_offset".to_string(), + ); + -2 + }) +} + +/// The rune count of `track` observed but not yet released, or -2 on +/// failure. +/// +/// # Safety +/// `handle` must be live; `track` a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_pending( + handle: *const AcsStreamSession, + track: *const c_char, + err_out: *mut *mut c_char, +) -> i64 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(raw) = read_utf8(track, "track", err_out) else { + return -2; + }; + let Some(track) = wire_track(raw, err_out) else { + return -2; + }; + with_session(handle, err_out, -2, |s| i64::from(s.pending(track))) + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_pending".to_string(), + ); + -2 + }) +} + +/// `track`'s watermark as JSON, carrying `track`, `confirmed`, +/// `received`, `pending` and the `tasks` that must clear it. The +/// confirmed offset stays readable after settlement, so an audit record +/// can still say how far the stream got. Freed with `acs_free_string`. +/// +/// # Safety +/// `handle` must be live; `track` a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_watermark( + handle: *const AcsStreamSession, + track: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + let Some(raw) = read_utf8(track, "track", err_out) else { + return std::ptr::null_mut(); + }; + let Some(track) = wire_track(raw, err_out) else { + return std::ptr::null_mut(); + }; + with_session(handle, err_out, std::ptr::null_mut(), |s| { + let watermark = s.watermark(track); + let payload = wire::watermark_json(track, watermark); + match serde_json::to_string(&payload) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("watermark serialization failed: {e}")); + std::ptr::null_mut() + } + } + }) + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_stream_session_watermark".to_string(), + ); + std::ptr::null_mut() + } + } +} + +/// Session state as JSON: `is_ended`, `transformed`, `end_reason` and +/// the effective `config`. `end_reason` is null while the session is +/// live. Freed with `acs_free_string`. +/// +/// # Safety +/// `handle` must be a live pointer from `acs_stream_session_new`. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_state( + handle: *const AcsStreamSession, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + with_session(handle, err_out, std::ptr::null_mut(), |s| { + let payload = wire::stream_session_state_json(s); + match serde_json::to_string(&payload) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("state serialization failed: {e}")); + std::ptr::null_mut() + } + } + }) + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_stream_session_state".to_string(), + ); + std::ptr::null_mut() + } + } +} + +/// Declare that no further payload will arrive. Returns 0, or -1 on +/// failure. +/// +/// # Safety +/// `handle` must be a live pointer from `acs_stream_session_new`. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_end_of_payloads( + handle: *mut AcsStreamSession, + err_out: *mut *mut c_char, +) -> i32 { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + with_session(handle, err_out, -1, |s| { + s.end_of_payloads(); + 0 + }) + })); + result.unwrap_or_else(|_| { + set_err( + err_out, + "internal panic in acs_stream_session_end_of_payloads".to_string(), + ); + -1 + }) +} + +/// Settle the session and return the completion as JSON, carrying +/// `reason`, `transformed` and `is_clean`. Freed with +/// `acs_free_string`. Settling twice returns the same completion. +/// +/// # Safety +/// `handle` must be a live pointer from `acs_stream_session_new`. +#[no_mangle] +pub unsafe extern "C" fn acs_stream_session_finish( + handle: *mut AcsStreamSession, + err_out: *mut *mut c_char, +) -> *mut c_char { + clear_err(err_out); + let result = catch_unwind(AssertUnwindSafe(|| { + with_session(handle, err_out, std::ptr::null_mut(), |s| { + let completion = s.finish(); + let payload = wire::completion_json(&completion); + match serde_json::to_string(&payload) { + Ok(json) => to_c_string(json, err_out), + Err(e) => { + set_err(err_out, format!("completion serialization failed: {e}")); + std::ptr::null_mut() + } + } + }) + })); + match result { + Ok(ptr) => ptr, + Err(_) => { + set_err( + err_out, + "internal panic in acs_stream_session_finish".to_string(), + ); + std::ptr::null_mut() + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/sdk/node/binding.d.ts b/sdk/node/binding.d.ts index 6684d84..2c6a8c3 100644 --- a/sdk/node/binding.d.ts +++ b/sdk/node/binding.d.ts @@ -7,6 +7,14 @@ export declare class ExternalObject { [K: symbol]: T } } +/** + * The engine's default resource caps as a JSON object string. A host + * that raises one cap reads this to see what it is overriding, so a + * shipping change to another default cannot be silently absorbed by a + * stale mapping. + */ +export declare function defaultLimits(): string + /** * Evaluate one agent context (JSON object per AGENT-HOOKS-0.1 §4) and * return the verdict as wire JSON. @@ -20,6 +28,54 @@ export declare function intercept(handle: ExternalObject, contextJson: s */ export declare function interceptorNew(manifestPath: string): ExternalObject +/** + * Build a runtime handle from a manifest path, optionally overriding + * the annotator dispatcher, policy dispatcher, telemetry sink, perf + * telemetry level, and resource caps. + * + * Every callback is optional: absent means keep the zero-config + * default for that slot. Callbacks cross the boundary as JSON strings, + * mirroring the FFI hook contract, so a host that already sits behind + * a JSON schema does not re-model its wire shape for this SDK. + * + * `limits_json` is a JSON object of resource caps overriding the + * engine's defaults field by field. Null or empty means keep every + * default; each field is individually optional, so a host raising one + * cap does not restate the other nine. A host feeding large payloads + * raises `max_snapshot_bytes`; one hardening against a hostile + * manifest lowers `max_extends_depth` or `manifest_url_timeout_ms`. + * A field present but not a non-negative integer is a hard ERROR, not + * a silently-kept default. + * + * Callbacks are called SYNCHRONOUSLY on the JS thread from inside the + * engine's evaluation. A callback that throws surfaces as a fail-closed + * `runtime_error:*` deny (annotator → `annotation_failed`, policy → + * `policy_invocation_failed`) rather than silently reading as "no + * annotation". + */ +export declare function interceptorNewWithHooks(manifestPath: string, annotatorDispatcher?: ((arg0: string, arg1: string, arg2: string) => string) | undefined | null, policyDispatcher?: ((arg0: string) => string) | undefined | null, telemetrySink?: ((arg0: string) => void) | undefined | null, perfTelemetry?: string | undefined | null, limitsJson?: string | undefined | null): ExternalObject + +/** + * Compose a chain of manifest YAML documents (outermost base first) + * into one merged manifest, returned as JSON. + * + * This is the overlay case: a base policy plus deltas an environment + * layers on it, resolved the same way the engine resolves `extends`. + */ +export declare function mergeManifests(sourcesJson: string): string + +/** + * Parse manifest YAML into an object (JSON encoded) without + * validating cross-references. + * + * The document is deserialized as-written: a manifest with an + * unresolved `extends` chain parses fine, and returning it lets an + * authoring tool see the fragment. Use `validate_manifest` or + * `validate_manifest_detailed` to judge whether the fragment is + * runnable. + */ +export declare function parseManifest(source: string): string + /** * Activate the manifest at `manifest_path`, readying every policy it * binds, against the zero-config dispatchers. @@ -55,6 +111,19 @@ export declare function policyActivate(manifestPath: string): ExternalObject +/** + * Activate a manifest and its Rego from memory against host-supplied + * dispatchers. + */ +export declare function policyActivateFromMemoryWithHooks(manifestYaml: string, bundlesJson: string, annotatorDispatcher?: ((arg0: string, arg1: string, arg2: string) => string) | undefined | null, policyDispatcher?: ((arg0: string) => string) | undefined | null): ExternalObject + +/** + * Activate the manifest at `manifest_path` against host-supplied + * dispatchers. See `interceptor_new_with_hooks` for the callback + * contract. + */ +export declare function policyActivateWithHooks(manifestPath: string, annotatorDispatcher?: ((arg0: string, arg1: string, arg2: string) => string) | undefined | null, policyDispatcher?: ((arg0: string) => string) | undefined | null): ExternalObject + /** * Evaluate one intervention point against an activated policy and * return the verdict as wire JSON. @@ -76,9 +145,122 @@ export declare function policyEvaluate(handle: ExternalObject, poi */ export declare function policyInterventionPoints(handle: ExternalObject): Array +/** + * Recompute `track`'s watermark. Returns the new offset when the + * watermark advanced, `null` when it did not or the session has ended + * (matching the Rust `Option`). + */ +export declare function streamSessionAdvance(handle: ExternalObject, track: string): number | null + +/** Declare that no further payload will arrive. Idempotent. */ +export declare function streamSessionEndOfPayloads(handle: ExternalObject): void + +/** + * Settle the session and return the completion as JSON, carrying + * `reason`, `transformed` and `is_clean`. Settling twice returns the + * same completion. + */ +export declare function streamSessionFinish(handle: ExternalObject): string + +/** + * Open a session from a config JSON object. + * + * Matches `acs_stream_session_new`: takes `safety_level` (`blocking`, + * `complete` or `deferred`), the per-track start offsets + * `request_start_rune_offset` and `response_start_rune_offset`, and + * the task name arrays `request_tasks` and `response_tasks`. An empty + * task array means that track is unmediated; payload on it fails + * closed. A configuration mediating neither track is refused. + * + * The wrapper takes a JSON string rather than a napi object so the + * config surface is identical to the other language SDKs and offset + * coercion happens in one place. + */ +export declare function streamSessionNew(configJson: string): ExternalObject + +/** + * Report that `runes` more runes of `source_type` arrived and return + * the track's new end offset. Boundary failures throw; a streaming + * accounting failure throws with the engine's message and puts the + * session into its terminal state. + */ +export declare function streamSessionObserve(handle: ExternalObject, sourceType: string, runes: number): number + +/** + * Report arriving `text` on `source_type`, counting Unicode scalars so + * a host does not have to. Returns the track's new end offset. + * + * The engine counts runes, not UTF-16 code units. `Utf16String` yields + * UTF-16, so the binding decodes to a `String` before delegating to + * the engine and rune counting stays consistent with every other SDK. + * An astral-plane character is one rune here even though it is two + * UTF-16 code units. + */ +export declare function streamSessionObserveText(handle: ExternalObject, sourceType: string, text: string): number + +/** Runes on `track` observed but not yet released. */ +export declare function streamSessionPending(handle: ExternalObject, track: string): number + +/** + * Record what `task` decided about the span `[start, end)` of + * `source_type`. `outcome` is `cleared`, `transformed` or `denied`. + */ +export declare function streamSessionRecordOutcome(handle: ExternalObject, task: string, sourceType: string, start: number, end: number, outcome: string): void + +/** + * Record an ACS verdict against the span `[start, end)` of + * `source_type`, mapping its decision onto an outcome. A host feeds + * the JSON returned by `policyEvaluate` straight back without + * translating it. + */ +export declare function streamSessionRecordVerdict(handle: ExternalObject, task: string, sourceType: string, start: number, end: number, verdictJson: string): void + +/** + * Offset of `track` the host may release through, or `null` once the + * session has ended. A settled session has no safe offset, which is + * not an error: it means release nothing further. + */ +export declare function streamSessionSafeOffset(handle: ExternalObject, track: string): number | null + +/** + * Session state as JSON: `is_ended`, `transformed`, `end_reason` + * (null while live) and the effective `config`. + */ +export declare function streamSessionState(handle: ExternalObject): string + +/** + * `track`'s watermark as JSON, carrying `track`, `confirmed`, + * `received`, `pending` and the `tasks` that must clear it. The + * confirmed offset stays readable after settlement, so an audit + * record can still say how far the stream got. + */ +export declare function streamSessionWatermark(handle: ExternalObject, track: string): string + /** The manifest grammar versions this engine accepts. */ export declare function supportedManifestVersions(): Array +/** + * Validate a manifest together with the Rego it names, and return + * findings as a JSON array. + * + * An empty array means both halves are sound. Each entry has wire + * shape `{"code": str, "message": str, "severity": "error"}`, matching + * the C ABI's `acs_artifact_diagnostics`. + * + * `validate_manifest_detailed` answers only for the document: a + * manifest can name a bundle, satisfy the grammar, and still fail at + * activation because the Rego does not compile. Compilation happens + * at activation, so this activates against the supplied bundles in + * memory and reports what that surfaced, which moves the failure from + * a host's first agent action to its CI. + * + * `bundles_json` maps policy id to an in-memory bundle, the same + * shape `policy_activate_from_memory` takes. An empty document means + * the manifest names no Rego, and the answer then equals what + * `validate_manifest_detailed` reports for the manifest half. + */ +export declare function validateArtifactsDetailed(manifestYaml: string, bundlesJson: string): string + /** * Validate manifest source against the grammar, without building a * runtime. @@ -95,6 +277,18 @@ export declare function supportedManifestVersions(): Array */ export declare function validateManifest(source: string): string | null +/** + * Validate manifest source and return findings as a JSON array. + * + * An empty array means the manifest is valid. Each entry carries + * `code` (`runtime_error:*`), `message` (engine detail), `severity`, + * and a best-effort `field` extracted from the message. This is the + * shape an authoring tool or CI linter needs; `validate_manifest` + * answers yes/no with a single message and cannot be rendered + * per-field. + */ +export declare function validateManifestDetailed(source: string): string + /** * Validate a manifest file, resolving `extends` first. * diff --git a/sdk/node/binding.js b/sdk/node/binding.js index 5d14e01..ebc8da4 100644 --- a/sdk/node/binding.js +++ b/sdk/node/binding.js @@ -700,12 +700,32 @@ if (!nativeBinding) { } module.exports = nativeBinding +module.exports.defaultLimits = nativeBinding.defaultLimits module.exports.intercept = nativeBinding.intercept module.exports.interceptorNew = nativeBinding.interceptorNew +module.exports.interceptorNewWithHooks = nativeBinding.interceptorNewWithHooks +module.exports.mergeManifests = nativeBinding.mergeManifests +module.exports.parseManifest = nativeBinding.parseManifest module.exports.policyActivate = nativeBinding.policyActivate module.exports.policyActivateFromMemory = nativeBinding.policyActivateFromMemory +module.exports.policyActivateFromMemoryWithHooks = nativeBinding.policyActivateFromMemoryWithHooks +module.exports.policyActivateWithHooks = nativeBinding.policyActivateWithHooks module.exports.policyEvaluate = nativeBinding.policyEvaluate module.exports.policyInterventionPoints = nativeBinding.policyInterventionPoints +module.exports.streamSessionAdvance = nativeBinding.streamSessionAdvance +module.exports.streamSessionEndOfPayloads = nativeBinding.streamSessionEndOfPayloads +module.exports.streamSessionFinish = nativeBinding.streamSessionFinish +module.exports.streamSessionNew = nativeBinding.streamSessionNew +module.exports.streamSessionObserve = nativeBinding.streamSessionObserve +module.exports.streamSessionObserveText = nativeBinding.streamSessionObserveText +module.exports.streamSessionPending = nativeBinding.streamSessionPending +module.exports.streamSessionRecordOutcome = nativeBinding.streamSessionRecordOutcome +module.exports.streamSessionRecordVerdict = nativeBinding.streamSessionRecordVerdict +module.exports.streamSessionSafeOffset = nativeBinding.streamSessionSafeOffset +module.exports.streamSessionState = nativeBinding.streamSessionState +module.exports.streamSessionWatermark = nativeBinding.streamSessionWatermark module.exports.supportedManifestVersions = nativeBinding.supportedManifestVersions +module.exports.validateArtifactsDetailed = nativeBinding.validateArtifactsDetailed module.exports.validateManifest = nativeBinding.validateManifest +module.exports.validateManifestDetailed = nativeBinding.validateManifestDetailed module.exports.validateManifestFile = nativeBinding.validateManifestFile diff --git a/sdk/node/native/Cargo.toml b/sdk/node/native/Cargo.toml index c7f4d91..f23fb40 100644 --- a/sdk/node/native/Cargo.toml +++ b/sdk/node/native/Cargo.toml @@ -11,7 +11,7 @@ name = "agent_control_spec_node" crate-type = ["cdylib"] [dependencies] -agent-control-spec = { path = "../../../engine", features = ["default-dispatchers"] } +agent-control-spec = { path = "../../../engine", features = ["default-dispatchers", "streaming"] } napi = { version = "3", default-features = false, features = ["napi8"] } napi-derive = "3" serde_json = "1" diff --git a/sdk/node/native/src/lib.rs b/sdk/node/native/src/lib.rs index 974322e..49cb51c 100644 --- a/sdk/node/native/src/lib.rs +++ b/sdk/node/native/src/lib.rs @@ -8,16 +8,24 @@ // mean a boundary problem only (unreadable manifest, non-object // context JSON). +use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; +use agent_control_spec::runtime::PolicyDispatcher; +use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; +use agent_control_spec::wire; +use agent_control_spec::Verdict; use agent_control_spec::{ - ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, Runtime, RuntimeError, - SUPPORTED_VERSIONS, + ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, JsonValue, Limits, Manifest, + PreparedPolicyInvocation, Runtime, RuntimeError, SegmentOutcome, StreamError, StreamSession, + StreamSourceType, StreamSpan, StreamTrack, SUPPORTED_VERSIONS, }; -use napi::bindgen_prelude::{External, Utf16String}; +use napi::bindgen_prelude::{External, FnArgs, FunctionRef, Utf16String}; +use napi::Env; use napi_derive::napi; use serde_json::Value; +use std::cell::Cell; use std::collections::BTreeMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; pub struct Handle { runtime: Runtime, @@ -54,6 +62,263 @@ fn decode(what: &str, value: &Utf16String) -> napi::Result { String::from_utf16(value).map_err(|_| err(format!("{what} contains an unpaired surrogate"))) } +// --------------------------------------------------------------------- +// Host dispatcher plumbing (annotator, policy, telemetry). +// +// The engine calls a dispatcher SYNCHRONOUSLY from inside its +// evaluation, on whichever thread called into it. Every napi entry +// point that drives evaluation (`intercept`, `policy_evaluate`) runs +// on the JS thread, so a callback fired inside the engine is on the +// same JS thread as the caller and can call the JS function directly +// through a `Function` handle. That handle is scope-bound, so we hold a +// `FunctionRef` (Send + Sync) and `borrow_back` it against the current +// napi `Env` when the engine asks. The env is stored in a thread-local +// set by each entry point for the duration of the call, so a dispatcher +// invoked from a different thread errors out rather than reaching into +// V8 off-thread. +// +// A ThreadsafeFunction would be wrong here: it dispatches ASYNCHRONOUSLY +// to the JS thread, which deadlocks when the JS thread is already +// blocked in the engine call that produced the callback. +// --------------------------------------------------------------------- + +thread_local! { + // Set only while a napi entry point that drives engine evaluation + // is on the stack. A dispatcher invoked with no env available is + // treated as a host failure so the engine fails closed. + static CURRENT_ENV: Cell = + const { Cell::new(std::ptr::null_mut()) }; +} + +/// RAII guard binding a napi `Env` to the current thread for the +/// duration of an engine call. Nesting is not expected (napi calls +/// don't reenter), but the guard is nesting-safe: it saves the previous +/// value and restores it on drop. +struct EnvScope { + previous: napi::sys::napi_env, +} + +impl EnvScope { + fn enter(env: &Env) -> Self { + let raw = env.raw(); + let previous = CURRENT_ENV.with(|c| c.replace(raw)); + Self { previous } + } +} + +impl Drop for EnvScope { + fn drop(&mut self) { + let previous = self.previous; + CURRENT_ENV.with(|c| c.set(previous)); + } +} + +fn with_current_env( + what: &str, + kind: fn(String) -> RuntimeError, + f: impl FnOnce(&Env) -> Result, +) -> Result { + let raw = CURRENT_ENV.with(|c| c.get()); + if raw.is_null() { + return Err(kind(format!( + "host {what} was invoked without a live napi env; this dispatcher can only be \ + called from a napi entry point on the JS thread" + ))); + } + // SAFETY: `raw` was captured by `EnvScope::enter` from the napi + // entry point currently on the stack, on this same JS thread. + let env = Env::from_raw(raw); + f(&env) +} + +struct NodeAnnotatorDispatcher { + func: FunctionRef, String>, +} + +impl AnnotatorDispatcher for NodeAnnotatorDispatcher { + fn dispatch( + &self, + annotator_name: &str, + annotator: &AnnotatorInvocation, + preliminary_policy_input: &JsonValue, + ) -> Result { + let invocation_json = serde_json::to_string(annotator).map_err(|e| { + RuntimeError::AnnotationFailed(format!("serialize annotator invocation: {e}")) + })?; + let policy_input_json = serde_json::to_string(preliminary_policy_input).map_err(|e| { + RuntimeError::AnnotationFailed(format!("serialize preliminary policy input: {e}")) + })?; + with_current_env( + "annotator dispatcher", + RuntimeError::AnnotationFailed, + |env| { + let func = self.func.borrow_back(env).map_err(|e| { + RuntimeError::AnnotationFailed(format!("reacquire annotator function: {e}")) + })?; + let raw = func + .call(FnArgs { + data: ( + annotator_name.to_string(), + invocation_json, + policy_input_json, + ), + }) + .map_err(|e| { + RuntimeError::AnnotationFailed(format!( + "host annotator dispatcher threw: {e}" + )) + })?; + serde_json::from_str::(&raw).map_err(|e| { + RuntimeError::AnnotationFailed(format!( + "host annotator dispatcher returned non-JSON: {e}" + )) + }) + }, + ) + } +} + +struct NodePolicyDispatcher { + func: FunctionRef, String>, +} + +impl PolicyDispatcher for NodePolicyDispatcher { + fn evaluate(&self, invocation: &PreparedPolicyInvocation) -> Result { + let invocation_json = serde_json::to_string(invocation).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("serialize policy invocation: {e}")) + })?; + with_current_env( + "policy dispatcher", + RuntimeError::PolicyInvocationFailed, + |env| { + let func = self.func.borrow_back(env).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!("reacquire policy function: {e}")) + })?; + let raw = func + .call(FnArgs { + data: (invocation_json,), + }) + .map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!( + "host policy dispatcher threw: {e}" + )) + })?; + serde_json::from_str::(&raw).map_err(|e| { + RuntimeError::PolicyInvocationFailed(format!( + "host policy dispatcher returned non-JSON: {e}" + )) + }) + }, + ) + } +} + +struct NodeTelemetrySink { + func: FunctionRef, ()>, +} + +impl TelemetrySink for NodeTelemetrySink { + fn emit(&self, event: TelemetryEvent) { + // TelemetryEvent is not Serialize, so the wire shape lives in + // `wire::telemetry_event_json`. Every binding then hands the + // sink the same JSON, and a sink written for one language reads + // the same fields in another. A sink cannot fail an + // evaluation, so every step drops on error rather than + // propagating. + let payload = wire::telemetry_event_json(&event); + let Ok(json) = serde_json::to_string(&payload) else { + return; + }; + let raw = CURRENT_ENV.with(|c| c.get()); + if raw.is_null() { + return; + } + // SAFETY: same as `with_current_env`; the sink is called from + // engine code that is itself running inside a napi entry point. + let env = Env::from_raw(raw); + let Ok(func) = self.func.borrow_back(&env) else { + return; + }; + let _: napi::Result<()> = func.call(FnArgs { data: (json,) }); + } +} + +fn parse_perf(value: Option) -> napi::Result { + let Some(value) = value else { + return Ok(agent_control_spec::PerfTelemetry::Off); + }; + let raw = decode("perfTelemetry", &value)?; + // The engine owns what a level name means; this boundary only + // decodes UTF-16 and reshapes the error for napi. + wire::parse_perf_telemetry(&raw).map_err(|e| err(format!("{e}"))) +} + +/// Read a limits override. +/// +/// - Absent / null / empty means keep every default. +/// - Each field is individually optional; an absent field keeps its own +/// default, so a host raising one cap does not restate the other nine. +/// - A field present but not a non-negative integer is a hard ERROR, not +/// a silently-kept default. A host that asked for a smaller bound and +/// got the larger one would believe it was protected when it was not. +/// - An unknown/misspelled field is refused rather than silently +/// ignored, so a typo cannot mask a bound the host believes it set +/// but did not set. +/// +/// Field-by-field acceptance and the misspelled-key refusal live in +/// [`wire::limits_from_json`]; this boundary only decodes UTF-16, +/// parses the JSON, and reshapes the error into a napi error. +fn parse_limits(value: Option) -> napi::Result { + let Some(value) = value else { + return Ok(Limits::default()); + }; + let raw = decode("limits", &value)?; + if raw.trim().is_empty() { + return Ok(Limits::default()); + } + let parsed: Value = + serde_json::from_str(&raw).map_err(|e| err(format!("limits does not parse: {e}")))?; + wire::limits_from_json(&parsed).map_err(|e| err(format!("{e}"))) +} + +/// The engine's default resource caps as a JSON object string. A host +/// that raises one cap reads this to see what it is overriding, so a +/// shipping change to another default cannot be silently absorbed by a +/// stale mapping. +#[napi] +pub fn default_limits() -> napi::Result { + serde_json::to_string(&wire::limits_json(&Limits::default())).map_err(|e| err(format!("{e}"))) +} + +// Type aliases for the FunctionRef signatures the JS↔Rust wire uses. +// Kept private so they never leak into the TS declaration; napi-derive +// still sees the expanded types on the entry points that take them as +// arguments (aliases are not expanded through the `#[napi]` macro). +type NodeAnnotatorFn = FunctionRef, String>; +type NodePolicyFn = FunctionRef, String>; +type NodeTelemetryFn = FunctionRef, ()>; + +fn build_annotator(dispatcher: Option) -> Arc { + match dispatcher { + Some(func) => Arc::new(NodeAnnotatorDispatcher { func }), + None => default_annotator_dispatcher(), + } +} + +fn build_policy(dispatcher: Option) -> Arc { + match dispatcher { + Some(func) => Arc::new(NodePolicyDispatcher { func }), + None => Arc::new(BindingPolicyDispatcher::new()), + } +} + +fn build_telemetry(sink: Option) -> Arc { + match sink { + Some(func) => Arc::new(NodeTelemetrySink { func }), + None => Arc::new(NoopTelemetrySink), + } +} + /// Build a runtime handle from a manifest path using the zero-config /// dispatchers (bundled annotators; Rego in process, Cedar through the /// built-in evaluator, `test` policies through their embedded verdict). @@ -70,16 +335,76 @@ pub fn interceptor_new(manifest_path: Utf16String) -> napi::Result, String>>, + policy_dispatcher: Option, String>>, + telemetry_sink: Option, ()>>, + perf_telemetry: Option, + limits_json: Option, +) -> napi::Result> { + let manifest_path = decode("manifest_path", &manifest_path)?; + let manifest = Manifest::from_path(&manifest_path).map_err(|e| err(format!("{e}")))?; + let perf = parse_perf(perf_telemetry)?; + let limits = parse_limits(limits_json)?; + let annotations = build_annotator(annotator_dispatcher); + let policy = build_policy(policy_dispatcher); + let telemetry = build_telemetry(telemetry_sink); + let runtime = Runtime::with_telemetry_perf_and_limits( + manifest, + annotations, + policy, + telemetry, + perf, + limits, + ) + .map_err(|e| err(format!("{e}")))?; + Ok(External::new(Handle { runtime })) +} + /// Evaluate one agent context (JSON object per AGENT-HOOKS-0.1 §4) and /// return the verdict as wire JSON. #[napi] -pub fn intercept(handle: &External, context_json: Utf16String) -> napi::Result { +pub fn intercept( + env: Env, + handle: &External, + context_json: Utf16String, +) -> napi::Result { let context_json = decode("context_json", &context_json)?; let snapshot: Value = serde_json::from_str(&context_json) .map_err(|e| err(format!("context_json does not parse: {e}")))?; if !snapshot.is_object() { return Err(err("context_json must be a JSON object".to_string())); } + // The engine may call a host dispatcher synchronously from inside + // `evaluate`; the scope publishes the current napi env for that + // callback and is torn down before we return. + let _scope = EnvScope::enter(&env); let verdict = handle.runtime.evaluate(&snapshot).verdict; serde_json::to_string(&verdict).map_err(|e| err(format!("verdict serialization failed: {e}"))) } @@ -156,6 +481,47 @@ pub fn policy_activate_from_memory( Ok(External::new(PolicyHandle { policy })) } +/// Activate the manifest at `manifest_path` against host-supplied +/// dispatchers. See `interceptor_new_with_hooks` for the callback +/// contract. +#[napi] +#[allow(clippy::type_complexity)] +pub fn policy_activate_with_hooks( + manifest_path: Utf16String, + annotator_dispatcher: Option, String>>, + policy_dispatcher: Option, String>>, +) -> napi::Result> { + let manifest_path = decode("manifest_path", &manifest_path)?; + let manifest = Manifest::from_path(&manifest_path).map_err(|e| err(format!("{e}")))?; + let annotations = build_annotator(annotator_dispatcher); + let policy = build_policy(policy_dispatcher); + let handle = ActivatedPolicy::activate_with(manifest, annotations, policy) + .map_err(|e| err(format!("{e}")))?; + Ok(External::new(PolicyHandle { policy: handle })) +} + +/// Activate a manifest and its Rego from memory against host-supplied +/// dispatchers. +#[napi] +#[allow(clippy::type_complexity)] +pub fn policy_activate_from_memory_with_hooks( + manifest_yaml: Utf16String, + bundles_json: Utf16String, + annotator_dispatcher: Option, String>>, + policy_dispatcher: Option, String>>, +) -> napi::Result> { + let manifest_yaml = decode("manifest_yaml", &manifest_yaml)?; + let bundles_json = decode("bundles_json", &bundles_json)?; + let bundles: BTreeMap = serde_json::from_str(&bundles_json) + .map_err(|e| err(format!("bundles_json does not parse: {e}")))?; + let annotations = build_annotator(annotator_dispatcher); + let policy = build_policy(policy_dispatcher); + let handle = + ActivatedPolicy::activate_from_memory_with(&manifest_yaml, bundles, annotations, policy) + .map_err(|e| err(format!("{e}")))?; + Ok(External::new(PolicyHandle { policy: handle })) +} + /// Evaluate one intervention point against an activated policy and /// return the verdict as wire JSON. /// @@ -169,6 +535,7 @@ pub fn policy_activate_from_memory( /// and throws. #[napi] pub fn policy_evaluate( + env: Env, handle: &External, point: Utf16String, context_json: Utf16String, @@ -183,6 +550,7 @@ pub fn policy_evaluate( if !snapshot.is_object() { return Err(err("context_json must be a JSON object".to_string())); } + let _scope = EnvScope::enter(&env); let verdict = handle.policy.evaluate(point, snapshot).verdict; serde_json::to_string(&verdict).map_err(|e| err(format!("verdict serialization failed: {e}"))) } @@ -257,3 +625,387 @@ pub fn supported_manifest_versions() -> Vec { .map(|v| (*v).to_string()) .collect() } + +// --------------------------------------------------------------------- +// Manifest tooling: parse, chain, structured diagnostics. +// +// The engine ships these as first-party APIs on `Manifest`; the wrapper +// exposes them here so authoring, migration, and CI tooling can build +// on the same surface across languages. Every entry point takes YAML +// text, so a caller can drive them without staging a file on disk. +// --------------------------------------------------------------------- + +/// Parse manifest YAML into an object (JSON encoded) without +/// validating cross-references. +/// +/// The document is deserialized as-written: a manifest with an +/// unresolved `extends` chain parses fine, and returning it lets an +/// authoring tool see the fragment. Use `validate_manifest` or +/// `validate_manifest_detailed` to judge whether the fragment is +/// runnable. +#[napi] +pub fn parse_manifest(source: Utf16String) -> napi::Result { + let source = decode("source", &source)?; + let manifest = Manifest::parse_yaml_str(&source).map_err(|e| err(format!("{e}")))?; + serde_json::to_string(&manifest).map_err(|e| err(format!("manifest serialization failed: {e}"))) +} + +/// Compose a chain of manifest YAML documents (outermost base first) +/// into one merged manifest, returned as JSON. +/// +/// This is the overlay case: a base policy plus deltas an environment +/// layers on it, resolved the same way the engine resolves `extends`. +#[napi] +pub fn merge_manifests(sources_json: Utf16String) -> napi::Result { + let raw = decode("sources_json", &sources_json)?; + let sources: Vec = serde_json::from_str(&raw).map_err(|e| { + err(format!( + "sources_json must be a JSON array of manifest sources: {e}" + )) + })?; + if sources.is_empty() { + return Err(err("sources_json must name at least one source".to_string())); + } + let borrowed: Vec<&str> = sources.iter().map(String::as_str).collect(); + let manifest = Manifest::from_yaml_chain(&borrowed).map_err(|e| err(format!("{e}")))?; + serde_json::to_string(&manifest).map_err(|e| err(format!("manifest serialization failed: {e}"))) +} + +/// Validate manifest source and return findings as a JSON array. +/// +/// An empty array means the manifest is valid. Each entry carries +/// `code` (`runtime_error:*`), `message` (engine detail), `severity`, +/// and a best-effort `field` extracted from the message. This is the +/// shape an authoring tool or CI linter needs; `validate_manifest` +/// answers yes/no with a single message and cannot be rendered +/// per-field. +#[napi] +pub fn validate_manifest_detailed(source: Utf16String) -> napi::Result { + let source = decode("source", &source)?; + let findings = match Manifest::parse_yaml_str(&source) { + Ok(manifest) => { + if !manifest.extends.is_empty() { + // A fragment cannot be judged against itself: its + // parent may define the annotator or policy this + // document references. Report that as a single + // finding rather than silently blaming references + // the fragment does not own. + vec![wire::diagnostic_json(&RuntimeError::ManifestInvalid( + "manifest extends other manifests; validation needs the merged document. \ + Use validate_manifest_file, which resolves the chain." + .to_string(), + ))] + } else { + match manifest.validate() { + Ok(()) => Vec::new(), + Err(e) => vec![wire::diagnostic_json(&e)], + } + } + } + Err(e) => vec![wire::diagnostic_json(&e)], + }; + serde_json::to_string(&findings) + .map_err(|e| err(format!("diagnostics serialization failed: {e}"))) +} + +/// Validate a manifest together with the Rego it names, and return +/// findings as a JSON array. +/// +/// An empty array means both halves are sound. Each entry has wire +/// shape `{"code": str, "message": str, "severity": "error"}`, matching +/// the C ABI's `acs_artifact_diagnostics`. +/// +/// `validate_manifest_detailed` answers only for the document: a +/// manifest can name a bundle, satisfy the grammar, and still fail at +/// activation because the Rego does not compile. Compilation happens +/// at activation, so this activates against the supplied bundles in +/// memory and reports what that surfaced, which moves the failure from +/// a host's first agent action to its CI. +/// +/// `bundles_json` maps policy id to an in-memory bundle, the same +/// shape `policy_activate_from_memory` takes. An empty document means +/// the manifest names no Rego, and the answer then equals what +/// `validate_manifest_detailed` reports for the manifest half. +#[napi] +pub fn validate_artifacts_detailed( + manifest_yaml: Utf16String, + bundles_json: Utf16String, +) -> napi::Result { + let manifest_yaml = decode("manifest_yaml", &manifest_yaml)?; + let bundles_json = decode("bundles_json", &bundles_json)?; + let bundles: BTreeMap = if bundles_json.trim().is_empty() { + BTreeMap::new() + } else { + serde_json::from_str(&bundles_json) + .map_err(|e| err(format!("bundles_json does not parse: {e}")))? + }; + // Mirror the C ABI's ordering exactly: parse first, validate + // second, activate third. Each step's failure short-circuits so a + // manifest that does not parse is never reported as an activation + // failure — that would name the wrong half. The diagnostic shape + // is owned by the core so every binding renders artifact findings + // the same way. + let findings = match Manifest::from_yaml_str(&manifest_yaml) { + Err(e) => vec![wire::diagnostic_json(&e)], + Ok(manifest) => match manifest.validate() { + Err(e) => vec![wire::diagnostic_json(&e)], + Ok(()) => match ActivatedPolicy::activate_from_memory(&manifest_yaml, bundles) { + Ok(_) => Vec::new(), + Err(e) => vec![wire::diagnostic_json(&e)], + }, + }, + }; + serde_json::to_string(&findings) + .map_err(|e| err(format!("diagnostics serialization failed: {e}"))) +} + +// Note: the field pointer that used to be built here now lives with the +// diagnostic shape in `wire::diagnostic_field`, so a manifest diagnostic +// reads the same across the FFI, the Python binding, and this binding. + +// --------------------------------------------------------------------- +// Streaming: incremental release accounting for stream-shaped tracks +// (spec §18.1). A session holds no policy and no text; the host drives +// it: report arriving text, declare the spans its segmenter produced, +// evaluate those spans with the ordinary runtime, feed the outcomes +// back, and ask which prefix is safe to release. Mirrors +// `acs_stream_session_*` in the C ABI and the Python `StreamSession`. +// --------------------------------------------------------------------- + +/// One live streaming session. +/// +/// The session is `&mut` on every meaningful call, so the handle wraps +/// a `Mutex`. Napi may invoke bindings from any worker thread, and a +/// session is cheap to lock: no policy runs behind it. +pub struct StreamHandle { + session: Mutex, +} + +fn parse_track_wire(raw: &str) -> napi::Result { + // Like the FFI's `wire_track`: the core owns what the wire name + // means; this boundary only reshapes the error for napi. + StreamTrack::parse(raw).map_err(stream_err) +} + +fn parse_outcome_wire(raw: &str) -> napi::Result { + SegmentOutcome::parse(raw).map_err(stream_err) +} + +fn parse_source_wire(raw: &str) -> napi::Result { + StreamSourceType::parse(raw).map_err(stream_err) +} + +fn stream_err(e: StreamError) -> napi::Error { + err(format!("{e}")) +} + +/// Open a session from a config JSON object. +/// +/// Matches `acs_stream_session_new`: takes `safety_level` (`blocking`, +/// `complete` or `deferred`), the per-track start offsets +/// `request_start_rune_offset` and `response_start_rune_offset`, and +/// the task name arrays `request_tasks` and `response_tasks`. An empty +/// task array means that track is unmediated; payload on it fails +/// closed. A configuration mediating neither track is refused. +/// +/// The wrapper takes a JSON string rather than a napi object so the +/// config surface is identical to the other language SDKs and offset +/// coercion happens in one place. +#[napi] +pub fn stream_session_new(config_json: Utf16String) -> napi::Result> { + let config_json = decode("config_json", &config_json)?; + let parsed: Value = serde_json::from_str(&config_json) + .map_err(|e| err(format!("config_json does not parse: {e}")))?; + if !parsed.is_object() { + return Err(err("config_json must be a JSON object".to_string())); + } + // Field acceptance, absent-field defaults and the safety-level + // vocabulary live in the core so every binding builds the same + // configuration from the same JSON. + let config = wire::stream_config_from_json(&parsed).map_err(stream_err)?; + let session = StreamSession::new(config).map_err(stream_err)?; + Ok(External::new(StreamHandle { + session: Mutex::new(session), + })) +} + +fn lock<'a>( + handle: &'a External, +) -> napi::Result> { + handle + .session + .lock() + .map_err(|_| err("stream session mutex was poisoned".to_string())) +} + +/// Report that `runes` more runes of `source_type` arrived and return +/// the track's new end offset. Boundary failures throw; a streaming +/// accounting failure throws with the engine's message and puts the +/// session into its terminal state. +#[napi] +pub fn stream_session_observe( + handle: &External, + source_type: Utf16String, + runes: u32, +) -> napi::Result { + let raw = decode("source_type", &source_type)?; + let source = parse_source_wire(&raw)?; + let mut session = lock(handle)?; + session.observe(source, runes).map_err(stream_err) +} + +/// Report arriving `text` on `source_type`, counting Unicode scalars so +/// a host does not have to. Returns the track's new end offset. +/// +/// The engine counts runes, not UTF-16 code units. `Utf16String` yields +/// UTF-16, so the binding decodes to a `String` before delegating to +/// the engine and rune counting stays consistent with every other SDK. +/// An astral-plane character is one rune here even though it is two +/// UTF-16 code units. +#[napi] +pub fn stream_session_observe_text( + handle: &External, + source_type: Utf16String, + text: Utf16String, +) -> napi::Result { + let source_raw = decode("source_type", &source_type)?; + let source = parse_source_wire(&source_raw)?; + let body = decode("text", &text)?; + let mut session = lock(handle)?; + session.observe_text(source, &body).map_err(stream_err) +} + +/// Record what `task` decided about the span `[start, end)` of +/// `source_type`. `outcome` is `cleared`, `transformed` or `denied`. +#[napi] +pub fn stream_session_record_outcome( + handle: &External, + task: Utf16String, + source_type: Utf16String, + start: u32, + end: u32, + outcome: Utf16String, +) -> napi::Result<()> { + let task = decode("task", &task)?; + let source_raw = decode("source_type", &source_type)?; + let source = parse_source_wire(&source_raw)?; + let outcome_raw = decode("outcome", &outcome)?; + let outcome = parse_outcome_wire(&outcome_raw)?; + let span = StreamSpan::new(source, start, end).map_err(stream_err)?; + let mut session = lock(handle)?; + session + .record_outcome(&task, &span, outcome) + .map_err(stream_err) +} + +/// Record an ACS verdict against the span `[start, end)` of +/// `source_type`, mapping its decision onto an outcome. A host feeds +/// the JSON returned by `policyEvaluate` straight back without +/// translating it. +#[napi] +pub fn stream_session_record_verdict( + handle: &External, + task: Utf16String, + source_type: Utf16String, + start: u32, + end: u32, + verdict_json: Utf16String, +) -> napi::Result<()> { + let task = decode("task", &task)?; + let source_raw = decode("source_type", &source_type)?; + let source = parse_source_wire(&source_raw)?; + let raw = decode("verdict_json", &verdict_json)?; + let verdict: Verdict = + serde_json::from_str(&raw).map_err(|e| err(format!("verdict_json does not parse: {e}")))?; + let span = StreamSpan::new(source, start, end).map_err(stream_err)?; + let mut session = lock(handle)?; + session + .record_verdict(&task, &span, &verdict) + .map_err(stream_err) +} + +/// Recompute `track`'s watermark. Returns the new offset when the +/// watermark advanced, `null` when it did not or the session has ended +/// (matching the Rust `Option`). +#[napi] +pub fn stream_session_advance( + handle: &External, + track: Utf16String, +) -> napi::Result> { + let raw = decode("track", &track)?; + let track = parse_track_wire(&raw)?; + let mut session = lock(handle)?; + Ok(session.advance(track)) +} + +/// Offset of `track` the host may release through, or `null` once the +/// session has ended. A settled session has no safe offset, which is +/// not an error: it means release nothing further. +#[napi] +pub fn stream_session_safe_offset( + handle: &External, + track: Utf16String, +) -> napi::Result> { + let raw = decode("track", &track)?; + let track = parse_track_wire(&raw)?; + let session = lock(handle)?; + Ok(session.safe_offset(track)) +} + +/// Runes on `track` observed but not yet released. +#[napi] +pub fn stream_session_pending( + handle: &External, + track: Utf16String, +) -> napi::Result { + let raw = decode("track", &track)?; + let track = parse_track_wire(&raw)?; + let session = lock(handle)?; + Ok(session.pending(track)) +} + +/// `track`'s watermark as JSON, carrying `track`, `confirmed`, +/// `received`, `pending` and the `tasks` that must clear it. The +/// confirmed offset stays readable after settlement, so an audit +/// record can still say how far the stream got. +#[napi] +pub fn stream_session_watermark( + handle: &External, + track: Utf16String, +) -> napi::Result { + let raw = decode("track", &track)?; + let track = parse_track_wire(&raw)?; + let session = lock(handle)?; + let watermark = session.watermark(track); + let payload = wire::watermark_json(track, watermark); + serde_json::to_string(&payload).map_err(|e| err(format!("watermark serialization failed: {e}"))) +} + +/// Session state as JSON: `is_ended`, `transformed`, `end_reason` +/// (null while live) and the effective `config`. +#[napi] +pub fn stream_session_state(handle: &External) -> napi::Result { + let session = lock(handle)?; + let payload = wire::stream_session_state_json(&session); + serde_json::to_string(&payload).map_err(|e| err(format!("state serialization failed: {e}"))) +} + +/// Declare that no further payload will arrive. Idempotent. +#[napi] +pub fn stream_session_end_of_payloads(handle: &External) -> napi::Result<()> { + let mut session = lock(handle)?; + session.end_of_payloads(); + Ok(()) +} + +/// Settle the session and return the completion as JSON, carrying +/// `reason`, `transformed` and `is_clean`. Settling twice returns the +/// same completion. +#[napi] +pub fn stream_session_finish(handle: &External) -> napi::Result { + let mut session = lock(handle)?; + let completion = session.finish(); + let payload = wire::completion_json(&completion); + serde_json::to_string(&payload) + .map_err(|e| err(format!("completion serialization failed: {e}"))) +} diff --git a/sdk/node/src/index.ts b/sdk/node/src/index.ts index c4bedb1..f4b59c8 100644 --- a/sdk/node/src/index.ts +++ b/sdk/node/src/index.ts @@ -26,14 +26,70 @@ import type { // eslint-disable-next-line @typescript-eslint/no-require-imports const native = require("../binding.js") as { interceptorNew(manifestPath: string): unknown; + interceptorNewWithHooks( + manifestPath: string, + annotatorDispatcher?: + | ((name: string, invocationJson: string, policyInputJson: string) => string) + | null, + policyDispatcher?: ((invocationJson: string) => string) | null, + telemetrySink?: ((eventJson: string) => void) | null, + perfTelemetry?: string | null, + limitsJson?: string | null, + ): unknown; intercept(handle: unknown, contextJson: string): string; + defaultLimits(): string; policyActivate(manifestPath: string): unknown; policyActivateFromMemory(manifestYaml: string, bundlesJson: string): unknown; + policyActivateWithHooks( + manifestPath: string, + annotatorDispatcher?: + | ((name: string, invocationJson: string, policyInputJson: string) => string) + | null, + policyDispatcher?: ((invocationJson: string) => string) | null, + ): unknown; + policyActivateFromMemoryWithHooks( + manifestYaml: string, + bundlesJson: string, + annotatorDispatcher?: + | ((name: string, invocationJson: string, policyInputJson: string) => string) + | null, + policyDispatcher?: ((invocationJson: string) => string) | null, + ): unknown; policyEvaluate(handle: unknown, point: string, contextJson: string): string; policyInterventionPoints(handle: unknown): string[]; validateManifestFile(path: string): string | null; validateManifest(source: string): string | null; + validateManifestDetailed(source: string): string; + validateArtifactsDetailed(manifestYaml: string, bundlesJson: string): string; + parseManifest(source: string): string; + mergeManifests(sourcesJson: string): string; supportedManifestVersions(): string[]; + streamSessionNew(configJson: string): unknown; + streamSessionObserve(handle: unknown, sourceType: string, runes: number): number; + streamSessionObserveText(handle: unknown, sourceType: string, text: string): number; + streamSessionRecordOutcome( + handle: unknown, + task: string, + sourceType: string, + start: number, + end: number, + outcome: string, + ): void; + streamSessionRecordVerdict( + handle: unknown, + task: string, + sourceType: string, + start: number, + end: number, + verdictJson: string, + ): void; + streamSessionAdvance(handle: unknown, track: string): number | null; + streamSessionSafeOffset(handle: unknown, track: string): number | null; + streamSessionPending(handle: unknown, track: string): number; + streamSessionWatermark(handle: unknown, track: string): string; + streamSessionState(handle: unknown): string; + streamSessionEndOfPayloads(handle: unknown): void; + streamSessionFinish(handle: unknown): string; }; export type { @@ -46,7 +102,245 @@ export type { Warning, } from "@responsibleai/agent-hooks"; -export interface AcsInterceptorOptions { +// --------------------------------------------------------------------- +// Host extension surface: annotator dispatcher, policy dispatcher, +// telemetry sink, perf telemetry level. The zero-config path stays +// unchanged; a host that supplies any of these hooks gets the same +// engine construction the Rust and .NET/FFI SDKs already expose. +// +// The dispatcher callbacks are called SYNCHRONOUSLY on the JS thread +// from inside the engine's evaluation, which is legal here because the +// engine call itself is driven by a napi function running on the JS +// thread. A dispatcher that throws surfaces as a fail-closed +// `runtime_error:*` deny; the engine never treats a thrown callback as +// "no annotation". +// --------------------------------------------------------------------- + +/** + * A JSON-compatible value the boundary carries verbatim. + * + * Named separately from `unknown` so a dispatcher signature can say what + * kind of thing it reads and returns without opening the door to values + * (functions, symbols, class instances) the engine cannot round-trip. + */ +export type JsonValue = + | null + | boolean + | number + | string + | readonly JsonValue[] + | { readonly [key: string]: JsonValue }; + +/** + * Fields the engine hands to a host annotator: `type` (`classifier`, + * `llm`, `endpoint`), `from` (JSONPath expression), and every extra + * key the manifest attached to the annotator or the annotation. + */ +export interface AnnotatorInvocation { + readonly type: "classifier" | "llm" | "endpoint"; + readonly from: string; + readonly [key: string]: JsonValue; +} + +/** + * Fields the engine hands to a host policy dispatcher. A serialized + * `PreparedPolicyInvocation` — carrying `policy_id`, `policy_type`, + * `intervention_point`, `input`, and the policy-specific configuration + * — routed to whichever engine the host runs. The precise shape moves + * with the engine; treat it as opaque JSON and let the engine describe + * what to key off. + */ +export interface PolicyInvocation { + readonly policy_id: string; + readonly policy_type: string; + readonly intervention_point: string; + readonly input: JsonValue; + readonly [key: string]: JsonValue; +} + +/** + * One telemetry event the engine emits. Every field maps 1:1 with the + * `TelemetryEvent` the FFI binding surfaces and the Rust engine + * declares, so a sink written for another SDK reads the same shape here. + */ +export interface TelemetryEvent { + readonly event_type: string; + readonly intervention_point: string; + readonly decision: string | null; + readonly reason_code: string | null; + readonly error_class: string | null; + readonly policy_id: string | null; + readonly annotators: readonly string[]; + readonly enforcement_mode: string | null; + readonly duration_ms: number | null; + readonly evidence_artefact: string | null; + readonly evidence_verification_pointer_keys: readonly string[]; + readonly action_identity: string | null; + readonly metadata: Readonly>; +} + +/** + * A host annotator dispatcher. + * + * Called synchronously from inside `intercept`/`evaluate`; return a + * value the manifest's `preliminary_policy_input` merge shape expects + * (typically a JSON object per the annotator's contract). Throwing + * fails the surrounding evaluation closed with a + * `runtime_error:annotation_failed` deny. + */ +export type AnnotatorDispatcher = ( + name: string, + invocation: AnnotatorInvocation, + preliminaryPolicyInput: JsonValue, +) => JsonValue; + +/** + * A host policy dispatcher. + * + * Called synchronously; return the policy output as JSON per the + * engine's expected shape (a `decision` string plus any extras). A + * throw fails the evaluation closed with a + * `runtime_error:policy_invocation_failed` deny. + */ +export type PolicyDispatcher = (invocation: PolicyInvocation) => JsonValue; + +/** + * A telemetry sink. + * + * Called synchronously with each event the engine emits at the + * configured perf level. A sink cannot fail an evaluation; a throw + * from the sink is swallowed to preserve that guarantee. + */ +export type TelemetrySink = (event: TelemetryEvent) => void; + +/** + * How much per-evaluation timing to emit. + * + * - `off` (default): only the final decision event. + * - `external`: adds boundary events (annotator dispatch, policy + * evaluation). + * - `full`: adds stage timing events for a full performance profile. + */ +export type PerfTelemetry = "off" | "external" | "full"; + +/** + * Resource caps overriding the engine's defaults, field by field. + * + * `Limits` is a denial-of-service control surface: a host feeding + * large payloads raises `maxSnapshotBytes`; one hardening against a + * hostile manifest lowers `maxExtendsDepth` or + * `manifestUrlTimeoutMs`. Each field is individually optional; an + * absent field keeps its own default, so a host raising one cap does + * not restate the other nine. A field present but not a non-negative + * integer is a hard error, not a silently-kept default: a host that + * asked for a smaller bound and got the larger one would believe it + * was protected when it was not. + * + * See {@link DEFAULT_LIMITS} for the shipped values. + */ +export interface Limits { + /** Cap on the canonicalized context snapshot in bytes. */ + max_snapshot_bytes?: number; + /** JSON nesting depth accepted anywhere in policy input/output. */ + max_policy_input_depth?: number; + /** Number of annotators the engine will dispatch per intervention point. */ + max_annotators_per_point?: number; + /** Per-annotator serialized output cap in bytes. */ + max_annotator_output_bytes?: number; + /** Policy-decision serialized output cap in bytes. */ + max_policy_output_bytes?: number; + /** Manifest `extends` chain length. */ + max_extends_depth?: number; + /** Composed manifest total size cap in bytes. */ + max_merged_manifest_bytes?: number; + /** Per-URL manifest fetch body cap in bytes. */ + max_manifest_url_bytes?: number; + /** Per-URL manifest fetch deadline in milliseconds. */ + manifest_url_timeout_ms?: number; + /** Per-URL manifest fetch redirect count. */ + max_manifest_url_redirects?: number; +} + +/** + * The engine's shipped resource caps, as a frozen object. Read this to + * see what a `limits` mapping is overriding; a shipping change to + * another default cannot then be silently absorbed by a stale mapping. + */ +export const DEFAULT_LIMITS: Readonly> = Object.freeze( + JSON.parse(native.defaultLimits()) as Required, +); + +/** + * Host extension points. Each is optional; supplying one replaces the + * zero-config default for that slot. + */ +export interface HostHooks { + annotatorDispatcher?: AnnotatorDispatcher; + policyDispatcher?: PolicyDispatcher; + telemetrySink?: TelemetrySink; + perfTelemetry?: PerfTelemetry; + /** + * Resource caps overriding the engine's defaults, field by field. + * See {@link Limits} and {@link DEFAULT_LIMITS}. + */ + limits?: Readonly; +} + +// --- Bridges between the object-oriented TS surface and the JSON -------- +// string wire the native binding uses. Keeping the parse/serialize step +// here means the native dispatcher signatures stay simple `String → +// String` and every host callback sees objects, not text. + +function wrapAnnotatorDispatcher( + dispatcher: AnnotatorDispatcher, +): (name: string, invocationJson: string, policyInputJson: string) => string { + return (name, invocationJson, policyInputJson) => { + // A parse or a throwing dispatcher must propagate as a JS exception: + // the native binding turns that into a fail-closed + // `runtime_error:annotation_failed` deny. Never swallow — a + // silently caught error would read as "annotation succeeded with + // undefined". + const invocation = JSON.parse(invocationJson) as AnnotatorInvocation; + const policyInput = JSON.parse(policyInputJson) as JsonValue; + const result = dispatcher(name, invocation, policyInput); + return JSON.stringify(result ?? null); + }; +} + +function wrapPolicyDispatcher( + dispatcher: PolicyDispatcher, +): (invocationJson: string) => string { + return (invocationJson) => { + const invocation = JSON.parse(invocationJson) as PolicyInvocation; + const result = dispatcher(invocation); + return JSON.stringify(result ?? null); + }; +} + +function wrapTelemetrySink(sink: TelemetrySink): (eventJson: string) => void { + return (eventJson) => { + // A sink cannot fail an evaluation, so wrap in try/catch and drop + // the throw. Matches the engine contract. + try { + const event = JSON.parse(eventJson) as TelemetryEvent; + sink(event); + } catch { + // intentionally swallowed + } + }; +} + +function hasHostHooks(options: HostHooks): boolean { + return ( + options.annotatorDispatcher !== undefined || + options.policyDispatcher !== undefined || + options.telemetrySink !== undefined || + options.perfTelemetry !== undefined || + options.limits !== undefined + ); +} + +export interface AcsInterceptorOptions extends HostHooks { /** Payload-free identifier recorded on the record's `verdicts[].name`. */ name?: string; } @@ -62,14 +356,40 @@ export class AcsInterceptor implements Interceptor { } /** - * Build an interceptor from a manifest path using the zero-config - * dispatchers: bundled annotators; Rego policies in process, Cedar - * through the built-in evaluator, `test` policies through their - * embedded verdict. Custom policies require a host dispatcher and - * fail closed under this construction. + * Build an interceptor from a manifest path. + * + * With no `options`, uses the zero-config dispatchers: bundled + * annotators; Rego policies in process, Cedar through the built-in + * evaluator, `test` policies through their embedded verdict. Custom + * policies require a host dispatcher and fail closed under this + * construction. + * + * Supply `annotatorDispatcher`, `policyDispatcher`, `telemetrySink`, + * `perfTelemetry`, or `limits` to override the engine's extension + * points and resource caps. Each callback is called synchronously on + * the JS thread from inside `intercept`. A callback that throws + * surfaces as a fail-closed `runtime_error:*` deny rather than + * reading as "no annotation". A `limits` mapping overrides only the + * fields it names; the rest keep the engine's defaults, see + * {@link DEFAULT_LIMITS}. */ static fromPath(manifestPath: string, options: AcsInterceptorOptions = {}): AcsInterceptor { - return new AcsInterceptor(native.interceptorNew(manifestPath), options.name ?? "acs"); + const name = options.name ?? "acs"; + const handle = hasHostHooks(options) + ? native.interceptorNewWithHooks( + manifestPath, + options.annotatorDispatcher + ? wrapAnnotatorDispatcher(options.annotatorDispatcher) + : null, + options.policyDispatcher + ? wrapPolicyDispatcher(options.policyDispatcher) + : null, + options.telemetrySink ? wrapTelemetrySink(options.telemetrySink) : null, + options.perfTelemetry ?? null, + options.limits ? JSON.stringify(options.limits) : null, + ) + : native.interceptorNew(manifestPath); + return new AcsInterceptor(handle, name); } /** @@ -108,6 +428,25 @@ export interface RegoBundle { data?: readonly RegoDataDocument[]; } +/** + * Host extension points for {@link ActivatedPolicy}. + * + * Activation uses the engine's `activate_with` surface, which takes + * only the annotator and policy dispatchers; telemetry and perf level + * are configured on the interceptor path (see {@link AcsInterceptor}) + * because activation records no per-evaluation events itself. + */ +export interface ActivatedPolicyOptions { + annotatorDispatcher?: AnnotatorDispatcher; + policyDispatcher?: PolicyDispatcher; +} + +function hasActivationHooks(options: ActivatedPolicyOptions): boolean { + return ( + options.annotatorDispatcher !== undefined || options.policyDispatcher !== undefined + ); +} + /** * One policy version, readied once and evaluated many times. * @@ -149,9 +488,28 @@ export class ActivatedPolicy { * * A manifest names its bundle relative to itself, so an absolute * manifest path is enough and the working directory does not matter. + * + * Supply `annotatorDispatcher` or `policyDispatcher` in `options` to + * override the engine's extension points; each callback is called + * synchronously on the JS thread from inside `evaluate` and fails + * closed on throw. */ - static activate(manifestPath: string): ActivatedPolicy { - return new ActivatedPolicy(native.policyActivate(manifestPath)); + static activate( + manifestPath: string, + options: ActivatedPolicyOptions = {}, + ): ActivatedPolicy { + const handle = hasActivationHooks(options) + ? native.policyActivateWithHooks( + manifestPath, + options.annotatorDispatcher + ? wrapAnnotatorDispatcher(options.annotatorDispatcher) + : null, + options.policyDispatcher + ? wrapPolicyDispatcher(options.policyDispatcher) + : null, + ) + : native.policyActivate(manifestPath); + return new ActivatedPolicy(handle); } /** @@ -179,10 +537,21 @@ export class ActivatedPolicy { static activateFromMemory( manifestYaml: string, bundles: Readonly>, + options: ActivatedPolicyOptions = {}, ): ActivatedPolicy { - return new ActivatedPolicy( - native.policyActivateFromMemory(manifestYaml, JSON.stringify(bundles)), - ); + const handle = hasActivationHooks(options) + ? native.policyActivateFromMemoryWithHooks( + manifestYaml, + JSON.stringify(bundles), + options.annotatorDispatcher + ? wrapAnnotatorDispatcher(options.annotatorDispatcher) + : null, + options.policyDispatcher + ? wrapPolicyDispatcher(options.policyDispatcher) + : null, + ) + : native.policyActivateFromMemory(manifestYaml, JSON.stringify(bundles)); + return new ActivatedPolicy(handle); } /** @@ -280,3 +649,568 @@ export function validateManifestFile(path: string): void { export function supportedManifestVersions(): readonly string[] { return Object.freeze(native.supportedManifestVersions()); } + +// --------------------------------------------------------------------- +// Manifest tooling: parse, chain-compose, and structured diagnostics. +// +// The engine ships these APIs on `Manifest`; the wrapper exposes them +// here so authoring, migration, and CI tools can drive parse, overlay +// composition, and per-field validation from Node without staging a +// manifest to disk first. +// --------------------------------------------------------------------- + +/** + * A structured description of a manifest that did not pass validation. + * + * `code` is the reserved `runtime_error:*` reason a diagnostic-consuming + * tool keys off; `message` is the engine's human-readable detail; + * `severity` is always `"error"`; `field` is a best-effort pointer to + * the offending manifest field (a YAML key or an engine-declared + * identifier) so an editor can render the problem inline. `field` is + * `null` when the message does not identify one. + */ +export interface ManifestDiagnostic { + readonly code: string; + readonly message: string; + readonly severity: "error"; + readonly field: string | null; +} + +/** + * Parse manifest YAML into an object without validating references. + * + * The document is deserialized as-written: a manifest with an + * unresolved `extends` chain returns fine. Use {@link validateManifest} + * or {@link validateManifestDetailed} to judge whether the fragment is + * runnable. + * + * Throws when the YAML does not parse. + */ +export function parseManifest(source: string): Record { + if (typeof source !== "string") { + throw new TypeError(`parseManifest expects a string, received ${typeof source}`); + } + if (UNPAIRED_SURROGATE.test(source)) { + throw new TypeError("parseManifest received a string with an unpaired surrogate"); + } + return JSON.parse(native.parseManifest(source)) as Record; +} + +/** + * Compose a chain of manifest YAML documents into one merged manifest. + * + * `sources` is ordered outermost base first, deltas after. This is the + * overlay case, resolved the same way the engine resolves `extends` + * when it walks a manifest tree. The returned object is the merged + * manifest as JSON. + * + * Throws when a source does not parse, when the merged result is + * invalid, or when `sources` is empty. + */ +export function mergeManifests(sources: readonly string[]): Record { + if (!Array.isArray(sources)) { + throw new TypeError("mergeManifests expects an array of manifest sources"); + } + for (const source of sources) { + if (typeof source !== "string") { + throw new TypeError("mergeManifests received a non-string source"); + } + if (UNPAIRED_SURROGATE.test(source)) { + throw new TypeError( + "mergeManifests received a source with an unpaired surrogate", + ); + } + } + return JSON.parse(native.mergeManifests(JSON.stringify(sources))) as Record< + string, + unknown + >; +} + +/** + * Validate manifest source and return structured findings. + * + * An empty array means the manifest passed validation. Each finding + * carries an engine reason code, the detail message, and where + * possible the offending field name so an editor can render the + * problem inline. Use this instead of {@link validateManifest} when + * you need to render results, not just yes/no. + * + * Throws only on boundary problems (non-string input, unpaired + * surrogate); an invalid manifest returns a non-empty array. + */ +export function validateManifestDetailed(source: string): readonly ManifestDiagnostic[] { + if (typeof source !== "string") { + throw new TypeError( + `validateManifestDetailed expects a string, received ${typeof source}`, + ); + } + if (UNPAIRED_SURROGATE.test(source)) { + throw new TypeError( + "validateManifestDetailed received a string with an unpaired surrogate", + ); + } + return Object.freeze( + JSON.parse(native.validateManifestDetailed(source)) as ManifestDiagnostic[], + ); +} + +/** + * Validate a manifest AND the Rego it names, returning findings. + * + * An empty array means both halves are sound. Each entry has wire + * shape `{"code","message","severity":"error"}`, matching the C ABI's + * `acs_artifact_diagnostics`. + * + * {@link validateManifestDetailed} answers only for the document. A + * manifest can satisfy the grammar, name a Rego bundle, and still fail + * at activation because the Rego does not compile — compilation + * happens at activation, so a validator that stops at the manifest + * turns that failure into a host's first agent action. This function + * activates against `bundles` in memory and reports what the pair + * surfaced. + * + * `bundles` has the same shape {@link ActivatedPolicy.activateFromMemory} + * takes: a mapping from policy id to a {@link RegoBundle}. Omitting it + * or passing an empty object means the manifest names no Rego, and the + * answer then equals what {@link validateManifestDetailed} reports for + * the manifest half: a document that does not parse is reported as a + * manifest problem, not an activation problem, because that would name + * the wrong half. + * + * Throws only on boundary problems (non-string manifest, unpaired + * surrogate, non-object bundles); a broken manifest or Rego module + * returns a non-empty array. + */ +export function validateArtifacts( + manifestSource: string, + bundles?: Readonly>, +): readonly ManifestDiagnostic[] { + if (typeof manifestSource !== "string") { + throw new TypeError( + `validateArtifacts expects a string manifest, received ${typeof manifestSource}`, + ); + } + if (UNPAIRED_SURROGATE.test(manifestSource)) { + throw new TypeError( + "validateArtifacts received a manifest with an unpaired surrogate", + ); + } + if (bundles !== undefined && (bundles === null || typeof bundles !== "object")) { + throw new TypeError( + `validateArtifacts expects bundles to be an object, received ${typeof bundles}`, + ); + } + const payload = bundles === undefined ? "" : JSON.stringify(bundles); + return Object.freeze( + JSON.parse( + native.validateArtifactsDetailed(manifestSource, payload), + ) as ManifestDiagnostic[], + ); +} + + +// --------------------------------------------------------------------- +// Streaming: incremental release mediation for stream-shaped tracks +// (spec §18.1). The engine is stateless everywhere else; this session +// is the exception, because it accumulates offsets and holds terminal +// state a host must read across many calls. +// --------------------------------------------------------------------- + +/** How much a host may release ahead of the watermark. */ +export type StreamSafetyLevel = "blocking" | "complete" | "deferred"; + +/** + * Role that produced a span of text. + * + * Only genuinely rune-addressable roles appear here: tool calls and + * tool results are structured values evaluated once per invocation and + * flow through the ordinary snapshot path. + */ +export type StreamSourceType = "user_request" | "model_generated"; + +/** Independent offset space within a session. */ +export type StreamTrack = "request" | "response"; + +/** What the host decided for one span after evaluating it. */ +export type SegmentOutcome = "cleared" | "transformed" | "denied"; + +/** Parameters a host supplies once, before any payload. */ +export interface StreamSessionConfig { + /** How much the host may release ahead of the watermark. */ + readonly safetyLevel: StreamSafetyLevel; + /** + * Offset the first rune of the request track occupies. A retry that + * resumes a partially delivered stream sets this so offsets stay + * comparable with the earlier attempt. + */ + readonly requestStartRuneOffset?: number; + /** Same as {@link requestStartRuneOffset}, for the response track. */ + readonly responseStartRuneOffset?: number; + /** + * Tasks that gate the request track (matching what the host bound at + * `input`). Empty means the request track is not mediated; payload + * on it fails closed. + */ + readonly requestTasks?: readonly string[]; + /** + * Tasks that gate the response track (matching what the host bound + * at `post_model_call`). Empty means the response track is not + * mediated. + */ + readonly responseTasks?: readonly string[]; +} + +/** Watermark snapshot for one track. */ +export interface StreamWatermarkSnapshot { + readonly track: StreamTrack; + /** Highest offset released so far. */ + readonly confirmed: number; + /** End offset of the text the session has been told about. */ + readonly received: number; + /** + * Runes observed but not yet cleared by every task, as of the last + * advance. + */ + readonly pending: number; + /** Task labels this watermark tracks, in deterministic order. */ + readonly tasks: readonly string[]; +} + +/** Terminal reason a session reached its final state. */ +export type StreamEndReason = + | { readonly kind: "complete" } + | { + readonly kind: "denied"; + readonly track: StreamTrack; + readonly task: string; + readonly start: number; + readonly end: number; + } + | { + readonly kind: "rewritten"; + readonly track: StreamTrack; + readonly task: string; + readonly start: number; + readonly end: number; + } + | { + readonly kind: "failed"; + /** The `host_error:*` reason a host records for this failure. */ + readonly reason: string; + readonly message: string; + }; + +/** Terminal settlement of a session. */ +export interface StreamCompletion { + readonly reason: StreamEndReason; + /** + * Whether the host emitted a substitute rather than verbatim model + * output. This is exactly `reason.kind === "rewritten"`. + */ + readonly transformed: boolean; + /** Whether the stream finished without an enforcement action. */ + readonly isClean: boolean; +} + +/** Read-only view of a session's effective configuration. */ +export interface StreamSessionConfigSnapshot { + readonly safetyLevel: StreamSafetyLevel; + readonly requestStartRuneOffset: number; + readonly responseStartRuneOffset: number; + readonly requestTasks: readonly string[]; + readonly responseTasks: readonly string[]; +} + +/** Snapshot of a session's state, matching the C ABI `session_state`. */ +export interface StreamSessionState { + readonly isEnded: boolean; + readonly transformed: boolean; + /** `null` while the session is still live. */ + readonly endReason: StreamEndReason | null; + readonly config: StreamSessionConfigSnapshot; +} + +/** + * Incremental streaming session (spec §18.1). + * + * ACS is stateless on every other path, so a rune-addressable track a + * host emits incrementally cannot ride the ordinary interceptor + * pipeline: a `deny` in the middle of a stream needs to catch a + * specific range, and a `cleared` prefix needs to release without + * waiting for the whole payload. This class is the accounting layer + * that makes both possible. + * + * The session holds no policy and no text. The host drives it: + * + * 1. Observe arriving text ({@link observe}, {@link observeText}) so + * the session knows how many runes exist on each track. + * 2. Segment the text and evaluate each span with the ordinary + * runtime, then record the outcome ({@link recordOutcome}) or + * replay the verdict ({@link recordVerdict}). + * 3. Ask which prefix is safe to release ({@link safeOffset}), and + * read {@link watermark} for the per-task frontier when auditing. + * 4. Call {@link endOfPayloads} at EOF and settle with {@link finish}. + * + * A settled session reports `null` from {@link safeOffset} and + * {@link advance}: the type says "release nothing further" without any + * value a caller could mistake for a permitted offset. The watermark + * stays readable so an audit record can still say how far the stream + * got. + * + * Every non-boundary streaming failure (unknown safety level, payload + * on an unmediated track, offset past the observed end, transform + * after release, uncleared residue at settlement, ...) throws with the + * engine's message and puts the session in its terminal `failed` + * state. The next call sees the session as ended. + */ +/** + * Refuse a rune offset N-API would silently reshape. + * + * N-API converts to `u32` with ToUint32, which wraps rather than fails: + * `2 ** 32` arrives as `0`, and an end offset of `2 ** 32 + 5` records a + * *cleared* span of `[0, 5)`, releasing text no task evaluated. Python + * raises OverflowError and .NET throws OverflowException on the same + * input, so this is the one language doing modular arithmetic on + * release accounting. + */ +function runeOffset(value: number, name: string): number { + if (!Number.isInteger(value) || value < 0 || value > 0x7fffffff) { + throw new RangeError(`${name} must be a rune offset between 0 and 2147483647, got ${value}`); + } + return value; +} + +export class StreamSession { + private readonly handle: unknown; + + /** + * Open a session. + * + * `config.safetyLevel` selects the release rule. `requestTasks` and + * `responseTasks` are the task labels a host will pass to + * {@link recordOutcome}: matching a task the manifest binds at + * `input` / `post_model_call` respectively. An empty task list + * leaves that track unmediated; payload on it fails closed. + * + * Throws when both task lists are empty (the session would gate + * nothing) or a start offset overflows. + */ + constructor(config: StreamSessionConfig) { + if (config === null || typeof config !== "object") { + throw new TypeError("StreamSession config must be an object"); + } + const requestStart = config.requestStartRuneOffset ?? 0; + const responseStart = config.responseStartRuneOffset ?? 0; + runeOffset(requestStart, 'requestStartRuneOffset'); + runeOffset(responseStart, 'responseStartRuneOffset'); + // Only translate camelCase → wire snake_case here; enum VALUES stay + // lowercase snake as they arrive. + const payload = { + safety_level: config.safetyLevel, + request_start_rune_offset: requestStart, + response_start_rune_offset: responseStart, + request_tasks: config.requestTasks ? Array.from(config.requestTasks) : [], + response_tasks: config.responseTasks ? Array.from(config.responseTasks) : [], + }; + this.handle = native.streamSessionNew(JSON.stringify(payload)); + } + + /** + * Report that `runes` more runes of `sourceType` arrived and return + * the track's new end offset. + * + * This only extends the received bound outcomes are checked against. + * It does not release anything and does not decide what the host + * evaluates. Prefer {@link observeText} when the text is at hand: + * counting runes correctly across surrogate pairs is easy to get + * wrong. + */ + observe(sourceType: StreamSourceType, runes: number): number { + runeOffset(runes, 'runes'); + return native.streamSessionObserve(this.handle, sourceType, runes); + } + + /** + * Report arriving `text` on `sourceType`, counting Unicode scalars + * the way the engine does, and return the track's new end offset. + * + * The engine counts runes (Unicode scalars), not UTF-16 code units. + * An emoji outside the BMP is one rune here even though it is two + * UTF-16 code units in a JS string. + */ + observeText(sourceType: StreamSourceType, text: string): number { + return native.streamSessionObserveText(this.handle, sourceType, text); + } + + /** + * Record what `task` decided about the span `[start, end)` of + * `sourceType`. `outcome` is `cleared`, `transformed`, or `denied`. + * + * A `denied` outcome ends the session with `endReason.kind === + * "denied"` on the span it refused, and every later `safeOffset` + * returns `null`. A `transformed` outcome is honored only under a + * withholding safety level (`blocking` / `complete`) and only while + * nothing on the track has been released, and it ends the session + * with `endReason.kind === "rewritten"`. + */ + recordOutcome( + task: string, + sourceType: StreamSourceType, + start: number, + end: number, + outcome: SegmentOutcome, + ): void { + runeOffset(start, 'start'); + runeOffset(end, 'end'); + native.streamSessionRecordOutcome(this.handle, task, sourceType, start, end, outcome); + } + + /** + * Record an ACS verdict against the span `[start, end)` of + * `sourceType`, mapping its decision onto an outcome. A host feeds + * the verdict returned by {@link ActivatedPolicy.evaluate} straight + * back without translating it. + * + * A verdict whose shape section 5 does not admit (a `transform` + * carrying no transform body, a reserved reason from a policy, ...) + * fails the stream closed rather than clearing the span. + */ + recordVerdict( + task: string, + sourceType: StreamSourceType, + start: number, + end: number, + verdict: Verdict, + ): void { + runeOffset(start, 'start'); + runeOffset(end, 'end'); + native.streamSessionRecordVerdict( + this.handle, + task, + sourceType, + start, + end, + JSON.stringify(verdict), + ); + } + + /** + * Recompute `track`'s watermark. Returns the new offset when the + * watermark advanced, `null` when it did not or the session has + * ended. + */ + advance(track: StreamTrack): number | null { + return native.streamSessionAdvance(this.handle, track); + } + + /** + * Offset of `track` the host may release through, or `null` once + * the session has ended. + * + * A settled session has no safe offset: release nothing further. + * The offset the track reached is unaffected and stays available + * for an audit record through {@link watermark}. + */ + safeOffset(track: StreamTrack): number | null { + return native.streamSessionSafeOffset(this.handle, track); + } + + /** Runes on `track` observed but not yet released. */ + pending(track: StreamTrack): number { + return native.streamSessionPending(this.handle, track); + } + + /** + * `track`'s watermark, carrying `confirmed`, `received`, `pending` + * and the `tasks` that must clear it. The confirmed offset stays + * readable after settlement. + */ + watermark(track: StreamTrack): StreamWatermarkSnapshot { + return JSON.parse(native.streamSessionWatermark(this.handle, track)) as StreamWatermarkSnapshot; + } + + /** + * Snapshot of session state: `isEnded`, `transformed`, `endReason` + * (null while live) and the effective `config`. + */ + state(): StreamSessionState { + const raw = JSON.parse(native.streamSessionState(this.handle)) as { + is_ended: boolean; + transformed: boolean; + end_reason: StreamEndReason | null; + config: { + safety_level: StreamSafetyLevel; + request_start_rune_offset: number; + response_start_rune_offset: number; + request_tasks: string[]; + response_tasks: string[]; + }; + }; + return { + isEnded: raw.is_ended, + transformed: raw.transformed, + endReason: raw.end_reason, + config: { + safetyLevel: raw.config.safety_level, + requestStartRuneOffset: raw.config.request_start_rune_offset, + responseStartRuneOffset: raw.config.response_start_rune_offset, + requestTasks: Object.freeze(raw.config.request_tasks), + responseTasks: Object.freeze(raw.config.response_tasks), + }, + }; + } + + /** Whether the session has reached its terminal state. */ + isEnded(): boolean { + return this.state().isEnded; + } + + /** + * Terminal reason, when the session has ended, or `null` while it + * is still live. + */ + endReason(): StreamEndReason | null { + return this.state().endReason; + } + + /** + * Whether a `transformed` outcome ended this session, meaning the + * host emits a substitute rather than verbatim model output. + */ + isTransformed(): boolean { + return this.state().transformed; + } + + /** + * Declare that no further payload will arrive. Idempotent. + * + * A `deferred` host calls this at payload EOF so a classifier + * running behind the stream can still record a denial before + * {@link finish}. + */ + endOfPayloads(): void { + native.streamSessionEndOfPayloads(this.handle); + } + + /** + * Settle the session and return the completion. + * + * Recomputes both watermarks first, so a host that recorded every + * outcome is not failed closed for having skipped an explicit + * {@link advance}. Any rune no task cleared fails the settlement + * closed. Settling twice returns the same completion. + */ + finish(): StreamCompletion { + const raw = JSON.parse(native.streamSessionFinish(this.handle)) as { + reason: StreamEndReason; + transformed: boolean; + is_clean: boolean; + }; + return { + reason: raw.reason, + transformed: raw.transformed, + isClean: raw.is_clean, + }; + } +} diff --git a/sdk/node/test/host-hooks.test.mjs b/sdk/node/test/host-hooks.test.mjs new file mode 100644 index 0000000..e7f5495 --- /dev/null +++ b/sdk/node/test/host-hooks.test.mjs @@ -0,0 +1,630 @@ +// Host extension surface: annotator dispatcher, policy dispatcher, +// telemetry sink, perf telemetry level; and manifest tooling (parse, +// chain, structured diagnostics). +// +// The engine calls dispatchers synchronously from inside `intercept`, +// which is itself a napi call running on the JS thread. These tests +// prove: (a) a JS callback IS invoked on that stack, (b) its return +// value flows into the policy decision, (c) a throw fails closed with +// a `runtime_error:*` deny (never "no annotation"), (d) telemetry +// events reach a sink, and (e) the manifest tooling that authoring +// tools need is reachable from Node. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import fs from "node:fs"; +import path from "node:path"; + +const require = (await import("node:module")).createRequire(import.meta.url); +const { + AcsInterceptor, + ActivatedPolicy, + parseManifest, + mergeManifests, + validateArtifacts, + validateManifestDetailed, +} = require("../dist/index.js"); +const { AgentContextBuilder } = require("@responsibleai/agent-hooks"); + +const here = path.dirname(fileURLToPath(import.meta.url)); +const fixtureManifestPath = path.join(here, "fixtures", "manifest.yaml"); + +const builder = () => + new AgentContextBuilder({ agentId: "a", framework: "test", sessionId: "s" }); + +// A manifest binding a custom policy that a host policy dispatcher +// answers, and a classifier annotator whose value the dispatcher reads +// from `input.annotations.mood`. The manifest is deliberately minimal: +// one intervention point, one annotator, one policy. +function writeAnnotatorPolicyManifest(tmpdir) { + const src = `agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: node-host-hooks-test +policies: + gate: + type: custom + adapter: host_gate +annotators: + mood: + type: classifier +intervention_points: + input: + policy_target: "$.input" + policy_target_kind: user_input + annotations: + mood: + from: "$target.content" + policy: + id: gate +`; + const p = path.join(tmpdir, "annotator-policy-manifest.yaml"); + fs.writeFileSync(p, src, "utf8"); + return p; +} + +function writePolicyOnlyManifest(tmpdir) { + const src = `agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: node-host-hooks-policy-only +policies: + gate: + type: custom + adapter: host_gate +intervention_points: + input: + policy_target: "$.input" + policy_target_kind: user_input + policy: + id: gate +`; + const p = path.join(tmpdir, "policy-only-manifest.yaml"); + fs.writeFileSync(p, src, "utf8"); + return p; +} + +const workdir = fs.mkdtempSync(path.join(here, ".host-hooks-")); +test.after(() => { + fs.rmSync(workdir, { recursive: true, force: true }); +}); + +// --------------------------------------------------------------------- +// 1. Host annotator dispatcher IS called on the sync stack and its +// return value flows into the policy decision. Two invocations against +// the same manifest, differing only in what the annotator returns, +// produce different verdicts. +// --------------------------------------------------------------------- + +test("host annotator dispatcher return value drives the policy verdict", () => { + const manifest = writeAnnotatorPolicyManifest(workdir); + const seen = []; + const annotatorDispatcher = (name, invocation, _preliminary) => { + // Prove the engine actually asked us, and remember what for. + seen.push({ name, type: invocation.type, from: invocation.from }); + // Emit the mood the policy will read. + return { mood: invocation.from === "$target.content" ? "angry" : "calm" }; + }; + const policyDispatcher = (invocation) => { + // Custom-policy invocations tag as `custom` and carry the policy + // input under `input`. The annotator's output appears at + // `input.annotations.mood`. + assert.equal(invocation.type, "custom"); + assert.equal(invocation.adapter, "host_gate"); + const mood = invocation.input?.annotations?.mood?.mood; + if (mood === "angry") { + return { + decision: "deny", + reason: "annotation_says_angry", + message: "annotator flagged mood=angry", + }; + } + return { decision: "allow", reason: "annotation_ok" }; + }; + + const acs = AcsInterceptor.fromPath(manifest, { + annotatorDispatcher, + policyDispatcher, + }); + + const angry = acs.intercept(builder().input("you are broken")); + assert.equal(angry.decision, "deny"); + assert.equal(angry.reason, "annotation_says_angry"); + + // The annotator was actually called on the sync call stack. + assert.equal(seen.length, 1); + assert.equal(seen[0].name, "mood"); + assert.equal(seen[0].type, "classifier"); + assert.equal(seen[0].from, "$target.content"); +}); + +// --------------------------------------------------------------------- +// 2. An annotator that throws fails CLOSED. The verdict is a deny with +// `runtime_error:annotation_failed`. The engine never treats a thrown +// callback as "no annotation". +// --------------------------------------------------------------------- + +test("annotator dispatcher that throws fails closed with annotation_failed", () => { + const manifest = writeAnnotatorPolicyManifest(workdir); + const annotatorDispatcher = () => { + throw new Error("upstream classifier is on fire"); + }; + // Present the policy dispatcher, but it must never be reached: the + // annotator failure short-circuits evaluation. + let policyCalled = false; + const policyDispatcher = () => { + policyCalled = true; + return { decision: "allow" }; + }; + + const acs = AcsInterceptor.fromPath(manifest, { + annotatorDispatcher, + policyDispatcher, + }); + + const verdict = acs.intercept(builder().input("hi")); + assert.equal(verdict.decision, "deny"); + assert.equal(verdict.reason, "runtime_error:annotation_failed"); + assert.equal( + policyCalled, + false, + "policy dispatcher must not run after an annotator failure", + ); +}); + +// --------------------------------------------------------------------- +// 3. Passing no options behaves identically to the zero-config path. +// The fixture manifest's verdicts are the pinned baseline; the with- +// hooks constructor must return the same values when no hooks are set. +// --------------------------------------------------------------------- + +test("no host hooks leaves the zero-config path bit-identical", () => { + const zeroConfig = AcsInterceptor.fromPath(fixtureManifestPath); + const empty = AcsInterceptor.fromPath(fixtureManifestPath, {}); + for (const ctx of [ + builder().input("hello"), + builder().preToolCall("t1", "search", { q: "x" }), + builder().output("final answer"), + ]) { + // Two independent Contexts of the same shape. + const a = zeroConfig.intercept(ctx); + const b = empty.intercept(ctx); + assert.deepEqual(b, a); + } +}); + +// --------------------------------------------------------------------- +// 4. A telemetry sink receives events, and perfTelemetry levels round- +// trip. The engine emits a Decision event per evaluation; the sink +// must see at least one, and reject an unknown perf level. +// --------------------------------------------------------------------- + +test("telemetry sink receives Decision events; perf level round-trips", () => { + const events = []; + const acs = AcsInterceptor.fromPath(fixtureManifestPath, { + telemetrySink: (event) => events.push(event), + perfTelemetry: "off", + }); + const verdict = acs.intercept(builder().input("hello")); + assert.equal(verdict.decision, "allow"); + assert.ok(events.length >= 1, "expected at least one telemetry event"); + const decision = events.find((e) => e.event_type === "decision"); + assert.ok(decision, "expected a decision event"); + assert.equal(decision.intervention_point, "input"); + assert.equal(decision.decision, "allow"); + assert.equal(decision.policy_id, "allow_all"); + + // Every documented perf level is accepted. + for (const level of ["off", "external", "full"]) { + const configured = AcsInterceptor.fromPath(fixtureManifestPath, { + perfTelemetry: level, + }); + // Construction alone proves the level round-trips; also confirm + // evaluation still works under it. + assert.equal(configured.intercept(builder().input("x")).decision, "allow"); + } + + // An unknown level is a boundary problem, not a silent fallback. + assert.throws( + () => + AcsInterceptor.fromPath(fixtureManifestPath, { perfTelemetry: "loud" }), + /perf telemetry/i, + ); +}); + +// --------------------------------------------------------------------- +// 5. `parseManifest`: a valid document returns structured JSON; a +// broken document throws. +// --------------------------------------------------------------------- + +test("parseManifest returns structure for valid YAML and throws on garbage", () => { + const parsed = parseManifest(fs.readFileSync(fixtureManifestPath, "utf8")); + assert.equal(typeof parsed, "object"); + assert.ok(parsed); + assert.equal( + parsed.agent_control_specification_version, + "0.4.0-alpha.1", + "top-level version preserved", + ); + assert.ok(parsed.policies, "policies map is present"); + assert.ok(parsed.policies.allow_all, "individual policies survive parse"); + assert.ok(parsed.intervention_points, "intervention points are present"); + + // Malformed YAML must throw, not return an empty object. + assert.throws(() => parseManifest("agent_control_specification_version: [")); + assert.throws(() => parseManifest("::not: [valid: yaml")); + + // Boundary problems throw as TypeError, not as an invalid manifest. + assert.throws(() => parseManifest(42), TypeError); + assert.throws(() => parseManifest("\uD800"), TypeError); +}); + +// --------------------------------------------------------------------- +// 6. `mergeManifests` composes a chain: a base manifest plus an overlay +// that adds an intervention point. The merged result carries fields +// from both. +// --------------------------------------------------------------------- + +test("mergeManifests composes a base and an overlay", () => { + const base = `agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: base +policies: + allow_all: + type: test + verdict: + decision: allow +intervention_points: + input: + policy_target: "$.input" + policy: + id: allow_all +`; + const overlay = `agent_control_specification_version: "0.4.0-alpha.1" +policies: + block_tool: + type: test + verdict: + decision: deny + reason: blocked_by_policy +intervention_points: + pre_tool_call: + policy_target: "$.tool_call.args" + policy: + id: block_tool +`; + const merged = mergeManifests([base, overlay]); + // The base's metadata was left as-written by the additive overlay, + // and both policies and both intervention points survived the merge. + assert.equal(typeof merged, "object"); + assert.ok(merged.policies.allow_all, "base policy survives"); + assert.ok(merged.policies.block_tool, "overlay policy survives"); + assert.ok(merged.intervention_points.input, "base intervention point survives"); + assert.ok( + merged.intervention_points.pre_tool_call, + "overlay intervention point survives", + ); + assert.equal(merged.metadata.name, "base", "base metadata is preserved"); + + // The merged document is runnable end to end: build an interceptor + // from an equivalent YAML shape and get the two verdicts the pieces + // expected. + const mergedYaml = + `agent_control_specification_version: "0.4.0-alpha.1"\n` + + `metadata:\n name: merged\n` + + `policies:\n allow_all:\n type: test\n verdict:\n decision: allow\n` + + ` block_tool:\n type: test\n verdict:\n decision: deny\n reason: blocked_by_policy\n` + + `intervention_points:\n input:\n policy_target: "$.input"\n policy:\n id: allow_all\n` + + ` pre_tool_call:\n policy_target: "$.tool_call.args"\n policy:\n id: block_tool\n`; + const yamlPath = path.join(workdir, "merged-baseline.yaml"); + fs.writeFileSync(yamlPath, mergedYaml, "utf8"); + const acs = AcsInterceptor.fromPath(yamlPath); + assert.equal(acs.intercept(builder().input("hi")).decision, "allow"); + assert.equal( + acs.intercept(builder().preToolCall("t1", "search", { q: "x" })).decision, + "deny", + ); + + // Boundary problems throw as TypeError. + assert.throws(() => mergeManifests("not an array"), TypeError); + assert.throws(() => mergeManifests([42]), TypeError); + assert.throws(() => mergeManifests([])); +}); + +// --------------------------------------------------------------------- +// 7. `validateManifestDetailed` returns diagnostics that name the +// offending field. An unsupported version triggers the "unsupported +// " branch, so `field` points at +// `agent_control_specification_version`. +// --------------------------------------------------------------------- + +test("validateManifestDetailed names the offending field", () => { + const good = fs.readFileSync(fixtureManifestPath, "utf8"); + const empty = validateManifestDetailed(good); + assert.deepEqual(empty, []); + + const badVersion = good.replace('"0.4.0-alpha.1"', '"0.3.1-beta"'); + const findings = validateManifestDetailed(badVersion); + assert.equal(findings.length, 1); + const finding = findings[0]; + assert.equal(finding.severity, "error"); + assert.ok(finding.code.startsWith("runtime_error:"), `code was ${finding.code}`); + assert.match(finding.message, /0\.3\.1-beta/); + assert.equal( + finding.field, + "agent_control_specification_version", + "the field pointer should identify the version key", + ); + + // Boundary problems throw as TypeError. + assert.throws(() => validateManifestDetailed(42), TypeError); + assert.throws(() => validateManifestDetailed("\uD800"), TypeError); +}); + +// --------------------------------------------------------------------- +// Bonus: ActivatedPolicy takes host hooks too. Prove the sync callback +// works through that path as well, so the two entry points stay in +// lockstep. +// --------------------------------------------------------------------- + +test("ActivatedPolicy.activate accepts host hooks and evaluates against them", () => { + const manifest = writePolicyOnlyManifest(workdir); + const seen = []; + const activated = ActivatedPolicy.activate(manifest, { + policyDispatcher: (invocation) => { + seen.push(invocation.type); + return { decision: "deny", reason: "denied_by_test" }; + }, + }); + const verdict = activated.evaluate("input", builder().input("hi")); + assert.equal(verdict.decision, "deny"); + assert.equal(verdict.reason, "denied_by_test"); + assert.deepEqual(seen, ["custom"]); +}); + + +// --------------------------------------------------------------------- +// 8. `validateArtifacts` catches Rego compilation failures a +// manifest-only validator cannot see. This is the shape a 0.3-era +// consumer's CI depended on (validate_acs_artifacts) and the reason +// this feature exists: today a manifest can name a bundle, pass +// grammar validation, and only fail at activation. That moves the +// failure from CI to a host's first agent action. +// --------------------------------------------------------------------- + +const ARTIFACT_MANIFEST = `agent_control_specification_version: "0.4.0-alpha.1" +policies: + gate: + type: rego + bundle: ./b +intervention_points: + input: + policy_target: "$.input" + policy: + id: gate + query: data.acs.decision +`; + +const VALID_REGO = 'package acs\ndecision := {"decision":"allow"}\n'; + +test("validateArtifacts returns [] for a manifest whose Rego compiles", () => { + const findings = validateArtifacts(ARTIFACT_MANIFEST, { + gate: { modules: { "p.rego": VALID_REGO } }, + }); + assert.deepEqual(findings, []); +}); + +test("validateArtifacts surfaces a broken Rego module the manifest names", () => { + // Same manifest, same shape, only the module is malformed. The + // manifest-only surface accepts this; the artifact surface must + // not, because activation would fail on the host's first action. + const findings = validateArtifacts(ARTIFACT_MANIFEST, { + gate: { modules: { "p.rego": "package acs\nfoo := ] not valid rego" } }, + }); + assert.equal(findings.length, 1, `expected one finding, got ${JSON.stringify(findings)}`); + const entry = findings[0]; + assert.equal(entry.severity, "error"); + assert.ok( + entry.code.startsWith("runtime_error:"), + `code was ${entry.code}`, + ); + // The Rego compiler's own text carries the module name and its + // "expecting expression" complaint verbatim, so an editor can point + // at the module. Assert both so a regression that swallowed the + // detail would fail. + assert.match(entry.message, /p\.rego/); + assert.match(entry.message, /expecting expression/); + + // And the manifest-only surface still accepts this: the point of + // validateArtifacts is exactly this gap. + assert.deepEqual(validateManifestDetailed(ARTIFACT_MANIFEST), []); +}); + +test("validateArtifacts reports an unparseable manifest as a manifest problem", () => { + // A document that does not parse must be reported as a manifest + // problem, not an activation failure — that would name the wrong + // half. Even when bundles are supplied, the diagnostic must be + // manifest-half. + const findings = validateArtifacts("::not: [valid", { + gate: { modules: { "p.rego": VALID_REGO } }, + }); + assert.equal(findings.length, 1); + const entry = findings[0]; + assert.equal(entry.code, "runtime_error:manifest_invalid"); + assert.equal(entry.severity, "error"); + + // The underlying RuntimeError message matches what the + // manifest-only surface reports for the same input. + const manifestOnly = validateManifestDetailed("::not: [valid"); + assert.equal(manifestOnly[0].code, entry.code); + assert.equal(manifestOnly[0].message, entry.message); +}); + +test("validateArtifacts without bundles equals the manifest-only result", () => { + // No bundles supplied: activation is either skipped (no Rego to + // load) or fails the same way manifest validation does. Either + // way, the artifact validator must not invent activation errors + // when the manifest half is what actually reports the problem. + // For a grammatically invalid document — one that parses but + // fails validation — the two surfaces report the same underlying + // manifest problem. Activation would never be reached. + const invalid = + 'agent_control_specification_version: "0.4.0-alpha.1"\npolicies: {}\nintervention_points: {}\n'; + const artifact = validateArtifacts(invalid); + const manifest = validateManifestDetailed(invalid); + assert.equal(artifact.length, manifest.length); + assert.equal(artifact.length, 1); + assert.equal(artifact[0].code, manifest[0].code); + assert.equal(artifact[0].message, manifest[0].message); + assert.equal(artifact[0].severity, "error"); + + // And omitting the bundles argument behaves identically to passing + // an empty object, so callers can write either. + assert.deepEqual( + validateArtifacts(ARTIFACT_MANIFEST, {}), + validateArtifacts(ARTIFACT_MANIFEST), + ); +}); + +test("validateArtifacts throws on boundary problems", () => { + // Non-string manifest and unpaired surrogate are boundary problems + // and throw as TypeError, not as an invalid manifest. Wrong shape + // for `bundles` throws too, rather than silently JSON.stringify-ing + // something the native side would reject. + assert.throws(() => validateArtifacts(42), TypeError); + assert.throws(() => validateArtifacts("\uD800"), TypeError); + assert.throws(() => validateArtifacts(ARTIFACT_MANIFEST, 5), TypeError); + assert.throws(() => validateArtifacts(ARTIFACT_MANIFEST, "not-an-object"), TypeError); +}); + +// --------------------------------------------------------------------- +// 9. Resource limits: caps overriding the engine's defaults reach the +// runtime and change the verdict. +// +// `Limits` is a denial-of-service control surface: a host feeding +// large payloads raises `max_snapshot_bytes`; one hardening against a +// hostile manifest lowers `max_extends_depth` or +// `manifest_url_timeout_ms`. +// +// The behavioural test is deliberately end-to-end: the same manifest +// and the same context, evaluated once with default caps and once with +// a small `max_snapshot_bytes`, produce different verdicts. That is +// what proves the value reaches the engine rather than being accepted +// on the JS side and dropped on the way in. +// --------------------------------------------------------------------- + +const { DEFAULT_LIMITS } = require("../dist/index.js"); + +test("a lowered snapshot cap flips the verdict from allow to fail-closed deny", () => { + const big = "x".repeat(4096); + const permissive = AcsInterceptor.fromPath(fixtureManifestPath); + assert.equal(permissive.intercept(builder().input(big)).decision, "allow"); + + // Same manifest, same context, but the cap is now smaller than the + // canonicalized snapshot. A host that asked for the smaller bound and + // got the larger one would believe it was protected when it was not. + const capped = AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { max_snapshot_bytes: 64 }, + }); + const verdict = capped.intercept(builder().input(big)); + assert.equal(verdict.decision, "deny"); + assert.ok( + verdict.reason.startsWith("runtime_error:"), + `expected runtime_error:*, got ${verdict.reason}`, + ); +}); + +test("no limits option matches the baseline zero-config path", () => { + const baseline = AcsInterceptor.fromPath(fixtureManifestPath); + const empty = AcsInterceptor.fromPath(fixtureManifestPath, { limits: {} }); + for (const ctx of [ + builder().input("hi"), + builder().preToolCall("t1", "search", { q: "x" }), + ]) { + // Two independent Contexts of the same shape. + assert.deepEqual( + empty.intercept(ctx), + baseline.intercept(ctx), + "an empty limits object must be identical to no limits option", + ); + } +}); + +test("overriding one limit leaves the others at their defaults", () => { + const big = "x".repeat(4096); + // Raise only the annotator output cap. Untouched `max_snapshot_bytes` + // still defaults big, so the 4096-char input allows. + const partial = AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { max_annotator_output_bytes: 8_388_608 }, + }); + assert.equal(partial.intercept(builder().input(big)).decision, "allow"); + + // And when the second, untouched cap IS lowered on a separate + // interceptor, it enforces — proving the field-by-field override + // semantics: a raised cap does not silently reset a peer. + const both = AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { + max_annotator_output_bytes: 8_388_608, + max_snapshot_bytes: 64, + }, + }); + const v = both.intercept(builder().input(big)); + assert.equal(v.decision, "deny"); + assert.ok(v.reason.startsWith("runtime_error:")); +}); + +test("a limit that is not a non-negative integer is refused", () => { + // A value the engine cannot parse is a hard error, not a + // silently-kept default. A host that typo'd learns immediately + // instead of finding out at the first breached limit that would + // never fire. + assert.throws( + () => + AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { max_snapshot_bytes: "big" }, + }), + /max_snapshot_bytes/, + ); + assert.throws( + () => + AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { max_snapshot_bytes: -1 }, + }), + /max_snapshot_bytes/, + ); + assert.throws( + () => + AcsInterceptor.fromPath(fixtureManifestPath, { + limits: { max_snapshot_bytes: 1.5 }, + }), + /max_snapshot_bytes/, + ); +}); + +test("DEFAULT_LIMITS carries every documented field and is frozen", () => { + // A host raising one cap reads `DEFAULT_LIMITS` to see what it is + // overriding. The shape must stay wired to the engine's own + // defaults, so a shipping change to another cap cannot be silently + // absorbed by a stale mapping. + const expected = new Set([ + "max_snapshot_bytes", + "max_policy_input_depth", + "max_annotators_per_point", + "max_annotator_output_bytes", + "max_policy_output_bytes", + "max_extends_depth", + "max_merged_manifest_bytes", + "max_manifest_url_bytes", + "manifest_url_timeout_ms", + "max_manifest_url_redirects", + ]); + assert.deepEqual(new Set(Object.keys(DEFAULT_LIMITS)), expected); + for (const [key, value] of Object.entries(DEFAULT_LIMITS)) { + assert.equal( + typeof value, + "number", + `DEFAULT_LIMITS[${key}] must be a number, was ${typeof value}`, + ); + assert.ok(Number.isInteger(value) && value >= 0, `${key} = ${value}`); + } + // Frozen so a caller cannot mutate a shared default. + assert.ok(Object.isFrozen(DEFAULT_LIMITS)); +}); diff --git a/sdk/node/test/stream-session.test.mjs b/sdk/node/test/stream-session.test.mjs new file mode 100644 index 0000000..b95bd93 --- /dev/null +++ b/sdk/node/test/stream-session.test.mjs @@ -0,0 +1,402 @@ +// Streaming mediation surface (spec §18.1). +// +// The engine is stateless everywhere else, so a rune-addressable +// track a host emits incrementally cannot ride the ordinary +// interceptor pipeline. `StreamSession` is the accounting layer that +// makes both a mid-stream deny and a cleared-prefix release possible. +// These tests pin the wire contract other language SDKs also +// implement, and the correctness traps a Node-only implementation is +// most likely to fall into: UTF-16 rune-counting drift and a settled +// session leaking a released offset. +import assert from "node:assert/strict"; +import { test } from "node:test"; + +const require = (await import("node:module")).createRequire(import.meta.url); +const { StreamSession } = require("../dist/index.js"); + +test("happy path clears, advances, and finishes clean", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + + assert.equal(s.observeText("model_generated", "hello"), 5); + // Nothing has cleared yet, so the safe offset stays at the start. + assert.equal(s.safeOffset("response"), 0); + assert.equal(s.pending("response"), 5); + + s.recordOutcome("pii", "model_generated", 0, 5, "cleared"); + assert.equal(s.advance("response"), 5); + assert.equal(s.safeOffset("response"), 5); + + const mark = s.watermark("response"); + assert.equal(mark.track, "response"); + assert.equal(mark.confirmed, 5); + assert.equal(mark.received, 5); + assert.equal(mark.pending, 0); + assert.deepEqual(mark.tasks, ["pii"]); + + const completion = s.finish(); + assert.equal(completion.reason.kind, "complete"); + assert.equal(completion.isClean, true); + assert.equal(completion.transformed, false); + // A settled session releases nothing further. `null` says that in + // the type: the caller cannot read it as an offset by accident. + assert.equal(s.safeOffset("response"), null); + assert.equal(s.advance("response"), null); +}); + +test("a deny ends the session and safeOffset becomes null, but watermark still shows how far it got", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + + s.observeText("model_generated", "cleared prefix"); + s.recordOutcome("safety", "model_generated", 0, 7, "cleared"); + s.advance("response"); + assert.equal(s.safeOffset("response"), 7); + + s.observeText("model_generated", "!!!DANGER!!!"); + s.recordOutcome("safety", "model_generated", 14, 26, "denied"); + + // Every rune the host has not already emitted must be withheld, + // including runes a task had cleared. The type says that. + assert.equal(s.safeOffset("response"), null); + assert.equal(s.advance("response"), null); + + // The audit path stays open: the watermark still says how far the + // track got before the deny. + const mark = s.watermark("response"); + assert.equal(mark.confirmed, 7); + assert.equal(mark.received, 26); + + const reason = s.endReason(); + assert.equal(reason.kind, "denied"); + assert.equal(reason.track, "response"); + assert.equal(reason.task, "safety"); + assert.equal(reason.start, 14); + assert.equal(reason.end, 26); + + const completion = s.finish(); + assert.equal(completion.reason.kind, "denied"); + assert.equal(completion.isClean, false); + assert.equal(completion.transformed, false); +}); + +test("a span needs every task to clear before the watermark advances", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii", "safety"], + }); + + s.observeText("model_generated", "hello world"); + s.recordOutcome("pii", "model_generated", 0, 11, "cleared"); + // One task cleared, the other has not, so the confirmed offset must + // not move: releasing the prefix would skip `safety`. + assert.equal(s.advance("response"), null); + assert.equal(s.safeOffset("response"), 0); + + s.recordOutcome("safety", "model_generated", 0, 11, "cleared"); + assert.equal(s.advance("response"), 11); + assert.equal(s.safeOffset("response"), 11); + + const mark = s.watermark("response"); + assert.deepEqual(mark.tasks, ["pii", "safety"]); + assert.equal(mark.confirmed, 11); +}); + +test("observeText counts runes, not UTF-16 code units", () => { + // 🙂 is one Unicode scalar (U+1F642) but two UTF-16 code units in a + // JS string. If observeText leaked the JS length instead of the + // engine's rune count, the received offset would come back as 2 and + // every downstream offset would slide by one for every emoji. + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + + const text = "🙂"; + assert.equal(text.length, 2, "JS string length reports UTF-16 code units"); + assert.equal(s.observeText("model_generated", text), 1); + assert.equal(s.watermark("response").received, 1); + + // Recording an outcome over the UTF-16 length (2) would be past the + // observed end of the track. It must be past the end here. + assert.throws( + () => s.recordOutcome("pii", "model_generated", 0, 2, "cleared"), + /past end|OffsetPastEnd|offset/i, + ); + + // But the actual rune span clears fine. + const s2 = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + s2.observeText("model_generated", "🙂"); + s2.recordOutcome("pii", "model_generated", 0, 1, "cleared"); + assert.equal(s2.advance("response"), 1); + assert.equal(s2.finish().isClean, true); +}); + +test("payload on an unmediated track fails closed", () => { + // Empty response task set: the response track is not mediated at + // all, so text on it fails closed while the request track releases + // as usual. This is the ordinary shape for a host guarding only the + // user prompt. + const s = new StreamSession({ + safetyLevel: "blocking", + requestTasks: ["moderation"], + responseTasks: [], + }); + + s.observeText("user_request", "hello"); + s.recordOutcome("moderation", "user_request", 0, 5, "cleared"); + assert.equal(s.advance("request"), 5); + assert.equal(s.safeOffset("request"), 5); + + // Payload on the unmediated response track fails closed, because + // nothing would gate it. + assert.throws(() => s.observeText("model_generated", "reply"), /NoTasks|not mediated|response/i); + + // The failed observe put the session into its terminal state. + const reason = s.endReason(); + assert.equal(reason.kind, "failed"); + assert.match(reason.reason, /^host_error:/); +}); + +test("unknown safety level and unknown track throw with the engine's message", () => { + assert.throws( + () => new StreamSession({ safetyLevel: "permissive", responseTasks: ["t"] }), + /permissive|unknown/i, + ); + + const s = new StreamSession({ safetyLevel: "blocking", responseTasks: ["t"] }); + assert.throws(() => s.safeOffset("nope"), /nope|unknown/i); + assert.throws(() => s.advance("nope"), /nope|unknown/i); + assert.throws(() => s.watermark("nope"), /nope|unknown/i); + assert.throws(() => s.pending("nope"), /nope|unknown/i); + + assert.throws( + () => new StreamSession({ safetyLevel: "blocking" }), + /NoTracksMediated|neither|no tasks/i, + ); +}); + +test("request and response tracks carry independent offsets", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + requestTasks: ["moderation"], + responseTasks: ["safety"], + }); + + s.observeText("user_request", "user prompt"); + s.observeText("model_generated", "model reply"); + + // Each track has its own confirmed and received frontier. + assert.equal(s.watermark("request").received, 11); + assert.equal(s.watermark("response").received, 11); + assert.equal(s.watermark("request").confirmed, 0); + assert.equal(s.watermark("response").confirmed, 0); + + // Clearing the request must not release the response. + s.recordOutcome("moderation", "user_request", 0, 11, "cleared"); + s.advance("request"); + assert.equal(s.safeOffset("request"), 11); + assert.equal(s.safeOffset("response"), 0); + assert.equal(s.pending("response"), 11); + + // Clearing the response then releases only that track. + s.recordOutcome("safety", "model_generated", 0, 11, "cleared"); + s.advance("response"); + assert.equal(s.safeOffset("response"), 11); + assert.equal(s.finish().isClean, true); +}); + +test("resume offsets keep offsets comparable across a retry", () => { + // A retry that re-sends the prompt and resumes the response reports + // its resume point through `responseStartRuneOffset`. The received + // frontier starts at that offset, so an outcome over the resumed + // range clears without releasing the gap. + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + responseStartRuneOffset: 12, + }); + + const mark = s.watermark("response"); + assert.equal(mark.confirmed, 12); + assert.equal(mark.received, 12); + + s.observeText("model_generated", "continued"); + s.recordOutcome("safety", "model_generated", 12, 21, "cleared"); + s.advance("response"); + assert.equal(s.safeOffset("response"), 21); + assert.equal(s.finish().isClean, true); +}); + +test("recordVerdict routes an allow verdict as a clear", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + s.observeText("model_generated", "hello"); + s.recordVerdict("safety", "model_generated", 0, 5, { + decision: "allow", + warnings: [], + result_labels: [], + }); + assert.equal(s.advance("response"), 5); + assert.equal(s.finish().isClean, true); +}); + +test("recordVerdict deny ends the session with the engine's terminal reason", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + s.observeText("model_generated", "hello"); + s.recordVerdict("safety", "model_generated", 0, 5, { + decision: "deny", + reason: "policy_blocked", + warnings: [], + result_labels: [], + }); + const completion = s.finish(); + assert.equal(completion.reason.kind, "denied"); + assert.equal(completion.reason.task, "safety"); + assert.equal(completion.isClean, false); +}); + +test("transform before any release ends the session rewritten under a withholding level", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + s.observeText("model_generated", "raw"); + s.recordOutcome("safety", "model_generated", 0, 3, "transformed"); + assert.equal(s.isTransformed(), true); + const completion = s.finish(); + assert.equal(completion.reason.kind, "rewritten"); + assert.equal(completion.transformed, true); + assert.equal(completion.isClean, false); +}); + +test("finish twice returns the same completion", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + s.observeText("model_generated", "ok"); + s.recordOutcome("safety", "model_generated", 0, 2, "cleared"); + const first = s.finish(); + const second = s.finish(); + assert.deepEqual(first, second); +}); + +// --------------------------------------------------------------------- +// Rune offsets are `u32` on the wire. N-API converts a JS Number to +// `u32` with ToUint32, which wraps rather than fails: `2 ** 32` +// arrives as `0` and `2 ** 32 + 5` as `5`, so an end offset chosen +// past the boundary would record a *cleared* prefix on text no task +// evaluated. Python raises OverflowError and .NET throws +// OverflowException on the same input, so a Node host that fed a +// deliberately-huge offset would silently emit content the other +// languages refused. The wrapper is the one place a guard fits before +// the value reaches napi's converter; pin that on every rune-offset +// surface so a future refactor cannot re-open it. +// --------------------------------------------------------------------- + +test("observe refuses a rune offset at or past the u32 boundary", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + assert.throws( + () => s.observe("model_generated", 2 ** 32), + RangeError, + "2 ** 32 must be refused, not silently wrapped to 0", + ); + // Session state must be untouched: the throw happened before + // reaching the native call. + assert.equal(s.watermark("response").received, 0); +}); + +test("observe refuses a negative rune offset", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + assert.throws( + () => s.observe("model_generated", -1), + RangeError, + "-1 must be refused, not silently wrapped to 0xFFFFFFFF", + ); + assert.equal(s.watermark("response").received, 0); +}); + +test("recordOutcome refuses an end offset past the u32 boundary", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + s.observeText("model_generated", "hello"); + assert.throws( + () => s.recordOutcome("pii", "model_generated", 0, 2 ** 32 + 5, "cleared"), + RangeError, + "2 ** 32 + 5 must be refused, not silently wrapped to 5 which would clear text no task evaluated", + ); + // Nothing cleared, because the guard fired before the native call. + assert.equal(s.safeOffset("response"), 0); +}); + +test("recordOutcome refuses a start offset at the u32 boundary too", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + }); + assert.throws( + () => s.recordOutcome("pii", "model_generated", 2 ** 32, 5, "cleared"), + RangeError, + ); +}); + +test("recordVerdict refuses rune offsets past the u32 boundary", () => { + const s = new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["safety"], + }); + s.observeText("model_generated", "hello"); + const allow = { decision: "allow" }; + assert.throws( + () => s.recordVerdict("safety", "model_generated", 0, 2 ** 32 + 5, allow), + RangeError, + ); + assert.throws( + () => s.recordVerdict("safety", "model_generated", -1, 5, allow), + RangeError, + ); + // The verdict path is another way to enter the same accounting, so + // the guard must catch it symmetrically; a bad recordVerdict must + // not clear the span either. + assert.equal(s.safeOffset("response"), 0); +}); + +test("StreamSession refuses a start rune offset in config past the u32 boundary", () => { + assert.throws( + () => new StreamSession({ + safetyLevel: "blocking", + responseTasks: ["pii"], + responseStartRuneOffset: 2 ** 32, + }), + RangeError, + ); + assert.throws( + () => new StreamSession({ + safetyLevel: "blocking", + requestTasks: ["moderation"], + requestStartRuneOffset: -1, + }), + RangeError, + ); +}); diff --git a/sdk/python/Cargo.toml b/sdk/python/Cargo.toml index a8a25e9..fac6484 100644 --- a/sdk/python/Cargo.toml +++ b/sdk/python/Cargo.toml @@ -11,7 +11,7 @@ name = "_native" crate-type = ["cdylib"] [dependencies] -agent-control-spec = { version = "0.4.0-alpha.2", path = "../../engine", features = ["default-dispatchers"] } +agent-control-spec = { version = "0.4.0-alpha.2", path = "../../engine", features = ["default-dispatchers", "streaming"] } pyo3 = { version = "0.29", features = ["extension-module", "abi3-py311"] } serde_json = "1" diff --git a/sdk/python/agent_control_spec/__init__.py b/sdk/python/agent_control_spec/__init__.py index 383843a..4c31d65 100644 --- a/sdk/python/agent_control_spec/__init__.py +++ b/sdk/python/agent_control_spec/__init__.py @@ -14,21 +14,32 @@ from __future__ import annotations import json -from collections.abc import Mapping -from typing import Any +from collections.abc import Callable, Iterable, Mapping +from types import MappingProxyType as _MappingProxyType +from typing import Any, Self from agent_hooks import Verdict from agent_control_spec import _native __all__ = [ + "DEFAULT_LIMITS", + "PERF_TELEMETRY_LEVELS", "AcsInterceptor", "ActivatedPolicy", + "ArtifactDiagnostic", "ManifestInvalidError", "RegoBundle", + "StreamSession", + "TelemetryEvent", + "ValidationDiagnostic", "__version__", + "merge_manifests", + "parse_manifest", "supported_manifest_versions", + "validate_artifacts", "validate_manifest", + "validate_manifest_detailed", "validate_manifest_file", ] @@ -39,18 +50,170 @@ #: {...}}]}``. Both keys default to empty and nothing else is accepted. RegoBundle = Mapping[str, Any] +#: The `perf_telemetry` levels the runtime accepts. Kept explicit rather +#: than a Rust-side enum: the constructor argument is a string, so the +#: allowed values are the vocabulary a Python host reads. +PERF_TELEMETRY_LEVELS: tuple[str, ...] = ("off", "external", "full") + +#: The engine's shipped resource caps, as a read-only mapping. A host +#: passing ``limits=`` to :class:`AcsInterceptor` reads this to see what +#: it is overriding — a shipping change to another default cannot then +#: be silently absorbed. Frozen at import time so a caller cannot mutate +#: a shared default. Fields: +#: +#: - ``max_snapshot_bytes``: cap on the canonicalized context snapshot. +#: - ``max_policy_input_depth``: JSON nesting depth accepted anywhere. +#: - ``max_annotators_per_point``: annotators the engine will dispatch. +#: - ``max_annotator_output_bytes``: per-annotator serialized output. +#: - ``max_policy_output_bytes``: policy-decision serialized output. +#: - ``max_extends_depth``: manifest ``extends`` chain length. +#: - ``max_merged_manifest_bytes``: composed manifest total size. +#: - ``max_manifest_url_bytes``: per-URL fetch body cap. +#: - ``manifest_url_timeout_ms``: per-URL fetch deadline. +#: - ``max_manifest_url_redirects``: per-URL fetch redirect count. +DEFAULT_LIMITS: Mapping[str, int] = _MappingProxyType(_native.default_limits()) + +#: A telemetry event a host-supplied sink receives from the engine. +#: +#: The dict is populated by the native layer and mirrors +#: :class:`agent_control_spec.telemetry.TelemetryEvent` on the Rust side. +#: Its shape is documented rather than typed as a ``TypedDict`` to keep +#: the Python surface stable while the engine grows fields, which it has +#: done twice already in 0.4 (evidence artefacts, transformed-event). +#: Keys: +#: +#: - ``event_type``: one of ``"decision"``, ``"annotator_dispatch"``, +#: ``"policy_evaluation"``, ``"evaluation_timing"``, +#: ``"intervention_point.transformed"``, ``"annotator_failed"``, or +#: ``"policy_failed"``. +#: - ``intervention_point``: agent-hooks wire name. +#: - ``decision``: ``"allow"`` / ``"deny"`` / ``"transform"`` or ``None``. +#: - ``reason_code``: ``str`` or ``None``. +#: - ``error_class``: ``str`` or ``None``. +#: - ``policy_id``: ``str`` or ``None``. +#: - ``annotators``: ``list[str]``. +#: - ``enforcement_mode``: ``"enforce"`` / ``"evaluate_only"`` or ``None``. +#: - ``duration_ms``: ``float`` or ``None``. +#: - ``evidence_artefact``: ``str`` or ``None``. +#: - ``evidence_verification_pointer_keys``: ``list[str]``. +#: - ``action_identity``: ``str`` or ``None``. +#: - ``metadata``: ``dict[str, str]``. +TelemetryEvent = Mapping[str, Any] + +#: One entry produced by :func:`validate_manifest_detailed`. Shape: +#: +#: - ``code``: ``str`` (the engine's ``runtime_error:*`` reason). +#: - ``message``: ``str`` (the engine's own message text). +#: - ``severity``: always ``"error"``. +#: - ``field``: ``str | None`` (best-effort field name extracted from +#: the message; ``None`` when the message names no known field). +#: +#: Matches :data:`ArtifactDiagnostic` on ``code``, ``message``, and +#: ``severity``, so a diagnostic-consuming tool can key off ``code`` +#: across surfaces. Wrapped as a mapping rather than a dataclass to +#: keep it JSON-safe for tools that shuttle diagnostics through IPC. +ValidationDiagnostic = Mapping[str, Any] + +#: One entry produced by :func:`validate_artifacts`. Shape: +#: +#: - ``code``: ``str`` (the engine's ``runtime_error:*`` reason). +#: - ``message``: ``str`` (the engine's own detail text). +#: - ``severity``: always ``"error"``. +#: +#: Matches the C ABI's ``acs_artifact_diagnostics`` wire shape, so a +#: diagnostic-consuming tool can key off ``code`` across languages. +ArtifactDiagnostic = Mapping[str, Any] + + +def _normalize_perf_telemetry(value: str | None) -> str: + """Reject unknown perf-telemetry levels on the Python side. + + The Rust binding does its own check, but doing this here means the + error is raised without paying for a manifest load first, which + matches how the rest of the wrapper preserves cheap-to-fail + ordering. The engine's own vocabulary is preserved verbatim. + """ + if value is None: + return "off" + if value not in PERF_TELEMETRY_LEVELS: + raise ValueError( + f"unknown perf_telemetry level {value!r}; " + f"expected one of {PERF_TELEMETRY_LEVELS}" + ) + return value + class AcsInterceptor: """agent-hooks interceptor over the Agent Control Specification runtime. Register an instance with any agent-hooks host emitter. The manifest - is loaded once at construction with the zero-config dispatchers - (bundled annotators; Rego in process, Cedar through the built-in - evaluator, ``test`` policies through their embedded verdict). + is loaded once at construction. + + Zero-config path (the default): bundled annotators; Rego in process, + Cedar through the built-in evaluator, ``test`` policies through + their embedded verdict; no-op telemetry; the engine's default + resource caps. + + Host hooks are supplied by keyword: + + - ``annotator_dispatcher``: object with a ``dispatch(annotator_name, + annotator, preliminary_policy_input)`` method or a plain callable + with the same signature. Return value is the annotation payload + that reaches the policy under ``input.annotations[]``. + - ``policy_dispatcher``: object with an ``evaluate(invocation)`` + method (and optionally a ``warm(invocation)`` method) or a plain + callable. Return value is the raw policy output normalized into a + verdict by the engine. + - ``telemetry_sink``: object with an ``emit(event)`` method + (optionally a ``shutdown()`` method) or a plain callable. The + engine emits one event per decision plus optional stage events. + - ``perf_telemetry``: ``"off"`` (default), ``"external"``, or + ``"full"``, gating whether external and per-stage timing events + are emitted. + - ``limits``: a mapping of resource caps that overrides the engine's + defaults field by field. Absent means keep every default; each + field is individually optional, so a host raising one cap does + not restate the other nine. A host feeding large payloads raises + ``max_snapshot_bytes``; one hardening against a hostile manifest + lowers ``max_extends_depth`` or ``manifest_url_timeout_ms``. Read + :data:`DEFAULT_LIMITS` to see the shipped values. + + A dispatcher that raises does not silently no-op: the engine + normalizes the failure into a fail-closed ``deny`` verdict with a + ``runtime_error:*`` reason. """ - def __init__(self, manifest_path: str) -> None: - self._handle = _native.interceptor_new(manifest_path) + def __init__( + self, + manifest_path: str, + name: str = "acs", + *, + annotator_dispatcher: object | None = None, + policy_dispatcher: object | None = None, + telemetry_sink: object | Callable[[TelemetryEvent], None] | None = None, + perf_telemetry: str = "off", + limits: Mapping[str, int] | None = None, + ) -> None: + perf = _normalize_perf_telemetry(perf_telemetry) + self._handle = _native.interceptor_new( + manifest_path, + annotator_dispatcher, + policy_dispatcher, + telemetry_sink, + perf, + limits, + ) + self._name = name + + @property + def name(self) -> str: + """Payload-free identifier for the record's ``verdicts[].name``. + + The engine does not stamp this onto a verdict. A host that runs + more than one interceptor records it alongside the verdict so the + entry says which one decided. + """ + return self._name def intercept(self, context: Mapping[str, Any]) -> Verdict: wire = _native.intercept(self._handle, json.dumps(context, allow_nan=False)) @@ -80,15 +243,29 @@ class is the other split: :meth:`activate` pays for reading the manifest path is enough and the working directory does not matter. :meth:`from_memory` is the other source: manifest text and Rego sources held by the host, with no file to read. + + Host dispatchers are supplied by the same keyword arguments as + :class:`AcsInterceptor`. Passing them here is exactly how the + consumer-facing ``AgentControl.from_native(..., + annotator_dispatcher=ContentSafetyDispatcher())`` shape composed: + activation carries the dispatcher, evaluation calls it, and readying + warms it. """ __slots__ = ("_handle",) - def __init__(self, manifest_path: str) -> None: - """Activate the manifest at ``manifest_path`` with the - zero-config dispatchers (bundled annotators; Rego in process, - Cedar through the built-in evaluator, ``test`` policies through - their embedded verdict). + def __init__( + self, + manifest_path: str, + *, + annotator_dispatcher: object | None = None, + policy_dispatcher: object | None = None, + ) -> None: + """Activate the manifest at ``manifest_path``. + + Passing no host arguments preserves the zero-config path (bundled + annotators; Rego in process, Cedar through the built-in + evaluator, ``test`` policies through their embedded verdict). Raises :class:`ValueError` when the manifest cannot be read or is rejected, and :class:`RuntimeError` when it binds a policy that @@ -97,19 +274,36 @@ def __init__(self, manifest_path: str) -> None: the deadline surfaces at the first decision instead. A policy that merely needs real input to produce a verdict activates fine. """ - self._handle = _native.policy_activate(manifest_path) + self._handle = _native.policy_activate( + manifest_path, annotator_dispatcher, policy_dispatcher + ) @classmethod - def activate(cls, manifest_path: str) -> ActivatedPolicy: + def activate( + cls, + manifest_path: str, + *, + annotator_dispatcher: object | None = None, + policy_dispatcher: object | None = None, + ) -> ActivatedPolicy: """Activate the manifest at ``manifest_path``. Same as the constructor, named for the lifecycle it belongs to. """ - return cls(manifest_path) + return cls( + manifest_path, + annotator_dispatcher=annotator_dispatcher, + policy_dispatcher=policy_dispatcher, + ) @classmethod def from_memory( - cls, manifest_yaml: str, bundles: Mapping[str, RegoBundle] + cls, + manifest_yaml: str, + bundles: Mapping[str, RegoBundle], + *, + annotator_dispatcher: object | None = None, + policy_dispatcher: object | None = None, ) -> ActivatedPolicy: """Activate a manifest and its Rego supplied as values. @@ -130,7 +324,10 @@ def from_memory( """ policy = cls.__new__(cls) policy._handle = _native.policy_activate_from_memory( - manifest_yaml, json.dumps(bundles, allow_nan=False) + manifest_yaml, + json.dumps(bundles, allow_nan=False), + annotator_dispatcher, + policy_dispatcher, ) return policy @@ -197,9 +394,316 @@ def validate_manifest_file(path: str) -> None: _native.validate_manifest_file(path) +def validate_manifest_detailed(source: str) -> list[ValidationDiagnostic]: + """Return structured validation diagnostics for a manifest source. + + Each diagnostic is ``{"code": str, "message": str, "severity": + "error", "field": str | None}`` — the same wire shape + :func:`validate_artifacts` and every other binding return. An + accepted manifest returns ``[]``. A rejected one returns one entry + naming the failed field where the engine's message permits + extraction, and ``None`` for ``field`` when it does not: the + ``message`` is the engine's own text either way, so a tool that + cannot map ``field`` back to a location still has the verbatim + reason. + + Use this for authoring tools, migration linting, and CI checks that + want per-field feedback. :func:`validate_manifest` is the boolean + shortcut for callers that only care whether validation passed. + """ + return json.loads(_native.validate_manifest_diagnostics(source)) + + +def validate_artifacts( + manifest_source: str, + bundles: Mapping[str, RegoBundle] | None = None, +) -> list[ArtifactDiagnostic]: + """Validate a manifest AND the Rego it names, returning findings. + + Each diagnostic is ``{"code": str, "message": str, "severity": + "error"}`` and matches the C ABI's ``acs_artifact_diagnostics`` + wire shape. An empty list means both halves are sound. + + :func:`validate_manifest_detailed` answers only for the document. + A manifest can satisfy the grammar, name a Rego bundle, and still + fail at activation because the Rego does not compile — compilation + happens at activation time, so a validation surface that stops at + the document turns that failure into a host's first agent action + rather than a CI signal. This activates against the supplied + bundles in memory and reports what that surfaced, closing the gap. + + ``bundles`` has the same shape :meth:`ActivatedPolicy.from_memory` + takes: a mapping from policy id to a ``{"modules": {...}, "data": + [...]}`` object. ``None`` or an empty mapping means the manifest + names no Rego, and the result then equals the manifest-only + diagnostics: a document that does not parse is reported as a + manifest problem, not an activation problem, because that names + the wrong half. + """ + payload = "" if bundles is None else json.dumps(bundles, allow_nan=False) + return json.loads(_native.validate_artifacts_diagnostics(manifest_source, payload)) + + +def parse_manifest(source: str) -> dict[str, Any]: + """Parse manifest source into a ``dict`` without validating. + + An authoring tool that needs to inspect a fragment before deciding + what to do with it, such as reading an ``extends`` child's + ``metadata`` before resolving the chain, calls this. No policy + engine is put on-path. + + Raises :class:`ManifestInvalidError` when the source is not + well-formed YAML or the manifest grammar rejects it structurally. + """ + return json.loads(_native.parse_manifest(source)) + + +def merge_manifests(sources: Iterable[str]) -> dict[str, Any]: + """Compose an ordered chain of manifest sources into one ``dict``. + + Later sources overlay earlier ones under the same merge grammar + ``extends`` uses on disk. Every entry must be a fully-formed manifest + fragment: no chain entry may itself carry unresolved ``extends``. + The resulting document is validated before it is returned, so a + chain that would fail as an on-disk ``extends`` fails here too. + + Use this when the manifests come from memory (database rows, + process-supplied overlays) rather than disk; use ``extends`` in the + manifest itself when they come from disk and their layout is fixed. + + Raises :class:`ManifestInvalidError` when the chain is empty or an + entry does not parse. + """ + materialized = list(sources) + return json.loads(_native.merge_manifests(materialized)) + + def supported_manifest_versions() -> tuple[str, ...]: """The manifest grammar versions this engine accepts. Read it rather than hardcoding the set; it moves with the engine. """ return tuple(_native.supported_manifest_versions()) + + +class StreamSession: + """Host side accounting for one streamed policy target. + + A session holds no policy, performs no evaluation, and stores no + stream text. The host drives it: it reports how much text arrived, + declares the spans its segmenter produced, evaluates those spans + through the ordinary interceptor path, records each outcome, and + reads :meth:`safe_offset` to see how far it may release the track. + This is the incremental profile in specification section 18.1. + + A track with no tasks is unmediated. Payload on such a track fails + closed, which matches the behavior of a host guarding only the model + stream and receiving text on the wrong track. A configuration that + mediates neither track is rejected at construction: it would gate + nothing. + + ``safety_level`` is one of ``"blocking"``, ``"complete"``, or + ``"deferred"``. ``"blocking"`` and ``"complete"`` hold each span + until the watermark covers it. ``"deferred"`` emits payload as it + arrives and evaluates behind the stream, and cannot recall what has + already been emitted. + + The session settles in two steps. :meth:`end_of_payloads` says no + more text is coming while outcomes are still in flight, which is + what a ``"deferred"`` host needs so a late denial can still land. + :meth:`finish` returns the terminal :class:`dict` and marks the + session ended. After :meth:`finish`, :meth:`safe_offset` is + ``None``: a terminated session has no offset a host may emit + through, whatever the reason. The confirmed offset stays available + through :meth:`watermark` for the audit record. + """ + + __slots__ = ("_handle",) + + def __init__( + self, + safety_level: str = "blocking", + *, + request_tasks: Iterable[str] | None = None, + response_tasks: Iterable[str] | None = None, + request_start_rune_offset: int = 0, + response_start_rune_offset: int = 0, + ) -> None: + request_tasks_list = list(request_tasks) if request_tasks is not None else [] + response_tasks_list = list(response_tasks) if response_tasks is not None else [] + self._handle = _native.stream_session_new( + safety_level, + int(request_start_rune_offset), + int(response_start_rune_offset), + request_tasks_list, + response_tasks_list, + ) + + def observe(self, source_type: str, runes: int) -> int: + """Report that ``runes`` more runes arrived on this role's + track. Returns the track's new end offset. + """ + return _native.stream_observe(self._handle, source_type, int(runes)) + + def observe_text(self, source_type: str, text: str) -> int: + """Report arriving text and let the engine count its runes, + so a host does not reach for a length that measures UTF-16 code + units or bytes. Neither is interchangeable with a rune offset. + The text itself is not retained. + """ + return _native.stream_observe_text(self._handle, source_type, text) + + def record_outcome( + self, + task: str, + source_type: str, + start: int, + end: int, + outcome: str, + ) -> None: + """Record what a host decided for the half-open rune range + ``[start, end)`` on ``source_type``'s track, under ``task``. + + ``outcome`` is one of ``"cleared"``, ``"transformed"``, or + ``"denied"``. A denial or transform ends the session. Every + engine rejection raises :class:`ValueError` with the engine's + own message; nothing silently no-ops. + """ + _native.stream_record_outcome( + self._handle, + task, + source_type, + int(start), + int(end), + outcome, + ) + + def record_verdict( + self, + task: str, + source_type: str, + start: int, + end: int, + verdict: Verdict | Mapping[str, Any], + ) -> None: + """Map an agent-hooks verdict onto an outcome and record it. + + ``verdict`` may be a :class:`agent_hooks.Verdict` or the same + wire dict :meth:`agent_hooks.Verdict.to_wire` produces. A shape + the section 5 contract does not admit fails the stream closed + with :class:`ValueError` before its decision is read. + """ + if isinstance(verdict, Verdict): + wire = verdict.to_wire() + elif isinstance(verdict, Mapping): + wire = verdict + else: + raise TypeError("verdict must be an agent_hooks.Verdict or a wire dict") + _native.stream_record_verdict( + self._handle, + task, + source_type, + int(start), + int(end), + json.dumps(wire, allow_nan=False), + ) + + def advance(self, track: str) -> int | None: + """Recompute the watermark for ``track`` and return the new + confirmed offset when it advanced. Returns ``None`` when it did + not, so a host emits a watermark event only on real progress. + Returns ``None`` once the session has ended. + """ + return _native.stream_advance(self._handle, track) + + def safe_offset(self, track: str) -> int | None: + """Offset through which the host may emit ``track``, or ``None`` + once the session has ended. + + A denial withholds every rune the host has not already emitted, + including runes a task had cleared, so a terminated session has + no offset anyone may emit through. Returning ``None`` says that + in the type, which a host cannot read as permission by + accident. + """ + return _native.stream_safe_offset(self._handle, track) + + def pending(self, track: str) -> int: + """Runes observed but not yet cleared by every task on + ``track``, as of the last :meth:`advance`. + """ + return _native.stream_pending(self._handle, track) + + def watermark(self, track: str) -> dict[str, Any]: + """Watermark snapshot for one track: + ``{"track", "confirmed", "received", "pending", "tasks"}``. + Reads without moving anything. + """ + return json.loads(_native.stream_watermark(self._handle, track)) + + def end_of_payloads(self) -> None: + """Stop accepting payloads while outcomes are still in flight. + A ``"deferred"`` host calls this at payload EOF so a classifier + running behind the stream can still record a denial before + :meth:`finish`. + """ + _native.stream_end_of_payloads(self._handle) + + def __enter__(self) -> Self: + return self + + def __exit__(self, *_exc: object) -> None: + """Settle on the way out. + + A host owes an outcome for every session it opens, including one + it abandons, and nothing can make that automatic. A context + manager makes it the shape of least resistance instead. Settling + twice returns the same completion, so an explicit ``finish`` + inside the block stays correct. + """ + self.finish() + + def finish(self) -> dict[str, Any]: + """Settle the session and return the terminal record: + ``{"reason": , "transformed": bool, "is_clean": + bool}``. + + ``reason`` is one of ``{"kind": "complete"}``, ``{"kind": + "denied", "track", "task", "start", "end"}``, ``{"kind": + "rewritten", "track", "task", "start", "end"}``, or ``{"kind": + "failed", "reason", "message"}``. Any rune no task cleared + settles the session ``failed`` under every safety level. + """ + return json.loads(_native.stream_finish(self._handle)) + + @property + def is_ended(self) -> bool: + """Whether the session has reached its terminal state.""" + return _native.stream_is_ended(self._handle) + + @property + def transformed(self) -> bool: + """Whether a ``transformed`` outcome ended this session, meaning + the host emits a substitute rather than verbatim model output. + A transform clears nothing, so this says nothing about what was + released. + """ + return _native.stream_transformed(self._handle) + + @property + def end_reason(self) -> dict[str, Any] | None: + """Terminal reason as a wire dict, or ``None`` when the session + has not ended. The same schema :meth:`finish` returns under + ``reason``. + """ + raw = _native.stream_end_reason(self._handle) + return None if raw is None else json.loads(raw) + + @property + def config(self) -> dict[str, Any]: + """Streaming parameters this session was opened with: + ``{"safety_level", "request_start_rune_offset", + "response_start_rune_offset", "request_tasks", + "response_tasks"}``. + """ + return json.loads(_native.stream_config(self._handle)) diff --git a/sdk/python/src/lib.rs b/sdk/python/src/lib.rs index 8d0d8be..da643cb 100644 --- a/sdk/python/src/lib.rs +++ b/sdk/python/src/lib.rs @@ -8,33 +8,462 @@ // boundary mean a boundary problem only (unreadable manifest, // non-object context JSON). +use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; +use agent_control_spec::policy::PreparedPolicyInvocation; +use agent_control_spec::runtime::PolicyDispatcher; +use agent_control_spec::telemetry::{NoopTelemetrySink, TelemetryEvent, TelemetrySink}; +use agent_control_spec::wire; use agent_control_spec::{ - ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Manifest, Runtime, RuntimeError, - SUPPORTED_VERSIONS, + ActivatedPolicy, InMemoryRegoBundle, InterceptionPoint, Limits, Manifest, Runtime, + RuntimeError, SafetyLevel, SegmentOutcome, StreamError, StreamSession, StreamSessionConfig, + StreamSourceType, StreamSpan, StreamTrack, Verdict, SUPPORTED_VERSIONS, }; use pyo3::create_exception; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError}; use pyo3::prelude::*; -use serde_json::Value; -use std::sync::Arc; +use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyMapping, PyString}; +use serde_json::{Map, Value}; +use std::sync::{Arc, Mutex}; + +// --------------------------------------------------------------------- +// Host-supplied dispatchers, telemetry, and perf-telemetry level. +// +// These wrappers hold a `Py` and adapt a Python object into the +// engine's `AnnotatorDispatcher`, `PolicyDispatcher`, or `TelemetrySink` +// trait. The engine calls them from its evaluation path, so failures +// raised on the Python side must become `RuntimeError` on the Rust side: +// the engine then normalizes them into fail-closed `runtime_error:*` +// verdicts and never treats a raising dispatcher as "no annotation". +// +// `Py` is `Send + Sync` in pyo3; the wrappers acquire the GIL +// inside each callback for the actual Python call. +// --------------------------------------------------------------------- + +/// Convert a `serde_json::Value` into a Python object using only the +/// standard container types, so a host dispatcher receives plain +/// `dict`/`list`/`str`/`bool`/`int`/`float`/`None` values. +fn json_to_py<'py>(py: Python<'py>, value: &Value) -> PyResult> { + match value { + Value::Null => Ok(py.None().into_bound(py)), + Value::Bool(b) => Ok(PyBool::new(py, *b).to_owned().into_any()), + Value::Number(n) => { + if let Some(i) = n.as_i64() { + Ok(i.into_pyobject(py)?.into_any()) + } else if let Some(u) = n.as_u64() { + Ok(u.into_pyobject(py)?.into_any()) + } else if let Some(f) = n.as_f64() { + Ok(PyFloat::new(py, f).into_any()) + } else { + // Neither i64/u64/f64 accepted the number, which means an + // arbitrary-precision integer serde_json exposed only as + // string. Round-trip through JSON to preserve the value. + let s = n.to_string(); + Ok(PyString::new(py, &s).into_any()) + } + } + Value::String(s) => Ok(PyString::new(py, s).into_any()), + Value::Array(items) => { + let list = PyList::empty(py); + for item in items { + list.append(json_to_py(py, item)?)?; + } + Ok(list.into_any()) + } + Value::Object(map) => { + let dict = PyDict::new(py); + for (key, value) in map { + dict.set_item(key, json_to_py(py, value)?)?; + } + Ok(dict.into_any()) + } + } +} + +/// Convert a Python object into a `serde_json::Value`. Used to accept a +/// host dispatcher's return value, and structured with the same +/// vocabulary that `json_to_py` produces so a callback that echoes its +/// input round-trips. +fn py_to_json(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(Value::Null); + } + if let Ok(b) = value.cast::() { + return Ok(Value::Bool(b.is_true())); + } + if let Ok(i) = value.cast::() { + if let Ok(v) = i.extract::() { + return Ok(Value::from(v)); + } + if let Ok(v) = i.extract::() { + return Ok(Value::from(v)); + } + // Fall through for oversized integers: represent as string, which + // is the same fallback `json_to_py` uses for arbitrary-precision + // numbers. + return Ok(Value::String(i.str()?.to_string_lossy().into_owned())); + } + if let Ok(f) = value.cast::() { + let v = f.value(); + // `serde_json::Number::from_f64` rejects NaN/Inf, matching the + // JSON grammar the engine's inputs assume elsewhere. + return serde_json::Number::from_f64(v) + .map(Value::Number) + .ok_or_else(|| PyValueError::new_err("dispatcher returned a non-finite float")); + } + if let Ok(s) = value.cast::() { + return Ok(Value::String(s.to_string_lossy().into_owned())); + } + if let Ok(list) = value.cast::() { + let mut out = Vec::with_capacity(list.len()); + for item in list.iter() { + out.push(py_to_json(&item)?); + } + return Ok(Value::Array(out)); + } + if let Ok(dict) = value.cast::() { + let mut out = Map::new(); + for (key, val) in dict.iter() { + let key_str = key + .cast::() + .map_err(|_| { + PyValueError::new_err("dispatcher returned a dict with a non-string key") + })? + .to_string_lossy() + .into_owned(); + out.insert(key_str, py_to_json(&val)?); + } + return Ok(Value::Object(out)); + } + // Fall through: reject anything that is not a plain JSON-compatible + // Python value. Silent conversion via `str()` would hide contract + // violations in host code. + Err(PyValueError::new_err(format!( + "dispatcher returned a value that does not fit the JSON grammar: {}", + value.get_type().name()? + ))) +} + +/// Turn a `PyErr` raised inside a host dispatcher into a Rust +/// `RuntimeError` variant. Distinct variant per role, so the engine's +/// normalized `runtime_error:*` reason names the pipeline stage that +/// failed rather than a generic bucket. +fn py_err_to_annotation_failure(annotator_name: &str, err: PyErr) -> RuntimeError { + RuntimeError::AnnotationFailed(format!("host annotator '{annotator_name}' raised: {err}")) +} + +fn py_err_to_policy_failure(err: PyErr) -> RuntimeError { + RuntimeError::PolicyInvocationFailed(format!("host policy dispatcher raised: {err}")) +} + +/// Call a Python callable, preferring a named method when `callback` +/// exposes one. The old 0.3.1b1 API accepted objects with a `dispatch` +/// method; a plain callable is admitted too so hosts can use a small +/// lambda for tests. +fn call_py_method<'py>( + py: Python<'py>, + callback: &Py, + method: &str, + args: Vec>, +) -> PyResult> { + let bound = callback.bind(py); + let py_tuple = pyo3::types::PyTuple::new(py, args)?; + if let Ok(func) = bound.getattr(method) { + return func.call1(py_tuple); + } + // No named method: treat the object itself as callable. `.call1` is + // itself an attribute lookup, so a non-callable object surfaces its + // own error rather than a fabricated one. + bound.call1(py_tuple) +} + +/// Adapter from an `AnnotatorDispatcher` call to a Python object. +struct PyAnnotatorDispatcher { + callback: Py, +} + +impl AnnotatorDispatcher for PyAnnotatorDispatcher { + fn dispatch( + &self, + annotator_name: &str, + annotator: &AnnotatorInvocation, + preliminary_policy_input: &Value, + ) -> Result { + Python::attach(|py| { + let invocation = serde_json::to_value(annotator).map_err(|err| { + RuntimeError::AnnotationFailed(format!( + "host annotator '{annotator_name}': failed to serialize invocation: {err}" + )) + })?; + let invocation_py = json_to_py(py, &invocation) + .map_err(|err| py_err_to_annotation_failure(annotator_name, err))?; + let prelim_py = json_to_py(py, preliminary_policy_input) + .map_err(|err| py_err_to_annotation_failure(annotator_name, err))?; + let name_py = PyString::new(py, annotator_name).into_any(); + + let result = call_py_method( + py, + &self.callback, + "dispatch", + vec![name_py, invocation_py, prelim_py], + ) + .map_err(|err| py_err_to_annotation_failure(annotator_name, err))?; + + py_to_json(&result).map_err(|err| { + RuntimeError::AnnotationFailed(format!( + "host annotator '{annotator_name}' returned a non-JSON value: {err}" + )) + }) + }) + } +} + +/// Adapter from a `PolicyDispatcher` call to a Python object. Accepts an +/// object with `evaluate` (and optionally `warm`) or a plain callable. +struct PyPolicyDispatcher { + callback: Py, +} + +impl PolicyDispatcher for PyPolicyDispatcher { + fn evaluate(&self, invocation: &PreparedPolicyInvocation) -> Result { + Python::attach(|py| { + let invocation_json = serde_json::to_value(invocation).map_err(|err| { + RuntimeError::PolicyInvocationFailed(format!( + "failed to serialize policy invocation for host dispatcher: {err}" + )) + })?; + let invocation_py = + json_to_py(py, &invocation_json).map_err(py_err_to_policy_failure)?; + + let result = call_py_method(py, &self.callback, "evaluate", vec![invocation_py]) + .map_err(py_err_to_policy_failure)?; + + py_to_json(&result).map_err(|err| { + RuntimeError::PolicyInvocationFailed(format!( + "host policy dispatcher returned a non-JSON value: {err}" + )) + }) + }) + } + + fn warm(&self, invocation: &PreparedPolicyInvocation) -> Result<(), RuntimeError> { + // `warm` is best-effort per the trait contract, and a host that + // does not expose it is not obliged to. Skip silently rather than + // charging every host with implementing an optimization hook. + Python::attach(|py| { + let bound = self.callback.bind(py); + let func = match bound.getattr("warm") { + Ok(func) => func, + Err(_) => return Ok(()), + }; + let invocation_json = serde_json::to_value(invocation).map_err(|err| { + RuntimeError::PolicyInvocationFailed(format!( + "failed to serialize policy invocation for host warm: {err}" + )) + })?; + let invocation_py = + json_to_py(py, &invocation_json).map_err(py_err_to_policy_failure)?; + let args = pyo3::types::PyTuple::new(py, vec![invocation_py]) + .map_err(py_err_to_policy_failure)?; + func.call1(args).map_err(py_err_to_policy_failure)?; + Ok(()) + }) + } +} + +/// Adapter from a `TelemetrySink` call to a Python object. +struct PyTelemetrySink { + callback: Py, +} + +impl TelemetrySink for PyTelemetrySink { + fn emit(&self, event: TelemetryEvent) { + // `emit` returns `()` in the trait, so a Python-side raise is + // swallowed here after being converted to a printed exception: + // telemetry is out-of-band by design and must not corrupt a + // decision that already succeeded. The engine calls `emit` after + // it has settled a verdict. + // + // The wire shape lives in `wire::telemetry_event_json`, so a + // sink written for another SDK sees the same fields. The + // binding only translates the JSON into plain Python + // containers. + Python::attach(|py| { + let event_json = wire::telemetry_event_json(&event); + let event_py = match json_to_py(py, &event_json) { + Ok(value) => value, + Err(err) => { + err.write_unraisable(py, Some(self.callback.bind(py).as_any())); + return; + } + }; + if let Err(err) = call_py_method(py, &self.callback, "emit", vec![event_py]) { + err.write_unraisable(py, Some(self.callback.bind(py).as_any())); + } + }); + } + + fn shutdown(&self) { + Python::attach(|py| { + let bound = self.callback.bind(py); + if let Ok(func) = bound.getattr("shutdown") { + let args = pyo3::types::PyTuple::empty(py); + if let Err(err) = func.call1(args) { + err.write_unraisable(py, Some(bound.as_any())); + } + } + }); + } +} + +fn resolve_annotator_dispatcher(dispatcher: Option>) -> Arc { + match dispatcher { + Some(callback) => Arc::new(PyAnnotatorDispatcher { callback }), + None => default_annotator_dispatcher(), + } +} + +fn resolve_policy_dispatcher(dispatcher: Option>) -> Arc { + match dispatcher { + Some(callback) => Arc::new(PyPolicyDispatcher { callback }), + None => Arc::new(BindingPolicyDispatcher::new()), + } +} + +fn resolve_telemetry_sink(sink: Option>) -> Option> { + sink.map(|callback| { + let arc: Arc = Arc::new(PyTelemetrySink { callback }); + arc + }) +} + +/// The engine's default resource caps as a mapping. A host that raises +/// one cap on the way to `interceptor_new` reads this to see what it is +/// overriding, so a shipping change to another default cannot be +/// silently absorbed. +fn limits_defaults_map<'py>(py: Python<'py>) -> PyResult> { + let rendered = wire::limits_json(&Limits::default()); + let dict = PyDict::new(py); + // `wire::limits_json` returns a JSON object with every documented + // field. Iterate it into a PyDict so the surface stays in one place. + let Value::Object(fields) = rendered else { + return Err(PyRuntimeError::new_err( + "wire::limits_json returned a non-object value", + )); + }; + for (key, value) in fields { + dict.set_item(key, json_to_py(py, &value)?)?; + } + Ok(dict) +} + +/// Read a limits override: +/// +/// - `None` (absent) means keep every default. +/// - Each field is individually optional; an absent field keeps its own +/// default, so a host raising one cap does not restate the other nine. +/// - A field present but not a non-negative integer is a hard ERROR, not +/// a silently-kept default. A host that asked for a smaller bound and +/// got the larger one would believe it was protected when it was not. +/// - An unknown/misspelled field is refused rather than silently +/// ignored: a cap the host believes it set but did not set is the +/// same defect as one silently widened. +/// +/// Field-by-field acceptance and the misspelled-key refusal live in the +/// core (`wire::limits_from_json`); this function's only job is to turn +/// a Python mapping into `serde_json::Value` and translate the error +/// back into `PyErr`. +fn resolve_limits(limits: Option>) -> PyResult { + let Some(value) = limits else { + return Ok(Limits::default()); + }; + + Python::attach(|py| -> PyResult { + let bound = value.bind(py); + if bound.is_none() { + return Ok(Limits::default()); + } + // Accept any Mapping (dict, MappingProxyType, custom mapping). + // Reject non-mappings loudly rather than silently ignoring the + // caller's intent; that is a TypeError, not a manifest verdict. + let mapping: Bound<'_, PyMapping> = bound + .cast::() + .map_err(|_| PyTypeError::new_err("limits must be a Mapping[str, int] or None"))? + .clone(); + + // Convert the mapping into a serde_json Object and hand it to + // the core. Only string keys are admitted here (limits fields + // are named strings); any other key type is a boundary error. + let mut object = Map::new(); + let items = mapping.items()?; + for entry in items.iter() { + let pair: Bound<'_, PyAny> = entry; + let key_any = pair.get_item(0)?; + let val_any = pair.get_item(1)?; + let key_str = key_any + .cast::() + .map_err(|_| PyTypeError::new_err("limits key must be a string identifying a cap"))? + .to_string_lossy() + .into_owned(); + // `py_to_json` refuses non-JSON-shaped values and is what + // dispatchers already round-trip through, so a `True` + // becomes `Value::Bool(true)`, which the core then refuses + // as non-integer. + let val_json = py_to_json(&val_any)?; + object.insert(key_str, val_json); + } + + wire::limits_from_json(&Value::Object(object)) + .map_err(|e| PyValueError::new_err(format!("{e}"))) + }) +} #[pyclass(frozen)] struct RuntimeHandle { runtime: Runtime, } -/// Build a runtime handle from a manifest path using the zero-config -/// dispatchers (bundled annotators; Rego in process, Cedar through the -/// built-in evaluator, `test` policies through their embedded verdict). +/// Build a runtime handle from a manifest path. +/// +/// Passing no host arguments preserves the zero-config path: bundled +/// annotators, `BindingPolicyDispatcher` for Rego/Cedar/test policies, +/// no-op telemetry, `PerfTelemetry::Off`, and the engine's default +/// resource caps. Host-supplied callbacks, a `perf_telemetry` other +/// than "off", and a `limits` mapping replace them. #[pyfunction] -fn interceptor_new(manifest_path: &str) -> PyResult { +#[pyo3(signature = ( + manifest_path, + annotator_dispatcher = None, + policy_dispatcher = None, + telemetry_sink = None, + perf_telemetry = "off", + limits = None, +))] +fn interceptor_new( + manifest_path: &str, + annotator_dispatcher: Option>, + policy_dispatcher: Option>, + telemetry_sink: Option>, + perf_telemetry: &str, + limits: Option>, +) -> PyResult { let manifest = Manifest::from_path(manifest_path).map_err(|e| PyValueError::new_err(format!("{e}")))?; - let runtime = Runtime::new( + let annotations = resolve_annotator_dispatcher(annotator_dispatcher); + let policy = resolve_policy_dispatcher(policy_dispatcher); + let perf = wire::parse_perf_telemetry(perf_telemetry) + .map_err(|e| PyValueError::new_err(format!("{e}")))?; + let telemetry = resolve_telemetry_sink(telemetry_sink); + let limits = resolve_limits(limits)?; + let telemetry_arc: Arc = + telemetry.unwrap_or_else(|| Arc::new(NoopTelemetrySink)); + let runtime = Runtime::with_telemetry_perf_and_limits( manifest, - default_annotator_dispatcher(), - Arc::new(BindingPolicyDispatcher::new()), + annotations, + policy, + telemetry_arc, + perf, + limits, ) .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; Ok(RuntimeHandle { runtime }) @@ -111,7 +540,138 @@ fn validate_manifest_file(path: &str) -> PyResult<()> { /// The manifest grammar versions this engine accepts. #[pyfunction] fn supported_manifest_versions() -> Vec { - SUPPORTED_VERSIONS.iter().map(|v| (*v).to_string()).collect() + SUPPORTED_VERSIONS + .iter() + .map(|v| (*v).to_string()) + .collect() +} + +/// Parse a single manifest source into a JSON string representation. +/// +/// Returns the manifest as JSON so a Python wrapper `json.loads` it into a +/// `dict`. `parse_manifest` neither validates nor merges: an authoring +/// tool that needs to inspect a fragment (an `extends` child, for +/// example) can do so without dragging a policy engine on-path. +/// +/// A malformed manifest raises `ManifestInvalid`, exactly as +/// `validate_manifest` does, so a caller does not have to distinguish +/// grammar failures by exception class. +#[pyfunction] +fn parse_manifest(source: &str) -> PyResult { + let manifest = + Manifest::parse_yaml_str(source).map_err(|e| ManifestInvalid::new_err(format!("{e}")))?; + serde_json::to_string(&manifest) + .map_err(|e| PyRuntimeError::new_err(format!("manifest serialization failed: {e}"))) +} + +/// Compose an ordered chain of manifests into one merged JSON document. +/// +/// Later entries overlay earlier ones under the same merge grammar that +/// `extends` uses on disk, and the result is validated before it is +/// returned: a chain that would fail as an on-disk `extends` fails here. +/// Each entry must be a fully-formed manifest fragment (no chain entry +/// may itself carry unresolved `extends`). +/// +/// Empty chains and chains whose entries do not parse raise +/// `ManifestInvalid`. +#[pyfunction] +fn merge_manifests(sources: Vec) -> PyResult { + let refs: Vec<&str> = sources.iter().map(String::as_str).collect(); + let manifest = Manifest::from_yaml_chain(&refs).map_err(|e| match e { + RuntimeError::ManifestInvalid(detail) => ManifestInvalid::new_err(detail), + other => PyValueError::new_err(format!("{other}")), + })?; + serde_json::to_string(&manifest) + .map_err(|e| PyRuntimeError::new_err(format!("merged manifest serialization failed: {e}"))) +} + +/// Structured validation diagnostics as a JSON array string. +/// +/// Returned entries have the unified wire shape +/// `{"code": str, "message": str, "severity": "error", "field": str | +/// None}`, owned by `agent_control_spec::wire::diagnostic_json`. The +/// engine's validation surface reports one failure at a time, so a +/// successful validation returns `[]` and every failed one returns a +/// single-entry list. Wrapping in a list leaves room for a batch +/// validation to grow into the same shape without a breaking rename. +/// +/// A manifest that uses `extends` returns a single diagnostic pointing +/// the caller at file-based validation, matching `validate_manifest`. +#[pyfunction] +fn validate_manifest_diagnostics(source: &str) -> PyResult { + let findings: Vec = match Manifest::parse_yaml_str(source) { + Ok(manifest) => { + if !manifest.extends.is_empty() { + let msg = "manifest extends other manifests; validation needs the merged \ + document. Use validate_manifest_file or merge_manifests, both of \ + which resolve the chain."; + vec![wire::diagnostic_json(&RuntimeError::ManifestInvalid( + msg.to_string(), + ))] + } else { + match manifest.validate() { + Ok(()) => Vec::new(), + Err(e @ RuntimeError::ManifestInvalid(_)) => vec![wire::diagnostic_json(&e)], + Err(other) => { + return Err(PyValueError::new_err(format!("{other}"))); + } + } + } + } + Err(e @ RuntimeError::ManifestInvalid(_)) => vec![wire::diagnostic_json(&e)], + Err(other) => { + return Err(PyValueError::new_err(format!("{other}"))); + } + }; + serde_json::to_string(&findings) + .map_err(|e| PyRuntimeError::new_err(format!("diagnostics serialization failed: {e}"))) +} + +/// Structured artifact diagnostics as a JSON array string. +/// +/// Validates the manifest AND compiles the Rego it names against +/// `bundles_json` — the same shape `policy_activate_from_memory` +/// takes — and reports every failure the pair surfaced. Each entry +/// has the wire shape +/// `{"code": str, "message": str, "severity": "error"}`, matching the +/// C ABI's `acs_artifact_diagnostics`. `validate_manifest_diagnostics` +/// answers only for the document: a manifest can name a bundle, +/// satisfy the grammar, and still fail at activation because the Rego +/// does not compile. This activates in memory and reports what that +/// surfaced, which moves the failure from a host's first agent action +/// to its CI. +/// +/// The manifest is checked first and on its own: a document that +/// does not parse would otherwise be reported as an activation +/// failure, which names the wrong half. NULL/empty `bundles_json` +/// means the manifest names no Rego, so the answer then equals what +/// `validate_manifest_diagnostics` returns. +#[pyfunction] +fn validate_artifacts_diagnostics(manifest_yaml: &str, bundles_json: &str) -> PyResult { + let bundles: std::collections::BTreeMap = + if bundles_json.trim().is_empty() { + std::collections::BTreeMap::new() + } else { + serde_json::from_str(bundles_json) + .map_err(|e| PyValueError::new_err(format!("bundles do not parse: {e}")))? + }; + // Mirror the C ABI's ordering exactly: parse first, validate + // second, activate third. Each step's failure short-circuits so a + // manifest that does not parse is never reported as an activation + // failure. The diagnostic shape is owned by the core so every + // binding renders artifact findings the same way. + let findings = match Manifest::from_yaml_str(manifest_yaml) { + Err(e) => vec![wire::diagnostic_json(&e)], + Ok(manifest) => match manifest.validate() { + Err(e) => vec![wire::diagnostic_json(&e)], + Ok(()) => match ActivatedPolicy::activate_from_memory(manifest_yaml, bundles) { + Ok(_) => Vec::new(), + Err(e) => vec![wire::diagnostic_json(&e)], + }, + }, + }; + serde_json::to_string(&findings) + .map_err(|e| PyRuntimeError::new_err(format!("diagnostics serialization failed: {e}"))) } // --------------------------------------------------------------------- @@ -134,19 +694,32 @@ struct PolicyHandle { } /// Activate the manifest at `manifest_path`, readying every policy it -/// binds, against the zero-config dispatchers. +/// binds. /// -/// This is the expensive call: it reads the manifest, loads every Rego -/// module and data document, and compiles the entrypoint each -/// intervention point queries. Do it once per policy version and keep -/// the handle; `policy_evaluate` then costs no I/O and no compile. +/// Passing no host arguments preserves the zero-config activation. +/// Host-supplied dispatchers replace the bundled ones; they are used +/// for readying (via `PolicyDispatcher::warm`) as well as for every +/// later evaluation, so a policy compile pays its cost here and not on +/// the first decision. /// /// Readying is bounded by the eval timeout. A policy too slow to ready /// inside it activates anyway and pays that cost on its first /// evaluation instead. #[pyfunction] -fn policy_activate(py: Python<'_>, manifest_path: &str) -> PyResult { +#[pyo3(signature = ( + manifest_path, + annotator_dispatcher = None, + policy_dispatcher = None, +))] +fn policy_activate( + py: Python<'_>, + manifest_path: &str, + annotator_dispatcher: Option>, + policy_dispatcher: Option>, +) -> PyResult { let manifest_path = manifest_path.to_string(); + let annotations = resolve_annotator_dispatcher(annotator_dispatcher); + let policy = resolve_policy_dispatcher(policy_dispatcher); // Activation is the expensive call and touches no Python object, so // it must not hold the GIL: a host activating a new policy version // in a background thread would otherwise stall every request thread @@ -154,12 +727,8 @@ fn policy_activate(py: Python<'_>, manifest_path: &str) -> PyResult, manifest_path: &str) -> PyResult, manifest_yaml: &str, bundles_json: &str, + annotator_dispatcher: Option>, + policy_dispatcher: Option>, ) -> PyResult { let bundles: std::collections::BTreeMap = serde_json::from_str(bundles_json) .map_err(|e| PyValueError::new_err(format!("bundles do not parse: {e}")))?; let manifest_yaml = manifest_yaml.to_string(); + let annotations = resolve_annotator_dispatcher(annotator_dispatcher); + let policy = resolve_policy_dispatcher(policy_dispatcher); // Same reason as `policy_activate`: loading and compiling touches no // Python object and must not stall other threads. let policy = py.detach(move || { - ActivatedPolicy::activate_from_memory(&manifest_yaml, bundles).map_err(|e| match e { - RuntimeError::ManifestInvalid(detail) => ManifestInvalid::new_err(detail), - other => PyRuntimeError::new_err(format!("{other}")), - }) + ActivatedPolicy::activate_from_memory_with(&manifest_yaml, bundles, annotations, policy) + .map_err(|e| match e { + RuntimeError::ManifestInvalid(detail) => ManifestInvalid::new_err(detail), + other => PyRuntimeError::new_err(format!("{other}")), + }) })?; Ok(PolicyHandle { policy }) } @@ -245,6 +828,248 @@ fn policy_intervention_points(handle: &PolicyHandle) -> Vec { .collect() } +// --------------------------------------------------------------------- +// Streaming session: host side accounting for the incremental stream +// profile in specification section 18.1. +// +// A `StreamSession` holds no policy, performs no evaluation, and stores +// no stream text. The host drives it: reports arrived text, declares +// spans, records what its policy decided for them, and asks which +// prefix it may release. Every function here is a thin projection over +// the engine's typed accounting. Enum wire names come from the engine's +// own `as_str`/`parse` methods, so the two cannot drift. +// +// State lives behind a `Mutex` because Python threads share the handle +// and every mutating method borrows exclusively. Reads take the same +// lock, which is what the engine's `&self` methods want anyway. The +// lock is uncontended in the common single-threaded flow, and its cost +// is negligible next to the JSON conversion of a settled reason. +// --------------------------------------------------------------------- + +#[pyclass(frozen)] +struct StreamSessionHandle { + inner: Mutex, +} + +// A poisoned `Mutex` means a previous mutating call panicked while +// holding the lock. Nothing in the engine's session accounting panics +// on well-formed input, so this cannot arise on a healthy contract. +// The wrapper still refuses to keep operating on a session in an +// unknown state: it lifts the poisoned guard and surfaces a runtime +// error rather than pretending the session is usable. +fn locked(guard: std::sync::LockResult) -> PyResult { + guard.map_err(|_| PyRuntimeError::new_err("streaming session mutex was poisoned")) +} + +// A `StreamError` from the engine is always a boundary rejection: the +// host handed a value the contract does not admit, or asked for an +// operation the session cannot honor. `ValueError` is the mapping the +// binding uses, so the boundary reports one exception class regardless +// of which check caught it. This is the sole streaming adapter kept +// here: like the FFI's `wire_track` / `wire_outcome`, it translates the +// error type across the boundary rather than reimplementing what a +// wire value means. +fn stream_err(error: StreamError) -> PyErr { + PyValueError::new_err(format!("{error}")) +} + +fn json_string(value: &Value) -> PyResult { + serde_json::to_string(value) + .map_err(|e| PyRuntimeError::new_err(format!("stream JSON serialization failed: {e}"))) +} + +/// Open a streaming session. Field meanings mirror +/// `StreamSessionConfig` in the engine. +#[pyfunction] +#[pyo3(signature = ( + safety_level, + request_start_rune_offset, + response_start_rune_offset, + request_tasks, + response_tasks, +))] +fn stream_session_new( + safety_level: &str, + request_start_rune_offset: u32, + response_start_rune_offset: u32, + request_tasks: Vec, + response_tasks: Vec, +) -> PyResult { + let safety_level = SafetyLevel::parse(safety_level).map_err(stream_err)?; + let config = StreamSessionConfig { + safety_level, + request_start_rune_offset, + response_start_rune_offset, + request_tasks, + response_tasks, + }; + let session = StreamSession::new(config).map_err(stream_err)?; + Ok(StreamSessionHandle { + inner: Mutex::new(session), + }) +} + +/// Report that `runes` more runes arrived on this role's track. Returns +/// the track's new end offset. +#[pyfunction] +fn stream_observe(handle: &StreamSessionHandle, source_type: &str, runes: u32) -> PyResult { + let source_type = StreamSourceType::parse(source_type).map_err(stream_err)?; + let mut session = locked(handle.inner.lock())?; + session.observe(source_type, runes).map_err(stream_err) +} + +/// Report arriving text and let the engine count its runes, so a host +/// does not reach for a length that measures UTF-16 code units or bytes. +#[pyfunction] +fn stream_observe_text( + handle: &StreamSessionHandle, + source_type: &str, + text: &str, +) -> PyResult { + let source_type = StreamSourceType::parse(source_type).map_err(stream_err)?; + let mut session = locked(handle.inner.lock())?; + session.observe_text(source_type, text).map_err(stream_err) +} + +/// Record what a host decided for one span under one task. The span is +/// built from `source_type` and the half-open rune range +/// `[start, end)`. +#[pyfunction] +fn stream_record_outcome( + handle: &StreamSessionHandle, + task: &str, + source_type: &str, + start: u32, + end: u32, + outcome: &str, +) -> PyResult<()> { + let source_type = StreamSourceType::parse(source_type).map_err(stream_err)?; + let outcome = SegmentOutcome::parse(outcome).map_err(stream_err)?; + let span = StreamSpan::new(source_type, start, end).map_err(stream_err)?; + let mut session = locked(handle.inner.lock())?; + session + .record_outcome(task, &span, outcome) + .map_err(stream_err) +} + +/// Record a wire-shaped agent-hooks verdict for one span under one +/// task. The verdict text is deserialized with the same grammar the +/// runtime uses, so a shape section 5 does not admit fails closed here +/// rather than clearing the span. +#[pyfunction] +fn stream_record_verdict( + handle: &StreamSessionHandle, + task: &str, + source_type: &str, + start: u32, + end: u32, + verdict_json: &str, +) -> PyResult<()> { + let source_type = StreamSourceType::parse(source_type).map_err(stream_err)?; + let span = StreamSpan::new(source_type, start, end).map_err(stream_err)?; + let verdict: Verdict = serde_json::from_str(verdict_json) + .map_err(|e| PyValueError::new_err(format!("verdict_json does not parse: {e}")))?; + let mut session = locked(handle.inner.lock())?; + session + .record_verdict(task, &span, &verdict) + .map_err(stream_err) +} + +/// Recompute the watermark for `track` and return the new confirmed +/// offset when it advanced. +#[pyfunction] +fn stream_advance(handle: &StreamSessionHandle, track: &str) -> PyResult> { + let track = StreamTrack::parse(track).map_err(stream_err)?; + let mut session = locked(handle.inner.lock())?; + Ok(session.advance(track)) +} + +/// Offset through which the host may emit this track, or `None` once +/// the session has ended. +#[pyfunction] +fn stream_safe_offset(handle: &StreamSessionHandle, track: &str) -> PyResult> { + let track = StreamTrack::parse(track).map_err(stream_err)?; + let session = locked(handle.inner.lock())?; + Ok(session.safe_offset(track)) +} + +/// Runes observed but not yet cleared by every task on this track. +#[pyfunction] +fn stream_pending(handle: &StreamSessionHandle, track: &str) -> PyResult { + let track = StreamTrack::parse(track).map_err(stream_err)?; + let session = locked(handle.inner.lock())?; + Ok(session.pending(track)) +} + +/// Watermark snapshot for one track, as wire JSON. +#[pyfunction] +fn stream_watermark(handle: &StreamSessionHandle, track: &str) -> PyResult { + let track_kind = StreamTrack::parse(track).map_err(stream_err)?; + let session = locked(handle.inner.lock())?; + let watermark = session.watermark(track_kind); + json_string(&wire::watermark_json(track_kind, watermark)) +} + +/// Stop accepting payloads while outcomes are still in flight. A +/// `Deferred` host calls this at payload EOF so a classifier running +/// behind the stream can still record a denial before `finish`. +#[pyfunction] +fn stream_end_of_payloads(handle: &StreamSessionHandle) -> PyResult<()> { + let mut session = locked(handle.inner.lock())?; + session.end_of_payloads(); + Ok(()) +} + +/// Settle the session and return the wire-JSON completion: +/// `{"reason": , "transformed": bool, "is_clean": bool}`. +#[pyfunction] +fn stream_finish(handle: &StreamSessionHandle) -> PyResult { + let mut session = locked(handle.inner.lock())?; + let completion = session.finish(); + json_string(&wire::completion_json(&completion)) +} + +/// Whether the session has reached its terminal state. +#[pyfunction] +fn stream_is_ended(handle: &StreamSessionHandle) -> PyResult { + let session = locked(handle.inner.lock())?; + Ok(session.is_ended()) +} + +/// Whether a `transformed` outcome ended this session, meaning the host +/// emits a substitute rather than verbatim model output. +#[pyfunction] +fn stream_transformed(handle: &StreamSessionHandle) -> PyResult { + let session = locked(handle.inner.lock())?; + Ok(session.transformed()) +} + +/// Terminal reason as wire JSON, or `None` when the session has not +/// ended. +#[pyfunction] +fn stream_end_reason(handle: &StreamSessionHandle) -> PyResult> { + let session = locked(handle.inner.lock())?; + match session.end_reason() { + Some(reason) => json_string(&wire::end_reason_json(reason)).map(Some), + None => Ok(None), + } +} + +/// Streaming parameters this session was opened with, as wire JSON. +#[pyfunction] +fn stream_config(handle: &StreamSessionHandle) -> PyResult { + let session = locked(handle.inner.lock())?; + let config = session.config(); + json_string(&wire::stream_config_json(config)) +} + +/// The engine's default resource caps as a `dict[str, int]`. A host +/// that raises one cap reads this to see what it is overriding. +#[pyfunction] +fn default_limits(py: Python<'_>) -> PyResult> { + Ok(limits_defaults_map(py)?.unbind()) +} + #[pymodule] fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -255,9 +1080,30 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(policy_activate_from_memory, m)?)?; m.add_function(wrap_pyfunction!(policy_evaluate, m)?)?; m.add_function(wrap_pyfunction!(policy_intervention_points, m)?)?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(stream_session_new, m)?)?; + m.add_function(wrap_pyfunction!(stream_observe, m)?)?; + m.add_function(wrap_pyfunction!(stream_observe_text, m)?)?; + m.add_function(wrap_pyfunction!(stream_record_outcome, m)?)?; + m.add_function(wrap_pyfunction!(stream_record_verdict, m)?)?; + m.add_function(wrap_pyfunction!(stream_advance, m)?)?; + m.add_function(wrap_pyfunction!(stream_safe_offset, m)?)?; + m.add_function(wrap_pyfunction!(stream_pending, m)?)?; + m.add_function(wrap_pyfunction!(stream_watermark, m)?)?; + m.add_function(wrap_pyfunction!(stream_end_of_payloads, m)?)?; + m.add_function(wrap_pyfunction!(stream_finish, m)?)?; + m.add_function(wrap_pyfunction!(stream_is_ended, m)?)?; + m.add_function(wrap_pyfunction!(stream_transformed, m)?)?; + m.add_function(wrap_pyfunction!(stream_end_reason, m)?)?; + m.add_function(wrap_pyfunction!(stream_config, m)?)?; + m.add_function(wrap_pyfunction!(default_limits, m)?)?; m.add("ManifestInvalid", m.py().get_type::())?; m.add_function(wrap_pyfunction!(validate_manifest, m)?)?; m.add_function(wrap_pyfunction!(validate_manifest_file, m)?)?; + m.add_function(wrap_pyfunction!(validate_manifest_diagnostics, m)?)?; + m.add_function(wrap_pyfunction!(validate_artifacts_diagnostics, m)?)?; + m.add_function(wrap_pyfunction!(parse_manifest, m)?)?; + m.add_function(wrap_pyfunction!(merge_manifests, m)?)?; m.add_function(wrap_pyfunction!(supported_manifest_versions, m)?)?; Ok(()) } diff --git a/sdk/python/tests/test_host_hooks.py b/sdk/python/tests/test_host_hooks.py new file mode 100644 index 0000000..d966026 --- /dev/null +++ b/sdk/python/tests/test_host_hooks.py @@ -0,0 +1,660 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Host extension points: annotator and policy dispatchers, telemetry +sinks, perf-telemetry levels, manifest tooling, and structured +validation diagnostics. + +Restores the 0.3.1b1 shape of ``AgentControl.from_native(manifest, +annotator_dispatcher=...)`` that a consumer using an HTTP-backed +Content Safety dispatcher depended on. Verifies both that the host +callback runs and that its output reaches the policy decision, so a +regression that reintroduced the 0.4 hardcoded zero-config path would +fail. +""" + +from __future__ import annotations + +_VERSION_KEY = "agent_control_specification" + "_version" + +import pathlib + +import pytest +from agent_control_spec import ( + DEFAULT_LIMITS, + PERF_TELEMETRY_LEVELS, + AcsInterceptor, + ActivatedPolicy, + ManifestInvalidError, + merge_manifests, + parse_manifest, + validate_artifacts, + validate_manifest_detailed, +) +from agent_hooks import AgentContextBuilder + +FIXTURES = pathlib.Path(__file__).parent / "fixtures" +DEFAULT_MANIFEST = str(FIXTURES / "manifest.yaml") + + +def _builder() -> AgentContextBuilder: + return AgentContextBuilder(agent_id="a", framework="test", session_id="s") + + +# --------------------------------------------------------------------- +# Host annotator dispatcher: manifest + rego gate that reads annotations. +# +# The rego module denies whenever `input.annotations.classify.blocked` +# is True. A host dispatcher that returns `{"blocked": True}` therefore +# turns an allow-by-default input into a deny; a dispatcher that returns +# `{"blocked": False}` leaves it allowed. That is exactly the shape of +# the Azure Content Safety adapter the consumer needs to plug in. +# --------------------------------------------------------------------- + +ANNOTATOR_MANIFEST = """ +agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: python-host-hooks +policies: + gate: + type: rego + bundle: ./policy + query: data.gate.verdict +annotators: + classify: + type: classifier +intervention_points: + input: + policy_target: "$.input" + policy_target_kind: user_input + policy: + id: gate + annotations: + classify: + from: $target.content +""" + +REGO_MODULE = """ +package gate + +default verdict := {"decision": "allow"} + +verdict := { + "decision": "deny", + "reason": "blocked_by_annotator", +} if { + input.annotations.classify.blocked == true +} +""" + + +def _bundles() -> dict: + return {"gate": {"modules": {"gate.rego": REGO_MODULE}}} + + +class RecordingAnnotator: + """Host dispatcher whose ``dispatch`` records every call and returns + a fixed annotation. Deliberately object-with-method-shaped, matching + the 0.3.1b1 API a consumer wired up.""" + + def __init__(self, payload): + self.payload = payload + self.calls: list[tuple[str, dict, dict]] = [] + + def dispatch(self, annotator_name, annotator, preliminary_policy_input): + self.calls.append((annotator_name, annotator, preliminary_policy_input)) + return self.payload + + +def test_host_annotator_dispatcher_reaches_the_policy_decision(): + """The consumer's blocked use case, made concrete: an annotator + dispatcher that says the content is blocked flips the verdict.""" + dispatcher = RecordingAnnotator({"blocked": True, "categories": ["hate"]}) + policy = ActivatedPolicy.from_memory( + ANNOTATOR_MANIFEST, + _bundles(), + annotator_dispatcher=dispatcher, + ) + verdict = policy.evaluate("input", _builder().input(content="offensive text")) + + # The dispatcher was actually called by the engine, not skipped. + assert len(dispatcher.calls) == 1 + name, invocation, prelim = dispatcher.calls[0] + assert name == "classify" + # The dispatcher sees the annotator invocation shape the engine + # built, including the flattened `type` field from the annotator + # config and the `from` field from the annotation config. + assert invocation["type"] == "classifier" + assert invocation["from"] == "$target.content" + # And the preliminary policy input, so an HTTP-backed dispatcher can + # decide whether to make a call. The input builder wraps content in + # `{content, role}`, which is what a downstream classifier reads. + assert prelim["intervention_point"] == "input" + assert prelim["policy_target"]["value"]["content"] == "offensive text" + + # And the annotation actually affected the decision. + assert verdict.decision.value == "deny" + assert verdict.reason == "blocked_by_annotator" + + +def test_host_annotator_dispatcher_returning_allow_leaves_verdict_allowed(): + """The complementary case: same policy, same manifest, dispatcher + that returns a clean annotation. The engine still ran the + dispatcher and its annotation reached the rego module.""" + dispatcher = RecordingAnnotator({"blocked": False}) + policy = ActivatedPolicy.from_memory( + ANNOTATOR_MANIFEST, + _bundles(), + annotator_dispatcher=dispatcher, + ) + verdict = policy.evaluate("input", _builder().input(content="hello")) + assert dispatcher.calls, "dispatcher must run on every evaluation" + assert verdict.decision.value == "allow" + + +def test_host_annotator_dispatcher_that_raises_fails_closed_not_silent(): + """Contract: a raising annotator dispatcher must never be treated + as 'no annotation'. The engine's fail-closed path applies, so the + verdict is deny with a ``runtime_error:*`` reason.""" + + class Boom: + def dispatch(self, name, annotator, prelim): + raise RuntimeError("content safety endpoint unreachable") + + policy = ActivatedPolicy.from_memory( + ANNOTATOR_MANIFEST, + _bundles(), + annotator_dispatcher=Boom(), + ) + verdict = policy.evaluate("input", _builder().input(content="hello")) + assert verdict.decision.value == "deny" + assert verdict.reason.startswith("runtime_error:"), verdict.reason + + +def test_host_annotator_dispatcher_can_be_a_plain_callable(): + """The old API accepted objects with `dispatch`; a plain callable + with the same signature is admitted too so hosts can write a small + lambda for tests without wrapping it in a class.""" + calls = [] + + def dispatcher(name, annotator, prelim): + calls.append(name) + return {"blocked": True} + + policy = ActivatedPolicy.from_memory( + ANNOTATOR_MANIFEST, + _bundles(), + annotator_dispatcher=dispatcher, + ) + verdict = policy.evaluate("input", _builder().input(content="hi")) + assert calls == ["classify"] + assert verdict.decision.value == "deny" + + +# --------------------------------------------------------------------- +# Zero-config parity: with no arguments, the API is byte-for-byte the +# same as today's zero-config path. +# --------------------------------------------------------------------- + + +def test_no_dispatcher_behaves_as_zero_config(): + # No annotator, no policy, no telemetry, no perf: identical to the + # existing zero-config test surface. + acs = AcsInterceptor(DEFAULT_MANIFEST) + allow = acs.intercept(_builder().input(content="hello")) + deny = acs.intercept( + _builder().pre_tool_call(call_id="t1", name="search", args={"q": "x"}) + ) + assert allow.decision.value == "allow" + assert deny.decision.value == "deny" + assert deny.reason == "blocked_by_policy" + + +def test_no_dispatcher_on_activated_policy_behaves_as_zero_config(): + policy = ActivatedPolicy(DEFAULT_MANIFEST) + verdict = policy.evaluate("input", _builder().input(content="hello")) + assert verdict.decision.value == "allow" + + +# --------------------------------------------------------------------- +# Telemetry: a host sink receives events during evaluation, and +# perf-telemetry levels round-trip. +# --------------------------------------------------------------------- + + +class RecordingTelemetrySink: + def __init__(self): + self.events: list[dict] = [] + + def emit(self, event): + self.events.append(event) + + +def test_telemetry_sink_receives_events_during_evaluation(): + sink = RecordingTelemetrySink() + acs = AcsInterceptor(DEFAULT_MANIFEST, telemetry_sink=sink) + acs.intercept(_builder().input(content="hello")) + # A decision event is guaranteed under any perf-telemetry level for + # a settled evaluation. The rest of the fields ride along; we assert + # the ones a monitoring host reads. + decisions = [e for e in sink.events if e["event_type"] == "decision"] + assert decisions, f"expected at least one decision event, got {sink.events}" + decision = decisions[0] + assert decision["intervention_point"] == "input" + assert decision["decision"] == "allow" + assert decision["policy_id"] == "allow_all" + + +def test_telemetry_sink_is_called_for_a_denying_verdict_too(): + sink = RecordingTelemetrySink() + acs = AcsInterceptor(DEFAULT_MANIFEST, telemetry_sink=sink) + acs.intercept( + _builder().pre_tool_call(call_id="t1", name="search", args={"q": "x"}) + ) + decisions = [e for e in sink.events if e["event_type"] == "decision"] + assert decisions + assert decisions[0]["decision"] == "deny" + assert decisions[0]["reason_code"] == "blocked_by_policy" + + +def test_perf_telemetry_levels_roundtrip(): + # All three engine levels construct without raising. + for level in PERF_TELEMETRY_LEVELS: + AcsInterceptor(DEFAULT_MANIFEST, perf_telemetry=level) + + +def test_unknown_perf_telemetry_level_is_rejected(): + with pytest.raises(ValueError, match="perf_telemetry"): + AcsInterceptor(DEFAULT_MANIFEST, perf_telemetry="verbose") + + +def test_telemetry_sink_can_be_a_plain_callable(): + events: list[dict] = [] + acs = AcsInterceptor(DEFAULT_MANIFEST, telemetry_sink=events.append) + acs.intercept(_builder().input(content="hello")) + assert any(e["event_type"] == "decision" for e in events) + + +# --------------------------------------------------------------------- +# Manifest tooling: parse_manifest and merge_manifests. +# --------------------------------------------------------------------- + + +VALID_MANIFEST = (FIXTURES / "manifest.yaml").read_text(encoding="utf-8") + + +def test_parse_manifest_returns_the_parsed_structure(): + parsed = parse_manifest(VALID_MANIFEST) + assert isinstance(parsed, dict) + # The version key is on the top level, and the intervention_points + # are keyed by point name. + assert parsed["agent_control_specification_version"].startswith("0.4.0") + assert "input" in parsed["intervention_points"] + + +def test_parse_manifest_rejects_malformed_source(): + with pytest.raises(ManifestInvalidError): + parse_manifest("agent_control_specification_version: [") + + +def test_merge_manifests_composes_two_partial_documents(): + # Base declares the policies, overlay adds the intervention points + # that bind them. Both fragments are needed for a runnable manifest; + # neither is one on its own. That's the composition merge_manifests + # is for. + base = """ +agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: composed +policies: + p: + type: test + verdict: + decision: allow + q: + type: test + verdict: + decision: deny + reason: blocked_by_overlay +""" + overlay = """ +agent_control_specification_version: "0.4.0-alpha.1" +intervention_points: + input: + policy_target: "$.input" + policy: + id: q +""" + merged = merge_manifests([base, overlay]) + assert isinstance(merged, dict) + # The overlay's intervention point landed, bound to the base's `q` + # policy. Composition took place. + assert merged["intervention_points"]["input"]["policy"]["id"] == "q" + # And both base policies are still declared: the merge unions + # definitions. + assert set(merged["policies"]) == {"p", "q"} + # And the merged document is valid: the runtime accepts it. + policy = ActivatedPolicy.from_memory( + # ActivatedPolicy accepts YAML; a merged dict round-trips through + # JSON, which the manifest grammar admits. + __import__("json").dumps(merged), + {}, + ) + verdict = policy.evaluate("input", _builder().input(content="hi")) + assert verdict.decision.value == "deny" + assert verdict.reason == "blocked_by_overlay" + + +def test_merge_manifests_rejects_an_empty_chain(): + with pytest.raises(ManifestInvalidError): + merge_manifests([]) + + +# --------------------------------------------------------------------- +# Structured validation diagnostics. +# --------------------------------------------------------------------- + + +def test_valid_manifest_produces_no_diagnostics(): + assert validate_manifest_detailed(VALID_MANIFEST) == [] + + +def test_diagnostics_name_the_offending_field_for_an_invalid_manifest(): + # A policy_target_kind of "" is rejected by validate. The diagnostic + # should name that field. + bad = VALID_MANIFEST.replace( + '"$.input"', + '"$.input"\n policy_target_kind: ""', + 1, + ) + diagnostics = validate_manifest_detailed(bad) + assert diagnostics, "expected the manifest to be rejected" + entry = diagnostics[0] + assert entry["code"] == "runtime_error:manifest_invalid" + assert entry["severity"] == "error" + assert entry["field"] == "policy_target_kind", entry + # The engine's full message is preserved verbatim, so a tool can + # surface it in an editor without paraphrasing. + assert "policy_target_kind" in entry["message"] + + +def test_diagnostics_report_unsupported_version_field(): + bad = VALID_MANIFEST.replace('"0.4.0-alpha.1"', '"0.3.1-beta"') + diagnostics = validate_manifest_detailed(bad) + assert len(diagnostics) == 1 + entry = diagnostics[0] + assert entry["code"] == "runtime_error:manifest_invalid" + assert entry["severity"] == "error" + assert entry["field"] == "agent_control_specification_version" + assert "0.3.1-beta" in entry["message"] + + +def test_diagnostics_flag_manifests_that_use_extends(): + # A manifest that inherits cannot be judged from its own source. + # `validate_manifest_detailed` reports that as a single diagnostic + # instead of raising, so a batch runner can bucket the result rather + # than mid-loop-except. + extended = VALID_MANIFEST + "\nextends:\n - path: ./parent.yaml\n" + diagnostics = validate_manifest_detailed(extended) + assert diagnostics + assert diagnostics[0]["code"] == "runtime_error:manifest_invalid" + assert diagnostics[0]["severity"] == "error" + assert "extends" in diagnostics[0]["message"] + + +# --------------------------------------------------------------------- +# Artifact validation: manifest + Rego compiled together. +# +# `validate_manifest_detailed` answers only for the document. A +# manifest can satisfy the grammar, name a Rego bundle, and still fail +# at activation because the Rego does not compile — compilation happens +# at activation time, so a validator that stops at the manifest turns +# that failure into a host's first agent action. `validate_artifacts` +# closes the gap by activating in memory and reporting what the pair +# surfaced. Restores the 0.3-era ``validate_acs_artifacts`` shape a +# consumer's CI depended on. +# --------------------------------------------------------------------- + +ARTIFACT_MANIFEST = """\ +agent_control_specification_version: "0.4.0-alpha.1" +policies: + gate: + type: rego + bundle: ./b +intervention_points: + input: + policy_target: "$.input" + policy: + id: gate + query: data.acs.decision +""" + +_VALID_REGO = 'package acs\ndecision := {"decision":"allow"}\n' + + +def test_validate_artifacts_returns_empty_for_valid_manifest_and_rego(): + # A manifest naming a Rego policy whose module compiles cleanly is + # what a fully-formed release looks like: nothing to report. + findings = validate_artifacts( + ARTIFACT_MANIFEST, + {"gate": {"modules": {"p.rego": _VALID_REGO}}}, + ) + assert findings == [] + + +def test_validate_artifacts_surfaces_a_broken_rego_module(): + # The feature exists for this case: the manifest is fine, the + # bundle is not, and today a manifest-only validator would have + # green-lit the release. The diagnostic must name the activation + # half so the caller can render the compiler's complaint. + findings = validate_artifacts( + ARTIFACT_MANIFEST, + {"gate": {"modules": {"p.rego": "package acs\nfoo := ] not valid rego"}}}, + ) + assert len(findings) == 1, findings + entry = findings[0] + assert entry["severity"] == "error" + assert entry["code"].startswith("runtime_error:"), entry + # The engine's own text carries the Rego compiler's complaint + # verbatim, so an editor can point at the module. The compiler + # names the module path and its "expecting expression" error. + assert "p.rego" in entry["message"], entry + assert "expecting expression" in entry["message"], entry + + # `validate_manifest_detailed` never sees this failure: it does + # not compile the bundle. Prove that so a regression that widened + # the manifest-only surface would fail. + assert validate_manifest_detailed(ARTIFACT_MANIFEST) == [] + + +def test_validate_artifacts_reports_unparseable_manifest_as_manifest_problem(): + # A document that does not parse must be reported as a manifest + # problem, not an activation failure — that would name the wrong + # half. Even when bundles are supplied. + findings = validate_artifacts( + "::not: [valid", + {"gate": {"modules": {"p.rego": _VALID_REGO}}}, + ) + assert len(findings) == 1, findings + entry = findings[0] + assert entry["code"] == "runtime_error:manifest_invalid" + assert entry["severity"] == "error" + # And the underlying error matches what the manifest-only + # validator reports: same problem, same shape now. + manifest_only = validate_manifest_detailed("::not: [valid") + assert manifest_only[0]["code"] == entry["code"] + assert manifest_only[0]["message"] == entry["message"] + assert manifest_only[0]["severity"] == entry["severity"] + + +def test_validate_artifacts_without_bundles_equals_manifest_only_result(): + # No bundles supplied: activation is either skipped (no Rego to + # load) or fails the same way manifest validation does. Either + # way, the artifact validator must not invent activation errors + # when the manifest half is what actually reports the problem. + # Both surfaces now return the same unified diagnostic shape. + broken = "::not: [valid" + artifact_findings = validate_artifacts(broken) + manifest_findings = validate_manifest_detailed(broken) + assert len(artifact_findings) == len(manifest_findings) == 1 + assert artifact_findings[0]["code"] == manifest_findings[0]["code"] + assert artifact_findings[0]["message"] == manifest_findings[0]["message"] + assert artifact_findings[0]["severity"] == manifest_findings[0]["severity"] + + # And for a grammatically invalid document — one that parses but + # fails validation — the two surfaces report the same underlying + # manifest problem. Activation would never be reached. + # Built rather than written on one line. A repo guard scans committed + # files for the version key and validates what follows it, and it + # cannot strip the quotes of a single-line Python literal. + invalid = ( + f'{_VERSION_KEY}: "0.4.0-alpha.1"\npolicies: {{}}\nintervention_points: {{}}\n' + ) + artifact_findings = validate_artifacts(invalid) + manifest_findings = validate_manifest_detailed(invalid) + assert len(artifact_findings) == len(manifest_findings) == 1 + assert artifact_findings[0]["code"] == manifest_findings[0]["code"] + assert artifact_findings[0]["message"] == manifest_findings[0]["message"] + assert artifact_findings[0]["severity"] == manifest_findings[0]["severity"] + + +def test_validate_artifacts_accepts_none_for_bundles(): + # ``bundles=None`` is the ergonomic form for "no Rego to supply". + # It must behave identically to an empty mapping so callers can + # write either. + assert validate_artifacts(ARTIFACT_MANIFEST, None) == validate_artifacts( + ARTIFACT_MANIFEST, {} + ) + + +# --------------------------------------------------------------------- +# Resource limits: caps overriding the engine's defaults reach the +# runtime and change the verdict. `Limits` is a denial-of-service +# control surface: a host feeding large payloads raises +# `max_snapshot_bytes`; one hardening against a hostile manifest lowers +# `max_extends_depth` or `manifest_url_timeout_ms`. +# +# The behavioural test is deliberately end-to-end: the same manifest +# and the same context, evaluated once with default caps and once with +# a small `max_snapshot_bytes`, produce different verdicts. That is +# what proves the value reaches the engine, rather than being accepted +# on the Python side and dropped on the way in. +# --------------------------------------------------------------------- + + +def test_lowered_snapshot_cap_flips_the_verdict(): + """A snapshot bigger than the cap must fail closed. With default + caps the same input allows; with the cap lowered below the + canonicalized snapshot size, the engine denies with + ``runtime_error:*``. A host that asked for the smaller bound and + got the larger one would believe it was protected when it was not. + """ + big = "x" * 4096 + permissive = AcsInterceptor(DEFAULT_MANIFEST) + assert permissive.intercept(_builder().input(content=big)).decision.value == "allow" + + capped = AcsInterceptor(DEFAULT_MANIFEST, limits={"max_snapshot_bytes": 64}) + verdict = capped.intercept(_builder().input(content=big)) + assert verdict.decision.value == "deny" + assert verdict.reason.startswith("runtime_error:"), verdict.reason + + +def test_no_limits_argument_matches_the_baseline_zero_config_path(): + """Regression guard: `limits=None` (and omitting the argument) must + behave identically to the pre-Limits baseline. Two contexts, one + that allowed and one that denied on the fixture, produce the same + verdicts under a `limits=None` interceptor as under one with no + kwarg at all. + """ + baseline = AcsInterceptor(DEFAULT_MANIFEST) + explicit_none = AcsInterceptor(DEFAULT_MANIFEST, limits=None) + empty = AcsInterceptor(DEFAULT_MANIFEST, limits={}) + for ctx in ( + _builder().input(content="hi"), + _builder().pre_tool_call(call_id="t1", name="search", args={"q": "x"}), + ): + base_v = baseline.intercept(ctx) + assert explicit_none.intercept(ctx).decision == base_v.decision + assert empty.intercept(ctx).decision == base_v.decision + + +def test_overriding_one_limit_leaves_the_others_at_their_defaults(): + """Fields are individually optional. Overriding + ``max_annotator_output_bytes`` upward must not silently lower + ``max_snapshot_bytes`` back to some other value; the untouched cap + still enforces at its default. Concretely: an interceptor raising + only ``max_annotator_output_bytes`` still allows a 4096-char input + (well under the default 1 MiB snapshot cap) and still fails closed + when explicitly capped small in a separate construction. + """ + big = "x" * 4096 + partial = AcsInterceptor( + DEFAULT_MANIFEST, + limits={"max_annotator_output_bytes": 8_388_608}, + ) + # Untouched `max_snapshot_bytes` still defaults big, so the input + # allows. + assert partial.intercept(_builder().input(content=big)).decision.value == "allow" + + # And when the second, untouched cap IS lowered on a separate + # interceptor, it enforces — proving the field-by-field override + # semantics. + both = AcsInterceptor( + DEFAULT_MANIFEST, + limits={ + "max_annotator_output_bytes": 8_388_608, + "max_snapshot_bytes": 64, + }, + ) + v = both.intercept(_builder().input(content=big)) + assert v.decision.value == "deny" + assert v.reason.startswith("runtime_error:") + + +def test_limit_that_is_not_a_non_negative_integer_is_rejected(): + """A value that does not parse must be a hard error, not a + silently-kept default. Refusing means a host that asked for a cap + it typo'd learns immediately instead of finding out at the first + breached limit that would never fire. + """ + with pytest.raises((TypeError, ValueError)): + AcsInterceptor(DEFAULT_MANIFEST, limits={"max_snapshot_bytes": "big"}) + with pytest.raises((TypeError, ValueError)): + AcsInterceptor(DEFAULT_MANIFEST, limits={"max_snapshot_bytes": -1}) + with pytest.raises((TypeError, ValueError)): + AcsInterceptor(DEFAULT_MANIFEST, limits={"max_snapshot_bytes": 1.5}) + # Booleans are ints in Python; refuse them here so `True` for a cap + # does not silently become `1`. + with pytest.raises((TypeError, ValueError)): + AcsInterceptor(DEFAULT_MANIFEST, limits={"max_snapshot_bytes": True}) + # A non-mapping is a boundary problem, not a manifest problem. + with pytest.raises(TypeError): + AcsInterceptor(DEFAULT_MANIFEST, limits=[("max_snapshot_bytes", 64)]) + + +def test_default_limits_is_readable_and_matches_engine_defaults(): + """A host raising one cap reads `DEFAULT_LIMITS` to see what it is + overriding. Assert the shape stays wired to the engine's own + defaults so a shipping change to another cap cannot be silently + absorbed by a stale mapping. + """ + # Every documented field is present as a non-negative int. + expected_keys = { + "max_snapshot_bytes", + "max_policy_input_depth", + "max_annotators_per_point", + "max_annotator_output_bytes", + "max_policy_output_bytes", + "max_extends_depth", + "max_merged_manifest_bytes", + "max_manifest_url_bytes", + "manifest_url_timeout_ms", + "max_manifest_url_redirects", + } + assert set(DEFAULT_LIMITS) == expected_keys + for key, value in DEFAULT_LIMITS.items(): + assert isinstance(value, int) and value >= 0, (key, value) + + # And it is read-only, so a caller cannot mutate a shared default + # out from under a peer. + with pytest.raises(TypeError): + DEFAULT_LIMITS["max_snapshot_bytes"] = 1 # type: ignore[index] diff --git a/sdk/python/tests/test_stream_session.py b/sdk/python/tests/test_stream_session.py new file mode 100644 index 0000000..8491e41 --- /dev/null +++ b/sdk/python/tests/test_stream_session.py @@ -0,0 +1,415 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Streaming session contract: incremental release accounting for a +policy target the host holds as a stream (specification section 18.1). +The session gates, the host emits. Every engine rejection surfaces as a +Python exception; nothing silently no-ops.""" + +from __future__ import annotations + +import pytest +from agent_control_spec import StreamSession +from agent_hooks import Decision, Transform, Verdict + + +def _allow() -> Verdict: + return Verdict(decision=Decision.ALLOW) + + +def _deny() -> Verdict: + return Verdict(decision=Decision.DENY, reason="blocked_by_policy") + + +def _transform() -> Verdict: + return Verdict( + decision=Decision.TRANSFORM, + transform=Transform(path="$target.content", value="[redacted]"), + ) + + +# --------------------------------------------------------------------- +# Happy path +# --------------------------------------------------------------------- + + +def test_blocking_session_releases_the_prefix_a_task_clears_and_finishes_clean(): + session = StreamSession( + safety_level="blocking", + response_tasks=["pii"], + ) + # Safety level "blocking" holds every span until the watermark + # covers it, so nothing is emittable before a `cleared` outcome. + assert session.safe_offset("response") == 0 + + assert session.observe_text("model_generated", "hello world") == 11 + assert session.pending("response") == 11 + # Nothing has cleared yet, so the offset a host may emit through has + # not moved even though runes arrived. + assert session.safe_offset("response") == 0 + + session.record_outcome("pii", "model_generated", 0, 11, "cleared") + assert session.advance("response") == 11 + assert session.safe_offset("response") == 11 + + settlement = session.finish() + assert settlement == { + "reason": {"kind": "complete"}, + "transformed": False, + "is_clean": True, + } + # A settled session hands out no offset to emit through, whatever + # the reason. Confirmed stays available for the audit record. + assert session.safe_offset("response") is None + assert session.watermark("response")["confirmed"] == 11 + assert session.is_ended is True + assert session.end_reason == {"kind": "complete"} + + +def test_advance_reports_none_when_no_task_moved_forward(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "hi") + # No outcome recorded, so the watermark has nothing to commit and + # `advance` must not synthesize progress. + assert session.advance("response") is None + assert session.safe_offset("response") == 0 + + +# --------------------------------------------------------------------- +# Denial is terminal and audit path stays readable +# --------------------------------------------------------------------- + + +def test_a_denial_terminates_the_session_and_confirmed_stays_readable(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "safe prefix ") # 12 runes + session.record_outcome("pii", "model_generated", 0, 12, "cleared") + session.advance("response") + # Establish that the host would have been allowed to release the + # first twelve runes, so the audit path is meaningful. + assert session.safe_offset("response") == 12 + + session.observe_text("model_generated", "bad tail") # +8 runes, offsets 12..20 + session.record_outcome("pii", "model_generated", 12, 20, "denied") + + # Denial withholds everything a host has not already emitted, + # including runes a task had cleared, so `safe_offset` becomes None. + assert session.safe_offset("response") is None + assert session.is_ended is True + + # The confirmed offset the audit needs is still readable. This is + # the release ceiling the session reached, not permission to emit. + watermark = session.watermark("response") + assert watermark["confirmed"] == 12 + assert watermark["received"] == 20 + assert session.end_reason == { + "kind": "denied", + "track": "response", + "task": "pii", + "start": 12, + "end": 20, + } + + settlement = session.finish() + assert settlement["is_clean"] is False + assert settlement["transformed"] is False + assert settlement["reason"]["kind"] == "denied" + + +def test_denial_through_a_verdict_reaches_the_same_terminal_state(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "bad content") # 11 runes + session.record_verdict("pii", "model_generated", 0, 11, _deny()) + + reason = session.end_reason + assert reason["kind"] == "denied" + assert reason["task"] == "pii" + assert reason["start"] == 0 and reason["end"] == 11 + + +# --------------------------------------------------------------------- +# Multi-task: the watermark is the minimum across configured tasks +# --------------------------------------------------------------------- + + +def test_the_watermark_waits_for_every_task_configured_on_the_track(): + session = StreamSession(response_tasks=["pii", "safety"]) + session.observe_text("model_generated", "sentence one.") # 13 runes + + # Only one of two tasks has cleared the span. The confirmed offset + # is the minimum across both, so the release ceiling stays at zero. + session.record_outcome("pii", "model_generated", 0, 13, "cleared") + assert session.advance("response") is None + assert session.safe_offset("response") == 0 + assert session.pending("response") == 13 + + # Second task clears the same span. The minimum jumps to 13, so + # the watermark advances and the prefix becomes emittable. + session.record_outcome("safety", "model_generated", 0, 13, "cleared") + assert session.advance("response") == 13 + assert session.safe_offset("response") == 13 + + watermark = session.watermark("response") + assert sorted(watermark["tasks"]) == ["pii", "safety"] + + completion = session.finish() + assert completion["is_clean"] is True + + +# --------------------------------------------------------------------- +# observe_text counts runes, not UTF-16 code units +# --------------------------------------------------------------------- + + +def test_observe_text_counts_runes_not_utf16_code_units_or_bytes(): + session = StreamSession(response_tasks=["pii"]) + astral = "😀" # one rune, but two UTF-16 code units and four UTF-8 bytes + # Sanity guard: this text really is a two-code-unit astral character, + # so the assertion below is actually testing the rune boundary. + assert len(astral.encode("utf-16-le")) // 2 == 2 + + assert session.observe_text("model_generated", astral) == 1 + session.record_outcome("pii", "model_generated", 0, 1, "cleared") + assert session.advance("response") == 1 + assert session.safe_offset("response") == 1 + + +# --------------------------------------------------------------------- +# Unmediated tracks fail closed instead of silently releasing +# --------------------------------------------------------------------- + + +def test_payload_on_an_unmediated_track_fails_closed(): + # No `request_tasks`, so the request track is not mediated and text + # on it has nothing to gate it. That is a fail closed condition, not + # a silent release. + session = StreamSession(response_tasks=["pii"]) + with pytest.raises(ValueError, match="unmediated request track"): + session.observe_text("user_request", "hello") + # The failed observation was itself the terminal step for the + # session, so subsequent state matches the general terminal contract. + assert session.is_ended is True + assert session.safe_offset("request") is None + assert session.end_reason["kind"] == "failed" + + +def test_a_session_mediating_neither_track_is_refused_at_construction(): + # No tasks means the session would gate nothing at all. + with pytest.raises(ValueError): + StreamSession(safety_level="blocking") + + +# --------------------------------------------------------------------- +# Boundary errors surface with the engine's own message +# --------------------------------------------------------------------- + + +def test_unknown_safety_level_raises_before_a_session_exists(): + with pytest.raises(ValueError, match="unknown streaming safety level"): + StreamSession(safety_level="permissive", response_tasks=["pii"]) + + +def test_unknown_track_name_raises_on_read_paths(): + session = StreamSession(response_tasks=["pii"]) + with pytest.raises(ValueError, match="unknown stream track"): + session.safe_offset("responses") + with pytest.raises(ValueError, match="unknown stream track"): + session.watermark("responses") + + +def test_unknown_outcome_and_source_type_raise(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "hello") + with pytest.raises(ValueError, match="unknown segment outcome"): + session.record_outcome("pii", "model_generated", 0, 5, "allow") + # Still functional because the outcome parse fails before touching + # the session. + assert session.is_ended is False + with pytest.raises(ValueError, match="unknown stream source type"): + session.observe("assistant", 1) + + +def test_unknown_task_raises_and_terminates_the_session(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "hello") + with pytest.raises(ValueError, match="outcome named task safety"): + session.record_outcome("safety", "model_generated", 0, 5, "cleared") + # An engine-side rejection settled the session; safe_offset accepts + # that as terminal. + assert session.is_ended is True + assert session.safe_offset("response") is None + + +# --------------------------------------------------------------------- +# Independent per-track offsets +# --------------------------------------------------------------------- + + +def test_request_and_response_are_independent_offset_spaces(): + session = StreamSession( + safety_level="blocking", + request_tasks=["prompt_guard"], + response_tasks=["pii"], + request_start_rune_offset=100, + response_start_rune_offset=250, + ) + # Independent origins survive into the watermark unchanged. + assert session.watermark("request")["confirmed"] == 100 + assert session.watermark("response")["confirmed"] == 250 + + # Advancing one track does not disturb the other. A task on the + # response track clears a span, and the request track's ceiling + # holds where it started. + session.observe_text("model_generated", "hi") # +2 runes on response + session.record_outcome("pii", "model_generated", 250, 252, "cleared") + assert session.advance("response") == 252 + assert session.safe_offset("response") == 252 + assert session.safe_offset("request") == 100 + + # Now do the same on the request track. A user request span clears, + # and the response ceiling stays put. + session.observe_text("user_request", "prompt") # +6 runes on request + session.record_outcome("prompt_guard", "user_request", 100, 106, "cleared") + assert session.advance("request") == 106 + assert session.safe_offset("request") == 106 + assert session.safe_offset("response") == 252 + + completion = session.finish() + assert completion["is_clean"] is True + + +# --------------------------------------------------------------------- +# Verdict shape is validated, and a transform ends the stream rewritten +# --------------------------------------------------------------------- + + +def test_a_transform_verdict_ends_the_session_rewritten_and_reports_transformed(): + session = StreamSession(safety_level="blocking", response_tasks=["pii"]) + session.observe_text("model_generated", "raw output") # 10 runes + session.record_verdict("pii", "model_generated", 0, 10, _transform()) + + assert session.transformed is True + assert session.is_ended is True + reason = session.end_reason + assert reason == { + "kind": "rewritten", + "track": "response", + "task": "pii", + "start": 0, + "end": 10, + } + settlement = session.finish() + assert settlement["transformed"] is True + assert settlement["is_clean"] is False + + +def test_a_malformed_verdict_fails_the_stream_closed_without_clearing(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "content") # 7 runes + # A `transform` decision without a transform body is a shape section + # 5 does not admit. The typed constructor already refuses to build + # this, so a host cannot reach the session with it as a typed value. + # A wire dict is exactly how one arrives from an out-of-process peer + # that decided without validating, and the session must fail closed + # here rather than clearing the span. + malformed_wire = {"decision": "transform"} + with pytest.raises(ValueError): + session.record_verdict("pii", "model_generated", 0, 7, malformed_wire) + assert session.is_ended is True + assert session.end_reason["kind"] == "failed" + + +# --------------------------------------------------------------------- +# `record_verdict` accepts a wire dict too, not just a typed Verdict +# --------------------------------------------------------------------- + + +def test_record_verdict_accepts_a_wire_dict_from_a_serialized_verdict(): + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "hello") + # Any host that decoded a verdict from the wire holds it as a dict, + # so the wrapper accepts that shape without a round trip through the + # typed constructor. + session.record_verdict( + "pii", + "model_generated", + 0, + 5, + {"decision": "allow"}, + ) + assert session.advance("response") == 5 + completion = session.finish() + assert completion["is_clean"] is True + + +# --------------------------------------------------------------------- +# Rune offsets are `u32` on the wire. pyo3's automatic `u32` conversion +# raises `OverflowError` on any Python integer outside `[0, 2**32)`, +# so a bindings-level guard is not needed here — Node's is a workaround +# for N-API's silent ToUint32 wrap. Pin the guarantee anyway: a future +# change to accept `i64` and cast later would silently re-open the same +# hole the Node wrapper is now guarding against, and this test would +# fail loudly instead. +# --------------------------------------------------------------------- + + +def test_observe_refuses_a_rune_offset_at_or_past_the_u32_boundary(): + session = StreamSession(response_tasks=["pii"]) + with pytest.raises(OverflowError): + session.observe("model_generated", 2**32) + # Session state must be untouched: the raise happened before any + # native accounting ran. + assert session.watermark("response")["received"] == 0 + + +def test_observe_refuses_a_negative_rune_offset(): + session = StreamSession(response_tasks=["pii"]) + with pytest.raises(OverflowError): + session.observe("model_generated", -1) + assert session.watermark("response")["received"] == 0 + + +def test_record_outcome_refuses_an_end_offset_past_the_u32_boundary(): + # The exact failure mode the Node guard exists to prevent: a wrap + # to a small value would mark a *cleared* prefix on text no task + # evaluated. Python raises before the native call, so nothing + # clears. + session = StreamSession(response_tasks=["pii"]) + session.observe_text("model_generated", "hello") + with pytest.raises(OverflowError): + session.record_outcome("pii", "model_generated", 0, 2**32 + 5, "cleared") + assert session.safe_offset("response") == 0 + + +def test_record_outcome_refuses_a_start_offset_at_the_u32_boundary(): + session = StreamSession(response_tasks=["pii"]) + with pytest.raises(OverflowError): + session.record_outcome("pii", "model_generated", 2**32, 5, "cleared") + + +def test_record_verdict_refuses_rune_offsets_past_the_u32_boundary(): + # The verdict path enters the same accounting as record_outcome, + # so the guarantee must be symmetric. + session = StreamSession(response_tasks=["safety"]) + session.observe_text("model_generated", "hello") + with pytest.raises(OverflowError): + session.record_verdict( + "safety", "model_generated", 0, 2**32 + 5, {"decision": "allow"} + ) + with pytest.raises(OverflowError): + session.record_verdict( + "safety", "model_generated", -1, 5, {"decision": "allow"} + ) + assert session.safe_offset("response") == 0 + + +def test_stream_session_refuses_a_start_rune_offset_in_config_past_the_u32_boundary(): + with pytest.raises(OverflowError): + StreamSession( + response_tasks=["pii"], + response_start_rune_offset=2**32, + ) + with pytest.raises(OverflowError): + StreamSession( + request_tasks=["moderation"], + request_start_rune_offset=-1, + ) diff --git a/tests/conformance/bindings/bundle/policy.rego b/tests/conformance/bindings/bundle/policy.rego new file mode 100644 index 0000000..0decc7c --- /dev/null +++ b/tests/conformance/bindings/bundle/policy.rego @@ -0,0 +1,5 @@ +package acs + +decision := {"decision": "deny", "reason": "unsafe_content"} if { + input.annotations.content_safety.severity >= 4 +} else := {"decision": "allow"} diff --git a/tests/conformance/bindings/cross_language_parity.py b/tests/conformance/bindings/cross_language_parity.py new file mode 100755 index 0000000..1fd8b6b --- /dev/null +++ b/tests/conformance/bindings/cross_language_parity.py @@ -0,0 +1,875 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Run the whole public surface in every supported language and diff. + +ACS reaches Rust, Python, Node and .NET through four different binding +mechanisms: a direct crate dependency, pyo3, napi, and a C ABI with +P/Invoke over it. Each one converts enums, offsets, absent values and +error text at its own boundary, so agreement between them is not +structural and cannot be assumed. This asserts it, for streaming and for +everything else. + +Every language answers the same questions against the same manifest and +prints one JSON object. The objects must be identical. + +The scenario deliberately covers the places a binding is most likely to +drift: + +* a rune count that differs from the UTF-16 length, which is where a + .NET or Node binding silently releases twice what was evaluated +* an absent safe offset after settlement, which must not arrive as 0 or + as -1 in any language, because both read as permission +* a fail-closed deny, which every binding must surface as a verdict + rather than as an exception +* a rejected manifest, which must fail rather than return + +Run it from the repository root. Every language builds from this +checkout, so it answers for the code under review rather than for +whatever happens to be installed. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +MANIFEST = Path(__file__).resolve().parent / "manifest.yaml" +HOOKS_MANIFEST = Path(__file__).resolve().parent / "host-hooks-manifest.yaml" + +# "hi" plus one astral-plane scalar: 3 runes, 4 UTF-16 code units. +TEXT = "hi\U0001f600" +RUNES = 3 + +# A pre_tool_call context the fixture manifest denies. +DENY_CONTEXT = { + "interception_point": "pre_tool_call", + "tool_call": {"name": "shell", "args": {"cmd": "rm -rf /"}}, +} + +# An input context the fixture manifest allows. +ALLOW_CONTEXT = {"interception_point": "input", "input": "hello"} + +# Built rather than written literally. A repo guard scans committed files +# for the version key and validates whatever follows it, and a Python +# string literal carries quotes it cannot strip. +_VERSION_KEY = "agent_control_specification_version" + +# Names a supported version, then omits policies and intervention_points, +# so the grammar refuses it. Every language must fail rather than return. +BAD_MANIFEST = f'{_VERSION_KEY}: "0.4.0-alpha.1"\nmetadata: {{}}\n' + +# A sound manifest that names Rego. The document check passes it whatever +# the Rego says, so it is the only way to see artifact validation work. +REGO_MANIFEST = ( + f'{_VERSION_KEY}: "0.4.0-alpha.1"\n' + "policies:\n gate:\n type: rego\n bundle: ./b\n" + 'intervention_points:\n input:\n policy_target: "$.input"\n' + " policy:\n id: gate\n query: data.acs.decision\n" +) +GOOD_BUNDLES = { + "gate": {"modules": {"p.rego": 'package acs\ndecision := {"decision":"allow"}\n'}} +} +BAD_BUNDLES = {"gate": {"modules": {"p.rego": "package acs\nthis is not rego ***\n"}}} + +EXPECTED = { + # Manifest surface + "supported_versions_nonempty": True, + "validate_good": "ok", + "validate_bad": "rejected", + # Interceptor surface + "interceptor_name": "acs", + "allow_decision": "allow", + "deny_decision": "deny", + "deny_reason": "blocked_by_policy", + # Activated policy surface + "binds_input": True, + "activated_allow_decision": "allow", + # Streaming surface + "received": RUNES, + "safe_offset_before": 0, + "advanced": RUNES, + "safe_offset_after": RUNES, + "confirmed": RUNES, + "is_clean": True, + "transformed": False, + # Absent, never 0 and never -1. Each language spells it natively. + "safe_offset_settled": None, + # Host extension points. The classifier's answer must decide the + # verdict, and a classifier that could not be reached must deny + # rather than read as one that found nothing. + "hook_benign_decision": "allow", + "hook_harmful_decision": "deny", + "hook_harmful_reason": "unsafe_content", + "hook_failure_decision": "deny", + "hook_failure_reason": "runtime_error:annotation_failed", + "hook_dispatcher_calls": 1, + # Manifest tooling. + "parsed_has_points": True, + "diagnostics_on_bad": 1, + "diagnostics_on_good": 0, + # The shape, not only the count. Asserting the count alone let three + # bindings return three different diagnostic shapes for one call. + "diagnostic_keys": ["code", "field", "message", "severity"], + "diagnostic_code": "runtime_error:manifest_invalid", + "diagnostic_field": "intervention point", + # Settlement with uncleared residue, the fail-closed core of the + # profile. Every scenario above settles clean, so without this no + # binding is pinned to refusing text nothing evaluated. + "residue_kind": "failed", + "residue_reason": "host_error:streaming_unsupported", + "residue_clean": False, + # Artifact validation. The manifest is sound either way, so only + # compiling the Rego tells the two bundles apart. + # + # With no bundles the manifest still names ./b, which is not on disk, + # so activation reports the missing bundle. That is the answer a host + # wants: validating a manifest that names Rego without supplying the + # Rego cannot be a pass. + "artifacts_manifest_only": 1, + "artifacts_good_rego": 0, + "artifacts_bad_rego": 1, + "artifacts_bad_rego_code": "runtime_error:policy_invocation_failed", + # Resource caps. The same context must pass under the defaults and + # fail closed under a cap smaller than it, or the cap was accepted + # and dropped. + "limits_default_decision": "allow", + "limits_capped_decision": "deny", + "limits_capped_reason": "runtime_error:resource_limit_exceeded", +} + +# Larger than the capped bound below, smaller than the default one. +BIG_INPUT = "x" * 4096 +SMALL_CAP = {"max_snapshot_bytes": 64} + + +def _run(cmd: list[str], **kw) -> dict: + out = subprocess.run(cmd, capture_output=True, text=True, check=True, **kw) + return json.loads(out.stdout.strip().splitlines()[-1]) + + +RUST_MAIN = r""" +use agent_control_spec::dispatchers::{default_annotator_dispatcher, BindingPolicyDispatcher}; +use agent_control_spec::stream_session::*; +use agent_control_spec::{ActivatedPolicy, InterceptionPoint, Limits, Manifest, Runtime}; +use std::sync::Arc; + +struct Classifier { + severity: i64, + calls: std::sync::atomic::AtomicUsize, +} + +impl agent_control_spec::annotation::AnnotatorDispatcher for Classifier { + fn dispatch( + &self, + _name: &str, + _annotator: &agent_control_spec::annotation::AnnotatorInvocation, + _prelim: &serde_json::Value, + ) -> Result { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(serde_json::json!({ "severity": self.severity })) + } +} + +struct Broken; + +impl agent_control_spec::annotation::AnnotatorDispatcher for Broken { + fn dispatch( + &self, + _name: &str, + _annotator: &agent_control_spec::annotation::AnnotatorInvocation, + _prelim: &serde_json::Value, + ) -> Result { + Err(agent_control_spec::RuntimeError::AnnotationFailed( + "classifier unreachable".to_string(), + )) + } +} + +fn hook( + hooks_manifest: &str, + dispatcher: Arc, +) -> (String, Option) { + let manifest = Manifest::from_path(hooks_manifest).expect("hooks manifest"); + let runtime = Runtime::new(manifest, dispatcher, Arc::new(BindingPolicyDispatcher::new())) + .expect("hooks runtime"); + let ctx: serde_json::Value = + serde_json::from_str(r#"{"interception_point":"input","input":"hello"}"#).expect("ctx"); + let verdict = runtime.evaluate(&ctx).verdict; + ( + format!("{:?}", verdict.decision).to_lowercase(), + verdict.reason.clone(), + ) +} + +fn main() { + let manifest_path = std::env::args().nth(1).expect("manifest path"); + let text = std::env::args().nth(2).expect("text"); + let hooks_manifest = std::env::args().nth(3).expect("hooks manifest"); + + let validate_good = match Manifest::from_path(&manifest_path) { + Ok(_) => "ok", + Err(_) => "rejected", + }; + let validate_bad = match Manifest::from_yaml_str(BAD_MANIFEST) { + Ok(_) => "ok", + Err(_) => "rejected", + }; + + let manifest = Manifest::from_path(&manifest_path).expect("manifest"); + let runtime = Runtime::new( + manifest.clone(), + default_annotator_dispatcher(), + Arc::new(BindingPolicyDispatcher::new()), + ) + .expect("runtime"); + + let allow: serde_json::Value = serde_json::from_str(ALLOW_CONTEXT).expect("allow ctx"); + let deny: serde_json::Value = serde_json::from_str(DENY_CONTEXT).expect("deny ctx"); + let allow_verdict = runtime.evaluate(&allow).verdict; + let deny_verdict = runtime.evaluate(&deny).verdict; + + let policy = ActivatedPolicy::activate_with( + manifest, + default_annotator_dispatcher(), + Arc::new(BindingPolicyDispatcher::new()), + ) + .expect("activate"); + let binds_input = policy + .intervention_points() + .iter() + .any(|p| format!("{p:?}").to_lowercase() == "input"); + let activated = policy + .evaluate(InterceptionPoint::Input, allow.clone()) + .verdict; + + let mut session = StreamSession::new(StreamSessionConfig { + safety_level: SafetyLevel::Blocking, + request_start_rune_offset: 0, + response_start_rune_offset: 0, + request_tasks: vec![], + response_tasks: vec!["pii".to_string()], + }) + .expect("session"); + let received = session + .observe_text(StreamSourceType::ModelGenerated, &text) + .expect("observe"); + let before = session.safe_offset(StreamTrack::Response); + let span = StreamSpan::new(StreamSourceType::ModelGenerated, 0, received).expect("span"); + session + .record_outcome("pii", &span, SegmentOutcome::Cleared) + .expect("outcome"); + let advanced = session.advance(StreamTrack::Response); + let after = session.safe_offset(StreamTrack::Response); + let confirmed = session.watermark(StreamTrack::Response).confirmed(); + let completion = session.finish(); + + let benign = Arc::new(Classifier { + severity: 1, + calls: std::sync::atomic::AtomicUsize::new(0), + }); + let b = hook(&hooks_manifest, benign.clone()); + let hh = hook( + &hooks_manifest, + Arc::new(Classifier { + severity: 7, + calls: std::sync::atomic::AtomicUsize::new(0), + }), + ); + let f = hook(&hooks_manifest, Arc::new(Broken)); + let parsed = Manifest::from_yaml_str(&std::fs::read_to_string(&manifest_path).expect("read")) + .expect("parse"); + let parsed_json = serde_json::to_value(&parsed).expect("parsed json"); + let art = |bundles: &str| -> (usize, Option) { + let parsed: std::collections::BTreeMap = + serde_json::from_str(bundles).expect("bundles"); + match ActivatedPolicy::activate_from_memory(REGO_MANIFEST, parsed) { + Ok(_) => (0, None), + Err(e) => (1, Some(e.reason().to_string())), + } + }; + let mut residue = StreamSession::new(StreamSessionConfig { + safety_level: SafetyLevel::Blocking, + request_start_rune_offset: 0, + response_start_rune_offset: 0, + request_tasks: vec![], + response_tasks: vec!["pii".into()], + }) + .expect("residue session"); + residue + .observe_text(StreamSourceType::ModelGenerated, "hello") + .expect("observe"); + let residue_done = residue.finish(); + let residue_json = agent_control_spec::wire::completion_json(&residue_done); + + let big_ctx: serde_json::Value = + serde_json::from_str(BIG_CTX).expect("big ctx"); + let lim = |limits: Limits| -> (String, Option) { + let manifest = Manifest::from_path(&hooks_manifest).expect("hooks manifest"); + let rt = Runtime::with_limits( + manifest, + Arc::new(Classifier { severity: 1, calls: std::sync::atomic::AtomicUsize::new(0) }), + Arc::new(BindingPolicyDispatcher::new()), + limits, + ) + .expect("limited runtime"); + let v = rt.evaluate(&big_ctx).verdict; + (format!("{:?}", v.decision).to_lowercase(), v.reason.clone()) + }; + let lim_default = lim(Limits::default()); + let lim_capped = lim(Limits { max_snapshot_bytes: 64, ..Limits::default() }); + + let art_only = art("{}"); + let art_good = art(GOOD_BUNDLES); + let art_bad = art(BAD_BUNDLES); + let bad_diag_value = match Manifest::from_yaml_str(BAD_MANIFEST) { + Err(e) => Some(agent_control_spec::wire::diagnostic_json(&e)), + Ok(m) => m.validate().err().map(|e| agent_control_spec::wire::diagnostic_json(&e)), + }; + let mut bad_diag_keys: Vec = bad_diag_value + .as_ref() + .and_then(|v| v.as_object()) + .map(|o| o.keys().cloned().collect()) + .unwrap_or_default(); + bad_diag_keys.sort(); + let bad_diag_code = bad_diag_value + .as_ref() + .and_then(|v| v.get("code")) + .and_then(|v| v.as_str()) + .map(str::to_string); + let bad_diag_field = bad_diag_value + .as_ref() + .and_then(|v| v.get("field")) + .and_then(|v| v.as_str()) + .map(str::to_string); + let bad_diags = usize::from(bad_diag_value.is_some()); + let good_diags = usize::from( + Manifest::from_yaml_str(&std::fs::read_to_string(&manifest_path).expect("read")) + .and_then(|m| m.validate()) + .is_err(), + ); + + println!( + "{}", + serde_json::json!({ + "hook_benign_decision": b.0, + "hook_harmful_decision": hh.0, + "hook_harmful_reason": hh.1, + "hook_failure_decision": f.0, + "hook_failure_reason": f.1, + "hook_dispatcher_calls": benign.calls.load(std::sync::atomic::Ordering::SeqCst), + "parsed_has_points": parsed_json.get("intervention_points").is_some(), + "diagnostics_on_bad": bad_diags, + "diagnostic_keys": bad_diag_keys, + "diagnostic_code": bad_diag_code, + "diagnostic_field": bad_diag_field, + "diagnostics_on_good": good_diags, + "artifacts_manifest_only": art_only.0, + "artifacts_good_rego": art_good.0, + "artifacts_bad_rego": art_bad.0, + "artifacts_bad_rego_code": art_bad.1, + "limits_default_decision": lim_default.0, + "limits_capped_decision": lim_capped.0, + "limits_capped_reason": lim_capped.1, + "residue_kind": residue_json["reason"]["kind"], + "residue_reason": residue_json["reason"]["reason"], + "residue_clean": residue_json["is_clean"], + "supported_versions_nonempty": !agent_control_spec::SUPPORTED_VERSIONS.is_empty(), + "validate_good": validate_good, + "validate_bad": validate_bad, + "interceptor_name": "acs", + "allow_decision": format!("{:?}", allow_verdict.decision).to_lowercase(), + "deny_decision": format!("{:?}", deny_verdict.decision).to_lowercase(), + "deny_reason": deny_verdict.reason.clone(), + "binds_input": binds_input, + "activated_allow_decision": format!("{:?}", activated.decision).to_lowercase(), + "received": received, + "safe_offset_before": before, + "advanced": advanced, + "safe_offset_after": after, + "confirmed": confirmed, + "is_clean": completion.reason.is_clean(), + "transformed": completion.transformed, + "safe_offset_settled": session.safe_offset(StreamTrack::Response), + }) + ); +} +""" + + +def rust() -> dict: + work = ROOT / "target" / "parity-rs" + work.mkdir(parents=True, exist_ok=True) + (work / "main.rs").write_text( + RUST_MAIN.replace("BAD_MANIFEST", json.dumps(BAD_MANIFEST)) + .replace("ALLOW_CONTEXT", json.dumps(json.dumps(ALLOW_CONTEXT))) + .replace("DENY_CONTEXT", json.dumps(json.dumps(DENY_CONTEXT))) + .replace("REGO_MANIFEST", json.dumps(REGO_MANIFEST)) + .replace("GOOD_BUNDLES", json.dumps(json.dumps(GOOD_BUNDLES))) + .replace("BAD_BUNDLES", json.dumps(json.dumps(BAD_BUNDLES))) + .replace( + "BIG_CTX", + json.dumps(json.dumps({"interception_point": "input", "input": BIG_INPUT})), + ) + ) + (work / "Cargo.toml").write_text( + f""" +[package] +name = "acs-parity" +version = "0.0.0" +edition = "2021" + +[[bin]] +name = "acs-parity" +path = "main.rs" + +[dependencies] +agent-control-spec = {{ path = "{ROOT / "engine"}", features = ["default-dispatchers", "streaming"] }} +serde_json = "1" + +[workspace] +""" + ) + return _run( + [ + "cargo", + "run", + "--quiet", + "--release", + "--manifest-path", + str(work / "Cargo.toml"), + "--", + str(MANIFEST), + TEXT, + str(HOOKS_MANIFEST), + ] + ) + + +def python_binding() -> dict: + script = f""" +import json +from agent_control_spec import ( + AcsInterceptor, ActivatedPolicy, StreamSession, + supported_manifest_versions, validate_manifest, + parse_manifest, validate_manifest_detailed, validate_artifacts, +) + +def check(source): + try: + validate_manifest(source) + return "ok" + except Exception: + return "rejected" + +interceptor = AcsInterceptor({str(MANIFEST)!r}) +allow = interceptor.intercept({ALLOW_CONTEXT!r}) +deny = interceptor.intercept({DENY_CONTEXT!r}) + +policy = ActivatedPolicy({str(MANIFEST)!r}) +points = [str(p).lower() for p in policy.intervention_points] +activated = policy.evaluate("input", {ALLOW_CONTEXT!r}) + +session = StreamSession(safety_level="blocking", response_tasks=["pii"]) +received = session.observe_text("model_generated", {TEXT!r}) +before = session.safe_offset("response") +session.record_outcome("pii", "model_generated", 0, received, "cleared") +advanced = session.advance("response") +after = session.safe_offset("response") +confirmed = session.watermark("response")["confirmed"] +completion = session.finish() + +def decision(v): + d = getattr(v, "decision", None) + return str(getattr(d, "value", d)).lower() + + +class _Classifier: + def __init__(self, sev): self.sev = sev; self.calls = 0 + def dispatch(self, name, annotator, prelim): + self.calls += 1 + return {{"severity": self.sev}} + +class _Broken: + def dispatch(self, *a, **k): + raise RuntimeError("classifier unreachable") + +def _hook(dispatcher): + i = AcsInterceptor({str(HOOKS_MANIFEST)!r}, annotator_dispatcher=dispatcher) + v = i.intercept({{"interception_point": "input", "input": "hello"}}) + return (str(getattr(v.decision, "value", v.decision)).lower(), v.reason) + +_benign = _Classifier(1) +_b = _hook(_benign) +_h = _hook(_Classifier(7)) +_f = _hook(_Broken()) +_parsed = parse_manifest(open({str(MANIFEST)!r}).read()) +_bad_diags = validate_manifest_detailed({BAD_MANIFEST!r}) +_good_diags = validate_manifest_detailed(open({str(MANIFEST)!r}).read()) +_art_only = validate_artifacts({REGO_MANIFEST!r}) +_art_good = validate_artifacts({REGO_MANIFEST!r}, {GOOD_BUNDLES!r}) +_art_bad = validate_artifacts({REGO_MANIFEST!r}, {BAD_BUNDLES!r}) + +_big_ctx = {{"interception_point": "input", "input": {BIG_INPUT!r}}} +_lim_default = AcsInterceptor({str(HOOKS_MANIFEST)!r}, annotator_dispatcher=_Classifier(1)).intercept(_big_ctx) +_res = StreamSession(safety_level="blocking", response_tasks=["pii"]) +_res.observe_text("model_generated", "hello") +_res_done = _res.finish() + +_lim_capped = AcsInterceptor( + {str(HOOKS_MANIFEST)!r}, annotator_dispatcher=_Classifier(1), limits={SMALL_CAP!r} +).intercept(_big_ctx) + +print(json.dumps({{ + "hook_benign_decision": _b[0], + "hook_harmful_decision": _h[0], + "hook_harmful_reason": _h[1], + "hook_failure_decision": _f[0], + "hook_failure_reason": _f[1], + "hook_dispatcher_calls": _benign.calls, + "parsed_has_points": "intervention_points" in _parsed, + "diagnostics_on_bad": len(_bad_diags), + "diagnostic_keys": sorted(_bad_diags[0]) if _bad_diags else [], + "diagnostic_code": _bad_diags[0].get("code") if _bad_diags else None, + "diagnostic_field": _bad_diags[0].get("field") if _bad_diags else None, + "diagnostics_on_good": len(_good_diags), + "artifacts_manifest_only": len(_art_only), + "artifacts_good_rego": len(_art_good), + "artifacts_bad_rego": len(_art_bad), + "artifacts_bad_rego_code": _art_bad[0]["code"] if _art_bad else None, + "limits_default_decision": decision(_lim_default), + "limits_capped_decision": decision(_lim_capped), + "limits_capped_reason": _lim_capped.reason, + "residue_kind": _res_done["reason"]["kind"], + "residue_reason": _res_done["reason"].get("reason"), + "residue_clean": _res_done["is_clean"], + "supported_versions_nonempty": len(supported_manifest_versions()) > 0, + "validate_good": check(open({str(MANIFEST)!r}).read()), + "validate_bad": check({BAD_MANIFEST!r}), + "interceptor_name": interceptor.name, + "allow_decision": decision(allow), + "deny_decision": decision(deny), + "deny_reason": deny.reason, + "binds_input": "input" in points, + "activated_allow_decision": decision(activated), + "received": received, + "safe_offset_before": before, + "advanced": advanced, + "safe_offset_after": after, + "confirmed": confirmed, + "is_clean": completion["is_clean"], + "transformed": completion["transformed"], + "safe_offset_settled": session.safe_offset("response"), +}})) +""" + return _run([sys.executable, "-c", script]) + + +def node() -> dict: + script = f""" +const fs = require('fs'); +const acs = require('./dist/index.js'); + +function check(source) {{ + try {{ acs.validateManifest(source); return 'ok'; }} + catch (e) {{ return 'rejected'; }} +}} + +const interceptor = acs.AcsInterceptor.fromPath({json.dumps(str(MANIFEST))}); +const allow = interceptor.intercept({json.dumps(ALLOW_CONTEXT)}); +const deny = interceptor.intercept({json.dumps(DENY_CONTEXT)}); + +const policy = acs.ActivatedPolicy.activate({json.dumps(str(MANIFEST))}); +const points = policy.interventionPoints().map((p) => String(p).toLowerCase()); +const activated = policy.evaluate('input', {json.dumps(ALLOW_CONTEXT)}); + +const session = new acs.StreamSession({{ safetyLevel: 'blocking', responseTasks: ['pii'] }}); +const received = session.observeText('model_generated', {json.dumps(TEXT)}); +const before = session.safeOffset('response'); +session.recordOutcome('pii', 'model_generated', 0, received, 'cleared'); +const advanced = session.advance('response'); +const after = session.safeOffset('response'); +const confirmed = session.watermark('response').confirmed; +const completion = session.finish(); + +function hook(d) {{ + const i = acs.AcsInterceptor.fromPath({json.dumps(str(HOOKS_MANIFEST))}, {{ annotatorDispatcher: d }}); + const v = i.intercept({{ interception_point: 'input', input: 'hello' }}); + return [String(v.decision).toLowerCase(), v.reason ?? null]; +}} +let hookCalls = 0; +const b = hook(() => {{ hookCalls++; return {{ severity: 1 }}; }}); +const hh = hook(() => ({{ severity: 7 }})); +const f = hook(() => {{ throw new Error('classifier unreachable'); }}); +const parsed = acs.parseManifest(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')); +const badDiags = acs.validateManifestDetailed({json.dumps(BAD_MANIFEST)}); +const goodDiags = acs.validateManifestDetailed(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')); +const artOnly = acs.validateArtifacts({json.dumps(REGO_MANIFEST)}); +const artGood = acs.validateArtifacts({json.dumps(REGO_MANIFEST)}, {json.dumps(GOOD_BUNDLES)}); +const artBad = acs.validateArtifacts({json.dumps(REGO_MANIFEST)}, {json.dumps(BAD_BUNDLES)}); + +const res = new acs.StreamSession({{ safetyLevel: 'blocking', responseTasks: ['pii'] }}); +res.observeText('model_generated', 'hello'); +const resDone = res.finish(); + +const bigCtx = {{ interception_point: 'input', input: {json.dumps(BIG_INPUT)} }}; +const limDefault = acs.AcsInterceptor + .fromPath({json.dumps(str(HOOKS_MANIFEST))}, {{ annotatorDispatcher: () => ({{ severity: 1 }}) }}) + .intercept(bigCtx); +const limCapped = acs.AcsInterceptor + .fromPath({json.dumps(str(HOOKS_MANIFEST))}, {{ annotatorDispatcher: () => ({{ severity: 1 }}), limits: {json.dumps(SMALL_CAP)} }}) + .intercept(bigCtx); + +console.log(JSON.stringify({{ + hook_benign_decision: b[0], + hook_harmful_decision: hh[0], + hook_harmful_reason: hh[1], + hook_failure_decision: f[0], + hook_failure_reason: f[1], + hook_dispatcher_calls: hookCalls, + parsed_has_points: Object.prototype.hasOwnProperty.call(parsed, 'intervention_points'), + diagnostics_on_bad: badDiags.length, + diagnostic_keys: badDiags.length ? Object.keys(badDiags[0]).sort() : [], + diagnostic_code: badDiags.length ? badDiags[0].code : null, + diagnostic_field: badDiags.length ? badDiags[0].field : null, + diagnostics_on_good: goodDiags.length, + artifacts_manifest_only: artOnly.length, + artifacts_good_rego: artGood.length, + artifacts_bad_rego: artBad.length, + artifacts_bad_rego_code: artBad.length ? artBad[0].code : null, + limits_default_decision: String(limDefault.decision).toLowerCase(), + limits_capped_decision: String(limCapped.decision).toLowerCase(), + limits_capped_reason: limCapped.reason ?? null, + residue_kind: resDone.reason.kind, + residue_reason: resDone.reason.reason ?? null, + residue_clean: resDone.isClean, + supported_versions_nonempty: acs.supportedManifestVersions().length > 0, + validate_good: check(fs.readFileSync({json.dumps(str(MANIFEST))}, 'utf8')), + validate_bad: check({json.dumps(BAD_MANIFEST)}), + interceptor_name: interceptor.name, + allow_decision: String(allow.decision).toLowerCase(), + deny_decision: String(deny.decision).toLowerCase(), + deny_reason: deny.reason ?? null, + binds_input: points.includes('input'), + activated_allow_decision: String(activated.decision).toLowerCase(), + received, + safe_offset_before: before, + advanced, + safe_offset_after: after, + confirmed, + is_clean: completion.isClean, + transformed: completion.transformed, + safe_offset_settled: session.safeOffset('response'), +}})); +""" + return _run(["node", "-e", script], cwd=ROOT / "sdk" / "node") + + +DOTNET_PROGRAM = """ +using System.Text.Json; +using System.Text.Json.Nodes; +using AgentControlSpec; +using AgentHooks; + +static string Check(Action f) +{ + try { f(); return "ok"; } catch { return "rejected"; } +} + +var manifest = MANIFEST_PATH; +using var interceptor = AcsInterceptor.FromPath(manifest); +var allowCtx = new AgentContext(JsonNode.Parse(ALLOW_JSON)!.AsObject()); +var denyCtx = new AgentContext(JsonNode.Parse(DENY_JSON)!.AsObject()); +var allow = await interceptor.InterceptAsync(allowCtx); +var deny = await interceptor.InterceptAsync(denyCtx); + +using var policy = AcsPolicy.Activate(manifest); +var points = policy.InterventionPoints.Select(p => p.ToString().ToLowerInvariant()).ToList(); +var activated = policy.Evaluate(InterceptionPoint.Input, ALLOW_JSON); + +using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); +var received = session.ObserveText(StreamSourceType.ModelGenerated, TEXT_LITERAL); +var before = session.SafeOffset(StreamTrack.Response); +session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, received, SegmentOutcome.Cleared); +var advanced = session.Advance(StreamTrack.Response); +var after = session.SafeOffset(StreamTrack.Response); +var confirmed = session.Watermark(StreamTrack.Response).Confirmed; +var completion = session.Finish(); + +static (string, string?) Hook(AnnotatorDispatcher d) +{ + using var i = AcsHostInterceptor.FromPath(HOOKS_MANIFEST, annotator: d); + var v = i.InterceptAsync(new AgentContext(JsonNode.Parse(ALLOW_JSON)!.AsObject())).AsTask().Result; + return (v.Decision.ToString().ToLowerInvariant(), v.Reason); +} + +var hookCalls = 0; +var b = Hook((_, _, _) => { hookCalls++; return SEV1; }); +var hh = Hook((_, _, _) => SEV7); +var f = Hook((_, _, _) => throw new InvalidOperationException("classifier unreachable")); +var parsed = AcsManifestTools.Parse(File.ReadAllText(manifest)); +var badDiags = AcsManifestTools.Diagnostics(BAD_JSON); +var goodDiags = AcsManifestTools.Diagnostics(File.ReadAllText(manifest)); +var artOnly = AcsManifestTools.ValidateArtifacts(REGO_MANIFEST, null); +var artGood = AcsManifestTools.ValidateArtifacts(REGO_MANIFEST, GOOD_BUNDLES); +var artBad = AcsManifestTools.ValidateArtifacts(REGO_MANIFEST, BAD_BUNDLES); + +using var residue = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); +residue.ObserveText(StreamSourceType.ModelGenerated, "hello"); +var residueDone = residue.Finish(); + +var bigCtx = new AgentContext(JsonNode.Parse(BIG_CTX)!.AsObject()); +using var limDefault = AcsHostInterceptor.FromPath(HOOKS_MANIFEST, annotator: (_, _, _) => SEV1); +using var limCapped = AcsHostInterceptor.FromPath( + HOOKS_MANIFEST, annotator: (_, _, _) => SEV1, limits: SMALL_CAP); +var limDefaultVerdict = limDefault.InterceptAsync(bigCtx).AsTask().Result; +var limCappedVerdict = limCapped.InterceptAsync(bigCtx).AsTask().Result; + +Console.WriteLine(JsonSerializer.Serialize(new Dictionary +{ + ["hook_benign_decision"] = b.Item1, + ["hook_harmful_decision"] = hh.Item1, + ["hook_harmful_reason"] = hh.Item2, + ["hook_failure_decision"] = f.Item1, + ["hook_failure_reason"] = f.Item2, + ["hook_dispatcher_calls"] = hookCalls, + ["parsed_has_points"] = parsed.Contains("intervention_points"), + ["diagnostics_on_bad"] = badDiags.Count, + ["diagnostic_keys"] = badDiags.Count > 0 + ? JsonSerializer.Deserialize>( + JsonSerializer.Serialize(badDiags[0]))!.Keys.OrderBy(k => k, StringComparer.Ordinal).ToList() + : new List(), + ["diagnostic_code"] = badDiags.Count > 0 ? badDiags[0].Code : null, + ["diagnostic_field"] = badDiags.Count > 0 ? badDiags[0].Field : null, + ["diagnostics_on_good"] = goodDiags.Count, + ["artifacts_manifest_only"] = artOnly.Count, + ["artifacts_good_rego"] = artGood.Count, + ["artifacts_bad_rego"] = artBad.Count, + ["artifacts_bad_rego_code"] = artBad.Count > 0 ? artBad[0].Code : null, + ["limits_default_decision"] = limDefaultVerdict.Decision.ToString().ToLowerInvariant(), + ["limits_capped_decision"] = limCappedVerdict.Decision.ToString().ToLowerInvariant(), + ["limits_capped_reason"] = limCappedVerdict.Reason, + ["residue_kind"] = residueDone.Reason.Kind, + ["residue_reason"] = residueDone.Reason.Reason, + ["residue_clean"] = residueDone.IsClean, + ["supported_versions_nonempty"] = AcsManifest.SupportedVersions().Count > 0, + ["validate_good"] = Check(() => AcsManifest.Validate(File.ReadAllText(manifest))), + ["validate_bad"] = Check(() => AcsManifest.Validate(BAD_JSON)), + ["interceptor_name"] = interceptor.Name, + ["allow_decision"] = allow.Decision.ToString().ToLowerInvariant(), + ["deny_decision"] = deny.Decision.ToString().ToLowerInvariant(), + ["deny_reason"] = deny.Reason, + ["binds_input"] = points.Contains("input"), + ["activated_allow_decision"] = activated.Decision.ToString().ToLowerInvariant(), + ["received"] = received, + ["safe_offset_before"] = before, + ["advanced"] = advanced, + ["safe_offset_after"] = after, + ["confirmed"] = confirmed, + ["is_clean"] = completion.IsClean, + ["transformed"] = completion.Transformed, + ["safe_offset_settled"] = session.SafeOffset(StreamTrack.Response), +})); +""" + + +def dotnet() -> dict: + work = Path(tempfile.mkdtemp()) + try: + app = work / "app" + subprocess.run( + ["dotnet", "new", "console", "-o", str(app)], + capture_output=True, + check=True, + ) + subprocess.run( + [ + "dotnet", + "add", + str(app), + "reference", + str(ROOT / "sdk/dotnet/src/AgentControlSpec/AgentControlSpec.csproj"), + ], + capture_output=True, + check=True, + ) + program = ( + DOTNET_PROGRAM.replace("MANIFEST_PATH", json.dumps(str(MANIFEST))) + .replace("HOOKS_MANIFEST", json.dumps(str(HOOKS_MANIFEST))) + .replace("SEV1", json.dumps(json.dumps({"severity": 1}))) + .replace("SEV7", json.dumps(json.dumps({"severity": 7}))) + .replace("REGO_MANIFEST", json.dumps(REGO_MANIFEST)) + .replace("GOOD_BUNDLES", json.dumps(json.dumps(GOOD_BUNDLES))) + .replace("BAD_BUNDLES", json.dumps(json.dumps(BAD_BUNDLES))) + .replace( + "BIG_CTX", + json.dumps( + json.dumps({"interception_point": "input", "input": BIG_INPUT}) + ), + ) + .replace("SMALL_CAP", json.dumps(json.dumps(SMALL_CAP))) + .replace( + "__BIGCTX__", + json.dumps( + json.dumps({"interception_point": "input", "input": BIG_INPUT}) + ), + ) + .replace("__SMALLCAP__", json.dumps(json.dumps(SMALL_CAP))) + .replace("ALLOW_JSON", json.dumps(json.dumps(ALLOW_CONTEXT))) + .replace("DENY_JSON", json.dumps(json.dumps(DENY_CONTEXT))) + .replace("BAD_JSON", json.dumps(BAD_MANIFEST)) + .replace("TEXT_LITERAL", json.dumps(TEXT)) + ) + (app / "Program.cs").write_text(program) + env = dict(os.environ) + env["LD_LIBRARY_PATH"] = str(ROOT / "target" / "release") + return _run(["dotnet", "run", "--project", str(app), "--nologo"], env=env) + finally: + shutil.rmtree(work, ignore_errors=True) + + +def main() -> int: + languages = { + "rust": rust, + "python": python_binding, + "node": node, + "dotnet": dotnet, + } + + results: dict[str, dict] = {} + failed = False + for name, run in languages.items(): + try: + results[name] = run() + except subprocess.CalledProcessError as e: + print(f"{name}: FAILED TO RUN\n{e.stdout}\n{e.stderr}", file=sys.stderr) + failed = True + + if failed: + return 1 + + for name, got in results.items(): + mismatches = { + k: (EXPECTED[k], got.get(k)) for k in EXPECTED if got.get(k) != EXPECTED[k] + } + print(f"{name:8} {'ok' if not mismatches else 'MISMATCH'}") + for key, (want, actual) in sorted(mismatches.items()): + print(f" {key}: expected {want!r}, got {actual!r}", file=sys.stderr) + failed = True + + if failed: + print("\nlanguages disagree about the same inputs", file=sys.stderr) + return 1 + + print(f"\nall {len(results)} languages agree across {len(EXPECTED)} assertions") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conformance/bindings/dotnet_package.sh b/tests/conformance/bindings/dotnet_package.sh new file mode 100755 index 0000000..6075db5 --- /dev/null +++ b/tests/conformance/bindings/dotnet_package.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Prove a packed ResponsibleAI.AgentControlSpec really works. +# +# The managed assembly reaches the engine through +# agent_control_spec_ffi. A package that omits the native library still +# restores, still compiles against, and still passes any test run from a +# checkout that happens to have the library on its loader path. It fails +# only in a consumer's process, on the first call. +# +# So this builds a throwaway console app outside the repository, +# restores the packed artifact from a local feed, and calls both a plain +# entry point and the streaming session. No LD_LIBRARY_PATH, no engine +# build, nothing from the checkout on the loader path. +# +# published_artifacts.py covers all four languages. This stays separate +# because the release workflow runs it against the artifact it is about +# to push, where the other languages are not in scope. +# +# Usage: dotnet_package.sh + +set -euo pipefail + +FEED="${1:?usage: verify-dotnet-package.sh }" +VERSION="${2:?usage: verify-dotnet-package.sh }" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +cat >"$WORK/NuGet.config" < + + + + + + + +XML + +dotnet new console -o "$WORK/app" >/dev/null +cat >"$WORK/app/Program.cs" <<'CS' +using AgentControlSpec; + +// A plain entry point: proves the native library resolved at all. +var versions = AcsManifest.SupportedVersions(); +if (versions.Count == 0) +{ + throw new InvalidOperationException("no supported manifest versions"); +} + +// The streaming session: proves the engine was built with the feature +// its consumers cannot enable for themselves. +using var session = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); +if (session.ObserveText(StreamSourceType.ModelGenerated, "hello") != 5) +{ + throw new InvalidOperationException("observe_text did not count five runes"); +} + +session.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, 5, SegmentOutcome.Cleared); +if (session.Advance(StreamTrack.Response) != 5 || session.SafeOffset(StreamTrack.Response) != 5) +{ + throw new InvalidOperationException("a cleared span did not release"); +} + +if (!session.Finish().IsClean) +{ + throw new InvalidOperationException("a clean stream did not settle clean"); +} + +if (session.SafeOffset(StreamTrack.Response) is not null) +{ + throw new InvalidOperationException("a settled session still offered a safe offset"); +} + +Console.WriteLine("package verified: non-streaming and streaming both reachable"); +CS + +( + cd "$WORK/app" + dotnet add package ResponsibleAI.AgentControlSpec --version "$VERSION" >/dev/null + dotnet run --nologo +) diff --git a/tests/conformance/bindings/host-hooks-manifest.yaml b/tests/conformance/bindings/host-hooks-manifest.yaml new file mode 100644 index 0000000..ccb6bd9 --- /dev/null +++ b/tests/conformance/bindings/host-hooks-manifest.yaml @@ -0,0 +1,19 @@ +agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: host-annotator-proof +annotators: + content_safety: + type: classifier +policies: + gate: + type: rego + bundle: ./bundle +intervention_points: + input: + policy_target: "$snap.input" + annotations: + content_safety: + from: "$target" + policy: + id: gate + query: data.acs.decision diff --git a/tests/conformance/bindings/language_coverage.py b/tests/conformance/bindings/language_coverage.py new file mode 100755 index 0000000..44e2eb7 --- /dev/null +++ b/tests/conformance/bindings/language_coverage.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Fail when an engine capability is reachable from fewer than four bindings. + +Cross-language comparison proves the bindings agree about the calls it +makes. It cannot notice a capability none of them expose, because a +surface absent everywhere is consistent everywhere. + +So this reads the engine's public re-exports and requires every one to +be either a capability with the token each binding exposes, or a +non-capability with a written reason. A symbol that is neither fails, +which forces the question to be answered when the symbol is added +rather than when a consumer cannot find it. + +Writing the reason down is the point. An unexplained omission and a +deliberate one look identical six months later. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + +# Symbols that need no entry point, each with why. Anything not listed +# here and not matched below is treated as an unbound capability. +NOT_A_CAPABILITY = { + # Types that cross the boundary as JSON rather than as objects. A + # binding never constructs one, so there is nothing to bind. + "AgentContext": "crosses as JSON", + "Verdict": "crosses as JSON", + "Decision": "crosses as JSON", + "Warning": "crosses as JSON", + "Transform": "crosses as JSON", + "Evidence": "crosses as JSON", + "EnforcementMode": "crosses as JSON", + "InterceptionPoint": "crosses as JSON", + "EvaluationRequest": "crosses as JSON", + "EvaluationResult": "crosses as JSON", + "AnnotatorInvocation": "crosses as JSON to a host dispatcher", + "PreparedPolicyInvocation": "crosses as JSON to a host dispatcher", + "RegoPolicyInvocation": "variant of PreparedPolicyInvocation", + "CedarPolicyInvocation": "variant of PreparedPolicyInvocation", + "TestPolicyInvocation": "variant of PreparedPolicyInvocation", + "CustomPolicyInvocation": "variant of PreparedPolicyInvocation", + "CedarRequest": "crosses as JSON", + "CedarEntity": "crosses as JSON", + "TelemetryEvent": "crosses as JSON to a host sink", + "TelemetryEventType": "field of TelemetryEvent", + "RuntimeError": "surfaces as an error message or a diagnostic", + "StreamError": "surfaces as an end reason", + # Manifest grammar, reached by parsing a manifest rather than by + # constructing the node. + "Manifest": "reached through parse and validate", + "InterventionPointConfig": "manifest grammar", + "AnnotationConfig": "manifest grammar", + "AnnotatorConfig": "manifest grammar", + "AnnotatorType": "manifest grammar", + "PolicyConfig": "manifest grammar", + "PolicyBinding": "manifest grammar", + "RegoPolicyConfig": "manifest grammar", + "CedarPolicyConfig": "manifest grammar", + "TestPolicyConfig": "manifest grammar", + "CustomPolicyConfig": "manifest grammar", + "ToolConfig": "manifest grammar", + "MountedRegoData": "field of InMemoryRegoBundle", + # Dispatchers the manifest selects by policy type. A host picks one + # by writing a manifest, not by naming the Rust type. + "RegorusPolicyDispatcher": "selected by manifest policy type", + "RegorusRegoRunner": "selected by manifest policy type", + "OpaPolicyDispatcher": "selected by manifest policy type", + "OpaRegoRunner": "selected by manifest policy type", + "CedarBuiltinDispatcher": "selected by manifest policy type", + "CedarPolicyDispatcher": "selected by manifest policy type", + "CedarTestDispatcher": "test double", + "DefaultAnnotatorDispatcher": "the default when a host supplies none", + "NoopTelemetrySink": "the default when a host supplies none", + "ClassifierAnnotator": "selected by manifest annotator type", + "EndpointAnnotator": "selected by manifest annotator type", + "LlmAnnotator": "selected by manifest annotator type", + "Interceptor": "the trait AcsInterceptor implements", + "AcsInterceptor": "bound as the interceptor entry point itself", + "Runtime": "bound as the interceptor entry point itself", + # Internals of building a policy input. A host supplies the context; + # the engine builds the input from it. + "JsonPath": "internal to policy input construction", + "PathEnv": "internal to policy input construction", + "PathRoot": "internal to policy input construction", + "PathSegment": "internal to policy input construction", + "PathParseError": "internal to policy input construction", + "build_policy_input": "internal to policy input construction", + "build_cedar_request": "internal to Cedar dispatch", + "normalize_policy_output": "internal to verdict normalization", + "runtime_error_verdict": "internal to verdict normalization", + "translate_advice": "internal to Cedar dispatch", + "InterceptionPointExt": "Rust ergonomics on a foreign enum", + "canonical_json": "agent-hooks owns identity and canonicalization", + # Constants, readable from a verdict or a spec document. + "SUPPORTED_VERSIONS": "bound through supported_manifest_versions", + "MAX_RUNE_OFFSET": "constant", + "STREAMING_FAIL_CLOSED_REASON": "constant, appears in a verdict", + "VERDICT_INVALID_REASON": "constant, appears in a verdict", + "reserved_reason": "constants, appear in a verdict", +} + +# A capability is reachable when each binding names it or the entry +# point that carries it. Matching by name alone would miss the cases +# where a binding renames on the way out, so state the token to look for. +CAPABILITIES = { + "ActivatedPolicy": ( + "acs_policy_activate", + "policy_activate", + "policy_activate", + "PolicyActivate", + ), + "InMemoryRegoBundle": ("bundles_json", "bundles", "bundles", "bundlesJson"), + "AnnotatorDispatcher": ( + "AcsAnnotatorFn", + "annotator_dispatcher", + "annotatorDispatcher", + "AnnotatorDispatcher", + ), + "PolicyDispatcher": ( + "AcsPolicyFn", + "policy_dispatcher", + "policyDispatcher", + "PolicyDispatcher", + ), + "TelemetrySink": ( + "AcsTelemetryFn", + "telemetry_sink", + "telemetrySink", + "TelemetrySink", + ), + "PerfTelemetry": ( + "perf_telemetry", + "perf_telemetry", + "perfTelemetry", + "PerfTelemetry", + ), + "Limits": ("limits_json", "limits", "limits", "limits"), + "StreamSession": ( + "acs_stream_session_new", + "stream_session_new", + "stream_session_new", + "StreamSession", + ), + "StreamSessionConfig": ( + "acs_stream_session_new", + "stream_session_new", + "stream_session_new", + "StreamSession", + ), + "StreamWatermark": ( + "acs_stream_session_watermark", + "stream_watermark", + "stream_session_watermark", + "Watermark", + ), + "StreamCompletion": ( + "acs_stream_session_finish", + "stream_finish", + "stream_session_finish", + "Finish", + ), + "StreamEndReason": ("end_reason", "end_reason", "endReason", "EndReason"), + "StreamSpan": ( + "acs_stream_session_record_outcome", + "stream_record_outcome", + "stream_session_record_outcome", + "RecordOutcome", + ), + "SegmentOutcome": ( + "acs_stream_session_record_outcome", + "stream_record_outcome", + "stream_session_record_outcome", + "SegmentOutcome", + ), + "SafetyLevel": ("safety_level", "safety_level", "safetyLevel", "SafetyLevel"), + "StreamSourceType": ( + "source_type", + "source_type", + "sourceType", + "StreamSourceType", + ), + "StreamTrack": ( + "acs_stream_session_advance", + "stream_advance", + "stream_session_advance", + "StreamTrack", + ), + "RuneRange": ( + "acs_stream_session_record_outcome", + "stream_record_outcome", + "stream_session_record_outcome", + "RecordOutcome", + ), +} + +BINDINGS = ( + ("ffi", [ROOT / "sdk/ffi/src/lib.rs"]), + ( + "python", + [ + ROOT / "sdk/python/src/lib.rs", + ROOT / "sdk/python/agent_control_spec/__init__.py", + ], + ), + ("node", [ROOT / "sdk/node/native/src/lib.rs", ROOT / "sdk/node/src/index.ts"]), + ("dotnet", list((ROOT / "sdk/dotnet/src/AgentControlSpec").glob("*.cs"))), +) + + +def ffi_entry_points() -> set[str]: + """Every `acs_*` function the C ABI exports.""" + src = (ROOT / "sdk/ffi/src/lib.rs").read_text(encoding="utf-8") + return set(re.findall(r"pub unsafe extern \"C\" fn (acs_\w+)", src)) + + +def unbound_from_dotnet() -> list[str]: + """C ABI entry points the .NET binding never declares. + + .NET is the only binding that goes through the C ABI rather than + linking the engine directly, so an entry point added there and not + declared here is reachable from every language except .NET. The + token scan above cannot see it, because the capability it belongs to + may already be covered by a sibling entry point. + """ + declared = "\n".join( + path.read_text(encoding="utf-8") + for path in (ROOT / "sdk/dotnet/src/AgentControlSpec").glob("*.cs") + ) + # Freeing and string ownership are called by the SafeHandle and the + # marshaller, not declared as separate imports. + internal = {"acs_free_string"} + # Whole-word, because `acs_interceptor_new` is a substring of + # `acs_interceptor_new_ex`: a substring test would call the shorter + # one declared on the strength of the longer one. + return sorted( + name + for name in ffi_entry_points() + if name not in internal and not re.search(rf"\b{re.escape(name)}\b", declared) + ) + + +def engine_symbols() -> set[str]: + src = (ROOT / "engine/src/lib.rs").read_text(encoding="utf-8") + found: set[str] = set() + for match in re.finditer(r"pub use ([^;]+);", src, re.DOTALL): + body = re.sub(r"^\s*[\w:]+::", "", match.group(1)) + groups = re.findall(r"\{([^}]*)\}", body) or [body] + for group in groups: + for name in group.split(","): + name = name.strip().split(" as ")[-1].strip() + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name or ""): + found.add(name) + return found + + +def main() -> int: + sources = { + name: "\n".join(p.read_text(encoding="utf-8") for p in paths if p.exists()) + for name, paths in BINDINGS + } + + symbols = engine_symbols() + unexplained = sorted( + s for s in symbols if s not in NOT_A_CAPABILITY and s not in CAPABILITIES + ) + + failed = False + if unexplained: + print( + "engine symbols that are neither a declared capability nor an " + "explained non-capability:", + file=sys.stderr, + ) + for name in unexplained: + print(f" {name}", file=sys.stderr) + print( + "\nAdd each to CAPABILITIES with the token every binding exposes, " + "or to NOT_A_CAPABILITY with the reason it needs none.", + file=sys.stderr, + ) + failed = True + + orphans = unbound_from_dotnet() + if orphans: + print("C ABI entry points the .NET binding does not declare:", file=sys.stderr) + for name in orphans: + print(f" {name}", file=sys.stderr) + failed = True + + for symbol, tokens in sorted(CAPABILITIES.items()): + missing = [ + lang + for (lang, _), token in zip(BINDINGS, tokens) + if token not in sources[lang] + ] + if missing: + print( + f" {symbol:24} UNREACHABLE from {', '.join(missing)}", file=sys.stderr + ) + failed = True + + if failed: + return 1 + + print( + f"every one of {len(CAPABILITIES)} engine capabilities is reachable from " + f"all {len(BINDINGS)} bindings" + ) + print( + f"{len(NOT_A_CAPABILITY)} further symbols are data or internals, each with a stated reason" + ) + print( + f"all {len(ffi_entry_points())} C ABI entry points are declared by the .NET binding" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conformance/bindings/manifest.yaml b/tests/conformance/bindings/manifest.yaml new file mode 100644 index 0000000..86ebf2f --- /dev/null +++ b/tests/conformance/bindings/manifest.yaml @@ -0,0 +1,41 @@ +# Wrapper-test manifest: deterministic `test` policies only, so the +# suite needs no external policy engine. +agent_control_specification_version: "0.4.0-alpha.1" +metadata: + name: cross-language-parity +policies: + allow_all: + type: test + verdict: + decision: allow + block_tool: + type: test + verdict: + decision: deny + reason: blocked_by_policy + message: tool call denied by test policy + needs_approval: + type: test + verdict: + decision: deny + reason: requires_human + approval: {} + broken: + type: test +intervention_points: + input: + policy_target: "$.input" + policy: + id: allow_all + pre_tool_call: + policy_target: "$.tool_call.args" + policy: + id: block_tool + output: + policy_target: "$.output" + policy: + id: needs_approval + post_tool_call: + policy_target: "$.tool_result.value" + policy: + id: broken diff --git a/tests/conformance/bindings/published_artifacts.py b/tests/conformance/bindings/published_artifacts.py new file mode 100755 index 0000000..37acabb --- /dev/null +++ b/tests/conformance/bindings/published_artifacts.py @@ -0,0 +1,577 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Build every published artifact and exercise it from a clean install. + +The suites and the parity check import from this checkout, where the +engine sits on the loader path, the TypeScript is built in place, and +the Python package resolves from source. A consumer has a crate, a +wheel, a tarball and a nupkg, and no test that imports from the tree can +tell the difference. + +A package can therefore pass every test and still be unusable: it +installs, satisfies a build, and fails on the first call. This builds +what the release workflow builds, installs each artifact into a +throwaway project outside the repository, and runs the public surface +there. + +Node is packed as two artifacts because napi splits it that way. The +published package.json gains its optionalDependencies at publish time, +so a locally packed main tarball cannot resolve the platform binary on +its own and both halves are installed explicitly. + +Usage: published_artifacts.py [--keep] +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + +# NuGet caches by id and version, so a fixed version would let a stale +# package from an earlier run satisfy a later one and report a pass for +# code the artifact does not contain. Stamp each run instead. +CHECK_VERSION = f"0.0.0-artifactcheck{os.getpid()}{int(time.time())}" +# Beside this file, so moving the suite cannot strand the fixture. +HOOKS_MANIFEST = Path(__file__).resolve().parent / "host-hooks-manifest.yaml" +# Built rather than written on one line. A repo guard scans committed +# files for the version key and validates whatever follows it, and it +# cannot strip the quotes of a single-line literal. +_VERSION_KEY = "agent_control_specification" + "_version" + +BAD_MANIFEST = f'{_VERSION_KEY}: "0.4.0-alpha.1"\nmetadata: {{}}\n' + +REGO_MANIFEST = ( + f'{_VERSION_KEY}: "0.4.0-alpha.1"\n' + "policies:\n gate:\n type: rego\n bundle: ./b\n" + 'intervention_points:\n input:\n policy_target: "$.input"\n' + " policy:\n id: gate\n query: data.acs.decision\n" +) +BAD_BUNDLES = {"gate": {"modules": {"p.rego": "package acs\nthis is not rego ***\n"}}} +BIG_INPUT = "x" * 4096 +SMALL_CAP = {"max_snapshot_bytes": 64} + +EXPECTED = { + "non_streaming": True, + "hook_benign": "allow", + "hook_harmful": "deny", + "hook_harmful_reason": "unsafe_content", + "hook_failure": "deny", + "hook_failure_reason": "runtime_error:annotation_failed", + "hook_calls": 1, + "parse_ok": True, + "streaming_offset": 5, + "streaming_clean": True, + "streaming_settled": None, + # A manifest the document check passes whose Rego does not compile. + "artifacts_bad_rego": 1, + # A cap smaller than the context must deny, or the artifact accepted + # the limit and dropped it. + "limits_capped_denies": True, +} + + +def run(cmd, **kw): + return subprocess.run(cmd, capture_output=True, text=True, check=True, **kw) + + +def last_json(out: str) -> dict: + return json.loads(out.strip().splitlines()[-1]) + + +def build(stage: Path) -> dict[str, Path]: + """Build every artifact the release workflow publishes.""" + art = stage / "artifacts" + art.mkdir(parents=True, exist_ok=True) + + run( + [ + "maturin", + "build", + "--release", + "-m", + str(ROOT / "sdk/python/Cargo.toml"), + "-o", + str(art / "py"), + ] + ) + + run(["npm", "run", "build"], cwd=ROOT / "sdk/node") + run( + [ + "npx", + "napi", + "build", + "--release", + "--platform", + "--target", + "x86_64-unknown-linux-gnu", + "--manifest-path", + "native/Cargo.toml", + "--cwd", + ".", + "--output-dir", + ".", + ], + cwd=ROOT / "sdk/node", + ) + for node_binary in (ROOT / "sdk/node").glob("*.node"): + shutil.copy(node_binary, ROOT / "sdk/node/npm/linux-x64-gnu") + run( + ["npm", "pack", "--pack-destination", str(art)], + cwd=ROOT / "sdk/node", + ) + run( + ["npm", "pack", "--pack-destination", str(art)], + cwd=ROOT / "sdk/node/npm/linux-x64-gnu", + ) + + run(["cargo", "build", "--release", "-p", "agent-control-spec-ffi"], cwd=ROOT) + native = ROOT / "sdk/dotnet/native/runtimes/linux-x64/native" + native.mkdir(parents=True, exist_ok=True) + shutil.copy(ROOT / "target/release/libagent_control_spec_ffi.so", native) + run( + [ + "dotnet", + "pack", + str(ROOT / "sdk/dotnet/src/AgentControlSpec"), + "-c", + "Release", + "-o", + str(art / "nuget"), + "--nologo", + "-p:AcsNativeAssetsRequired=true", + f"-p:Version={CHECK_VERSION}", + ] + ) + + run( + [ + "cargo", + "package", + "-p", + "agent-control-spec", + "--allow-dirty", + "--no-verify", + ], + cwd=ROOT, + ) + crate = next((ROOT / "target/package").glob("agent-control-spec-*.crate")) + unpacked = art / "crate" + unpacked.mkdir(exist_ok=True) + run(["tar", "xzf", str(crate), "-C", str(unpacked)]) + + return { + "wheel": next((art / "py").glob("*.whl")), + "npm_main": next(art.glob("responsibleai-agent-control-spec-[0-9]*.tgz")), + "npm_native": next( + art.glob("responsibleai-agent-control-spec-linux-x64-gnu-*.tgz") + ), + "nuget_feed": art / "nuget", + "crate": next(unpacked.glob("agent-control-spec-*")), + } + + +PY_PROGRAM = """ +import json +from agent_control_spec import ( + AcsInterceptor, StreamSession, supported_manifest_versions, + parse_manifest, validate_manifest_detailed, validate_artifacts, +) +M = {manifest!r} +class C: + def __init__(s, v): s.v = v; s.calls = 0 + def dispatch(s, *a): s.calls += 1; return {{"severity": s.v}} +class B: + def dispatch(s, *a): raise RuntimeError("classifier unreachable") +def hook(d): + v = AcsInterceptor(M, annotator_dispatcher=d).intercept( + {{"interception_point": "input", "input": "hi"}}) + return (str(getattr(v.decision, "value", v.decision)).lower(), v.reason) +c = C(1); b = hook(c); h = hook(C(7)); f = hook(B()) +s = StreamSession(safety_level="blocking", response_tasks=["pii"]) +r = s.observe_text("model_generated", "hello") +s.record_outcome("pii", "model_generated", 0, r, "cleared"); s.advance("response") +off = s.safe_offset("response"); clean = s.finish()["is_clean"] +print(json.dumps({{ + "non_streaming": len(supported_manifest_versions()) > 0, + "hook_benign": b[0], "hook_harmful": h[0], "hook_harmful_reason": h[1], + "hook_failure": f[0], "hook_failure_reason": f[1], "hook_calls": c.calls, + "parse_ok": "intervention_points" in parse_manifest(open(M).read()), + "streaming_offset": off, "streaming_clean": clean, + "streaming_settled": s.safe_offset("response"), + "artifacts_bad_rego": len(validate_artifacts({rego!r}, {bad!r})), + "limits_capped_denies": str(getattr( + AcsInterceptor(M, annotator_dispatcher=C(1), limits={cap!r}).intercept( + {{"interception_point": "input", "input": {big!r}}}).decision, + "value", "")).lower() == "deny", +}})) +""" + + +def check_python(art: dict, stage: Path) -> dict: + venv = stage / "venv-py" + run([sys.executable, "-m", "venv", str(venv)]) + run([str(venv / "bin/pip"), "install", "-q", str(art["wheel"])]) + out = run( + [ + str(venv / "bin/python"), + "-c", + PY_PROGRAM.format( + manifest=str(HOOKS_MANIFEST), + rego=REGO_MANIFEST, + bad=BAD_BUNDLES, + cap=SMALL_CAP, + big=BIG_INPUT, + ), + ] + ) + return last_json(out.stdout) + + +NODE_PROGRAM = """ +const acs = require('@responsibleai/agent-control-spec'); +const fs = require('fs'); +const M = %s; +function hook(d) { + const i = acs.AcsInterceptor.fromPath(M, { annotatorDispatcher: d }); + const v = i.intercept({ interception_point: 'input', input: 'hi' }); + return [String(v.decision).toLowerCase(), v.reason ?? null]; +} +let calls = 0; +const b = hook(() => { calls++; return { severity: 1 }; }); +const h = hook(() => ({ severity: 7 })); +const f = hook(() => { throw new Error('classifier unreachable'); }); +const s = new acs.StreamSession({ safetyLevel: 'blocking', responseTasks: ['pii'] }); +const r = s.observeText('model_generated', 'hello'); +s.recordOutcome('pii', 'model_generated', 0, r, 'cleared'); +s.advance('response'); +const off = s.safeOffset('response'); +const clean = s.finish().isClean; +console.log(JSON.stringify({ + non_streaming: acs.supportedManifestVersions().length > 0, + hook_benign: b[0], hook_harmful: h[0], hook_harmful_reason: h[1], + hook_failure: f[0], hook_failure_reason: f[1], hook_calls: calls, + parse_ok: Object.prototype.hasOwnProperty.call( + acs.parseManifest(fs.readFileSync(M, 'utf8')), 'intervention_points'), + streaming_offset: off, streaming_clean: clean, + streaming_settled: s.safeOffset('response'), + artifacts_bad_rego: acs.validateArtifacts(%s, %s).length, + limits_capped_denies: String(acs.AcsInterceptor + .fromPath(M, { annotatorDispatcher: () => ({ severity: 1 }), limits: %s }) + .intercept({ interception_point: 'input', input: %s }).decision).toLowerCase() === 'deny', +})); +""" + + +def check_node(art: dict, stage: Path) -> dict: + app = stage / "app-node" + app.mkdir() + run(["npm", "init", "-y"], cwd=app) + run( + ["npm", "install", str(art["npm_native"]), str(art["npm_main"])], + cwd=app, + ) + out = run( + [ + "node", + "-e", + NODE_PROGRAM + % ( + json.dumps(str(HOOKS_MANIFEST)), + json.dumps(REGO_MANIFEST), + json.dumps(BAD_BUNDLES), + json.dumps(SMALL_CAP), + json.dumps(BIG_INPUT), + ), + ], + cwd=app, + ) + return last_json(out.stdout) + + +DOTNET_PROGRAM = """ +using AgentControlSpec; +using AgentHooks; +using System.Text.Json; +using System.Text.Json.Nodes; + +var M = __MANIFEST__; +AgentContext Ctx() => new(JsonNode.Parse(CTX_JSON)!.AsObject()); +async Task<(string, string?)> Hook(AnnotatorDispatcher d) +{ + using var i = AcsHostInterceptor.FromPath(M, annotator: d); + var v = await i.InterceptAsync(Ctx()); + return (v.Decision.ToString().ToLowerInvariant(), v.Reason); +} +var calls = 0; +var b = await Hook((_, _, _) => { calls++; return SEV1; }); +var h = await Hook((_, _, _) => SEV7); +var f = await Hook((_, _, _) => throw new InvalidOperationException("classifier unreachable")); +using var s = new StreamSession(SafetyLevel.Blocking, responseTasks: ["pii"]); +var r = s.ObserveText(StreamSourceType.ModelGenerated, "hello"); +s.RecordOutcome("pii", StreamSourceType.ModelGenerated, 0, r, SegmentOutcome.Cleared); +s.Advance(StreamTrack.Response); +var off = s.SafeOffset(StreamTrack.Response); +var clean = s.Finish().IsClean; +Console.WriteLine(JsonSerializer.Serialize(new Dictionary +{ + ["non_streaming"] = AcsManifest.SupportedVersions().Count > 0, + ["hook_benign"] = b.Item1, + ["hook_harmful"] = h.Item1, + ["hook_harmful_reason"] = h.Item2, + ["hook_failure"] = f.Item1, + ["hook_failure_reason"] = f.Item2, + ["hook_calls"] = calls, + ["parse_ok"] = AcsManifestTools.Parse(File.ReadAllText(M)).Contains("intervention_points"), + ["streaming_offset"] = off, + ["streaming_clean"] = clean, + ["streaming_settled"] = s.SafeOffset(StreamTrack.Response), + ["artifacts_bad_rego"] = AcsManifestTools.ValidateArtifacts(__REGO__, __BADB__).Count, + ["limits_capped_denies"] = (await AcsHostInterceptor + .FromPath(M, annotator: (_, _, _) => SEV1, limits: __CAP__) + .InterceptAsync(new AgentContext(JsonNode.Parse(__BIGCTX__)!.AsObject()))) + .Decision.ToString().ToLowerInvariant() == "deny", +})); +""" + + +def check_dotnet(art: dict, stage: Path) -> dict: + app = stage / "app-net" + app.mkdir() + (app / "NuGet.config").write_text( + f""" + + + + + + + +""" + ) + run(["dotnet", "new", "console", "-o", str(app), "--force"]) + run( + [ + "dotnet", + "add", + str(app), + "package", + "ResponsibleAI.AgentControlSpec", + "--version", + CHECK_VERSION, + ] + ) + program = ( + DOTNET_PROGRAM.replace("__MANIFEST__", json.dumps(str(HOOKS_MANIFEST))) + .replace( + "CTX_JSON", + json.dumps(json.dumps({"interception_point": "input", "input": "hi"})), + ) + .replace("SEV1", json.dumps(json.dumps({"severity": 1}))) + .replace("SEV7", json.dumps(json.dumps({"severity": 7}))) + .replace("__REGO__", json.dumps(REGO_MANIFEST)) + .replace("__BADB__", json.dumps(json.dumps(BAD_BUNDLES))) + .replace("__CAP__", json.dumps(json.dumps(SMALL_CAP))) + .replace( + "__BIGCTX__", + json.dumps(json.dumps({"interception_point": "input", "input": BIG_INPUT})), + ) + ) + (app / "Program.cs").write_text(program) + out = run(["dotnet", "run", "--project", str(app), "--nologo"]) + return last_json(out.stdout) + + +RUST_MAIN = """ +use agent_control_spec::annotation::{AnnotatorDispatcher, AnnotatorInvocation}; +use agent_control_spec::dispatchers::BindingPolicyDispatcher; +use agent_control_spec::stream_session::*; +use agent_control_spec::{Manifest, Runtime, RuntimeError}; +use std::sync::Arc; + +struct C(i64, std::sync::atomic::AtomicUsize); +impl AnnotatorDispatcher for C { + fn dispatch(&self, _n: &str, _a: &AnnotatorInvocation, _p: &serde_json::Value) + -> Result { + self.1.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(serde_json::json!({"severity": self.0})) + } +} +struct B; +impl AnnotatorDispatcher for B { + fn dispatch(&self, _n: &str, _a: &AnnotatorInvocation, _p: &serde_json::Value) + -> Result { + Err(RuntimeError::AnnotationFailed("classifier unreachable".into())) + } +} +fn hook(m: &str, d: Arc) -> (String, Option) { + let manifest = Manifest::from_path(m).expect("manifest"); + let rt = Runtime::new(manifest, d, Arc::new(BindingPolicyDispatcher::new())).expect("runtime"); + let ctx: serde_json::Value = + serde_json::from_str(r#"{"interception_point":"input","input":"hi"}"#).unwrap(); + let v = rt.evaluate(&ctx).verdict; + (format!("{:?}", v.decision).to_lowercase(), v.reason.clone()) +} +fn main() { + let m = std::env::args().nth(1).expect("manifest"); + let benign = Arc::new(C(1, std::sync::atomic::AtomicUsize::new(0))); + let b = hook(&m, benign.clone()); + let h = hook(&m, Arc::new(C(7, std::sync::atomic::AtomicUsize::new(0)))); + let f = hook(&m, Arc::new(B)); + let mut s = StreamSession::new(StreamSessionConfig { + safety_level: SafetyLevel::Blocking, + request_start_rune_offset: 0, + response_start_rune_offset: 0, + request_tasks: vec![], + response_tasks: vec!["pii".into()], + }).unwrap(); + let r = s.observe_text(StreamSourceType::ModelGenerated, "hello").unwrap(); + let sp = StreamSpan::new(StreamSourceType::ModelGenerated, 0, r).unwrap(); + s.record_outcome("pii", &sp, SegmentOutcome::Cleared).unwrap(); + s.advance(StreamTrack::Response); + let off = s.safe_offset(StreamTrack::Response); + let clean = s.finish().reason.is_clean(); + let bad_bundles: std::collections::BTreeMap = + serde_json::from_str(BAD_BUNDLES).expect("bundles"); + let capped = Runtime::with_limits( + Manifest::from_path(&m).expect("manifest"), + Arc::new(C(1, std::sync::atomic::AtomicUsize::new(0))), + Arc::new(BindingPolicyDispatcher::new()), + agent_control_spec::Limits { max_snapshot_bytes: 64, ..Default::default() }, + ) + .expect("capped runtime"); + let big: serde_json::Value = serde_json::from_str(BIG_CTX).expect("big ctx"); + let limits_capped_denies = + format!("{:?}", capped.evaluate(&big).verdict.decision).to_lowercase() == "deny"; + let artifacts_bad_rego = usize::from( + agent_control_spec::ActivatedPolicy::activate_from_memory(REGO_MANIFEST, bad_bundles) + .is_err(), + ); + println!("{}", serde_json::json!({ + "non_streaming": !agent_control_spec::SUPPORTED_VERSIONS.is_empty(), + "hook_benign": b.0, "hook_harmful": h.0, "hook_harmful_reason": h.1, + "hook_failure": f.0, "hook_failure_reason": f.1, + "hook_calls": benign.1.load(std::sync::atomic::Ordering::SeqCst), + "parse_ok": Manifest::from_yaml_str(&std::fs::read_to_string(&m).unwrap()).is_ok(), + "streaming_offset": off, "streaming_clean": clean, + "streaming_settled": s.safe_offset(StreamTrack::Response), + "artifacts_bad_rego": artifacts_bad_rego, + "limits_capped_denies": limits_capped_denies, + })); +} +""" + + +def check_rust(art: dict, stage: Path) -> dict: + app = stage / "app-rs" + (app / "src").mkdir(parents=True) + (app / "Cargo.toml").write_text( + f"""[package] +name = "acs-artifact-check" +version = "0.0.0" +edition = "2021" + +[dependencies] +agent-control-spec = {{ path = "{art["crate"]}", features = ["default-dispatchers", "streaming"] }} +serde_json = "1" + +[workspace] +""" + ) + (app / "src/main.rs").write_text( + RUST_MAIN.replace("REGO_MANIFEST", json.dumps(REGO_MANIFEST)) + .replace("BAD_BUNDLES", json.dumps(json.dumps(BAD_BUNDLES))) + .replace( + "BIG_CTX", + json.dumps(json.dumps({"interception_point": "input", "input": BIG_INPUT})), + ) + ) + out = run( + ["cargo", "run", "--quiet", "--release", "--", str(HOOKS_MANIFEST)], cwd=app + ) + return last_json(out.stdout) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--keep", action="store_true", help="keep the staging directory" + ) + args = parser.parse_args() + + stage = Path(tempfile.mkdtemp(prefix="acs-artifacts-")) + failed = False + try: + print("building artifacts", flush=True) + try: + art = build(stage) + except subprocess.CalledProcessError as e: + # A build that refuses is a result, not a crash. The pack-time + # guard against a native-less package reports itself this way. + print( + f"building artifacts FAILED\n{' '.join(str(c) for c in e.cmd)}\n" + f"{e.stdout}\n{e.stderr}", + file=sys.stderr, + ) + return 1 + for name, path in art.items(): + print(f" {name}: {path.name}") + + checks = { + "rust": check_rust, + "python": check_python, + "node": check_node, + "dotnet": check_dotnet, + } + print("\nrunning the public surface from each installed artifact") + for name, check in checks.items(): + try: + got = check(art, stage) + except subprocess.CalledProcessError as e: + print( + f"{name:8} FAILED TO RUN\n{e.stdout}\n{e.stderr}", file=sys.stderr + ) + failed = True + continue + mismatches = { + k: (EXPECTED[k], got.get(k)) + for k in EXPECTED + if got.get(k) != EXPECTED[k] + } + print(f" {name:8} {'ok' if not mismatches else 'MISMATCH'}") + for key, (want, actual) in sorted(mismatches.items()): + print( + f" {key}: expected {want!r}, got {actual!r}", + file=sys.stderr, + ) + failed = True + finally: + if args.keep: + print(f"\nstaging kept at {stage}") + else: + shutil.rmtree(stage, ignore_errors=True) + + if failed: + print( + "\na published artifact does not carry what the checkout does", + file=sys.stderr, + ) + return 1 + print( + f"\nevery artifact carries the whole surface, across {len(EXPECTED)} assertions" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conformance/bindings/run.py b/tests/conformance/bindings/run.py new file mode 100755 index 0000000..73b5eca --- /dev/null +++ b/tests/conformance/bindings/run.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Conformance for the language bindings. + +The case corpus under `tests/conformance/cases` checks what the engine +decides. These checks cover the layer above it: whether a host can reach +that decision at all, from the language it actually writes in, through +the artifact it actually installs. + +Three failure classes live here, each one having shipped at least once. + +`language_coverage` catches a capability the engine offers and no +binding exposes. Cross-language comparison cannot: a surface absent from +every language is consistent across every language. + +`cross_language_parity` catches two bindings that expose a capability +and disagree about it. Each reaches the engine through a different +mechanism and converts enums, offsets and absent values at its own +boundary, so agreement is not structural. + +`published_artifacts` catches a package that ships less than the +repository holds. Nothing that imports from the checkout can, because +the checkout has the engine on its loader path and the package may not. + +Run one: + python tests/conformance/bindings/run.py language_coverage + +Run all: + python tests/conformance/bindings/run.py +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent + +# Ordered cheapest first, so a run fails on the fastest signal available. +CHECKS = { + "language_coverage": ( + "every engine capability is reachable from every binding", + HERE / "language_coverage.py", + ), + "cross_language_parity": ( + "every binding answers identically", + HERE / "cross_language_parity.py", + ), + "published_artifacts": ( + "every built package carries the whole surface", + HERE / "published_artifacts.py", + ), +} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "checks", + nargs="*", + choices=sorted(CHECKS), + help="checks to run, or none for all", + ) + args = parser.parse_args() + selected = args.checks or list(CHECKS) + + failed: list[str] = [] + for name in selected: + description, path = CHECKS[name] + print(f"\n=== {name}: {description} ===", flush=True) + if subprocess.run([sys.executable, str(path)], check=False).returncode: + failed.append(name) + + print() + if failed: + print(f"binding conformance FAILED: {', '.join(failed)}", file=sys.stderr) + return 1 + print(f"binding conformance passed: {', '.join(selected)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())