diff --git a/CHANGELOG.md b/CHANGELOG.md index 4187633..d82253e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ User-visible changes to the spec and SDKs. Versioning rules: ## Unreleased +- **CTK: declared `tool_seam_host_error` posture (§13.1).** A harness declares `continue` (default) or `terminate`, and `expect.run_outcome_by_posture` resolves the 13 tool-seam `host_error:*` vectors to the single outcome that declared surface must produce — the §6.2 terminate clause is now claimable (#68). +- **CTK: AH-CTK-100 asserts §6.1 substance, not transcript cosmetics.** New `context_must_contain`/`context_must_not_contain` interception assertions pin non-incorporation of the denied tool result and the deny surfacing to the model in some form, leaving message layout and payload shape to the host (#69). - **The §12.1 incremental exception is CTK-testable.** The vectors the alpha.5 entry below left as future work exist: a `streaming/incremental` part (`AH-CTK-110`–`AH-CTK-113`) exercises diff --git a/conformance/CLAIMS.md b/conformance/CLAIMS.md index f2619fb..c2365c1 100644 --- a/conformance/CLAIMS.md +++ b/conformance/CLAIMS.md @@ -16,7 +16,11 @@ or baseline profiles — the claim attaches the CTK's **per-part report** (runner results grouped by each vector's `part` tag), which communicates *what was exercised*, not a tier name. -A claim with `identity_provider: null` MUST state that its records and +A claim with a non-default posture (§13.1) MUST state it (e.g. +`tool_seam_host_error: terminate` — the host terminates the turn on a +`host_error:*` deny at the tool seam, which §6.2 permits); the report's +passing vectors attest that posture's outcomes, not the default's. A +claim with `identity_provider: null` MUST state that its records and approvals are identity-unbound (§10.1). A claim with a host-defined provider MUST disclose whether the provider is **content-derived** (a pure function of the projected context, like `jcs-sha256`) or not — @@ -49,7 +53,9 @@ artefacts, in the PR: wires the framework — specifically confirming it drives the framework's **production dispatch path** with only model/tool I/O mocked (a harness that re-implements dispatch attests nothing). -3. **Disclosure flags** where applicable: `identity_provider: null` → +3. **Disclosure flags** where applicable: a non-default posture + (`tool_seam_host_error: terminate`) → the claim states it; + `identity_provider: null` → the claim states records/approvals are identity-unbound; custom provider → content-derived or not; `buffered_output: false` → the claim states a deny at `output` cannot retract streamed diff --git a/conformance/HARNESS.md b/conformance/HARNESS.md index e1fc81e..da0b969 100644 --- a/conformance/HARNESS.md +++ b/conformance/HARNESS.md @@ -125,6 +125,32 @@ Non-finite floats (NaN/Infinity) and lone surrogates cannot be expressed in a JSON vector at all — those §4.4 marshalling guards are pinned by per-SDK unit tests, not vectors. +## Postures + +Where the spec permits two host behaviors, the harness **declares** +which one its host implements and the runner selects the single +expected outcome for that declared surface — a vector never accepts +"either outcome", so a pass always attests one specific behavior. + +`tool_seam_host_error: continue | terminate` (default `continue`) +declares what the host does with the run after a `host_error:*` deny +at `pre_tool_call`/`post_tool_call` (§6.2): `continue` surfaces a tool +error to the model and keeps the loop going; `terminate` means the +host's own semantics terminate the turn — the posture §6.2's "unless +the host's own semantics terminate the turn" clause permits. Vectors +whose run ends in such a deny carry `expect.run_outcome_by_posture`, +and the runner resolves it against this declaration (forwarded in the +run-record wire as `postures.tool_seam_host_error`). + +Declare it per SDK convention: a `tool_seam_host_error` attribute +(Python), `toolSeamHostError` (TypeScript), the optional +`ToolSeamHostErrorDeclarer` interface (Go), the `ToolSeamHostError` +property (default interface member, .NET), or the +`tool_seam_host_error()` trait method (Rust). Omitting it declares +`continue` — the posture every in-tree reference harness implements. +The declaration belongs in the host's §13.3 claim alongside its +capabilities. + ## Incremental mediation Vectors in the `streaming/incremental` part carry a chunked mock diff --git a/conformance/RUNNER.md b/conformance/RUNNER.md index a2f7065..f95345e 100644 --- a/conformance/RUNNER.md +++ b/conformance/RUNNER.md @@ -41,7 +41,11 @@ for each vector file in conformance/vectors/*.json: {outcome:rr.outcome, final_output:rr.final_output, tool_invocations:rr.tool_invocations, error:rr.error, identities:rr.identities, # (input, enforced) per emission - records:rr.records}) # wire-shaped §10.3 records + records:rr.records, # wire-shaped §10.3 records + postures:{ # harness *declarations* (§13.1), + tool_seam_host_error: # not observed behavior — they + harness.tool_seam_host_error # select run_outcome_by_posture + ?? "continue"}}) # (HARNESS.md "Postures") ``` ## The conformance report (§13.1) diff --git a/conformance/vectors.schema.json b/conformance/vectors.schema.json index 6b18c20..b0be8ce 100644 --- a/conformance/vectors.schema.json +++ b/conformance/vectors.schema.json @@ -394,6 +394,20 @@ "context_must_validate": { "type": "boolean", "default": true + }, + "context_must_contain": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Substrings that MUST appear somewhere in the serialization of the recorded AgentContext — shape-agnostic: pins that content surfaced to this interception point in SOME form (e.g. a deny reason surfaced to the model per §6.2) without prescribing message layout or payload format. Needles should avoid characters JSON escapes (quotes, backslashes, control characters)." + }, + "context_must_not_contain": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Substrings that MUST NOT appear anywhere in the serialization of the recorded AgentContext — pins non-incorporation (§6.1): a discarded result must not surface here in any form. Same needle caveat as context_must_contain." } } } @@ -449,7 +463,50 @@ "completed", "blocked", "error" - ] + ], + "description": "The run outcome for a host with the spec-default posture. When run_outcome_by_posture is present, this MUST equal its outcomes entry for the default posture value (the engine enforces the equality), so pre-posture runners resolve the same expectation for default-posture hosts." + }, + "run_outcome_by_posture": { + "type": "object", + "additionalProperties": false, + "required": [ + "posture", + "outcomes" + ], + "properties": { + "posture": { + "type": "string", + "enum": [ + "tool_seam_host_error" + ], + "description": "The declared host posture (§13.1) that selects the expected outcome. tool_seam_host_error: what the host does with the run after a host_error:* deny at pre_tool_call/post_tool_call — \"continue\" (the §6.2 default: surface a tool error to the model and keep the loop going) or \"terminate\" (the host's own semantics terminate the turn, which §6.2 explicitly permits)." + }, + "outcomes": { + "type": "object", + "additionalProperties": false, + "required": [ + "continue", + "terminate" + ], + "properties": { + "continue": { + "enum": [ + "completed", + "blocked", + "error" + ] + }, + "terminate": { + "enum": [ + "completed", + "blocked", + "error" + ] + } + } + } + }, + "description": "Posture-conditional run outcome, for behavior the spec permits either way (§6.2). The runner forwards the harness's declared posture and the engine selects the single outcome that declared surface must produce — each (vector, declared surface) pair stays single-valued, so a pass never means \"one of several outcomes happened\"." }, "identities_equal": { "type": "boolean", diff --git a/conformance/vectors/AH-CTK-070-interceptor-raises.json b/conformance/vectors/AH-CTK-070-interceptor-raises.json index 4d52a52..4226445 100644 --- a/conformance/vectors/AH-CTK-070-interceptor-raises.json +++ b/conformance/vectors/AH-CTK-070-interceptor-raises.json @@ -21,6 +21,10 @@ ], "interceptions_absent": ["post_tool_call"], "tool_not_invoked": ["http_get"], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-071-malformed-verdict.json b/conformance/vectors/AH-CTK-071-malformed-verdict.json index d77007b..de26363 100644 --- a/conformance/vectors/AH-CTK-071-malformed-verdict.json +++ b/conformance/vectors/AH-CTK-071-malformed-verdict.json @@ -21,6 +21,10 @@ ], "interceptions_absent": ["post_tool_call"], "tool_not_invoked": ["http_get"], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-072-resolver-identity-mismatch.json b/conformance/vectors/AH-CTK-072-resolver-identity-mismatch.json index 0df497b..72ea17f 100644 --- a/conformance/vectors/AH-CTK-072-resolver-identity-mismatch.json +++ b/conformance/vectors/AH-CTK-072-resolver-identity-mismatch.json @@ -91,6 +91,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-073-resolver-raises.json b/conformance/vectors/AH-CTK-073-resolver-raises.json index 9140218..41616c1 100644 --- a/conformance/vectors/AH-CTK-073-resolver-raises.json +++ b/conformance/vectors/AH-CTK-073-resolver-raises.json @@ -86,6 +86,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-085-parallel-transform-conflict-deny.json b/conformance/vectors/AH-CTK-085-parallel-transform-conflict-deny.json index 071963b..dd1e863 100644 --- a/conformance/vectors/AH-CTK-085-parallel-transform-conflict-deny.json +++ b/conformance/vectors/AH-CTK-085-parallel-transform-conflict-deny.json @@ -96,6 +96,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-087-unanimous-disagreement-deny.json b/conformance/vectors/AH-CTK-087-unanimous-disagreement-deny.json index 123b64b..a337af2 100644 --- a/conformance/vectors/AH-CTK-087-unanimous-disagreement-deny.json +++ b/conformance/vectors/AH-CTK-087-unanimous-disagreement-deny.json @@ -93,6 +93,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-092-evidence-cap.json b/conformance/vectors/AH-CTK-092-evidence-cap.json index 3dbf6dc..1aa12ae 100644 --- a/conformance/vectors/AH-CTK-092-evidence-cap.json +++ b/conformance/vectors/AH-CTK-092-evidence-cap.json @@ -80,6 +80,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-094-failure-deny-attribution.json b/conformance/vectors/AH-CTK-094-failure-deny-attribution.json index 8868ae9..96b9489 100644 --- a/conformance/vectors/AH-CTK-094-failure-deny-attribution.json +++ b/conformance/vectors/AH-CTK-094-failure-deny-attribution.json @@ -86,6 +86,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-095-postfold-domain-rejection.json b/conformance/vectors/AH-CTK-095-postfold-domain-rejection.json index 5cba465..9fc7643 100644 --- a/conformance/vectors/AH-CTK-095-postfold-domain-rejection.json +++ b/conformance/vectors/AH-CTK-095-postfold-domain-rejection.json @@ -85,6 +85,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-097-runall-failure-slot.json b/conformance/vectors/AH-CTK-097-runall-failure-slot.json index 47759a2..4c8ea3f 100644 --- a/conformance/vectors/AH-CTK-097-runall-failure-slot.json +++ b/conformance/vectors/AH-CTK-097-runall-failure-slot.json @@ -72,7 +72,11 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } }, "composition": { "profile": "sequential/run_all" diff --git a/conformance/vectors/AH-CTK-098-approve-carrying-deny.json b/conformance/vectors/AH-CTK-098-approve-carrying-deny.json index e472434..2f2c0d8 100644 --- a/conformance/vectors/AH-CTK-098-approve-carrying-deny.json +++ b/conformance/vectors/AH-CTK-098-approve-carrying-deny.json @@ -91,6 +91,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-100-deny-post-tool-non-incorporation.json b/conformance/vectors/AH-CTK-100-deny-post-tool-non-incorporation.json index a31a6d3..8a65837 100644 --- a/conformance/vectors/AH-CTK-100-deny-post-tool-non-incorporation.json +++ b/conformance/vectors/AH-CTK-100-deny-post-tool-non-incorporation.json @@ -1,6 +1,6 @@ { "id": "AH-CTK-100", - "title": "deny at post_tool_call: the tool result is discarded, not incorporated into agent state (§6.1)", + "title": "deny at post_tool_call: the tool result is discarded, not incorporated into agent state; the deny surfaces to the model in some form, transcript shape unprescribed (§6.1)", "spec_clause": "6.1, 6.2", "part": "enforcement/post_action_deny", "capabilities": [ @@ -67,9 +67,15 @@ }, { "interception_point": "pre_model_call", - "context": { - "messages[1].role": "tool", - "messages[1].content": "blocked: ctk:tainted-result" + "context_must_contain": ["ctk:tainted-result"], + "context_must_not_contain": ["SECRET-RESULT"] + } + ], + "tool_invocations": [ + { + "name": "http_get", + "args": { + "url": "https://x" } } ], diff --git a/conformance/vectors/AH-CTK-102-runall-failed-transform-shortcircuit.json b/conformance/vectors/AH-CTK-102-runall-failed-transform-shortcircuit.json index e70f931..eb52771 100644 --- a/conformance/vectors/AH-CTK-102-runall-failed-transform-shortcircuit.json +++ b/conformance/vectors/AH-CTK-102-runall-failed-transform-shortcircuit.json @@ -96,6 +96,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/conformance/vectors/AH-CTK-103-approve-carrying-failing-transform.json b/conformance/vectors/AH-CTK-103-approve-carrying-failing-transform.json index c433a5c..e253ebc 100644 --- a/conformance/vectors/AH-CTK-103-approve-carrying-failing-transform.json +++ b/conformance/vectors/AH-CTK-103-approve-carrying-failing-transform.json @@ -94,6 +94,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/dotnet/src/AgentHooks.Conformance/IHarness.cs b/sdk/dotnet/src/AgentHooks.Conformance/IHarness.cs index 2d21f26..a0aa501 100644 --- a/sdk/dotnet/src/AgentHooks.Conformance/IHarness.cs +++ b/sdk/dotnet/src/AgentHooks.Conformance/IHarness.cs @@ -114,6 +114,16 @@ public interface IHarness string Name { get; } IReadOnlySet Capabilities { get; } + /// Declared §6.2 posture at the tool seam (§13.1): what the + /// host does with the run after a host_error:* deny at + /// pre_tool_call/post_tool_call. "continue" (the + /// default — surface a tool error to the model and keep the loop + /// going) or "terminate" (the host's own semantics terminate + /// the turn, which §6.2 explicitly permits). The runner forwards this + /// declaration so expect.run_outcome_by_posture vectors resolve + /// to the single outcome this surface must produce. + string ToolSeamHostError => "continue"; + /// Wire the scenario's mock model + tools into the framework, /// register the interceptors and resolver, set the enforcement mode, /// the vector's composition profile (§7.1), and its identity provider diff --git a/sdk/dotnet/src/AgentHooks.Conformance/Runner.cs b/sdk/dotnet/src/AgentHooks.Conformance/Runner.cs index a5945b3..074006b 100644 --- a/sdk/dotnet/src/AgentHooks.Conformance/Runner.cs +++ b/sdk/dotnet/src/AgentHooks.Conformance/Runner.cs @@ -115,7 +115,7 @@ public static async Task RunVectorAsync( var recordedJson = new JsonArray( recorded.Select(c => (JsonNode)c).ToArray()).ToJsonString(Compact); - var rrJson = RunRecordToWire(rr); + var rrJson = RunRecordToWire(rr, harness.ToolSeamHostError); var result = (JsonObject)JsonNode.Parse( Native.CtkAssert(vectorJson, recordedJson, rrJson))!; return new VectorResult( @@ -126,7 +126,7 @@ public static async Task RunVectorAsync( (result["failures"] as JsonArray)?.Select(n => (string)n!).ToList() ?? []); } - private static string RunRecordToWire(RunRecord rr) + private static string RunRecordToWire(RunRecord rr, string toolSeamHostError) { var identities = new JsonArray(); foreach (var (i, e) in rr.Identities ?? []) @@ -146,6 +146,12 @@ private static string RunRecordToWire(RunRecord rr) ["error"] = rr.Error, ["identities"] = identities, ["records"] = records, + // Harness *declarations* (§13.1), not observed behavior: the + // engine selects expect.run_outcome_by_posture entries by them. + ["postures"] = new JsonObject + { + ["tool_seam_host_error"] = toolSeamHostError, + }, }; return o.ToJsonString(Compact); } diff --git a/sdk/go/conformance/harness.go b/sdk/go/conformance/harness.go index 6ed576a..6380f11 100644 --- a/sdk/go/conformance/harness.go +++ b/sdk/go/conformance/harness.go @@ -108,3 +108,16 @@ type Harness interface { Teardown() } + +// ToolSeamHostErrorDeclarer is an optional Harness extension declaring +// the host's §6.2 posture at the tool seam (§13.1): what the host does +// with the run after a host_error:* deny at pre_tool_call / +// post_tool_call. "continue" (the default — surface a tool error to +// the model and keep the loop going) or "terminate" (the host's own +// semantics terminate the turn, which §6.2 explicitly permits). A +// Harness that does not implement it declares the default. The runner +// forwards this declaration so expect.run_outcome_by_posture vectors +// resolve to the single outcome this surface must produce. +type ToolSeamHostErrorDeclarer interface { + ToolSeamHostError() string +} diff --git a/sdk/go/conformance/runner.go b/sdk/go/conformance/runner.go index 8c3e2bf..1725519 100644 --- a/sdk/go/conformance/runner.go +++ b/sdk/go/conformance/runner.go @@ -97,7 +97,7 @@ func mustJSON(v any) string { return string(b) } -func runRecordToWire(rr RunRecord) string { +func runRecordToWire(rr RunRecord, postures map[string]string) string { invs := make([]map[string]any, len(rr.ToolInvocations)) for i, t := range rr.ToolInvocations { invs[i] = map[string]any{"name": t.Name, "args": t.Args} @@ -122,6 +122,9 @@ func runRecordToWire(rr RunRecord) string { "error": rr.Err, "identities": ids, "records": records, + // Harness *declarations* (§13.1), not observed behavior: the + // engine selects expect.run_outcome_by_posture entries by them. + "postures": postures, }) } @@ -228,7 +231,14 @@ func RunVector(ctx context.Context, h Harness, vector map[string]any) (VectorRes if first != nil { recorded = first.recorded } - return agenthooks.CtkAssert(vectorJSON, recorded, runRecordToWire(rr)) + // §13.1 posture declaration; a Harness that does not implement the + // optional declarer interface declares the spec default. + posture := "continue" + if d, ok := h.(ToolSeamHostErrorDeclarer); ok { + posture = d.ToolSeamHostError() + } + postures := map[string]string{"tool_seam_host_error": posture} + return agenthooks.CtkAssert(vectorJSON, recorded, runRecordToWire(rr, postures)) } func scenarioFromWire(s map[string]any) Scenario { diff --git a/sdk/python/python/agent_hooks/ctk/harness.py b/sdk/python/python/agent_hooks/ctk/harness.py index cd733d3..a87b03d 100644 --- a/sdk/python/python/agent_hooks/ctk/harness.py +++ b/sdk/python/python/agent_hooks/ctk/harness.py @@ -128,6 +128,15 @@ class Harness(Protocol): name: str capabilities: set[Capability] + #: Declared §6.2 posture at the tool seam (§13.1): what the host does + #: with the run after a ``host_error:*`` deny at + #: ``pre_tool_call``/``post_tool_call``. ``"continue"`` (the default — + #: surface a tool error to the model and keep the loop going) or + #: ``"terminate"`` (the host's own semantics terminate the turn, which + #: §6.2 explicitly permits). The runner forwards this declaration so + #: ``expect.run_outcome_by_posture`` vectors resolve to the single + #: outcome this surface must produce. + tool_seam_host_error: str = "continue" def setup( self, diff --git a/sdk/python/python/agent_hooks/ctk/runner.py b/sdk/python/python/agent_hooks/ctk/runner.py index f3ca14a..0702252 100644 --- a/sdk/python/python/agent_hooks/ctk/runner.py +++ b/sdk/python/python/agent_hooks/ctk/runner.py @@ -71,7 +71,7 @@ def load_vectors(directory: str | pathlib.Path | None = None) -> list[dict[str, return vectors -def _run_record_to_wire(rr: RunRecord) -> str: +def _run_record_to_wire(rr: RunRecord, postures: dict[str, str]) -> str: return dumps( { "outcome": rr.outcome.value, @@ -80,6 +80,9 @@ def _run_record_to_wire(rr: RunRecord) -> str: "error": rr.error, "identities": [{"input_identity": i, "enforced_identity": e} for i, e in rr.identities], "records": rr.records, + # Harness *declarations* (§13.1), not observed behavior: the + # engine selects expect.run_outcome_by_posture entries by them. + "postures": postures, } ) @@ -130,11 +133,14 @@ async def run_vector(harness: Harness, vector: dict[str, Any]) -> VectorResult: finally: harness.teardown() + # §13.1 posture declaration; getattr keeps structural (non-subclass) + # Harness implementations working — absent means the spec default. + postures = {"tool_seam_host_error": getattr(harness, "tool_seam_host_error", "continue")} result = json.loads( _core.ctk_assert( vector_json, dumps(first.recorded if first else []), - _run_record_to_wire(rr), + _run_record_to_wire(rr, postures), ) ) return VectorResult( diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-070-interceptor-raises.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-070-interceptor-raises.json index 4d52a52..4226445 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-070-interceptor-raises.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-070-interceptor-raises.json @@ -21,6 +21,10 @@ ], "interceptions_absent": ["post_tool_call"], "tool_not_invoked": ["http_get"], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-071-malformed-verdict.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-071-malformed-verdict.json index d77007b..de26363 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-071-malformed-verdict.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-071-malformed-verdict.json @@ -21,6 +21,10 @@ ], "interceptions_absent": ["post_tool_call"], "tool_not_invoked": ["http_get"], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-072-resolver-identity-mismatch.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-072-resolver-identity-mismatch.json index 0df497b..72ea17f 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-072-resolver-identity-mismatch.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-072-resolver-identity-mismatch.json @@ -91,6 +91,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-073-resolver-raises.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-073-resolver-raises.json index 9140218..41616c1 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-073-resolver-raises.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-073-resolver-raises.json @@ -86,6 +86,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-085-parallel-transform-conflict-deny.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-085-parallel-transform-conflict-deny.json index 071963b..dd1e863 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-085-parallel-transform-conflict-deny.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-085-parallel-transform-conflict-deny.json @@ -96,6 +96,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-087-unanimous-disagreement-deny.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-087-unanimous-disagreement-deny.json index 123b64b..a337af2 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-087-unanimous-disagreement-deny.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-087-unanimous-disagreement-deny.json @@ -93,6 +93,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-092-evidence-cap.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-092-evidence-cap.json index 3dbf6dc..1aa12ae 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-092-evidence-cap.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-092-evidence-cap.json @@ -80,6 +80,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-094-failure-deny-attribution.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-094-failure-deny-attribution.json index 8868ae9..96b9489 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-094-failure-deny-attribution.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-094-failure-deny-attribution.json @@ -86,6 +86,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-095-postfold-domain-rejection.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-095-postfold-domain-rejection.json index 5cba465..9fc7643 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-095-postfold-domain-rejection.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-095-postfold-domain-rejection.json @@ -85,6 +85,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-097-runall-failure-slot.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-097-runall-failure-slot.json index 47759a2..4c8ea3f 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-097-runall-failure-slot.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-097-runall-failure-slot.json @@ -72,7 +72,11 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } }, "composition": { "profile": "sequential/run_all" diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-098-approve-carrying-deny.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-098-approve-carrying-deny.json index e472434..2f2c0d8 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-098-approve-carrying-deny.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-098-approve-carrying-deny.json @@ -91,6 +91,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-100-deny-post-tool-non-incorporation.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-100-deny-post-tool-non-incorporation.json index a31a6d3..8a65837 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-100-deny-post-tool-non-incorporation.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-100-deny-post-tool-non-incorporation.json @@ -1,6 +1,6 @@ { "id": "AH-CTK-100", - "title": "deny at post_tool_call: the tool result is discarded, not incorporated into agent state (§6.1)", + "title": "deny at post_tool_call: the tool result is discarded, not incorporated into agent state; the deny surfaces to the model in some form, transcript shape unprescribed (§6.1)", "spec_clause": "6.1, 6.2", "part": "enforcement/post_action_deny", "capabilities": [ @@ -67,9 +67,15 @@ }, { "interception_point": "pre_model_call", - "context": { - "messages[1].role": "tool", - "messages[1].content": "blocked: ctk:tainted-result" + "context_must_contain": ["ctk:tainted-result"], + "context_must_not_contain": ["SECRET-RESULT"] + } + ], + "tool_invocations": [ + { + "name": "http_get", + "args": { + "url": "https://x" } } ], diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-102-runall-failed-transform-shortcircuit.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-102-runall-failed-transform-shortcircuit.json index e70f931..eb52771 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-102-runall-failed-transform-shortcircuit.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-102-runall-failed-transform-shortcircuit.json @@ -96,6 +96,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-103-approve-carrying-failing-transform.json b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-103-approve-carrying-failing-transform.json index c433a5c..e253ebc 100644 --- a/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-103-approve-carrying-failing-transform.json +++ b/sdk/python/python/agent_hooks/ctk/vectors/AH-CTK-103-approve-carrying-failing-transform.json @@ -94,6 +94,10 @@ } } ], - "run_outcome": "completed" + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } } } diff --git a/sdk/rust/core/src/ctk.rs b/sdk/rust/core/src/ctk.rs index b02ef7f..185ad1c 100644 --- a/sdk/rust/core/src/ctk.rs +++ b/sdk/rust/core/src/ctk.rs @@ -170,6 +170,18 @@ pub trait Harness: Send { /// (`"model_calls"`, `"tool_calls"`, …). fn capabilities(&self) -> Vec; + /// Declared §6.2 posture at the tool seam (§13.1): what the host + /// does with the run after a `host_error:*` deny at + /// `pre_tool_call`/`post_tool_call`. `"continue"` (the default — + /// surface a tool error to the model and keep the loop going) or + /// `"terminate"` (the host's own semantics terminate the turn, + /// which §6.2 explicitly permits). The runner forwards this + /// declaration so `expect.run_outcome_by_posture` vectors resolve + /// to the single outcome this surface must produce. + fn tool_seam_host_error(&self) -> &str { + "continue" + } + /// Wire one vector into the framework: the scenario's mock model + /// tools, the interceptors and resolver, the enforcement mode, the /// vector's composition profile (§7.1), its identity provider @@ -285,9 +297,15 @@ pub async fn run_vector(harness: &mut dyn Harness, vector: &Value) -> VectorResu identity_provider, redact_for_approval, }); - let rr = harness.run().await; + let posture = harness.tool_seam_host_error().to_owned(); + let mut rr = harness.run().await; harness.teardown(); + // Forward the harness's declared posture (§13.1) so the engine can + // select the expected run_outcome where the spec permits both. + rr.postures + .insert("tool_seam_host_error".to_owned(), posture); + let recorded = recorded.lock().expect("recorder poisoned").clone(); assert_vector(vector, &recorded, &rr) } @@ -541,6 +559,8 @@ impl Harness for ReferenceHarness { .iter() .map(|r| serde_json::to_value(r).expect("record serializes")) .collect(), + // The runner overwrites this from the Harness declaration. + postures: Default::default(), } } diff --git a/sdk/rust/core/src/ctk_engine.rs b/sdk/rust/core/src/ctk_engine.rs index 0c61b26..5daa40c 100644 --- a/sdk/rust/core/src/ctk_engine.rs +++ b/sdk/rust/core/src/ctk_engine.rs @@ -164,7 +164,7 @@ pub struct IdentityPair { } /// Wire-shaped `RunRecord` the harness returns. -#[derive(Debug, Deserialize)] +#[derive(Debug, Default, Deserialize)] pub struct RunRecord { pub outcome: String, #[serde(default)] @@ -184,6 +184,14 @@ pub struct RunRecord { /// combined-verdict content). #[serde(default)] pub records: Vec, + /// Host-declared postures (§13.1), forwarded by the runner from the + /// harness *declaration* — never inferred from observed behavior. + /// Known key: `tool_seam_host_error` = `"continue"` (the §6.2 + /// default) or `"terminate"` (the host's own semantics terminate + /// the turn on a `host_error:*` deny at the tool seam). A missing + /// key means the spec-default posture. + #[serde(default)] + pub postures: std::collections::BTreeMap, } /// Result of one vector run. @@ -288,6 +296,41 @@ fn assert_interceptions(expect: &Value, recorded: &[Value], failures: &mut Vec Option<&'static str> { + match posture { + "tool_seam_host_error" => Some("continue"), + _ => None, + } +} + +/// Resolve the expected `run_outcome` for this harness's declared +/// postures. `expect.run_outcome` stays the single value for the +/// spec-default posture (so pre-posture runners keep working for +/// default-posture hosts); `expect.run_outcome_by_posture`, where +/// present, selects by the posture the runner forwarded in +/// `RunRecord.postures` where the spec permits both behaviors. +fn expected_outcome(expect: &Value, rr: &RunRecord, failures: &mut Vec) -> String { + let base = expect["run_outcome"].as_str().unwrap_or("").to_owned(); + let Some(by) = expect.get("run_outcome_by_posture") else { + return base; + }; + let posture = by.get("posture").and_then(Value::as_str).unwrap_or(""); + let Some(default) = posture_default(posture) else { + failures.push(format!( + "run_outcome_by_posture names unknown posture {posture:?}" + )); + return base; + }; + let outcomes = &by["outcomes"]; + // Vector-authoring integrity: the default-posture outcome MUST + // equal run_outcome, or pre-posture runners and posture-aware + // runners would disagree about the same default-posture host. + if outcomes.get(default).and_then(Value::as_str) != Some(base.as_str()) { + failures.push(format!( + "vector error: run_outcome_by_posture.outcomes.{default} must equal run_outcome {base:?}" + )); + } + let declared = rr + .postures + .get(posture) + .map(String::as_str) + .unwrap_or(default); + match outcomes.get(declared).and_then(Value::as_str) { + Some(o) => o.to_owned(), + None => { + failures.push(format!( + "declared posture {posture}={declared:?} has no outcome in run_outcome_by_posture" + )); + base + } + } +} + fn assert_record(expect: &Value, rr: &RunRecord, failures: &mut Vec) { - let want_outcome = expect["run_outcome"].as_str().unwrap_or(""); + let want_outcome = expected_outcome(expect, rr, failures); if rr.outcome != want_outcome { failures.push(format!( "run_outcome == {:?}, want {want_outcome:?}", @@ -577,6 +674,121 @@ mod tests { assert!(should_skip(&v, &["model_calls", "tool_calls"]).is_none()); } + fn posture_vector() -> Value { + json!({ + "id": "T", "title": "t", + "expect": { + "interceptions": [], + "run_outcome": "completed", + "run_outcome_by_posture": { + "posture": "tool_seam_host_error", + "outcomes": { "continue": "completed", "terminate": "blocked" } + } + } + }) + } + + fn rr_with(outcome: &str, posture: Option<&str>) -> RunRecord { + let mut rr = RunRecord { + outcome: outcome.into(), + ..Default::default() + }; + if let Some(p) = posture { + rr.postures.insert("tool_seam_host_error".into(), p.into()); + } + rr + } + + #[test] + fn run_outcome_selected_by_declared_posture() { + let v = posture_vector(); + // Default (undeclared) posture = continue. + assert_eq!( + assert_vector(&v, &[], &rr_with("completed", None)).status, + "pass" + ); + // Explicit continue. + let r = assert_vector(&v, &[], &rr_with("completed", Some("continue"))); + assert_eq!(r.status, "pass", "{:?}", r.failures); + // Declared terminate host: blocked is the expected outcome … + let r = assert_vector(&v, &[], &rr_with("blocked", Some("terminate"))); + assert_eq!(r.status, "pass", "{:?}", r.failures); + // … and completed now fails. + let r = assert_vector(&v, &[], &rr_with("completed", Some("terminate"))); + assert_eq!(r.status, "fail"); + // A continue host reporting blocked still fails. + let r = assert_vector(&v, &[], &rr_with("blocked", None)); + assert_eq!(r.status, "fail"); + } + + #[test] + fn run_outcome_by_posture_authoring_errors() { + // Unknown posture name fails loudly. + let mut v = posture_vector(); + v["expect"]["run_outcome_by_posture"]["posture"] = json!("no_such_posture"); + let r = assert_vector(&v, &[], &rr_with("completed", None)); + assert_eq!(r.status, "fail"); + assert!( + r.failures[0].contains("unknown posture"), + "{:?}", + r.failures + ); + // Default-posture outcome disagreeing with run_outcome fails. + let mut v = posture_vector(); + v["expect"]["run_outcome_by_posture"]["outcomes"]["continue"] = json!("blocked"); + let r = assert_vector(&v, &[], &rr_with("completed", None)); + assert_eq!(r.status, "fail"); + // A declared posture value the vector does not cover fails. + let v = posture_vector(); + let r = assert_vector(&v, &[], &rr_with("blocked", Some("halt"))); + assert_eq!(r.status, "fail"); + assert!( + r.failures.iter().any(|f| f.contains("has no outcome")), + "{:?}", + r.failures + ); + } + + #[test] + fn context_substring_assertions() { + let vector = json!({ + "id": "T", "title": "t", + "expect": { + "sequence_strict": false, + "interceptions": [{ + "interception_point": "pre_model_call", + "context_must_validate": false, + "context_must_contain": ["ctk:tainted-result"], + "context_must_not_contain": ["SECRET-RESULT"] + }], + "run_outcome": "completed" + } + }); + let ok = vec![json!({ + "interception_point": "pre_model_call", "sequence": 0, + "messages": [{"role": "tool", "content": {"error": true, "reason": "ctk:tainted-result"}}] + })]; + let r = assert_vector(&vector, &ok, &rr_with("completed", None)); + assert_eq!(r.status, "pass", "{:?}", r.failures); + + let leaky = vec![json!({ + "interception_point": "pre_model_call", "sequence": 0, + "messages": [{"role": "tool", "content": "SECRET-RESULT"}] + })]; + let r = assert_vector(&vector, &leaky, &rr_with("completed", None)); + assert_eq!(r.status, "fail"); + assert!( + r.failures.iter().any(|f| f.contains("MUST NOT surface")), + "{:?}", + r.failures + ); + assert!( + r.failures.iter().any(|f| f.contains("in any form")), + "{:?}", + r.failures + ); + } + #[test] fn assert_vector_pass() { let vector = json!({ @@ -596,11 +808,7 @@ mod tests { })]; let rr = RunRecord { outcome: "completed".into(), - final_output: Value::Null, - tool_invocations: vec![], - error: None, - identities: vec![], - records: vec![], + ..Default::default() }; let r = assert_vector(&vector, &recorded, &rr); assert_eq!(r.status, "pass", "{:?}", r.failures); diff --git a/sdk/rust/core/tests/ctk_external_harness.rs b/sdk/rust/core/tests/ctk_external_harness.rs index 7cd5adc..9282066 100644 --- a/sdk/rust/core/tests/ctk_external_harness.rs +++ b/sdk/rust/core/tests/ctk_external_harness.rs @@ -28,11 +28,7 @@ impl Harness for ExternalHarness { async fn run(&mut self) -> RunRecord { RunRecord { outcome: "completed".into(), - final_output: json!(null), - tool_invocations: Vec::new(), - error: None, - identities: Vec::new(), - records: Vec::new(), + ..Default::default() } } diff --git a/sdk/typescript/src/ctk/index.ts b/sdk/typescript/src/ctk/index.ts index 46d6dbd..c98e1fb 100644 --- a/sdk/typescript/src/ctk/index.ts +++ b/sdk/typescript/src/ctk/index.ts @@ -66,6 +66,16 @@ export interface Harness { readonly name: string; readonly capabilities: ReadonlySet; + /** Declared §6.2 posture at the tool seam (§13.1): what the host does + * with the run after a `host_error:*` deny at + * `pre_tool_call`/`post_tool_call`. `"continue"` (the default — + * surface a tool error to the model and keep the loop going) or + * `"terminate"` (the host's own semantics terminate the turn, which + * §6.2 explicitly permits). The runner forwards this declaration so + * `expect.run_outcome_by_posture` vectors resolve to the single + * outcome this surface must produce. */ + readonly toolSeamHostError?: "continue" | "terminate"; + /** Wire the scenario's mock model + tools into the framework, * register the interceptors and resolver, set the enforcement mode, * the vector's composition profile (§7.1), and its identity provider diff --git a/sdk/typescript/src/ctk/runner.ts b/sdk/typescript/src/ctk/runner.ts index df18262..e79eb3c 100644 --- a/sdk/typescript/src/ctk/runner.ts +++ b/sdk/typescript/src/ctk/runner.ts @@ -106,7 +106,7 @@ class ScriptedResolver { } } -function runRecordToWire(rr: RunRecord): string { +function runRecordToWire(rr: RunRecord, postures: Record): string { return JSON.stringify({ outcome: rr.outcome, final_output: rr.final_output ?? null, @@ -114,6 +114,9 @@ function runRecordToWire(rr: RunRecord): string { error: rr.error ?? null, identities: rr.identities.map(([i, e]) => ({ input_identity: i, enforced_identity: e })), records: rr.records, + // Harness *declarations* (§13.1), not observed behavior: the engine + // selects expect.run_outcome_by_posture entries by them. + postures, }); } @@ -184,8 +187,14 @@ export async function runVector(harness: Harness, vector: JsonValue): Promise/`. A conformance claim is the tuple `(, , agent-hooks/, , , , @)` plus the CTK report, recorded in `conformance/CLAIMS.md`. A claim with +a non-default posture (§13.1) MUST state it (e.g. +`tool_seam_host_error: terminate`) — the report's passing vectors +attest that posture's outcomes, not the default's. A claim with `identity_provider: null` MUST state that its approvals and records are identity-unbound. A claim with `buffered_output: false` MUST state that a `deny` at `output` cannot retract already-streamed content