diff --git a/provider/anthropicprovider/agent.go b/provider/anthropicprovider/agent.go index e349778e..8c699536 100644 --- a/provider/anthropicprovider/agent.go +++ b/provider/anthropicprovider/agent.go @@ -490,9 +490,14 @@ func buildMessageParam(msg *message.Message) (anthropic.MessageParam, error) { var content []anthropic.ContentBlockParamUnion for _, c := range msg.Contents { + var ( + block anthropic.ContentBlockParamUnion + built bool + ) switch c := c.(type) { case *message.TextContent: - content = append(content, anthropic.NewTextBlock(c.Text)) + block = anthropic.NewTextBlock(c.Text) + built = true case *message.FunctionCallContent: // Parse the JSON string arguments into a map for Anthropic SDK var args map[string]any @@ -507,7 +512,8 @@ func buildMessageParam(msg *message.Message) (anthropic.MessageParam, error) { // for a tool call with empty or absent arguments. args = map[string]any{} } - content = append(content, anthropic.NewToolUseBlock(c.CallID, args, c.Name)) + block = anthropic.NewToolUseBlock(c.CallID, args, c.Name) + built = true case *message.FunctionResultContent: resStr := "" switch { @@ -535,16 +541,28 @@ func buildMessageParam(msg *message.Message) (anthropic.MessageParam, error) { } } } - content = append(content, anthropic.NewToolResultBlock(c.CallID, resStr, c.Error != nil)) + block = anthropic.NewToolResultBlock(c.CallID, resStr, c.Error != nil) + built = true case *message.DataContent: if c.TopLevelMediaType() == "image" { mediaType := c.MediaType if mediaType == "" { mediaType = "image/jpeg" } - content = append(content, anthropic.NewImageBlockBase64(mediaType, c.Data)) + block = anthropic.NewImageBlockBase64(mediaType, c.Data) + built = true } } + if !built { + continue + } + // Opt-in cache_control: if the source content was marked via WithCacheControl, + // carry that breakpoint onto the block just built for it. A content with no marker + // is left untouched, so caching never turns itself on silently. This runs for every + // block type -- text, image, document, tool-use, tool-result -- because the marker + // lives on the message.Content, not on any particular Anthropic wire type. + applyContentCacheControl(c, &block) + content = append(content, block) } switch msg.Role { diff --git a/provider/anthropicprovider/agent_test.go b/provider/anthropicprovider/agent_test.go index fe71e111..4947f804 100644 --- a/provider/anthropicprovider/agent_test.go +++ b/provider/anthropicprovider/agent_test.go @@ -972,3 +972,143 @@ func TestStreamingClosesResponseBody(t *testing.T) { t.Fatal("streaming response body was not closed after early consumer exit") } } + +// --- Prompt caching (cache_control), BGB/PR #496 ---------------------------- +// +// These are black-box wire-format tests: they mark content with the exported +// WithCacheControl and assert the cache_control breakpoint on the captured +// outbound request, exercising the public agent path rather than internals. +// The exhaustive "every SDK union variant" invariant lives in +// caching_internal_test.go, which is unreachable through the public API (the +// message package produces only a handful of the SDK's block variants). + +// captureCacheRequest runs msgs through the agent against a stub server and +// returns the outbound Anthropic request decoded as JSON. +func captureCacheRequest(t *testing.T, msgs []*message.Message) map[string]any { + t.Helper() + bodyCh := make(chan []byte, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read request body: %v", err) + } + bodyCh <- body + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintln(w, `{"id":"msg","type":"message","role":"assistant","model":"m","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`) + })) + defer server.Close() + + a := newTestClient(t, server) + if _, err := a.Run(t.Context(), msgs).Collect(); err != nil { + t.Fatalf("run: %v", err) + } + var out map[string]any + if err := json.Unmarshal(<-bodyCh, &out); err != nil { + t.Fatalf("unmarshal request body: %v", err) + } + return out +} + +// lastMessageContentBlocks returns the decoded content blocks of the final +// request message. +func lastMessageContentBlocks(t *testing.T, out map[string]any) []any { + t.Helper() + msgs, _ := out["messages"].([]any) + if len(msgs) == 0 { + t.Fatal("expected at least one message") + } + blocks, _ := msgs[len(msgs)-1].(map[string]any)["content"].([]any) + if len(blocks) == 0 { + t.Fatal("expected content blocks on the last message") + } + return blocks +} + +// Opt-in, not default: an unmarked content must produce no cache_control anywhere. +// A cache WRITE costs more than a plain input token, so caching must never turn +// itself on and quietly raise the bill for a short single-turn agent. +func TestCacheControl_AbsentWhenUnmarked(t *testing.T) { + msg := &message.Message{Role: message.RoleUser, Contents: []message.Content{ + &message.TextContent{Text: "hi"}, + }} + out := captureCacheRequest(t, []*message.Message{msg}) + raw, _ := json.Marshal(out) + if strings.Contains(string(raw), "cache_control") { + t.Fatalf("cache_control present on an unmarked request:\n%s", raw) + } +} + +// A text content marked with WithCacheControl must produce a cache_control +// breakpoint on exactly the block built from it, defaulting to the 5-minute tier +// (no ttl on the wire). +func TestCacheControl_MarksTextBlock(t *testing.T) { + msg := &message.Message{Role: message.RoleUser, Contents: []message.Content{ + anthropicprovider.WithCacheControl(&message.TextContent{Text: "cache me"}), + }} + out := captureCacheRequest(t, []*message.Message{msg}) + + blocks := lastMessageContentBlocks(t, out) + last, _ := blocks[len(blocks)-1].(map[string]any) + cc, ok := last["cache_control"].(map[string]any) + if !ok { + t.Fatalf("no cache_control on the marked text block: %v", last) + } + if cc["type"] != "ephemeral" { + t.Fatalf("cache_control type = %v, want ephemeral", cc["type"]) + } + if _, present := cc["ttl"]; present { + t.Fatalf("default breakpoint should carry no ttl, got %v", cc["ttl"]) + } +} + +// TTL flows through: an explicit 1-hour marker lands ttl=1h on the wire, and the +// explicit 5-minute tier is a distinct real value. +func TestCacheControl_TTLFlowsThrough(t *testing.T) { + msg1h := &message.Message{Role: message.RoleUser, Contents: []message.Content{ + anthropicprovider.WithCacheControl(&message.TextContent{Text: "an hour"}, anthropicprovider.WithTTL(anthropicprovider.Ephemeral1h)), + }} + out1h := captureCacheRequest(t, []*message.Message{msg1h}) + blocks1h := lastMessageContentBlocks(t, out1h) + cc1h, _ := blocks1h[len(blocks1h)-1].(map[string]any)["cache_control"].(map[string]any) + if cc1h["ttl"] != "1h" { + t.Fatalf("cache_control ttl = %v, want 1h", cc1h["ttl"]) + } + + msg5m := &message.Message{Role: message.RoleUser, Contents: []message.Content{ + anthropicprovider.WithCacheControl(&message.TextContent{Text: "five minutes"}, anthropicprovider.WithTTL(anthropicprovider.Ephemeral5m)), + }} + out5m := captureCacheRequest(t, []*message.Message{msg5m}) + blocks5m := lastMessageContentBlocks(t, out5m) + cc5m, _ := blocks5m[len(blocks5m)-1].(map[string]any)["cache_control"].(map[string]any) + if cc5m["ttl"] != "5m" { + t.Fatalf("explicit 5m ttl = %v, want 5m", cc5m["ttl"]) + } +} + +// text and image blocks marked with WithCacheControl each carry the breakpoint on +// the wire. (tool_use and tool_result variants, and every other SDK union variant, +// are covered exhaustively by the internal union-coverage test.) +func TestCacheControl_MarksUserBlockTypes(t *testing.T) { + cases := []struct { + name string + content message.Content + wantType string + }{ + {"text", anthropicprovider.WithCacheControl(&message.TextContent{Text: "hi"}), "text"}, + {"image", anthropicprovider.WithCacheControl(&message.DataContent{MediaType: "image/png", Data: "aGVsbG8="}), "image"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + msg := &message.Message{Role: message.RoleUser, Contents: []message.Content{tc.content}} + out := captureCacheRequest(t, []*message.Message{msg}) + blocks := lastMessageContentBlocks(t, out) + last, _ := blocks[len(blocks)-1].(map[string]any) + if last["type"] != tc.wantType { + t.Fatalf("block type = %v, want %v", last["type"], tc.wantType) + } + if _, ok := last["cache_control"]; !ok { + t.Fatalf("no cache_control on the marked %s block: %v", tc.wantType, last) + } + }) + } +} diff --git a/provider/anthropicprovider/caching.go b/provider/anthropicprovider/caching.go new file mode 100644 index 00000000..478f6a6a --- /dev/null +++ b/provider/anthropicprovider/caching.go @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft. All rights reserved. + +package anthropicprovider + +import ( + "reflect" + + "github.com/anthropics/anthropic-sdk-go" + "github.com/microsoft/agent-framework-go/message" +) + +// Ephemeral5m and Ephemeral1h are the cache lifetimes Anthropic supports for a +// cache_control breakpoint. They are the anthropic-sdk-go values, surfaced here as +// named constants so callers do not have to reach for the SDK's verbose identifiers. +// +// Their type is the SDK's own anthropic.CacheControlEphemeralTTL, deliberately: this +// package already hands raw SDK types across its public surface (see MessageNewParams), +// so a provider-specific caching primitive stays consistent by doing the same rather +// than wrapping the SDK enum in a parallel type the caller would have to convert. +const ( + // Ephemeral5m caches for 5 minutes. This is also the default when no TTL is set. + Ephemeral5m = anthropic.CacheControlEphemeralTTLTTL5m + // Ephemeral1h caches for 1 hour. + Ephemeral1h = anthropic.CacheControlEphemeralTTLTTL1h +) + +// CacheControl describes an Anthropic cache_control breakpoint on a single content block. +type CacheControl struct { + // TTL is the cache lifetime. The zero value ("") means the Anthropic default of + // 5-minute ephemeral caching; Ephemeral1h selects the 1-hour tier. + TTL anthropic.CacheControlEphemeralTTL +} + +// CacheControlOption configures the CacheControl attached by WithCacheControl. +type CacheControlOption func(*CacheControl) + +// WithTTL sets the cache lifetime for the breakpoint. Passing the zero value leaves +// Anthropic's default (5 minutes) in effect. +func WithTTL(ttl anthropic.CacheControlEphemeralTTL) CacheControlOption { + return func(cc *CacheControl) { cc.TTL = ttl } +} + +// cacheControlKey is the package-private key under which a cache_control marker is stored +// in a content's AdditionalProperties bag. Unexported and package-qualified so it cannot +// collide with a key a caller sets for their own purposes. +const cacheControlKey = "anthropicprovider.cacheControl" + +// WithCacheControl attaches an Anthropic cache_control breakpoint to a single +// message.Content, in place, and returns it for chaining. Caching is expressed on the +// individual content block, not toggled agent-wide, so the caller decides exactly which +// blocks become cache breakpoints. +// +// The marker is stored in the content's ContentHeader.AdditionalProperties under a +// package-private key and read back in buildMessageParam, which transfers it onto whatever +// Anthropic block the content becomes: text, image, document, tool-use, or tool-result. A +// content whose block type has no cache_control field on the wire (the thinking and +// redacted_thinking variants) is a clean no-op rather than an error. +// +// Opt-in, deliberately. A cache WRITE costs more than a plain input token, so caching is a +// net loss for a short single-turn prompt; nothing here turns it on unless the caller asks. +// +// Breakpoint budget: Anthropic permits at most 4 cache_control breakpoints per request +// (the tool definitions, the system prompt, and message blocks all draw from the same +// budget). The caller is responsible for staying within it; this function does not dedupe +// or cap, and a request with too many breakpoints is rejected by the API. Higher-level +// automatic placement is intentionally left to a future change. +// +// For full control over the raw request, including breakpoints on tools or the system +// prompt, use MessageNewParams as the low-level escape hatch. +func WithCacheControl(c message.Content, opts ...CacheControlOption) message.Content { + var cc CacheControl + for _, opt := range opts { + opt(&cc) + } + markCacheControl(c, cc) + return c +} + +// markCacheControl writes the cache_control marker into a content's AdditionalProperties. +// +// Every message.Content implementation embeds message.ContentHeader by value, so each one +// already carries an AdditionalProperties bag for provider metadata. ContentHeader.Header() +// only hands back a COPY, though, so the field has to be reached through the content's own +// pointer to mutate the original. Reflection does that generically: it finds the promoted +// AdditionalProperties field on any content type, present and future, without a per-type +// switch that would silently miss a type the day it is added. +// +// TODO: this reflection dance can be dropped once message.Content exposes its ContentHeader +// by pointer (Header() returns a copy today). With a pointer accessor, markCacheControl can +// set AdditionalProperties on the original header directly, with no reflection. +func markCacheControl(c message.Content, cc CacheControl) { + v := reflect.ValueOf(c) + if v.Kind() != reflect.Pointer || v.IsNil() { + return + } + f := v.Elem().FieldByName("AdditionalProperties") + if !f.IsValid() || !f.CanSet() || f.Kind() != reflect.Map { + return + } + if f.IsNil() { + f.Set(reflect.MakeMap(f.Type())) + } + f.SetMapIndex(reflect.ValueOf(cacheControlKey), reflect.ValueOf(cc).Convert(f.Type().Elem())) +} + +// cacheControlOf reads back a marker set by WithCacheControl. Reading uses the public +// Header() copy -- no reflection needed, because reading a value does not require mutating +// the original. +func cacheControlOf(c message.Content) (CacheControl, bool) { + props := c.Header().AdditionalProperties + if props == nil { + return CacheControl{}, false + } + cc, ok := props[cacheControlKey].(CacheControl) + return cc, ok +} + +// applyContentCacheControl transfers a WithCacheControl marker from a message.Content onto +// the Anthropic block that was built from it. No marker means no change (opt-in). If the +// block type cannot carry a breakpoint, setCacheControl reports false and this is a no-op +// rather than a panic: degraded (that block simply uncached) beats a crash. +func applyContentCacheControl(c message.Content, block *anthropic.ContentBlockParamUnion) { + cc, ok := cacheControlOf(c) + if !ok { + return + } + setCacheControl(block, cc.TTL) +} + +var cacheControlType = reflect.TypeOf(anthropic.NewCacheControlEphemeralParam()) + +// setCacheControl marks a content block as a cache breakpoint with the given TTL, +// reporting whether it could. False means the populated variant has no cache_control field +// (thinking and redacted_thinking are the only two), so the block cannot carry one. +// +// This reflects over the union rather than switching on its variants, and that is a +// deliberate trade. ContentBlockParamUnion has 17 variants today; a hand-written switch +// covering the obvious five leaves the other twelve -- every server-tool result, which is +// to say the BIGGEST blocks in a modern agent's context -- silently uncached, and rots the +// moment the SDK adds an eighteenth. Reflection makes the SDK's own type definitions the +// source of truth: a block is cacheable exactly when its param struct carries the field. +// New variants are then handled the day they appear, correctly, with no edit here. +// +// The cost is one reflect walk over a 17-field struct per request. Next to a network call +// to an LLM, that is not a cost. +// +// An empty TTL leaves CacheControlEphemeralParam.TTL at its zero value, which the SDK +// omits from the wire, so Anthropic applies its 5-minute default. +func setCacheControl(b *anthropic.ContentBlockParamUnion, ttl anthropic.CacheControlEphemeralTTL) bool { + v := reflect.ValueOf(b).Elem() + for i := range v.NumField() { + f := v.Field(i) + if f.Kind() != reflect.Pointer || f.IsNil() { + continue + } + // Exactly one variant is non-nil, so this is the block's real type. + cc := f.Elem().FieldByName("CacheControl") + if !cc.IsValid() || !cc.CanSet() || cc.Type() != cacheControlType { + return false // e.g. thinking: no cache_control field exists on the wire type. + } + param := anthropic.NewCacheControlEphemeralParam() + param.TTL = ttl + cc.Set(reflect.ValueOf(param)) + return true + } + return false // empty union: nothing populated. +} diff --git a/provider/anthropicprovider/caching_internal_test.go b/provider/anthropicprovider/caching_internal_test.go new file mode 100644 index 00000000..bbb651e5 --- /dev/null +++ b/provider/anthropicprovider/caching_internal_test.go @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft. All rights reserved. + +package anthropicprovider + +// The behavioral wire-format cache_control tests are black-box and live in +// agent_test.go. What remains here is the one invariant that is inherently +// white-box: it drives itself off the Anthropic SDK's own ContentBlockParamUnion +// and calls the unexported reflective setCacheControl to prove EVERY SDK variant +// is handled. The public agent path can only produce a handful of the SDK's block +// variants, so this exhaustive guard cannot be expressed black-box without losing +// the coverage that is its entire point (catching a future SDK variant, or a +// hand-written switch that silently skips the twelve non-obvious cacheable +// variants). It therefore stays package-internal. + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/anthropics/anthropic-sdk-go" +) + +// Marking a block that has no cache_control field on the wire is a clean no-op. +// +// thinking and redacted_thinking are the only two content variants with no +// cache_control field, so a breakpoint cannot sit on them. setCacheControl must +// report false and leave the block untouched rather than panic -- degraded (that +// block uncached) beats a crash. +func TestSetCacheControl_NoFieldBlockIsCleanNoop(t *testing.T) { + thinking := anthropic.NewThinkingBlock("sig", "let me think") + if got := setCacheControl(&thinking, ""); got != false { + t.Fatalf("setCacheControl on a thinking block = %v, want false", got) + } + // The block must be unchanged and still marshal cleanly (no panic, no cache_control). + raw, err := json.Marshal(thinking) + if err != nil { + t.Fatalf("marshal thinking block: %v", err) + } + if strings.Contains(string(raw), "cache_control") { + t.Fatalf("thinking block gained a cache_control it cannot carry:\n%s", raw) + } +} + +// The invariant, guarded against a moving SDK. +// +// Every ContentBlockParamUnion variant whose param struct HAS a cache_control field +// must actually receive a breakpoint, and every variant that lacks one must report +// false. This drives itself off the union's own reflected shape, so when the SDK +// adds an 18th variant this test covers it the day it lands -- and if anyone ever +// replaces the reflective setCacheControl with a hand-written switch, it fails for +// whatever they forgot. That is the point: a switch covering the obvious five would +// silently skip twelve cacheable variants, including every server-tool result. +// +// It also proves the invariant this whole design rests on: for each variant exactly +// one pointer field of the union is non-nil, which is what lets setCacheControl +// treat "the first non-nil field" as the block's real type. +func TestSetCacheControl_CoversEveryCacheableVariant(t *testing.T) { + union := reflect.TypeOf(anthropic.ContentBlockParamUnion{}) + + var cacheable, skipped int + for i := range union.NumField() { + field := union.Field(i) + if field.Type.Kind() != reflect.Pointer { + continue // the embedded paramUnion marker + } + + // Build a union with just this variant populated. + block := anthropic.ContentBlockParamUnion{} + variant := reflect.New(field.Type.Elem()) + reflect.ValueOf(&block).Elem().Field(i).Set(variant) + + // Exactly one non-nil pointer = the variant. Guard that invariant directly. + if n := nonNilPointerFields(block); n != 1 { + t.Fatalf("%s: union has %d non-nil pointer fields, want exactly 1", field.Name, n) + } + + _, wantsCache := field.Type.Elem().FieldByName("CacheControl") + got := setCacheControl(&block, Ephemeral1h) + + if got != wantsCache { + t.Errorf("%s: setCacheControl = %v, but the wire type %s a cache_control field", + field.Name, got, map[bool]string{true: "HAS", false: "has no"}[wantsCache]) + continue + } + if wantsCache { + cacheable++ + cc := variant.Elem().FieldByName("CacheControl") + ephemeral := cc.Interface().(anthropic.CacheControlEphemeralParam) + if ephemeral.Type != "ephemeral" { + t.Errorf("%s: reported success but wrote no ephemeral breakpoint", field.Name) + } + // TTL must flow through the reflective setter, not get dropped. + if ephemeral.TTL != Ephemeral1h { + t.Errorf("%s: ttl = %q, want %q", field.Name, ephemeral.TTL, Ephemeral1h) + } + } else { + skipped++ + } + } + + // Guard the guard: if the union were ever read as empty, every assertion above + // would vacuously pass and this test would prove nothing. + if cacheable < 10 || skipped != 2 { + t.Fatalf("union shape looks wrong: %d cacheable, %d non-cacheable "+ + "(expected 10+ cacheable and exactly 2 non-cacheable: thinking, redacted_thinking)", + cacheable, skipped) + } +} + +// nonNilPointerFields counts the non-nil pointer fields of a ContentBlockParamUnion, +// the "exactly one variant is populated" invariant setCacheControl relies on. +func nonNilPointerFields(block anthropic.ContentBlockParamUnion) int { + v := reflect.ValueOf(block) + var n int + for i := range v.NumField() { + f := v.Field(i) + if f.Kind() == reflect.Pointer && !f.IsNil() { + n++ + } + } + return n +}