diff --git a/README.md b/README.md index d1fd9a8..ce6a1ce 100644 --- a/README.md +++ b/README.md @@ -146,9 +146,43 @@ See [docs/components.md](docs/components.md) and [docs/reference/presets.md](doc | `FORCE_MODEL` | — | overwrite the request `model` (eval-containers `EVAL_MODEL`) | Routes: `POST /openai/v1/chat/completions`, `POST /anthropic/v1/messages`, `GET /healthz`, -`GET /stats` (savings rollups), `GET /expand?id=` (recover an offloaded original). Per-request: header +`GET /stats` (savings rollups), `GET /expand?id=` (recover an offloaded original), and — with +`--dashboard` — `GET /dashboard/` plus `/api/*`. Per-request: header `x-context-guru-session` sets the session key; `x-context-guru-bypass: true` skips the pipeline. +## Dashboard + +`--dashboard` adds a persistent observability UI at `/dashboard/` plus a JSON/SSE API at +`/api/*`. It exists to answer the question the product exists to answer — **what value is +context-guru providing?** — and to make the answer falsifiable. + +```sh +context-guru-proxy --preset codesmart --dashboard +# open http://localhost:4000/dashboard/ +``` + +[![The context-guru dashboard](docs/img/dashboard/01-overview.jpg)](docs/dashboard.md) + +- **Four labelled savings denominators**, because a single "savings %" is a lie of + omission: of what we tried to compact · of new provider-billed input · of the whole + request (diluted) · unique-of-whole. Each one states what it divides by, and reports + **n/a** rather than a number it cannot compute. +- **Baseline vs actual cumulative cost**, with the saved area shaded, plus an honest + savings **waterfall** that will show a negative net if we spent more than we saved. +- **The cost of our own safety mechanisms beside their benefit** — cache-frozen tokens, + restorations, reverts, and context-guru's own latency and LLM spend. +- **Per-component economics**: unique vs gross savings, `overcount_ratio`, own latency, and + a verdict — so a component that burns wall time for nothing is obvious without a doc. +- **Sessions, requests, and the before/after Git-style diff** of exactly what was removed. +- **Benchmark ingestion** straight from `summary.json` + `rows-*.json`, with cost-vs-reward + per arm and per-task drill-down. + +Embedded via `go:embed` — no CDN, no npm, no build step, so it works air-gapped. Capture is +off the hot path (**~175 ns** per request, drops rather than blocks) and redaction happens +before anything reaches disk. `/stats` is unchanged. + +Full guide: **[docs/dashboard.md](docs/dashboard.md)**. + ## The pipeline Every component operates on tool-output messages. **Reformat** = lossless repack; **Offload** = drop @@ -212,8 +246,9 @@ Details in [docs/integrations.md](docs/integrations.md). ## Docs -- [docs/design.md](docs/design.md) — architecture: component model, fail-open pipeline, store, session, expand loop, metrics, operating modes. +- [docs/design.md](docs/design.md) — architecture: component model, fail-open pipeline, store, session, expand loop, metrics, operating modes, the dashboard's capture/store layer. - [docs/how-to/operating-modes.md](docs/how-to/operating-modes.md) — sync vs observe: when to use each, and how to read observe's projections. +- [docs/dashboard.md](docs/dashboard.md) — the persistent observability dashboard: metrics semantics, the diff view, storage, access gating, API. - [docs/components.md](docs/components.md) — every registered component: how it works, live before→after, lossiness, config, best use. - [docs/integrations.md](docs/integrations.md) — proxy gateway vs AuthBridge plugin, with request paths. - [docs/setup.md](docs/setup.md) — setup + a concrete SWE-bench run through the eval-containers gateway. diff --git a/apply/apply.go b/apply/apply.go index 194d7b5..e99c401 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -84,6 +84,33 @@ type slot struct { lossless bool // wholeMessage: does bifrost round-trip this message without dropping fields } +// Trace is the per-request record of what BodyFull actually did: the resolved +// session, the pipeline's own run report (per-component accounting), the +// before/after text of every rewritten message, and the cache-awareness facts +// that decided which messages were even eligible. It is the dashboard's capture +// input — the same material CONTEXT_GURU_DUMP writes to a file, handed to a +// caller instead. Purely observational: nothing on it affects the rewrite. +type Trace struct { + Session string + Bypassed bool + CacheAware bool + MaxCachedIdx int + // Messages is the normalized message count this request carried. + Messages int + // AttemptedTokens is the token count of the messages age/supersession + // offloaders were ALLOWED to touch (the uncached tail when cache-aware, the + // whole request otherwise). It is the honest denominator for + // "saved / attempted-to-compress"; TokensBefore−AttemptedTokens is the + // compaction our own cache-safety mechanism deliberately gave up. + AttemptedTokens int + // FrozenTokens is TokensBefore−AttemptedTokens: the cost of cache safety. + FrozenTokens int + // Run is the pipeline's aggregate report (nil when the pipeline never ran). + Run *components.RunReport + // Changes lists each rewritten message's before/after text (clipped). + Changes []Change +} + // Body runs the pipeline with no LLM clients available (deterministic components // only). See BodyWithModel to supply model clients for LLM-based components. func Body(ctx context.Context, pipe *components.Pipeline, st store.Store, provider bschemas.ModelProvider, body []byte, explicitSession string, bypass bool) ([]byte, bool) { @@ -121,10 +148,16 @@ func BodyFull(ctx context.Context, pipe *components.Pipeline, st store.Store, pr } // BodyOpts is the full entry point: everything BodyFull takes plus the operating mode -// (#31) and the per-session boundary tracker. Hosts that support modes call this; -// BodyFull is the positional shim every other caller keeps using. +// (#31), the per-session boundary tracker, and the observational Trace the dashboard's +// capture path reads. Hosts that support modes call this; BodyFull is the positional +// shim every other caller keeps using. +// +// The rewrite is byte-identical whether or not anyone reads the trace: every trace +// field is filled from a value the rewrite already computed, and nothing branches on it. func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o Opts) (res Result) { body, provider, bypass := o.Body, o.Provider, o.Bypass + tr := &res.Trace + tr.Bypassed = bypass // Top-level fail-open backstop: the per-component recover in pipeline.runOne only // covers component code. A panic anywhere else on the rewrite path (normalize, the // sjson splice, rebuildCountChanged, a marshal) must NOT 500 the client — forward @@ -143,7 +176,11 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o models := o.Models msgsRaw := gjson.GetBytes(body, "messages") if !msgsRaw.Exists() || !msgsRaw.IsArray() { - return Result{Body: body} + // Assign rather than return a fresh Result: res already carries the trace fields + // set above, and a bypassed request that also lacks a messages array must still + // report itself as bypassed rather than as "no messages". + res.Body = body + return res } // Volatile-tail split, before anything else touches the body. This is a @@ -166,7 +203,8 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o norm, slots := normalize(provider, msgsRaw.Array()) if len(norm) == 0 { - return Result{Body: body, Changed: systemSplit} // keep the split even with nothing to compact + res.Body, res.Changed = body, systemSplit // keep the split even with nothing to compact + return res } if debugTraffic { @@ -175,7 +213,6 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o chat := &bschemas.BifrostChatRequest{Provider: provider, Input: norm} sys, firstUser := systemAndFirstUser(norm) sessionID := session.Resolve(o.Session, sys, firstUser) - res.Session = sessionID cacheAware := resolveCacheAware(o.CacheMode, provider, body) maxCachedIdx := -1 if cacheAware && !bypass { @@ -211,6 +248,11 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o ExistingBreakpoints: wireBreakpoints(body), Mode: mode, } + tr.Session, tr.CacheAware, tr.MaxCachedIdx, tr.Messages = sessionID, cacheAware, maxCachedIdx, len(norm) + // The eligible (attempted) denominator: what age/supersession offloaders were + // allowed to touch. Everything before MaxCachedIdx is frozen for cache safety — + // the cost of that mechanism, reported next to its benefit. + tr.AttemptedTokens = attemptedTokens(norm, c) // Canonical form of each normalized message BEFORE the pipeline, so a // count-changing component (summarize) can be mapped back to the body. @@ -220,7 +262,13 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o } rr := pipe.Run(chat, c) - res.Run = rr + tr.Run = rr + if rr != nil { + tr.FrozenTokens = rr.TokensBefore - tr.AttemptedTokens + if tr.FrozenTokens < 0 { + tr.FrozenTokens = 0 + } + } // A component changed the message count (summarize restructures the transcript // to [msg0, , last-K]). Rebuild the messages array preserving each @@ -240,7 +288,7 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o // The tail split already rewrote `body`, so the result must be forwarded even // if no component changes a message. changed := systemSplit - var changes []change + var changes []Change // Per-message count of changes this writeback threw away, attributed back to the // components that made them once the loop is done. discarded := map[int]int{} @@ -297,6 +345,7 @@ func BodyOpts(ctx context.Context, pipe *components.Pipeline, st store.Store, o } } pipe.RecordDiscards(rr, discarded) + tr.Changes = changes if changed && dumpPath != "" { dumpChanges(c.Session, changes) } @@ -399,9 +448,24 @@ func putLen(st store.Store, session string, n int) { st.Put("cg:len:"+session, []byte(strconv.Itoa(n))) } -// change is one rewritten message, captured for the CONTEXT_GURU_DUMP trace so a -// human can see exactly what context-guru did to the wire. -type change struct { +// attemptedTokens sums the tokens of the messages an age/supersession offloader +// was allowed to touch this turn (Ctx.TailOnly). With cache-awareness off it is +// the whole request; with it on it is the uncached tail, and the difference is +// what cache safety cost us in foregone compaction. +func attemptedTokens(norm []bschemas.ChatMessage, c *components.Ctx) int { + n := 0 + for i := range norm { + if c.TailOnly(i) { + n += schema.TextTokens(schema.MessageText(norm[i])) + } + } + return n +} + +// Change is one rewritten message, captured for the CONTEXT_GURU_DUMP trace and +// for the dashboard's before/after diff view, so a human can see exactly what +// context-guru did to the wire. +type Change struct { Path string `json:"path"` BeforeTokens int `json:"before_tokens"` AfterTokens int `json:"after_tokens"` @@ -409,8 +473,8 @@ type change struct { After string `json:"after"` } -func mkChange(path, before, after string) change { - return change{ +func mkChange(path, before, after string) Change { + return Change{ Path: path, BeforeTokens: schema.TextTokens(before), AfterTokens: schema.TextTokens(after), Before: clip(before, 4000), After: clip(after, 4000), } @@ -430,7 +494,7 @@ func clip(s string, n int) string { var dumpPath = os.Getenv("CONTEXT_GURU_DUMP") // dumpChanges appends one JSON line describing this request's rewrites. -func dumpChanges(session string, changes []change) { +func dumpChanges(session string, changes []Change) { f, err := os.OpenFile(dumpPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return diff --git a/apply/opts.go b/apply/opts.go index ea66396..f13273d 100644 --- a/apply/opts.go +++ b/apply/opts.go @@ -33,15 +33,17 @@ type Opts struct { } // Result is BodyOpts' output. +// +// The embedded Trace carries everything observational: the resolved Session, the +// pipeline's Run report (which observe mode reads as its ONLY output, since the body +// is thrown away), the cache-awareness facts, and each rewritten message's +// before/after text for the dashboard. It is embedded rather than duplicated so +// there is exactly one Session and one Run in the codebase — two copies of the same +// value is how one of them goes stale. type Result struct { // Body is the body to forward. Always valid: on any trouble it is the input. Body []byte // Changed is false when Body is the untouched input. Changed bool - // Session is the resolved session id (the caller usually cannot compute it: it falls - // back to a content hash of system + first user message). - Session string - // Run is the pipeline's report for this request, nil when the pipeline did not run. - // Observe mode needs it: the run is the ONLY output, since the body is thrown away. - Run *components.RunReport + Trace } diff --git a/cmd/context-guru-proxy/main.go b/cmd/context-guru-proxy/main.go index cfc1d23..03c3927 100644 --- a/cmd/context-guru-proxy/main.go +++ b/cmd/context-guru-proxy/main.go @@ -16,11 +16,15 @@ import ( "log/slog" "net/http" "os" + "strconv" "strings" + "time" "github.com/rossoctl/context-guru/components" _ "github.com/rossoctl/context-guru/components/all" "github.com/rossoctl/context-guru/config" + "github.com/rossoctl/context-guru/dash" + "github.com/rossoctl/context-guru/internal/buildinfo" "github.com/rossoctl/context-guru/internal/cheapmodel" "github.com/rossoctl/context-guru/internal/modelinfo" "github.com/rossoctl/context-guru/metrics" @@ -37,6 +41,35 @@ func main() { bob = flag.String("bob-upstream", envOr("BOB_UPSTREAM", ""), "Bob (BobShell) backend base URL; enables the Bob gateway routes when set (e.g. https://api.us-east.bob.ibm.com)") storeFlag = flag.String("store", envOr("STORE", ""), "override state store: true|false (default: config store.enabled, else on)") modeFlag = flag.String("mode", envOr("MODE", ""), "operating mode: sync (default) | observe (overrides the config's mode:)") + + // Dashboard. Off by default so an existing deployment's behavior and route + // table are unchanged until asked for; on, it adds /dashboard/ + /api/*. + // NOTE: deliberately NO "disable observability in production" gate — for a + // tool whose value IS observability, that would be backwards. + dashOn = flag.Bool("dashboard", envBool("DASHBOARD", false), + "enable the persistent dashboard (embedded UI at /dashboard/, JSON+SSE at /api/*)") + dashDB = flag.String("dashboard-db", envOr("DASHBOARD_DB", "./context-guru-dashboard.db"), + "dashboard SQLite path; ':memory:' keeps history in RAM only (lost on restart)") + dashRetain = flag.Duration("dashboard-retention", envDuration("DASHBOARD_RETENTION", 7*24*time.Hour), + "drop dashboard rows older than this (0 = no age limit)") + dashMaxBytes = flag.Int64("dashboard-max-bytes", int64(envInt("DASHBOARD_MAX_BYTES", 512<<20)), + "cap the dashboard database size, dropping oldest rows first (0 = no size limit)") + // Content capture is opt-IN, not opt-out. The before/after diff is the dashboard's + // best view, but it is the one path that writes ARBITRARY agent output to disk, and + // arbitrary output cannot be allowlisted the way headers and config keys are — it + // gets pattern scrubbing, and a pattern denylist is always one unseen credential + // shape behind reality (a review of 22 realistic shapes found 11 leaking). So the + // default is the safe one and the operator turns it on for their own transcripts. + dashContent = flag.Bool("dashboard-content", envBool("DASHBOARD_CONTENT", false), + "capture before/after message text for the diff view; stores arbitrary agent output on disk (scrubbed of known credential shapes and size-capped first), so it is opt-in") + dashContentCap = flag.Int("dashboard-content-cap", envInt("DASHBOARD_CONTENT_CAP", 16<<10), + "maximum bytes stored per captured before/after blob") + dashQueue = flag.Int("dashboard-queue", envInt("DASHBOARD_QUEUE", 4096), + "capture channel depth; a full channel DROPS events (counted, and shown in the UI) rather than delaying a request") + dashCIDRs = flag.String("dashboard-trusted-cidrs", envOr("DASHBOARD_TRUSTED_CIDRS", ""), + "comma-separated CIDRs allowed to view per-request CONTENT and the effective config (loopback always is; aggregates are open)") + dashBench = flag.String("dashboard-bench-dirs", envOr("DASHBOARD_BENCH_DIRS", ""), + "comma-separated directories of benchmark runs (each with summary.json + rows-*.json) to ingest") ) flag.Parse() @@ -62,6 +95,46 @@ func main() { log.Fatalf("build pipeline: %v", err) } + windows := modelWindows() + + var rec *dash.Recorder + if *dashOn { + opts := dash.Options{ + DBPath: *dashDB, + RetentionAge: *dashRetain, + RetentionBytes: *dashMaxBytes, + CaptureContent: *dashContent, + ContentCap: *dashContentCap, + QueueSize: *dashQueue, + TrustedCIDRs: splitComma(*dashCIDRs), + BenchDirs: splitComma(*dashBench), + // The REAL mode, not a hardcoded "active". In observe mode nothing context-guru + // computed was ever enforced, so the dashboard must say so unmissably rather than + // present projections as achieved savings. + Mode: dashMode(mode), + Effective: effectiveConfig(cfg, addr, *openai, *anthropic, *bob, *dashDB, *dashContent, *dashCIDRs), + } + // A negative retention means "no limit"; a zero means "use the default". Map + // an explicit 0 from the flag to "no limit", which is what a user typing 0 means. + if *dashRetain == 0 { + opts.RetentionAge = -1 + } + if *dashMaxBytes == 0 { + opts.RetentionBytes = -1 + } + r, err := dash.NewRecorder(opts) + if err != nil { + log.Fatalf("dashboard: %v", err) + } + rec = r + defer rec.Close() + if runs, tasks := rec.DB().IngestBenchRoots(opts.BenchDirs); runs > 0 { + slog.Info("dashboard: ingested benchmark runs", "runs", runs, "tasks", tasks) + } + slog.Info("dashboard enabled", "url", "http://"+addr+"/dashboard/", "db", rec.DB().Path(), + "content_capture", *dashContent) + } + h := proxy.New(pipe, cfg.NewStore(), agg, proxy.Options{ OpenAIUpstream: *openai, AnthropicUpstream: *anthropic, @@ -74,8 +147,11 @@ func main() { CheapModel: cheapModelFromEnv(), // static "config"-source LLM for NeedsModel components InjectExpand: os.Getenv("INJECT_EXPAND"), // auto (default) | always | never CacheMode: os.Getenv("CACHE_MODE"), // auto (default) | on | off — cache-aware compaction - Windows: modelWindows(), // dynamic context-window resolver (fraction triggers) - Mode: mode, // sync (default) | observe — explicit, never inferred + Windows: windows, // dynamic context-window resolver (fraction triggers) + Prices: priceResolver(windows), // per-token rates, so each captured request is priced at write time + Preset: cfg.Preset, + Dashboard: rec, // nil unless --dashboard + Mode: mode, // sync (default) | observe — explicit, never inferred Observe: proxy.ObserveOptions{ MaxQueue: cfg.Observe.MaxQueue, Workers: cfg.Observe.Workers, @@ -117,6 +193,82 @@ func loadConfig(path, preset string) (*config.Config, error) { return config.LoadBytes([]byte("preset: " + preset + "\n")) } +// splitComma splits a comma-separated flag value into trimmed, non-empty items. +func splitComma(s string) []string { + var out []string + for _, p := range strings.Split(s, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + +// priceResolver returns the Pricer side of the window resolver, when it has one. +// A nil Pricer means "no rates known", and every captured row is then marked +// partially accounted rather than priced as free. +func priceResolver(r modelinfo.Resolver) modelinfo.Pricer { + p, _ := r.(modelinfo.Pricer) + return p +} + +// effectiveConfig assembles the RESOLVED configuration for the dashboard's config +// view — preset expanded, pipeline as actually built, upstream bases and dashboard +// settings included. It is key-allowlisted by dash.RedactConfig before serving, and +// deliberately carries no credential: keys are read from the environment at use +// time and never copied into this map. +func effectiveConfig(cfg *config.Config, addr, openai, anthropic, bob, dbPath string, content bool, cidrs string) map[string]any { + comps := map[string]any{} + for name, node := range cfg.Components { + var v any + if err := node.Decode(&v); err == nil { + comps[name] = v + } + } + return map[string]any{ + "preset": cfg.Preset, + "pipeline": cfg.Pipeline, + "components": comps, + "listen_addr": addr, + "openai_upstream": openai, + "anthropic_upstream": anthropic, + "bob_upstream": bob, + "force_model": os.Getenv("FORCE_MODEL"), + "cache_mode": envOr("CACHE_MODE", "auto"), + "inject_expand": envOr("INJECT_EXPAND", "auto"), + "cheap_model": os.Getenv("CHEAP_MODEL"), + "cheap_model_provider": envOr("CHEAP_MODEL_PROVIDER", "anthropic"), + "store": map[string]any{"ttl_seconds": cfg.Store.TTLSeconds, "max_entries": cfg.Store.MaxEntries}, + "dashboard": map[string]any{"db_path": dbPath, "capture_content": content, "trusted_cidrs": cidrs}, + "build_version": buildinfo.Version, + "build_commit": buildinfo.Commit, + } +} + +// envBool reads a permissive boolean environment variable. +func envBool(key string, def bool) bool { + if v, ok := parseBool(os.Getenv(key)); ok { + return v + } + return def +} + +// envInt reads an integer environment variable, falling back on anything unparseable. +func envInt(key string, def int) int { + if v, err := strconv.Atoi(strings.TrimSpace(os.Getenv(key))); err == nil { + return v + } + return def +} + +// envDuration reads a Go duration environment variable (e.g. "72h"). +func envDuration(key string, def time.Duration) time.Duration { + if d, err := time.ParseDuration(strings.TrimSpace(os.Getenv(key))); err == nil { + return d + } + return def +} + func envOr(key, def string) string { if v := os.Getenv(key); v != "" { return v @@ -136,6 +288,17 @@ func parseBool(s string) (v, ok bool) { return false, false } +// dashMode maps the operating mode onto the dashboard's own label. Two vocabularies +// exist because they answer different questions: `components.Mode` is "what does the +// pipeline do", while the dashboard's per-row mode also has to express `bypass`, which +// is a property of one request rather than of the deployment. +func dashMode(m components.Mode) string { + if m == components.ModeObserve { + return dash.ModeObserve + } + return dash.ModeActive +} + // modelWindows builds the dynamic context-window resolver used for fraction-based // triggers. Default chain: LiteLLM's public prices map (cached) -> small embedded // fallback. MODEL_INFO_URL overrides the map source; MODEL_INFO=off disables it diff --git a/dash/api.go b/dash/api.go new file mode 100644 index 0000000..2a5cd14 --- /dev/null +++ b/dash/api.go @@ -0,0 +1,265 @@ +package dash + +import ( + "encoding/json" + "net" + "net/http" + "strconv" + "strings" +) + +// API serves the dashboard: the JSON endpoints, the SSE stream, and the embedded +// UI. It holds only read access to the store plus the recorder's counters — it +// never writes a request row. +type API struct { + rec *Recorder + trust []*net.IPNet +} + +// NewAPI builds the HTTP surface for a recorder. Malformed CIDRs are skipped with +// no error: a typo in a trust list must not stop the proxy, and the failure mode +// (loopback-only) is the safe one. +func NewAPI(rec *Recorder) *API { + a := &API{rec: rec} + for _, c := range rec.Opts().TrustedCIDRs { + c = strings.TrimSpace(c) + if c == "" { + continue + } + if _, n, err := net.ParseCIDR(c); err == nil { + a.trust = append(a.trust, n) + } + } + return a +} + +// Mount registers every dashboard route on a mux under the given prefix +// (typically "/dashboard" for the UI and "/api" for the data). +func (a *API) Mount(m *http.ServeMux) { + m.HandleFunc("GET /dashboard", func(w http.ResponseWriter, r *http.Request) { + // One canonical URL: /dashboard and /dashboard/ must not be two pages. + http.Redirect(w, r, "/dashboard/", http.StatusMovedPermanently) + }) + m.Handle("GET /dashboard/", http.StripPrefix("/dashboard/", uiHandler())) + m.HandleFunc("GET /api/stats", a.stats) + m.HandleFunc("GET /api/series", a.series) + m.HandleFunc("GET /api/requests", a.requests) + m.HandleFunc("GET /api/requests/{id}", a.request) + m.HandleFunc("GET /api/sessions", a.sessions) + m.HandleFunc("GET /api/components", a.components) + m.HandleFunc("GET /api/facets", a.facets) + m.HandleFunc("GET /api/config", a.config) + m.HandleFunc("GET /api/benchmarks", a.benchmarks) + m.HandleFunc("GET /api/benchmarks/{id}/tasks", a.benchmarkTasks) + m.HandleFunc("GET /api/capture", a.capture) + m.HandleFunc("GET /api/events", a.rec.Hub().ServeHTTP) +} + +// trusted reports whether a request may see per-request CONTENT and the effective +// configuration. Loopback always may; otherwise the peer must be in a configured +// trusted CIDR. Aggregates are deliberately NOT gated — a proxy bound to 0.0.0.0 +// should still show its own numbers, and the point of this tool is observability. +// +// This is the one place headroom's gate is worth copying, and the one place it is +// not: we gate CONTENT (which can carry a user's source code), never metrics. +func (a *API) trusted(r *http.Request) bool { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host = r.RemoteAddr + } + ip := net.ParseIP(host) + if ip == nil { + return false + } + if ip.IsLoopback() { + return true + } + for _, n := range a.trust { + if n.Contains(ip) { + return true + } + } + return false +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + // The dashboard is same-origin only; no CORS header, so a random page cannot + // read a developer's transcripts out of a locally-bound proxy. + if err := json.NewEncoder(w).Encode(v); err != nil { + return + } +} + +func httpErr(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{"error": msg}) +} + +// filterFrom parses the shared filter query parameters. Unknown values are simply +// not matched — a filter is a view, so a bad value shows an empty list rather than +// a 400 the UI has to special-case. +func filterFrom(r *http.Request) Filter { + q := r.URL.Query() + f := Filter{ + Session: q.Get("session"), + Model: q.Get("model"), + Provider: q.Get("provider"), + Agent: q.Get("agent"), + Preset: q.Get("preset"), + Mode: q.Get("mode"), + Component: q.Get("component"), + Reason: q.Get("reason"), + Accounting: q.Get("accounting"), + Q: q.Get("q"), + } + f.Since = atoi64(q.Get("since")) + f.Until = atoi64(q.Get("until")) + return f +} + +func atoi64(s string) int64 { + n, _ := strconv.ParseInt(s, 10, 64) + return n +} + +func atoiDefault(s string, def int) int { + if s == "" { + return def + } + n, err := strconv.Atoi(s) + if err != nil { + return def + } + return n +} + +func (a *API) stats(w http.ResponseWriter, r *http.Request) { + o, err := a.rec.DB().Overview(filterFrom(r)) + if err != nil { + httpErr(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, o) +} + +func (a *API) series(w http.ResponseWriter, r *http.Request) { + bucket := atoi64(r.URL.Query().Get("bucket")) + b, err := a.rec.DB().Series(filterFrom(r), bucket) + if err != nil { + httpErr(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]any{"bucket_ms": bucket, "buckets": b}) +} + +func (a *API) requests(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + p, err := a.rec.DB().Requests(filterFrom(r), atoi64(q.Get("before")), atoiDefault(q.Get("limit"), 50)) + if err != nil { + httpErr(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, p) +} + +func (a *API) request(w http.ResponseWriter, r *http.Request) { + id := atoi64(r.PathValue("id")) + if id <= 0 { + httpErr(w, http.StatusBadRequest, "bad id") + return + } + trusted := a.trusted(r) + e, err := a.rec.DB().Request(id, trusted) + if err != nil { + httpErr(w, http.StatusNotFound, "no such request") + return + } + writeJSON(w, map[string]any{ + "request": e, + // Tell the UI WHY content is missing, so "no content" and "not allowed to + // see content" are never the same empty panel. + "content_visible": trusted, + "content_captured": a.rec.Opts().CaptureContent, + "content_cap_bytes": a.rec.Opts().ContentCap, + }) +} + +func (a *API) sessions(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + rows, total, err := a.rec.DB().Sessions(filterFrom(r), + atoiDefault(q.Get("limit"), 50), atoiDefault(q.Get("offset"), 0)) + if err != nil { + httpErr(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]any{"sessions": rows, "total": total}) +} + +func (a *API) components(w http.ResponseWriter, r *http.Request) { + rows, err := a.rec.DB().Components(filterFrom(r)) + if err != nil { + httpErr(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]any{"components": rows}) +} + +func (a *API) facets(w http.ResponseWriter, r *http.Request) { + f, err := a.rec.DB().Facets(filterFrom(r)) + if err != nil { + httpErr(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, f) +} + +func (a *API) config(w http.ResponseWriter, r *http.Request) { + if !a.trusted(r) { + httpErr(w, http.StatusForbidden, + "effective configuration is visible from loopback or a trusted CIDR only") + return + } + // Redact even for a trusted caller: nothing sensitive should be in here at all, + // and a defence that only applies to untrusted callers is one misconfiguration + // away from being no defence. + writeJSON(w, RedactConfig(a.rec.Opts().Effective)) +} + +func (a *API) benchmarks(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("refresh") == "1" { + runs, tasks := a.rec.DB().IngestBenchRoots(a.rec.Opts().BenchDirs) + writeJSON(w, map[string]any{"ingested_runs": runs, "ingested_tasks": tasks}) + return + } + runs, err := a.rec.DB().BenchRuns() + if err != nil { + httpErr(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]any{"runs": runs}) +} + +func (a *API) benchmarkTasks(w http.ResponseWriter, r *http.Request) { + rows, err := a.rec.DB().BenchTasks(atoi64(r.PathValue("id")), r.URL.Query().Get("arm")) + if err != nil { + httpErr(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, map[string]any{"tasks": rows}) +} + +// capture reports the capture pipeline's own health, drops included. Exposed as a +// first-class endpoint (and rendered in the UI) because a dashboard that hides its +// own coverage gaps cannot be trusted about anything else. +func (a *API) capture(w http.ResponseWriter, r *http.Request) { + s := a.rec.Stats() + writeJSON(w, map[string]any{ + "capture": s, + "description": "Captured is what the proxy handed to the capture channel; written is what " + + "reached the database; dropped is what a full channel discarded rather than " + + "delay a request. A non-zero drop count means the numbers above under-report — " + + "raise the queue size or lower the traffic before drawing conclusions.", + }) +} diff --git a/dash/api_test.go b/dash/api_test.go new file mode 100644 index 0000000..5c6e2cf --- /dev/null +++ b/dash/api_test.go @@ -0,0 +1,621 @@ +package dash + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +// newTestAPI wires a recorder + API with a few requests already persisted. +func newTestAPI(t *testing.T, opts Options) (*API, *Recorder) { + t.Helper() + if opts.DBPath == "" { + opts.DBPath = filepath.Join(t.TempDir(), "d.db") + } + opts.BatchSize, opts.FlushInterval = 1, time.Millisecond + rec, err := NewRecorder(opts) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { rec.Close() }) + return NewAPI(rec), rec +} + +// seed writes events straight through the store so the test does not race the +// writer goroutine. +func seed(t *testing.T, rec *Recorder, evs ...*Event) { + t.Helper() + if err := rec.DB().insertBatch(evs); err != nil { + t.Fatal(err) + } +} + +func get(t *testing.T, a *API, path, remote string) (*httptest.ResponseRecorder, map[string]any) { + t.Helper() + m := http.NewServeMux() + a.Mount(m) + req := httptest.NewRequest(http.MethodGet, path, nil) + if remote != "" { + req.RemoteAddr = remote + } + w := httptest.NewRecorder() + m.ServeHTTP(w, req) + var body map[string]any + if strings.Contains(w.Header().Get("Content-Type"), "json") { + _ = json.Unmarshal(w.Body.Bytes(), &body) + } + return w, body +} + +func TestAPIRoutesServeJSON(t *testing.T) { + a, rec := newTestAPI(t, Options{CaptureContent: true}) + e := mkEvent(time.Now().UnixMilli(), "sess-1", "aws/claude-sonnet-5", 1000, 800) + e.Content = []ContentRow{{Path: "messages.2", BeforeTokens: 200, AfterTokens: 0, + Before: "tool output\nline two\n", After: "tool output\n<>"}} + seed(t, rec, e) + + for _, path := range []string{ + "/api/stats", "/api/series?bucket=60000", "/api/requests", "/api/sessions", + "/api/components", "/api/facets", "/api/benchmarks", "/api/capture", + } { + w, body := get(t, a, path, "127.0.0.1:1234") + if w.Code != http.StatusOK { + t.Errorf("%s -> %d: %s", path, w.Code, w.Body.String()) + continue + } + if body == nil { + t.Errorf("%s returned no JSON object", path) + } + } +} + +// TestCaptureReportsModeAndObserveQueue covers what the observe banner reads. The +// issue required "You are currently in observe mode…" to be unmistakable, and the mode +// was previously hardcoded to "active" in main.go with dash.Options.Mode set and never +// read — so the banner could not have fired however the proxy was configured. +func TestCaptureReportsModeAndObserveQueue(t *testing.T) { + // Default: active, and no queue at all (a sync deployment must show no phantom one). + a, _ := newTestAPI(t, Options{}) + _, body := get(t, a, "/api/capture", "127.0.0.1:1") + c, _ := body["capture"].(map[string]any) + if c["mode"] != ModeActive { + t.Errorf("mode = %v; want %q", c["mode"], ModeActive) + } + if _, ok := c["observe_queue"]; ok { + t.Error("observe_queue is present with no pool running; the UI would render an empty queue") + } + + // Observe, with the host publishing its pool counters. + ao, reco := newTestAPI(t, Options{Mode: ModeObserve}) + reco.SetObserveQueue(func() QueueStats { + return QueueStats{Queued: 7, Pending: 2, Processed: 40, Dropped: 3, Errors: 1} + }) + _, body = get(t, ao, "/api/capture", "127.0.0.1:1") + c, _ = body["capture"].(map[string]any) + if c["mode"] != ModeObserve { + t.Errorf("mode = %v; want %q", c["mode"], ModeObserve) + } + q, ok := c["observe_queue"].(map[string]any) + if !ok { + t.Fatalf("observe_queue missing: %v", c) + } + // dropped is the counter that changes a reader's conclusion, so assert it explicitly. + if q["dropped"] != float64(3) || q["processed"] != float64(40) { + t.Errorf("observe_queue = %v; want processed=40 dropped=3", q) + } +} + +func TestAPIRequestDetailGating(t *testing.T) { + a, rec := newTestAPI(t, Options{CaptureContent: true, TrustedCIDRs: []string{"10.1.0.0/16"}}) + e := mkEvent(time.Now().UnixMilli(), "sess-1", "m", 1000, 800) + e.Content = []ContentRow{{Path: "messages.2", Before: "a customer's private source", After: "x"}} + seed(t, rec, e) + path := "/api/requests/1" + + // Loopback always sees content. + _, body := get(t, a, path, "127.0.0.1:5000") + if body["content_visible"] != true { + t.Error("loopback should see per-request content") + } + req := body["request"].(map[string]any) + if _, ok := req["content"]; !ok { + t.Error("loopback response carried no content") + } + + // A trusted CIDR sees content. + _, body = get(t, a, path, "10.1.2.3:5000") + if body["content_visible"] != true { + t.Error("a trusted CIDR should see per-request content") + } + + // Anything else gets the row but NOT the content, and is told why. + w, body := get(t, a, path, "203.0.113.9:5000") + if w.Code != http.StatusOK { + t.Fatalf("untrusted peer should still see the metrics row, got %d", w.Code) + } + if body["content_visible"] != false { + t.Error("an untrusted peer must not see content") + } + req = body["request"].(map[string]any) + if c, ok := req["content"]; ok { + t.Errorf("content leaked to an untrusted peer: %v", c) + } + // The aggregate row IS visible — a proxy bound to 0.0.0.0 should still show its + // own numbers; only content and config are gated. + if req["tokens_before"] == nil { + t.Error("metrics were withheld from an untrusted peer; only content should be") + } +} + +func TestAPIConfigIsGatedAndRedacted(t *testing.T) { + a, _ := newTestAPI(t, Options{ + TrustedCIDRs: []string{"10.1.0.0/16"}, + Effective: map[string]any{ + "preset": "codesmart", + "anthropic_api_key": fakeKey("CONFIGLEAK"), + "unknown_field": "unclassified", + }, + }) + + w, _ := get(t, a, "/api/config", "203.0.113.9:5000") + if w.Code != http.StatusForbidden { + t.Errorf("untrusted peer got %d for /api/config; want 403", w.Code) + } + + w, body := get(t, a, "/api/config", "127.0.0.1:5000") + if w.Code != http.StatusOK { + t.Fatalf("loopback got %d for /api/config", w.Code) + } + if body["preset"] != "codesmart" { + t.Errorf("preset = %v; want codesmart", body["preset"]) + } + // Redacted even for a trusted caller — a defence that only applies to untrusted + // callers is one misconfiguration from being no defence. + raw := w.Body.String() + if strings.Contains(raw, "CONFIGLEAK") { + t.Errorf("a credential reached a trusted caller: %s", raw) + } + if body["unknown_field"] != Redacted { + t.Errorf("unknown_field = %v; want redacted", body["unknown_field"]) + } +} + +func TestAPIFilterAndKeysetPaginationOverHTTP(t *testing.T) { + a, rec := newTestAPI(t, Options{}) + now := time.Now().UnixMilli() + var evs []*Event + for i := 0; i < 12; i++ { + e := mkEvent(now-int64(i)*1000, "sess-a", "model-a", 100, 90) + if i%2 == 0 { + e.SessionID, e.Model = "sess-b", "model-b" + } + evs = append(evs, e) + } + seed(t, rec, evs...) + + _, body := get(t, a, "/api/requests?session=sess-b", "127.0.0.1:1") + if int(body["total"].(float64)) != 6 { + t.Errorf("session filter total = %v; want 6", body["total"]) + } + + // Page through with the returned cursor. + _, body = get(t, a, "/api/requests?limit=5", "127.0.0.1:1") + cursor := int64(body["next_cursor"].(float64)) + if cursor == 0 { + t.Fatal("no next_cursor with 12 rows and limit 5") + } + first := body["requests"].([]any) + if len(first) != 5 { + t.Fatalf("page 1 had %d rows; want 5", len(first)) + } + _, body2 := get(t, a, "/api/requests?limit=5&before="+strconv.FormatInt(cursor, 10), "127.0.0.1:1") + second := body2["requests"].([]any) + if len(second) != 5 { + t.Fatalf("page 2 had %d rows; want 5", len(second)) + } + firstIDs := map[float64]bool{} + for _, r := range first { + firstIDs[r.(map[string]any)["id"].(float64)] = true + } + for _, r := range second { + if firstIDs[r.(map[string]any)["id"].(float64)] { + t.Error("page 2 repeated a row from page 1") + } + } +} + +func TestAPIServesEmbeddedUIWithNoNetworkFetches(t *testing.T) { + a, _ := newTestAPI(t, Options{}) + m := http.NewServeMux() + a.Mount(m) + + // /dashboard must land on the canonical /dashboard/. + req := httptest.NewRequest(http.MethodGet, "/dashboard", nil) + w := httptest.NewRecorder() + m.ServeHTTP(w, req) + if w.Code != http.StatusMovedPermanently { + t.Errorf("/dashboard -> %d; want a redirect to /dashboard/", w.Code) + } + + for _, path := range []string{"/dashboard/", "/dashboard/style.css", "/dashboard/app.js"} { + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + m.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("%s -> %d", path, w.Code) + } + if w.Body.Len() == 0 { + t.Errorf("%s served an empty body", path) + } + if csp := w.Header().Get("Content-Security-Policy"); !strings.Contains(csp, "default-src 'self'") { + t.Errorf("%s missing a self-only CSP: %q", path, csp) + } + } + + // The offline guarantee: no asset may reference an external origin. context-guru + // ships into air-gapped contexts, so a CDN tag is a shipping bug, not a nit. + for _, path := range []string{"/dashboard/", "/dashboard/style.css", "/dashboard/app.js"} { + req := httptest.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + m.ServeHTTP(w, req) + body := w.Body.String() + for _, bad := range []string{"https://", "http://", "//cdn.", "unpkg.com", "jsdelivr", "googleapis"} { + if strings.Contains(body, bad) { + // Allow the SVG namespace, which is a spec identifier and never fetched. + if bad == "http://" && strings.Count(body, bad) == strings.Count(body, "http://www.w3.org/2000/svg") { + continue + } + t.Errorf("%s references an external origin (%q); the dashboard must work offline", path, bad) + } + } + } +} + +func TestUIHasTestIDsForEveryStatTile(t *testing.T) { + // The visual layer is regression-tested by asserting the testids the Playwright + // checks and the docs screenshots depend on. If a tile is renamed, this fails + // before the screenshots silently go stale. + js, err := uiFS.ReadFile("ui/app.js") + if err != nil { + t.Fatal(err) + } + html, err := uiFS.ReadFile("ui/index.html") + if err != nil { + t.Fatal(err) + } + source := string(js) + string(html) + + // Stat tiles get their testid as "tile-"+key from the tile() helper, so assert on + // the call site: tile('', … . Renaming a tile then fails here, before the + // Playwright checks and the docs screenshots silently go stale. + for _, key := range []string{ + "requests", "tokens-before", "tokens-after", "saved-gross", "saved-unique", + "saved-adjusted", "overcount", "cost-baseline", "cost-actual", "cost-cg", + "saved-usd", "cache-read", "cache-write", "fresh-input", "output", + "cg-latency", "upstream-latency", "expands", "reverts", "passthroughs", + } { + if !strings.Contains(source, "tile('"+key+"'") { + t.Errorf("stat tile %q is not rendered (expected a tile('%s', …) call); "+ + "data-testid=tile-%s is asserted by the UI checks and the docs screenshots", key, key, key) + } + } + // Every other panel carries its testid literally. + for _, id := range []string{ + "denominators", "waterfall", "safety-cost", "cache-miss", "uncompressed-reasons", + "accounting", "chart-cost", "chart-tokens", "chart-cache", "chart-latency", + "chart-volume", "chart-components", "components-table", "sessions-table", + "requests-table", "requests-page", "requests-next", "requests-prev", + "bench-list", "bench-run", "bench-scatter", "bench-tasks", + "config-body", "capture-body", "capture-warning", "observe-banner", + "drawer-body", "diff-block", + "detail-summary", "detail-components", "live-table", "live-indicator", + "theme-toggle", "tab-overview", "tab-components", "tab-sessions", "tab-requests", + "tab-benchmarks", "tab-config", "filter-q", "filter-range", "filter-model", + "filter-provider", "filter-agent", "filter-preset", "filter-mode", + "filter-component", "filter-reason", "filter-accounting", "filter-clear", + "request-row", "diff-mode-git", "diff-mode-side", "drawer-close", + } { + if !strings.Contains(source, `"`+id+`"`) && !strings.Contains(source, "'"+id+"'") { + t.Errorf("data-testid %q is not produced by the UI; a check or screenshot depends on it", id) + } + } +} + +// TestBenchIngestCommitsNothingForARunWithNoTasks pins the counter/table agreement. +// IngestBenchDir used to INSERT the bench_runs row before it knew whether any rows-* +// file parsed, then return tasks=0 — so IngestBenchRoots did not count the run but the +// row was already committed. A real jobs root produced "runs=17" in the log and 42 rows +// from the API, 25 of them with no arms, padding the Benchmarks tab with empty shells +// and making the PR's own "42 runs ingested" claim wrong. +func TestBenchIngestCommitsNothingForARunWithNoTasks(t *testing.T) { + dir := t.TempDir() + write := func(sub, name, body string) { + if err := os.MkdirAll(filepath.Join(dir, sub), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, sub, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + // A summary with no rows files at all: the shape of an abandoned or in-progress run. + write("smoke-hd", "summary.json", `{"model":"m","dataset":"d"}`) + // A summary whose only rows file is truncated, so nothing parses. + write("dbg1", "summary.json", `{"model":"m","dataset":"d"}`) + write("dbg1", "rows-off.json", `[{"task":"t1",`) + // And one good run, so this proves selectivity rather than a blanket refusal. + write("real", "summary.json", `{"model":"m","dataset":"d"}`) + write("real", "rows-off.json", `[{"task":"t1","reward":1.0,"steps":3,"agent_cost":0.1}]`) + + db := openTestDB(t) + runs, tasks := db.IngestBenchRoots([]string{dir}) + if runs != 1 || tasks != 1 { + t.Errorf("ingested %d runs / %d tasks; want 1/1 (the two contentless dirs must not count)", runs, tasks) + } + + got, err := db.BenchRuns() + if err != nil { + t.Fatal(err) + } + // The assertion that actually failed before: the API's row count must EQUAL the + // counter the log reports. + if len(got) != runs { + names := make([]string, len(got)) + for i, r := range got { + names[i] = r.Name + } + t.Errorf("BenchRuns returned %d rows but the ingest counted %d runs: %v", len(got), runs, names) + } + for _, r := range got { + if len(r.Arms) == 0 { + t.Errorf("run %q was committed with no arms; the UI would render an empty row", r.Name) + } + } +} + +func TestBenchIngestFromRealHarnessArtifacts(t *testing.T) { + dir := t.TempDir() + run := filepath.Join(dir, "study-run") + if err := os.MkdirAll(run, 0o755); err != nil { + t.Fatal(err) + } + // Exactly the shape deploy/harbor/*.py writes. + summary := `{"model":"aws/claude-sonnet-5","dataset":"swe-bench-verified", + "price":[2e-06,1e-05,2e-07,2.5e-06],"tasks":2, + "configs":[{"config":"codesmart","solved":1,"solve_rate":0.5}]}` + rowsOff := `[{"task":"t1","reward":1.0,"steps":20,"prompt_tokens":1000,"completion_tokens":50, + "cache_read":900,"cache_write":90,"fresh_input":10,"agent_cost":0.5,"norm_cost":0.4, + "wall_s":100.0,"exception":false}, + {"task":"t2","reward":0.0,"steps":30,"prompt_tokens":2000,"completion_tokens":60, + "cache_read":1800,"cache_write":180,"fresh_input":20,"agent_cost":1.0,"norm_cost":0.8, + "wall_s":200.0,"exception":true}]` + rowsCG := `[{"task":"t1","reward":1.0,"steps":18,"prompt_tokens":800,"completion_tokens":45, + "cache_read":740,"cache_write":50,"fresh_input":10,"agent_cost":0.4,"norm_cost":0.32, + "wall_s":95.0,"exception":false}, + {"task":"t2","reward":1.0,"steps":25,"prompt_tokens":1600,"completion_tokens":55, + "cache_read":1500,"cache_write":90,"fresh_input":10,"agent_cost":0.8,"norm_cost":0.64, + "wall_s":180.0,"exception":false}]` + write := func(name, body string) { + if err := os.WriteFile(filepath.Join(run, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + write("summary.json", summary) + write("rows-off.json", rowsOff) + write("rows-codesmart.json", rowsCG) + // A directory that is not a run must be skipped silently, since a jobs root is + // full of in-progress and unrelated directories. + if err := os.MkdirAll(filepath.Join(dir, "not-a-run"), 0o755); err != nil { + t.Fatal(err) + } + // A truncated rows file must not abort the whole ingest. + write("rows-broken.json", `[{"task":"t1",`) + + db := openTestDB(t) + runs, tasks := db.IngestBenchRoots([]string{dir}) + if runs != 1 || tasks != 4 { + t.Fatalf("ingested %d runs / %d tasks; want 1/4", runs, tasks) + } + + got, err := db.BenchRuns() + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("BenchRuns = %d", len(got)) + } + r := got[0] + if r.Model != "aws/claude-sonnet-5" || r.Dataset != "swe-bench-verified" { + t.Errorf("run metadata: %+v", r) + } + if len(r.Arms) != 2 { + t.Fatalf("arms = %d; want 2 (off, codesmart)", len(r.Arms)) + } + byArm := map[string]BenchArm{} + for _, a := range r.Arms { + byArm[a.Arm] = a + } + off, cg := byArm["off"], byArm["codesmart"] + if off.Tasks != 2 || off.Solved != 1 || off.Exceptions != 1 { + t.Errorf("off arm = %+v", off) + } + // Solve rate is over SCORED trials — an exception is not a failed solve. + if off.Scored != 1 || off.SolveRate != 1 { + t.Errorf("off solve rate = %v over %d scored; an exception must not count as unsolved", + off.SolveRate, off.Scored) + } + if cg.Solved != 2 || cg.TotalCostUSD < 1.19 || cg.TotalCostUSD > 1.21 { + t.Errorf("codesmart arm = %+v", cg) + } + if cg.CacheHitRate <= 0 || cg.CacheHitRate >= 1 { + t.Errorf("cache hit rate = %v; want a fraction", cg.CacheHitRate) + } + + // Per-task drill-down. + tasksOut, err := db.BenchTasks(r.ID, "codesmart") + if err != nil { + t.Fatal(err) + } + if len(tasksOut) != 2 { + t.Fatalf("per-task rows = %d; want 2", len(tasksOut)) + } + all, err := db.BenchTasks(r.ID, "") + if err != nil { + t.Fatal(err) + } + if len(all) != 4 { + t.Errorf("all-arm rows = %d; want 4", len(all)) + } + + // Re-ingest must REPLACE, not duplicate — a proxy restart pointed at a jobs root + // would otherwise double every historical run. + runs2, tasks2 := db.IngestBenchRoots([]string{dir}) + if runs2 != 1 || tasks2 != 4 { + t.Fatalf("re-ingest = %d runs / %d tasks", runs2, tasks2) + } + after, _ := db.BenchRuns() + if len(after) != 1 { + t.Errorf("re-ingest duplicated the run: %d runs", len(after)) + } + allAfter, _ := db.BenchTasks(after[0].ID, "") + if len(allAfter) != 4 { + t.Errorf("re-ingest duplicated tasks: %d rows", len(allAfter)) + } +} + +func TestAPIBadRequestIDs(t *testing.T) { + a, _ := newTestAPI(t, Options{}) + for _, path := range []string{"/api/requests/0", "/api/requests/abc"} { + w, _ := get(t, a, path, "127.0.0.1:1") + if w.Code != http.StatusBadRequest { + t.Errorf("%s -> %d; want 400", path, w.Code) + } + } + w, _ := get(t, a, "/api/requests/999999", "127.0.0.1:1") + if w.Code != http.StatusNotFound { + t.Errorf("missing request -> %d; want 404", w.Code) + } +} + +func TestTrustedRejectsMalformedRemoteAddr(t *testing.T) { + a, _ := newTestAPI(t, Options{TrustedCIDRs: []string{"not-a-cidr", "10.0.0.0/8"}}) + // A malformed CIDR is skipped rather than failing startup, but the valid one works. + req := httptest.NewRequest(http.MethodGet, "/api/config", nil) + req.RemoteAddr = "10.0.0.5:1" + if !a.trusted(req) { + t.Error("a valid CIDR alongside a malformed one should still be honored") + } + req.RemoteAddr = "garbage" + if a.trusted(req) { + t.Error("an unparseable remote address must not be trusted") + } + req.RemoteAddr = "[::1]:4000" + if !a.trusted(req) { + t.Error("IPv6 loopback should be trusted") + } +} + +// TestUIScriptParses guards the failure that actually bit during development: a +// missing closing paren in app.js. Go's compiler cannot see it, every Go test +// passed, the HTML still served 200 — and the whole dashboard rendered blank. A +// syntax check is the cheapest possible guard. +// +// It uses node when available (CI images have it; the repo needs no npm and no +// package.json) and otherwise falls back to a paren/brace balance check over the +// source with strings and comments stripped, which is enough to catch this class +// of typo without a JS engine. +func TestUIScriptParses(t *testing.T) { + src, err := uiFS.ReadFile("ui/app.js") + if err != nil { + t.Fatal(err) + } + if node, lookErr := exec.LookPath("node"); lookErr == nil { + f := filepath.Join(t.TempDir(), "app.js") + if err := os.WriteFile(f, src, 0o600); err != nil { + t.Fatal(err) + } + out, err := exec.Command(node, "--check", f).CombinedOutput() + if err != nil { + t.Fatalf("app.js does not parse:\n%s", out) + } + return + } + t.Log("node not found; falling back to a bracket-balance check") + if line, ok := unbalancedAt(string(src)); !ok { + t.Errorf("app.js brackets do not balance (first imbalance around line %d)", line) + } +} + +func TestUnbalancedDetectorCatchesADroppedParen(t *testing.T) { + // The exact shape of the bug that rendered a blank dashboard during development: + // a nested el(...) chain whose outer appendChild lost its closing paren. + bad := "body.appendChild(el('tr', {},\n el('td', { text: x }),\n el('td', { text: y });\n" + if _, ok := unbalancedAt(bad); ok { + t.Error("the balance check missed a dropped closing paren") + } + // Brackets inside strings, comments and template literals must not confuse it. + good := "const a = '(((';\n// )))\n/* ((( */\nconst b = `${x} ) ( `;\nf(g(h()));\n" + if _, ok := unbalancedAt(good); !ok { + t.Error("the balance check false-positived on brackets inside strings/comments") + } +} + +// unbalancedAt scans JS source with strings, template literals, regexes and +// comments skipped, and reports whether ()[]{} balance. Deliberately simple: it +// only has to catch a dropped closing paren, not to be a parser. +func unbalancedAt(s string) (int, bool) { + var stack []byte + line := 1 + openLine := map[int]int{} + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c == '\n': + line++ + case c == '/' && i+1 < len(s) && s[i+1] == '/': + for i < len(s) && s[i] != '\n' { + i++ + } + line++ + case c == '/' && i+1 < len(s) && s[i+1] == '*': + i += 2 + for i+1 < len(s) && !(s[i] == '*' && s[i+1] == '/') { + if s[i] == '\n' { + line++ + } + i++ + } + i++ + case c == '\'' || c == '"' || c == '`': + quote := c + i++ + for i < len(s) && s[i] != quote { + if s[i] == '\\' { + i++ + } else if s[i] == '\n' { + line++ + } + i++ + } + case c == '(' || c == '[' || c == '{': + stack = append(stack, c) + openLine[len(stack)] = line + case c == ')' || c == ']' || c == '}': + want := map[byte]byte{')': '(', ']': '[', '}': '{'}[c] + if len(stack) == 0 || stack[len(stack)-1] != want { + return line, false + } + stack = stack[:len(stack)-1] + } + } + if len(stack) != 0 { + return openLine[len(stack)], false + } + return 0, true +} diff --git a/dash/bench.go b/dash/bench.go new file mode 100644 index 0000000..39bdae7 --- /dev/null +++ b/dash/bench.go @@ -0,0 +1,303 @@ +package dash + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// Benchmark ingestion reads the artifacts deploy/harbor/*.py already writes — +// summary.json plus one rows-.json per config — so there is no new export +// format to maintain and every historical run in /tmp/*-runs is already ingestible. +// +// A run is identified by its DIRECTORY NAME, and re-ingesting replaces it. That +// makes ingestion idempotent: pointing the proxy at a jobs root and restarting it +// cannot double-count a run. + +// harborSummary is the subset of summary.json we read. Unknown fields are kept +// verbatim in the stored blob so the UI can show anything the harness reports +// without a schema change here. +type harborSummary struct { + Model string `json:"model"` + Dataset string `json:"dataset"` + Configs []json.RawMessage `json:"configs"` +} + +// harborRow is one task's trial row from rows-.json. +type harborRow struct { + Task string `json:"task"` + Reward float64 `json:"reward"` + Steps int `json:"steps"` + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + CacheRead int64 `json:"cache_read"` + CacheWrite int64 `json:"cache_write"` + FreshInput int64 `json:"fresh_input"` + AgentCost float64 `json:"agent_cost"` + NormCost float64 `json:"norm_cost"` + WallS float64 `json:"wall_s"` + Exception bool `json:"exception"` +} + +// IngestBenchDir ingests one run directory (summary.json + rows-*.json). A +// directory with neither is skipped without error — scanning a jobs root full of +// in-progress runs must not fail. +func (d *DB) IngestBenchDir(dir string) (tasks int, err error) { + name := filepath.Base(strings.TrimRight(dir, string(os.PathSeparator))) + rowFiles, _ := filepath.Glob(filepath.Join(dir, "rows-*.json")) + summaryPath := filepath.Join(dir, "summary.json") + sumBytes, sumErr := os.ReadFile(summaryPath) + if sumErr != nil && len(rowFiles) == 0 { + return 0, nil // not a run directory + } + + var sum harborSummary + if len(sumBytes) > 0 { + if err := json.Unmarshal(sumBytes, &sum); err != nil { + return 0, fmt.Errorf("dash: %s/summary.json: %w", name, err) + } + } else { + sumBytes = []byte("{}") + } + ts := time.Now().UnixMilli() + if fi, err := os.Stat(summaryPath); err == nil { + ts = fi.ModTime().UnixMilli() + } + + tx, err := d.sql.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() //nolint:errcheck // no-op after Commit + + // Replace-on-reingest: delete the old run (cascading its tasks) then insert. + if _, err := tx.Exec(`DELETE FROM bench_runs WHERE name = ?`, name); err != nil { + return 0, err + } + res, err := tx.Exec(`INSERT INTO bench_runs(name, ts, dataset, model, summary) VALUES (?,?,?,?,?)`, + name, ts, sum.Dataset, sum.Model, string(sumBytes)) + if err != nil { + return 0, err + } + runID, err := res.LastInsertId() + if err != nil { + return 0, err + } + stmt, err := tx.Prepare(`INSERT INTO bench_tasks(run_id, arm, task, reward, steps, + prompt_tokens, completion_tokens, cache_read, cache_write, fresh_input, + cost_usd, norm_cost_usd, wall_s, exception) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) + if err != nil { + return 0, err + } + defer stmt.Close() + + sort.Strings(rowFiles) + for _, rf := range rowFiles { + arm := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(rf), "rows-"), ".json") + b, err := os.ReadFile(rf) + if err != nil { + continue + } + var rows []harborRow + if err := json.Unmarshal(b, &rows); err != nil { + continue // a partially-written rows file must not abort the whole ingest + } + for _, r := range rows { + if _, err := stmt.Exec(runID, arm, r.Task, r.Reward, r.Steps, + r.PromptTokens, r.CompletionTokens, r.CacheRead, r.CacheWrite, r.FreshInput, + r.AgentCost, r.NormCost, r.WallS, boolInt(r.Exception)); err != nil { + return 0, err + } + tasks++ + } + } + // Commit the run row ONLY if it actually gained tasks. Committing first and then + // returning tasks=0 is how the log line ("ingested benchmark runs runs=17") came to + // disagree with the API (42 rows, 25 of them with no arms at all): the callers below + // count a run only when tasks>0, but the row was already committed, so the + // Benchmarks tab filled up with contentless shells. The deferred Rollback discards + // it, which also means a directory that stops parsing does not silently replace a + // previously-good run with an empty one. + if tasks == 0 { + return 0, nil + } + if err := tx.Commit(); err != nil { + return 0, err + } + return tasks, nil +} + +// IngestBenchRoots scans each root one level deep for run directories and ingests +// every one it finds. Errors are collected per-directory and do not abort the scan. +func (d *DB) IngestBenchRoots(roots []string) (runs, tasks int) { + for _, root := range roots { + // A root may itself be a run directory. + if n, err := d.IngestBenchDir(root); err == nil && n > 0 { + runs++ + tasks += n + continue + } + entries, err := os.ReadDir(root) + if err != nil { + continue + } + for _, e := range entries { + if !e.IsDir() { + continue + } + n, err := d.IngestBenchDir(filepath.Join(root, e.Name())) + if err != nil || n == 0 { + continue + } + runs++ + tasks += n + } + } + return runs, tasks +} + +// BenchRun is one ingested run as served to the UI. +type BenchRun struct { + ID int64 `json:"id"` + Name string `json:"name"` + TS int64 `json:"ts"` + Dataset string `json:"dataset"` + Model string `json:"model"` + Summary json.RawMessage `json:"summary"` + Arms []BenchArm `json:"arms"` +} + +// BenchArm aggregates one arm (baseline / context-guru / headroom / rtk) of a run. +// This is the cost-vs-reward view: an arm that saves money by failing tasks is not +// saving money, so reward sits beside cost in the same row. +type BenchArm struct { + Arm string `json:"arm"` + Tasks int64 `json:"tasks"` + Scored int64 `json:"scored"` + Solved int64 `json:"solved"` + SolveRate float64 `json:"solve_rate"` + MeanReward float64 `json:"mean_reward"` + MeanSteps float64 `json:"mean_steps"` + TotalCostUSD float64 `json:"total_cost_usd"` + MeanCostUSD float64 `json:"mean_cost_usd"` + TotalNormCostUSD float64 `json:"total_norm_cost_usd"` + CacheRead int64 `json:"cache_read"` + CacheWrite int64 `json:"cache_write"` + FreshInput int64 `json:"fresh_input"` + Completion int64 `json:"completion_tokens"` + CacheHitRate float64 `json:"cache_hit_rate"` + MeanWallS float64 `json:"mean_wall_s"` + Exceptions int64 `json:"exceptions"` +} + +// BenchRuns returns every ingested run with its per-arm aggregates. +func (d *DB) BenchRuns() ([]*BenchRun, error) { + rows, err := d.sql.Query(`SELECT id, name, ts, dataset, model, summary FROM bench_runs ORDER BY ts DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []*BenchRun{} + for rows.Next() { + var r BenchRun + var summary string + if err := rows.Scan(&r.ID, &r.Name, &r.TS, &r.Dataset, &r.Model, &summary); err != nil { + return nil, err + } + r.Summary = json.RawMessage(summary) + out = append(out, &r) + } + if err := rows.Err(); err != nil { + return nil, err + } + for _, r := range out { + arms, err := d.benchArms(r.ID) + if err != nil { + return nil, err + } + r.Arms = arms + } + return out, nil +} + +func (d *DB) benchArms(runID int64) ([]BenchArm, error) { + rows, err := d.sql.Query(`SELECT arm, COUNT(*), + SUM(CASE WHEN exception = 0 THEN 1 ELSE 0 END), + SUM(CASE WHEN reward >= 1 THEN 1 ELSE 0 END), + AVG(reward), AVG(steps), SUM(cost_usd), AVG(cost_usd), SUM(norm_cost_usd), + SUM(cache_read), SUM(cache_write), SUM(fresh_input), SUM(completion_tokens), + AVG(wall_s), SUM(exception) + FROM bench_tasks WHERE run_id = ? GROUP BY arm ORDER BY arm`, runID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []BenchArm + for rows.Next() { + var a BenchArm + if err := rows.Scan(&a.Arm, &a.Tasks, &a.Scored, &a.Solved, &a.MeanReward, &a.MeanSteps, + &a.TotalCostUSD, &a.MeanCostUSD, &a.TotalNormCostUSD, + &a.CacheRead, &a.CacheWrite, &a.FreshInput, &a.Completion, + &a.MeanWallS, &a.Exceptions); err != nil { + return nil, err + } + if a.Scored > 0 { + a.SolveRate = float64(a.Solved) / float64(a.Scored) + } + if total := a.CacheRead + a.CacheWrite + a.FreshInput; total > 0 { + a.CacheHitRate = float64(a.CacheRead) / float64(total) + } + out = append(out, a) + } + return out, rows.Err() +} + +// BenchTask is one task row for the per-task drill-down. +type BenchTask struct { + Arm string `json:"arm"` + Task string `json:"task"` + Reward float64 `json:"reward"` + Steps int64 `json:"steps"` + CacheRead int64 `json:"cache_read"` + CacheWrite int64 `json:"cache_write"` + FreshInput int64 `json:"fresh_input"` + Completion int64 `json:"completion_tokens"` + CostUSD float64 `json:"cost_usd"` + NormCostUSD float64 `json:"norm_cost_usd"` + WallS float64 `json:"wall_s"` + Exception bool `json:"exception"` +} + +// BenchTasks returns a run's task rows, optionally restricted to one arm. +func (d *DB) BenchTasks(runID int64, arm string) ([]*BenchTask, error) { + q := `SELECT arm, task, reward, steps, cache_read, cache_write, fresh_input, + completion_tokens, cost_usd, norm_cost_usd, wall_s, exception + FROM bench_tasks WHERE run_id = ?` + args := []any{runID} + if arm != "" { + q += " AND arm = ?" + args = append(args, arm) + } + q += " ORDER BY task, arm" + rows, err := d.sql.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []*BenchTask{} + for rows.Next() { + var t BenchTask + var exc int + if err := rows.Scan(&t.Arm, &t.Task, &t.Reward, &t.Steps, &t.CacheRead, &t.CacheWrite, + &t.FreshInput, &t.Completion, &t.CostUSD, &t.NormCostUSD, &t.WallS, &exc); err != nil { + return nil, err + } + t.Exception = exc != 0 + out = append(out, &t) + } + return out, rows.Err() +} diff --git a/dash/capture.go b/dash/capture.go new file mode 100644 index 0000000..a2e7cd3 --- /dev/null +++ b/dash/capture.go @@ -0,0 +1,390 @@ +package dash + +import ( + "log/slog" + "sync" + "sync/atomic" + "time" +) + +// Options configures the dashboard. The zero value is usable: an in-memory +// database, content capture on, 7-day / 512 MiB retention, loopback-only access +// to per-request content and effective config. +type Options struct { + // DBPath is the SQLite file. "" or ":memory:" keeps everything in RAM (the + // no-persistence mode), which is also the automatic fallback when the path + // cannot be opened — the proxy must never fail to start over a dashboard. + DBPath string + // Retention bounds the store by age AND size. Zero values use the defaults + // below; a negative value disables that rule. + RetentionAge time.Duration + RetentionBytes int64 + // CaptureContent enables the before/after content capture the diff view needs. + // Opt-OUT: it is the headline feature, so it defaults on. ContentCap bounds each + // captured blob (default 16 KiB); ContentMaxPerRequest bounds how many + // rewritten messages are captured per request (default 24). + CaptureContent bool + ContentCap int + ContentMaxPerRequest int + // QueueSize is the capture channel's depth (default 4096). When it is full, + // events are DROPPED and counted rather than blocking a request. + QueueSize int + // BatchSize / FlushInterval control how the writer batches inserts. + BatchSize int + FlushInterval time.Duration + // TrustedCIDRs are the networks allowed to see per-request CONTENT and the + // effective configuration. Loopback is always allowed. Aggregates are open to + // everyone (a proxy people bind to 0.0.0.0 still wants its numbers visible). + TrustedCIDRs []string + // Mode is the proxy's operating mode ("active" | "observe"), served on /api/capture + // so the UI can render the observe banner. Empty means active. + // + // There is deliberately no Preset field here: per-row preset labelling comes from + // proxy.Options.Preset at the capture site (proxy/dashcapture.go), and a second copy + // of the same value in a second Options struct is a copy that goes stale. + Mode string + // Effective is the resolved, already-structured configuration to serve at + // /api/config. It is redacted before serving; nothing sensitive should be in + // here in the first place. + Effective map[string]any + // BenchDirs are directories scanned for harbor benchmark runs (summary.json + + // rows-*.json) at startup and on demand. + BenchDirs []string +} + +const ( + defaultQueueSize = 4096 + defaultBatchSize = 128 + defaultFlushInterval = 250 * time.Millisecond + defaultRetentionAge = 7 * 24 * time.Hour + defaultRetentionBytes = 512 << 20 + defaultContentCap = 16 << 10 + defaultContentPerReq = 24 + pruneInterval = 5 * time.Minute +) + +func (o *Options) withDefaults() { + if o.QueueSize <= 0 { + o.QueueSize = defaultQueueSize + } + if o.BatchSize <= 0 { + o.BatchSize = defaultBatchSize + } + if o.FlushInterval <= 0 { + o.FlushInterval = defaultFlushInterval + } + if o.RetentionAge == 0 { + o.RetentionAge = defaultRetentionAge + } + if o.RetentionBytes == 0 { + o.RetentionBytes = defaultRetentionBytes + } + if o.ContentCap == 0 { + o.ContentCap = defaultContentCap + } + if o.ContentMaxPerRequest == 0 { + o.ContentMaxPerRequest = defaultContentPerReq + } +} + +// Recorder is the capture pipeline: a buffered channel, one writer goroutine that +// batches inserts, and an SSE hub the writer fans summaries out to. +// +// The contract that matters: Record NEVER blocks and never returns an error. A +// full queue drops the event and increments a counter that the dashboard itself +// displays — an observability layer that silently lies about its own coverage is +// worse than one that admits a gap. This is why the dashboard cannot add request +// latency: the hot path does one channel send with a default branch. +type Recorder struct { + db *DB + opts Options + hub *Hub + + ch chan *Event + done chan struct{} + wg sync.WaitGroup + + captured atomic.Int64 + dropped atomic.Int64 + written atomic.Int64 + errors atomic.Int64 + + // observeQueue is the host's accessor for its off-path pool counters, or nil in + // sync mode. A func rather than a value so the counters are read at serve time. + observeQueue atomic.Pointer[func() QueueStats] + + // Cache-attribution state: the last time we saw each session and whether we + // have seen each model, so a cold start is never reported as a bust. + mu sync.Mutex + lastSeen map[string]int64 // session -> epoch ms of previous request + seenModel map[string]bool + // perComp accumulates unique-savings dedup keys so a per-request unique figure + // exists at capture time. Bounded; see markUnique. + seenKeys map[string]struct{} +} + +// NewRecorder opens the store and starts the writer goroutine. It never returns a +// fatal error for a bad path: an unopenable database degrades to in-memory, with a +// warning, because the proxy's job is to proxy. +func NewRecorder(opts Options) (*Recorder, error) { + opts.withDefaults() + db, err := Open(opts.DBPath) + if err != nil { + slog.Warn("dash: could not open the dashboard database; falling back to in-memory (history will not survive a restart)", + "path", opts.DBPath, "err", err) + db, err = Open(":memory:") + if err != nil { + return nil, err + } + } + r := &Recorder{ + db: db, + opts: opts, + hub: NewHub(), + ch: make(chan *Event, opts.QueueSize), + done: make(chan struct{}), + lastSeen: map[string]int64{}, + seenModel: map[string]bool{}, + seenKeys: map[string]struct{}{}, + } + r.wg.Add(1) + go r.run() + return r, nil +} + +// DB exposes the store for queries (read-only use by the API). +func (r *Recorder) DB() *DB { return r.db } + +// Opts exposes the effective options (read-only). +func (r *Recorder) Opts() Options { return r.opts } + +// Hub exposes the SSE fan-out. +func (r *Recorder) Hub() *Hub { return r.hub } + +// Record hands an event to the writer. It is safe from any goroutine, never +// blocks, and never fails: a full queue drops and counts. Callers must not touch +// the event afterwards — the writer owns it. +func (r *Recorder) Record(e *Event) { + if r == nil || e == nil { + return + } + if e.TS == 0 { + e.TS = time.Now().UnixMilli() + } + r.captured.Add(1) + select { + case r.ch <- e: + default: + r.dropped.Add(1) + } +} + +// Close stops the writer after draining what is already queued. +func (r *Recorder) Close() error { + if r == nil { + return nil + } + close(r.done) + r.wg.Wait() + r.hub.Close() + return r.db.Close() +} + +// Stats reports the capture pipeline's own health — including its drops, which is +// the number that keeps every other number honest. +type Stats struct { + Captured int64 `json:"captured"` + Written int64 `json:"written"` + Dropped int64 `json:"dropped"` + Errors int64 `json:"errors"` + Queued int `json:"queued"` + QueueCap int `json:"queue_cap"` + Clients int `json:"sse_clients"` + DBPath string `json:"db_path"` + DBBytes int64 `json:"db_bytes"` + // Mode is the proxy's operating mode ("active" | "observe"). The UI renders an + // unmissable banner in observe mode: every request was forwarded UNTOUCHED, so a + // reader who mistakes these figures for enforced savings has drawn exactly the wrong + // conclusion. That is worth a banner rather than a field on a detail tab. + Mode string `json:"mode"` + // ObserveQueue is the off-path measurement pool's counters, supplied by the host + // (the pool lives in `modes`, above this package). Omitted when no pool is running, + // so a sync deployment shows no phantom queue. Its `dropped` matters most: a drop is + // an observation given up, so the projection UNDERSTATES what compaction would save. + ObserveQueue *QueueStats `json:"observe_queue,omitempty"` +} + +// QueueStats mirrors metrics.QueueStats / modes.Stats. Declared here rather than +// imported because the dependency runs the other way. +type QueueStats struct { + Queued int64 `json:"queued"` + Pending int64 `json:"pending"` + Processed int64 `json:"processed"` + Dropped int64 `json:"dropped"` + Errors int64 `json:"errors"` +} + +// SetObserveQueue lets the host publish its off-path pool's counters. Safe from any +// goroutine and safe to call on a nil Recorder. +func (r *Recorder) SetObserveQueue(fn func() QueueStats) { + if r == nil { + return + } + r.observeQueue.Store(&fn) +} + +// Stats snapshots the pipeline counters. +func (r *Recorder) Stats() Stats { + if r == nil { + return Stats{} + } + size, _ := r.db.sizeBytes() + s := Stats{ + Captured: r.captured.Load(), Written: r.written.Load(), + Dropped: r.dropped.Load(), Errors: r.errors.Load(), + Queued: len(r.ch), QueueCap: cap(r.ch), + Clients: r.hub.Clients(), DBPath: r.db.Path(), DBBytes: size, + Mode: r.opts.Mode, + } + if s.Mode == "" { + s.Mode = ModeActive + } + if fn := r.observeQueue.Load(); fn != nil { + q := (*fn)() + s.ObserveQueue = &q + } + return s +} + +// run is the single writer goroutine: batch, insert in one transaction, fan out, +// and prune on a timer. Nothing else writes to the database. +func (r *Recorder) run() { + defer r.wg.Done() + batch := make([]*Event, 0, r.opts.BatchSize) + flush := time.NewTicker(r.opts.FlushInterval) + defer flush.Stop() + prune := time.NewTicker(pruneInterval) + defer prune.Stop() + + write := func() { + if len(batch) == 0 { + return + } + // Redact on THIS goroutine, before the insert. This is the expensive half of + // capture (nine regexes over up to ContentMaxPerRequest x 2 blobs) and it lives + // here rather than at the capture site deliberately: `finish` is called from the + // handler's defer, which runs before the handler returns, so redacting there makes + // a keep-alive client's next request wait on it (~53 ms measured, ~25% of a + // request). Nothing reaches the database unredacted either way — the security + // property is the ordering against the INSERT, which this preserves. + for _, e := range batch { + e.Redact() + } + if err := r.db.insertBatch(batch); err != nil { + r.errors.Add(int64(len(batch))) + slog.Warn("dash: dropping a batch of captured requests", "n", len(batch), "err", err) + } else { + r.written.Add(int64(len(batch))) + for _, e := range batch { + r.hub.Publish(e) + } + } + batch = batch[:0] + } + + for { + select { + case e := <-r.ch: + batch = append(batch, e) + if len(batch) >= r.opts.BatchSize { + write() + } + case <-flush.C: + write() + case <-prune.C: + if n, err := r.db.Prune(time.Now(), r.opts.RetentionAge, r.opts.RetentionBytes); err != nil { + slog.Warn("dash: retention prune failed", "err", err) + } else if n > 0 { + slog.Info("dash: pruned old dashboard rows", "requests", n) + } + case <-r.done: + // Drain whatever is queued so a clean shutdown does not lose the tail. + for { + select { + case e := <-r.ch: + batch = append(batch, e) + if len(batch) >= r.opts.BatchSize { + write() + } + continue + default: + } + break + } + write() + return + } + } +} + +// Observe records the session/model facts needed for cache attribution and +// returns them. Called on the request path (one map lookup under a short mutex), +// before Record. +func (r *Recorder) Observe(session, model string, now int64) (seenSession, seenModel bool, sinceLastMs int64) { + if r == nil { + return true, true, 0 + } + r.mu.Lock() + defer r.mu.Unlock() + prev, seenSession := r.lastSeen[session] + if seenSession { + sinceLastMs = now - prev + } + seenModel = r.seenModel[model] + // Bound both maps: a proxy runs for weeks and every distinct session id would + // otherwise be retained forever. Sessions are keyed by content hash or client + // id, so the working set is small; a reset just re-reports a cold start, which + // is honest (we genuinely no longer know). + // ponytail: crude clear-on-overflow; swap for an LRU if session churn ever matters. + if len(r.lastSeen) > 20000 { + r.lastSeen = map[string]int64{} + } + if len(r.seenModel) > 1000 { + r.seenModel = map[string]bool{} + } + r.lastSeen[session] = now + r.seenModel[model] = true + return seenSession, seenModel, sinceLastMs +} + +// MarkUnique attributes a component's savings to NEW content only, deduping by +// the content keys the component stashed — the same rule metrics.Aggregator uses, +// so the dashboard's unique figure and /stats' agree. Returns saved tokens +// attributable to content not seen before. +func (r *Recorder) MarkUnique(component string, keys []string, saved int) int { + if r == nil || saved <= 0 { + return 0 + } + if len(keys) == 0 { + return saved // no key to dedup on: count the run once (Aggregator's rule) + } + r.mu.Lock() + defer r.mu.Unlock() + // ponytail: clear-on-overflow, same reasoning as Observe. Over-reporting a + // repeat as unique after a reset is bounded and visible via overcount_ratio. + if len(r.seenKeys) > 200000 { + r.seenKeys = map[string]struct{}{} + } + newKeys := 0 + for _, k := range keys { + ck := component + "\x00" + k + if _, seen := r.seenKeys[ck]; !seen { + r.seenKeys[ck] = struct{}{} + newKeys++ + } + } + if newKeys == 0 { + return 0 + } + return saved * newKeys / len(keys) +} diff --git a/dash/capture_test.go b/dash/capture_test.go new file mode 100644 index 0000000..d13f391 --- /dev/null +++ b/dash/capture_test.go @@ -0,0 +1,463 @@ +package dash + +import ( + "math" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +// TestCaptureDropsRatherThanBlocks is the property that lets the dashboard exist +// at all: with the queue full and the writer wedged, Record must return +// immediately and count the loss. If this ever blocks, enabling observability +// becomes a latency incident. +func TestCaptureDropsRatherThanBlocks(t *testing.T) { + r, err := NewRecorder(Options{DBPath: ":memory:", QueueSize: 4, + BatchSize: 1000, FlushInterval: time.Hour}) // writer will not drain + if err != nil { + t.Fatal(err) + } + defer r.Close() + + const n = 500 + done := make(chan struct{}) + go func() { + for i := 0; i < n; i++ { + r.Record(&Event{SessionID: "s"}) + } + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Record blocked on a full queue; capture must never block a request") + } + + s := r.Stats() + if s.Captured != n { + t.Errorf("captured = %d; want %d", s.Captured, n) + } + if s.Dropped == 0 { + t.Fatal("a full queue dropped nothing; either the drop is not counted or the queue grew") + } + // With a depth-4 queue and a writer that never flushes, the overwhelming + // majority must be dropped — proving the queue is genuinely bounded and not + // quietly growing to absorb the burst. + if s.Dropped < int64(n)/2 { + t.Errorf("only %d of %d dropped through a depth-%d queue; is the queue unbounded?", + s.Dropped, n, s.QueueCap) + } + // The drop must be VISIBLE, not merely survived — every other number depends on + // the viewer knowing coverage was incomplete. + if s.QueueCap != 4 { + t.Errorf("queue cap = %d; want the configured 4", s.QueueCap) + } +} + +func TestCloseDrainsQueuedEvents(t *testing.T) { + path := filepath.Join(t.TempDir(), "d.db") + r, err := NewRecorder(Options{DBPath: path, QueueSize: 1024, + BatchSize: 10000, FlushInterval: time.Hour}) // nothing flushes until Close + if err != nil { + t.Fatal(err) + } + for i := 0; i < 25; i++ { + r.Record(mkEvent(int64(1000+i), "s", "m", 100, 90)) + } + if err := r.Close(); err != nil { + t.Fatal(err) + } + db, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + page, err := db.Requests(Filter{}, 0, 100) + if err != nil { + t.Fatal(err) + } + if page.Total != 25 { + t.Errorf("Close persisted %d of 25 queued events", page.Total) + } +} + +func TestObserveDetectsColdStartOncePerSessionAndModel(t *testing.T) { + r, err := NewRecorder(Options{DBPath: ":memory:"}) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + seenSess, seenModel, since := r.Observe("s1", "m1", 1000) + if seenSess || seenModel { + t.Error("first request for a session+model must report both unseen (cold start)") + } + if since != 0 { + t.Errorf("since = %d on a first request; want 0", since) + } + seenSess, seenModel, since = r.Observe("s1", "m1", 4000) + if !seenSess || !seenModel { + t.Error("second request must report the session and model as seen") + } + if since != 3000 { + t.Errorf("since = %d; want 3000", since) + } + // A NEW session on an already-seen model is still a session cold start. + if s, m, _ := r.Observe("s2", "m1", 5000); s || !m { + t.Errorf("new session on known model: seenSession=%v seenModel=%v; want false,true", s, m) + } +} + +func TestAttributeCacheBuckets(t *testing.T) { + const ttl = 300_000 + cases := []struct { + name string + read int64 + seenSession, seenModel bool + sinceMs int64 + prefixChanged bool + want string + }{ + {"a cache read is a hit", 5000, true, true, 1000, false, CacheHit}, + {"first request of a session is a cold start", 0, false, true, 0, true, CacheColdStart}, + {"first request for a model is a cold start", 0, true, false, 0, true, CacheColdStart}, + {"a gap past the TTL is expiry", 0, true, true, ttl + 1, false, CacheTTLExpiry}, + {"TTL wins the tie against a changed prefix", 0, true, true, ttl + 1, true, CacheTTLExpiry}, + {"a changed prefix inside the TTL is a bust", 0, true, true, 1000, true, CachePrefixChange}, + {"otherwise unknown", 0, true, true, 1000, false, CacheUnknown}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + e := &Event{CacheRead: tc.read} + e.AttributeCache(tc.seenSession, tc.seenModel, tc.sinceMs, ttl, tc.prefixChanged) + if e.CacheMissReason != tc.want { + t.Errorf("got %q, want %q", e.CacheMissReason, tc.want) + } + }) + } +} + +func TestMarkUniqueDedupsByContentKey(t *testing.T) { + r, err := NewRecorder(Options{DBPath: ":memory:"}) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + // First sighting: all of it is new. + if got := r.MarkUnique("extract", []string{"k1", "k2"}, 200); got != 200 { + t.Errorf("first sighting = %d; want 200", got) + } + // Same compaction re-sent on the next turn: nothing new. + if got := r.MarkUnique("extract", []string{"k1", "k2"}, 200); got != 0 { + t.Errorf("re-sent compaction = %d; want 0 (this is what stops gross from lying)", got) + } + // Half new: proportional attribution. + if got := r.MarkUnique("extract", []string{"k2", "k3"}, 200); got != 100 { + t.Errorf("half-new = %d; want 100", got) + } + // Keys are namespaced per component, so two components stashing the same content + // each get credit for their own work. + if got := r.MarkUnique("dedup", []string{"k1"}, 50); got != 50 { + t.Errorf("other component = %d; want 50 (keys must be per-component)", got) + } + // No keys at all: count the run once (the Aggregator's rule). + if got := r.MarkUnique("collapse", nil, 70); got != 70 { + t.Errorf("keyless run = %d; want 70", got) + } +} + +func TestFromTraceMapsPipelineOutcome(t *testing.T) { + tr := apply.Trace{ + Session: "s1", CacheAware: true, MaxCachedIdx: 3, Messages: 8, + AttemptedTokens: 400, FrozenTokens: 600, + Run: &components.RunReport{ + TokensBefore: 1000, TokensAfter: 800, + Components: []components.Report{ + {Component: "extract", Kind: "offload", TokensBefore: 1000, TokensAfter: 800}, + {Component: "dedup", Kind: "offload", TokensBefore: 800, TokensAfter: 800, Skipped: true}, + {Component: "boom", Kind: "offload", TokensBefore: 800, TokensAfter: 800, Reverted: true}, + }, + }, + Changes: []apply.Change{{Path: "messages.4", BeforeTokens: 200, AfterTokens: 0, + Before: "big output", After: ""}}, + } + var e Event + e.FromTrace(tr, map[string]int{"extract": 150}) + + if e.SessionID != "s1" || !e.CacheAware || e.Messages != 8 { + t.Errorf("header fields wrong: %+v", e) + } + if e.TokensBefore != 1000 || e.TokensAfter != 800 || e.Saved() != 200 { + t.Errorf("token accounting wrong: %d -> %d", e.TokensBefore, e.TokensAfter) + } + if e.AttemptedTokens != 400 || e.FrozenTokens != 600 { + t.Errorf("eligibility wrong: attempted=%d frozen=%d", e.AttemptedTokens, e.FrozenTokens) + } + if e.SavedUnique != 150 { + t.Errorf("unique saved = %d; want the deduped 150, not the gross 200", e.SavedUnique) + } + if e.Reverts != 1 { + t.Errorf("reverts = %d; want 1", e.Reverts) + } + if len(e.Components) != 3 { + t.Fatalf("component rows = %d; want 3", len(e.Components)) + } + if !e.Components[0].Acted || !e.Components[0].Mutated { + t.Error("the acting component was not marked acted+mutated") + } + if e.Components[1].Mutated || !e.Components[1].Skipped { + t.Error("a skipped component must not be marked mutated") + } + if e.Components[2].Mutated { + t.Error("a reverted component must not be marked mutated") + } + if len(e.Content) != 1 || e.Content[0].Path != "messages.4" { + t.Errorf("content not carried: %+v", e.Content) + } + if e.UncompressedReason != "" { + t.Errorf("a request that saved tokens must have no uncompressed reason, got %q", e.UncompressedReason) + } +} + +func TestUncompressedReasonBuckets(t *testing.T) { + cases := []struct { + name string + tr apply.Trace + want string + }{ + {"bypassed", apply.Trace{Bypassed: true}, ReasonBypassed}, + {"no run", apply.Trace{}, ReasonNoMessages}, + {"no messages", apply.Trace{Messages: 0, Run: &components.RunReport{}}, ReasonNoMessages}, + {"nothing triggered", apply.Trace{Messages: 3, Run: &components.RunReport{ + TokensBefore: 100, TokensAfter: 100, + Components: []components.Report{{Component: "extract", Skipped: true}}, + }}, ReasonBelowTrigger}, + {"all frozen", apply.Trace{Messages: 3, CacheAware: true, AttemptedTokens: 0, + Run: &components.RunReport{TokensBefore: 100, TokensAfter: 100, + Components: []components.Report{{Component: "extract"}}}}, ReasonAllFrozen}, + {"all reverted", apply.Trace{Messages: 3, Run: &components.RunReport{ + TokensBefore: 100, TokensAfter: 100, + Components: []components.Report{{Component: "extract", Reverted: true}}, + }}, ReasonReverted}, + {"ran, found nothing", apply.Trace{Messages: 3, AttemptedTokens: 100, + Run: &components.RunReport{TokensBefore: 100, TokensAfter: 100, + Components: []components.Report{{Component: "extract"}}}}, ReasonNoSavings}, + {"compacted", apply.Trace{Messages: 3, AttemptedTokens: 100, + Run: &components.RunReport{TokensBefore: 100, TokensAfter: 50, + Components: []components.Report{{Component: "extract", TokensBefore: 100, TokensAfter: 50}}}}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var e Event + e.FromTrace(tc.tr, nil) + if e.UncompressedReason != tc.want { + t.Errorf("reason = %q; want %q", e.UncompressedReason, tc.want) + } + }) + } +} + +func TestPriceNeverReportsUnknownCostAsFree(t *testing.T) { + p := modelinfo.Price{Input: 2e-06, Output: 1e-05, CacheRead: 2e-07, CacheWrite: 2.5e-06} + + // Complete accounting: cost, plus a baseline that prices the UNIQUE removed + // tokens at the cache-WRITE rate they would have entered as and the re-sent + // remainder at the cache-READ rate the provider would have served it from. Here + // every removed token is unique, so the whole 200 gets the write rate. + e := &Event{TokensBefore: 1000, TokensAfter: 800, SavedUnique: 200, FreshInput: 10, + CacheRead: 5000, CacheWrite: 500, OutputTokens: 100} + e.Price(p, true) + if e.TokenAccounting != AccountingComplete { + t.Errorf("accounting = %q; want complete", e.TokenAccounting) + } + wantCost := p.Cost(10, 5000, 500, 100) + if e.CostUSD != wantCost { + t.Errorf("cost = %v; want %v", e.CostUSD, wantCost) + } + if e.BaselineCostUSD != wantCost+200*p.CacheWrite { + t.Errorf("baseline = %v; want cost + 200 unique removed tokens at the cache-write rate", e.BaselineCostUSD) + } + if e.BaselineCostUSD <= e.CostUSD { + t.Error("baseline must exceed actual when tokens were removed") + } + + // No usage data: partial, and NOT priced — a cost we cannot compute must read as + // unknown, never as zero. + e2 := &Event{TokensBefore: 1000, TokensAfter: 800} + e2.Price(p, false) + if e2.TokenAccounting != AccountingPartial { + t.Errorf("accounting = %q; want partial", e2.TokenAccounting) + } + if e2.CostUSD != 0 || e2.BaselineCostUSD != 0 { + t.Error("an unpriceable request must leave costs at zero AND be flagged, not be priced") + } + + // No pricing table and no content either: missing. + e3 := &Event{} + e3.Price(modelinfo.Price{}, true) + if e3.TokenAccounting != AccountingMissing { + t.Errorf("accounting = %q; want missing", e3.TokenAccounting) + } +} + +// TestDollarsDeriveFromUniqueNotGrossSavings is the regression test for the defect +// this dashboard shipped with: `net_dollars_saved` was priced off GROSS savings, so a +// single compaction re-sent on every later turn was paid for once per turn. The tile +// read $7.00 beside an `overcount_ratio` of 13.1x — the dashboard displayed the +// correction factor for its own headline and did not apply it. +// +// The fixture is DELIBERATELY overcounted: one 1,000-token compaction, unique on turn +// 1, re-sent unchanged on nine further turns. Gross savings are 10,000 tokens; +// genuinely-never-sent content is 1,000. Both pricing bugs are pinned: +// +// - the denominator (unique, not gross), and +// - the tier (the re-sent remainder is a cache READ, not a cache WRITE; on this +// price table a write is 12.5x a read, so mispricing it inflates on top of the +// overcount). +// +// A gross+write implementation reports ~11.4x the correct figure here and fails. +func TestDollarsDeriveFromUniqueNotGrossSavings(t *testing.T) { + p := modelinfo.Price{Input: 2e-06, Output: 1e-05, CacheRead: 2e-07, CacheWrite: 2.5e-06} + const turns, saved, uniqueTurn = 10, 1000, 0 + + db := openTestDB(t) + var events []*Event + for i := range turns { + e := &Event{ + TS: int64(1000 + i), SessionID: "s1", Model: "m", + TokensBefore: 50_000, TokensAfter: 50_000 - saved, + // Unique only on the first turn: every later turn re-sends the same content, + // which is exactly what MarkUnique's dedup reports. + FreshInput: 10, CacheRead: 49_000, CacheWrite: 100, OutputTokens: 50, + } + if i == uniqueTurn { + e.SavedUnique = saved + } + e.Price(p, true) + events = append(events, e) + } + if err := db.insertBatch(events); err != nil { + t.Fatal(err) + } + + o, err := db.Overview(Filter{}) + if err != nil { + t.Fatal(err) + } + + // The fixture's own overcount factor, as the dashboard computes and displays it. + if o.SavedGross != turns*saved || o.SavedUnique != saved { + t.Fatalf("fixture: gross=%d unique=%d; want %d/%d", o.SavedGross, o.SavedUnique, turns*saved, saved) + } + if o.OvercountRatio != float64(turns) { + t.Fatalf("overcount_ratio = %v; want %v", o.OvercountRatio, float64(turns)) + } + + // The dollar figure the tile renders, from first principles: the unique 1,000 + // tokens would have been new input (cache-write), the 9,000 re-sent ones would + // have been served as cache reads. + wantDelta := float64(saved)*p.CacheWrite + float64((turns-1)*saved)*p.CacheRead + if got := o.BaselineCostUSD - o.CostUSD; math.Abs(got-wantDelta) > 1e-12 { + t.Errorf("baseline − actual = %.10f; want %.10f", got, wantDelta) + } + if math.Abs(o.NetSavedUSD-wantDelta) > 1e-12 { + t.Errorf("net_saved_usd = %.10f; want %.10f", o.NetSavedUSD, wantDelta) + } + + // And the explicit anti-regression: the figure the old code produced. Pricing all + // 10,000 gross tokens at the cache-write rate is 11.36x the honest number, so a + // reversion cannot slip through as a rounding difference. + grossWritePriced := float64(turns*saved) * p.CacheWrite + if math.Abs(o.NetSavedUSD-grossWritePriced) < 1e-9 { + t.Fatal("net_saved_usd still prices GROSS savings at the cache-write rate") + } + if o.NetSavedUSD >= grossWritePriced { + t.Errorf("net_saved_usd %.10f should be far below the gross-priced %.10f", + o.NetSavedUSD, grossWritePriced) + } +} + +// TestBaselineNeverPricesMoreThanTheRequestSaved guards the clamp: SavedUnique is +// attributed per component and can exceed a single request's own gross saving when +// several components stash the same content key. Without the clamp the "re-sent +// remainder" term goes negative and the baseline inflates. +func TestBaselineNeverPricesMoreThanTheRequestSaved(t *testing.T) { + p := modelinfo.Price{CacheRead: 2e-07, CacheWrite: 2.5e-06} + e := &Event{TokensBefore: 1000, TokensAfter: 900, SavedUnique: 5000} + e.Price(p, true) + if want := 100 * p.CacheWrite; math.Abs(e.BaselineCostUSD-e.CostUSD-want) > 1e-12 { + t.Errorf("baseline delta = %.12f; want %.12f (clamped to the 100 tokens actually removed)", + e.BaselineCostUSD-e.CostUSD, want) + } +} + +func TestAgentClassification(t *testing.T) { + cases := map[string]string{ + "claude-cli/2.0.14 (external, cli)": "claude-cli", + "claude-code/1.2.3": "claude-code", + "codex_cli_rs/0.4.0": "codex", + "Gemini-CLI/1.0": "gemini-cli", + "": "unknown", + "curl/8.5.0": "curl", + "SomeVeryLongUnknownAgentNameWithNoDelimiterAtAllThatKeepsGoingForever": "someverylongunknownagentnamewith", + } + for ua, want := range cases { + if got := AgentFor(ua); got != want { + t.Errorf("AgentFor(%q) = %q; want %q", ua, got, want) + } + } +} + +// TestConcurrentCaptureIsRaceFree drives every shared structure at once. Run under +// -race, this is the mandatory check on the capture path. +func TestConcurrentCaptureIsRaceFree(t *testing.T) { + r, err := NewRecorder(Options{DBPath: ":memory:", QueueSize: 256, BatchSize: 16, + FlushInterval: 2 * time.Millisecond, CaptureContent: true, ContentCap: 1024}) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for g := 0; g < 8; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < 200; i++ { + e := mkEvent(int64(1000+i), "sess", "model", 1000, 900) + e.Content = []ContentRow{{Path: "messages.1", Before: "before", After: "after"}} + r.Observe(e.SessionID, e.Model, e.TS) + r.MarkUnique("extract", []string{"k1", "k2"}, 100) + r.Record(e) + if i%20 == 0 { + _ = r.Stats() + } + } + }(g) + } + // Concurrent readers, racing the writer's commits. + for g := 0; g < 3; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 60; i++ { + if _, err := r.DB().Overview(Filter{}); err != nil { + t.Errorf("Overview during concurrent writes: %v", err) + return + } + if _, err := r.DB().Requests(Filter{}, 0, 10); err != nil { + t.Errorf("Requests during concurrent writes: %v", err) + return + } + } + }() + } + wg.Wait() + if err := r.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/dash/event.go b/dash/event.go new file mode 100644 index 0000000..15d3993 --- /dev/null +++ b/dash/event.go @@ -0,0 +1,338 @@ +package dash + +import ( + "strings" + + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +// Token-accounting honesty levels. A request is only `complete` when the provider +// told us all four token tiers; `partial` means we have content-token counts but +// no billed usage (so cost is an estimate at best); `missing` means we have +// neither. The UI must never render a partial row as exact — that is how a +// dashboard becomes unfalsifiable. +const ( + AccountingComplete = "complete" + AccountingPartial = "partial" + AccountingMissing = "missing" +) + +// Cache-miss attribution buckets. cold_start is NOT a failure: the first request +// of a session, or the first for a given model, has nothing to hit. TTL wins ties +// against prefix_change — a prefix that changed after the cache had already +// expired was not the cause. +const ( + CacheHit = "hit" + CacheColdStart = "cold_start" + CacheTTLExpiry = "ttl_expiry" + CachePrefixChange = "prefix_change" + CacheUnknown = "unknown" +) + +// "Why didn't you compact this?" — a first-class reason bucket, not an absence of +// data. An empty string means we DID compact. +const ( + ReasonBypassed = "bypassed" // x-context-guru-bypass on this request + ReasonNoMessages = "no_messages" // nothing to operate on + ReasonBelowTrigger = "below_trigger" // every component's trigger declined + ReasonAllFrozen = "cache_frozen" // eligible tail was empty (cache safety) + ReasonNoSavings = "found_nothing" // components ran, found nothing to remove + ReasonReverted = "reverted" // components acted but were all reverted +) + +// Operating modes, as rendered in the UI. +const ( + ModeActive = "active" + ModeBypass = "bypass" + ModeObserve = "observe" +) + +// Event is one captured request, as handed to the capture channel. It is built on +// the request goroutine from values the request path already computed (no extra +// token counting, no extra allocation of the transcript) and is then owned +// entirely by the writer goroutine. +type Event struct { + ID int64 `json:"id"` + TS int64 `json:"ts"` // epoch ms + SessionID string `json:"session_id"` + Model string `json:"model"` + Provider string `json:"provider"` + Agent string `json:"agent"` + Preset string `json:"preset"` + Mode string `json:"mode"` + Route string `json:"route"` + Status int `json:"status"` + + Bypassed bool `json:"bypassed"` + CacheAware bool `json:"cache_aware"` + Messages int `json:"messages"` + + TokensBefore int `json:"tokens_before"` + TokensAfter int `json:"tokens_after"` + AttemptedTokens int `json:"attempted_tokens"` + FrozenTokens int `json:"frozen_tokens"` + SavedUnique int `json:"saved_unique"` + + FreshInput int64 `json:"fresh_input"` + CacheRead int64 `json:"cache_read"` + CacheWrite int64 `json:"cache_write"` + OutputTokens int64 `json:"output_tokens"` + + CostUSD float64 `json:"cost_usd"` + BaselineCostUSD float64 `json:"baseline_cost_usd"` + CGLLMCostUSD float64 `json:"cg_llm_cost_usd"` + CGLatencyMs float64 `json:"cg_latency_ms"` + UpstreamMs float64 `json:"upstream_ms"` + + Expands int `json:"expands"` + ExpandTokens int `json:"expand_tokens"` + Reverts int `json:"reverts"` + + TokenAccounting string `json:"token_accounting"` + CacheMissReason string `json:"cache_miss_reason"` + UncompressedReason string `json:"uncompressed_reason"` + + Components []CompRow `json:"components,omitempty"` + Content []ContentRow `json:"content,omitempty"` + + // ContentCap is the per-blob byte cap Redact applies. Set at the capture site and + // consumed by the writer goroutine; not persisted (a knob, not a fact about the + // request). + ContentCap int `json:"-"` +} + +// Redact scrubs credential shapes from captured content and applies the size cap. The +// WRITER goroutine calls it immediately before the INSERT — never the request +// goroutine, where nine regexes over dozens of 16 KiB blobs cost ~53 ms, paid by the +// next request on a keep-alive connection. +// +// The placement is the whole security property: redaction happens before anything +// reaches the database, so a secret is never on disk and there is no redact-on-read +// filter to forget. Running it here rather than at the capture site changes WHICH +// GOROUTINE pays, not whether it runs. +// +// Idempotent, so a double call cannot corrupt a row. +func (e *Event) Redact() { + for i := range e.Content { + e.Content[i].Before = RedactContent(e.Content[i].Before, e.ContentCap) + e.Content[i].After = RedactContent(e.Content[i].After, e.ContentCap) + } +} + +// Saved is this request's gross content-token saving. +func (e *Event) Saved() int { + if e.TokensAfter > e.TokensBefore { + return 0 + } + return e.TokensBefore - e.TokensAfter +} + +// CompRow is one component's accounting on one request. +type CompRow struct { + Component string `json:"component"` + Kind string `json:"kind"` + Acted bool `json:"acted"` + Mutated bool `json:"mutated"` + Reverted bool `json:"reverted"` + Skipped bool `json:"skipped"` + SavedGross int `json:"saved_gross"` + SavedUnique int `json:"saved_unique"` + DurationMs float64 `json:"duration_ms"` + Err string `json:"err,omitempty"` +} + +// ContentRow is one rewritten message's before/after text (already redacted and +// size-capped by the caller). +type ContentRow struct { + Path string `json:"path"` + BeforeTokens int `json:"before_tokens"` + AfterTokens int `json:"after_tokens"` + Before string `json:"before,omitempty"` + After string `json:"after,omitempty"` +} + +// FromTrace fills the pipeline-derived half of an Event from an apply.Trace. +// Usage/cost/latency come from the response and are filled by the caller. +func (e *Event) FromTrace(tr apply.Trace, uniqueSaved map[string]int) { + e.SessionID = tr.Session + e.Bypassed = tr.Bypassed + e.CacheAware = tr.CacheAware + e.Messages = tr.Messages + e.AttemptedTokens = tr.AttemptedTokens + e.FrozenTokens = tr.FrozenTokens + if tr.Bypassed { + e.Mode = ModeBypass + } else if e.Mode == "" { + e.Mode = ModeActive + } + if tr.Run != nil { + e.TokensBefore, e.TokensAfter = tr.Run.TokensBefore, tr.Run.TokensAfter + for _, r := range tr.Run.Components { + row := CompRow{ + Component: r.Component, + Kind: r.Kind, + Reverted: r.Reverted, + Skipped: r.Skipped, + SavedGross: r.Saved(), + DurationMs: r.DurationMs, + } + row.Mutated = !r.Reverted && !r.Skipped + row.Acted = row.Mutated && row.SavedGross > 0 + if u, ok := uniqueSaved[r.Component]; ok { + row.SavedUnique = u + } + if r.Err != nil { + row.Err = r.Err.Error() + } + if r.Reverted { + e.Reverts++ + } + e.SavedUnique += row.SavedUnique + e.Components = append(e.Components, row) + } + } + for _, c := range tr.Changes { + e.Content = append(e.Content, ContentRow{ + Path: c.Path, BeforeTokens: c.BeforeTokens, AfterTokens: c.AfterTokens, + Before: c.Before, After: c.After, + }) + } + e.UncompressedReason = uncompressedReason(e, tr) +} + +// uncompressedReason answers "why didn't you compact this?" from what the trace +// shows. Empty means we did compact. +func uncompressedReason(e *Event, tr apply.Trace) string { + if tr.Bypassed { + return ReasonBypassed + } + if tr.Run == nil || tr.Messages == 0 { + return ReasonNoMessages + } + if e.Saved() > 0 { + return "" + } + if e.Reverts > 0 && e.Reverts == len(e.Components) { + return ReasonReverted + } + if tr.CacheAware && tr.AttemptedTokens == 0 && tr.Run.TokensBefore > 0 { + return ReasonAllFrozen + } + acted := 0 + for _, c := range e.Components { + if c.Mutated { + acted++ + } + } + if acted == 0 { + return ReasonBelowTrigger + } + return ReasonNoSavings +} + +// Price fills the cost columns AT WRITE TIME, from this request's four billed +// token tiers plus a baseline counterfactual. +// +// The baseline is what the SAME request would have cost had context-guru not +// removed anything. Getting this right is the whole point of the dashboard, and +// there are two ways to get it wrong, both of which inflate it: +// +// - Pricing GROSS savings. `Saved()` is tokens_before − tokens_after for THIS +// turn, and the agent re-sends its whole transcript every turn, so the same +// compaction is re-counted once per remaining turn. On a real 63-request +// window that is a 13.1x overcount — a factor this dashboard computes and +// displays as `overcount_ratio` right beside the dollar figure. Only +// SavedUnique is content that genuinely never reached the provider. +// - Pricing everything at the cache-WRITE rate. That rate (11.5x a read) is +// right for content entering the prompt for the first time. The re-sent +// remainder would have been served from the provider's cache, so the most it +// could have been billed at is the cache-READ rate. Pricing it as a write +// multiplies the overcount by another ~11.5. +// +// So: unique savings at the write rate, the re-sent remainder at the read rate. +// Restored (expanded) content is content we removed and then had to serve back, +// so it is added to the ACTUAL cost side, never subtracted from baseline. +// +// accountingComplete=false leaves every cost at zero and the row is marked +// partial/missing: a cost we cannot compute must read as unknown, not as free. +func (e *Event) Price(p modelinfo.Price, accountingComplete bool) { + if !accountingComplete || p.Zero() { + if e.TokensBefore > 0 { + e.TokenAccounting = AccountingPartial + } else { + e.TokenAccounting = AccountingMissing + } + return + } + e.TokenAccounting = AccountingComplete + e.CostUSD = p.Cost(e.FreshInput, e.CacheRead, e.CacheWrite, e.OutputTokens) + e.BaselineCostUSD = e.CostUSD + e.baselineDeltaUSD(p) +} + +// baselineDeltaUSD is what the removed content would have cost had it been sent: +// the unique part as new input (cache-write rate), the re-sent remainder as a +// cache read. +func (e *Event) baselineDeltaUSD(p modelinfo.Price) float64 { + unique := e.SavedUnique + gross := e.Saved() + // SavedUnique is attributed per component and can exceed the request's own + // gross saving when several components stash the same content key; clamp it, so + // the repeat term can never go negative and inflate the baseline. + if unique > gross { + unique = gross + } + if unique < 0 { + unique = 0 + } + return float64(unique)*p.CacheWrite + float64(gross-unique)*p.CacheRead +} + +// AttributeCache buckets this request's cache behavior. seenSession/seenModel say +// whether we have already seen a request for this session / this model — the +// first of either is a COLD START, which is not a failure and must never be +// reported as a bust (headroom's model-aware rule). TTL wins ties: a prefix that +// changed after the entry had already expired was not the cause. +func (e *Event) AttributeCache(seenSession, seenModel bool, sinceLastMs int64, ttlMs int64, prefixChanged bool) { + switch { + case e.CacheRead > 0: + e.CacheMissReason = CacheHit + case !seenSession || !seenModel: + e.CacheMissReason = CacheColdStart + case ttlMs > 0 && sinceLastMs > ttlMs: + e.CacheMissReason = CacheTTLExpiry + case prefixChanged: + e.CachePrefixChangeReason() + default: + e.CacheMissReason = CacheUnknown + } +} + +// CachePrefixChangeReason marks the miss as caused by a changed prefix. +func (e *Event) CachePrefixChangeReason() { e.CacheMissReason = CachePrefixChange } + +// AgentFor classifies a client User-Agent into an agent family so the dashboard +// can filter by application. +func AgentFor(ua string) string { return agentFromUserAgent(ua) } + +// agentFromUserAgent classifies the client into an agent family so the dashboard +// can filter by application. Unknown clients keep their raw first token rather +// than being lumped into "other" — a filter is useless if everything is "other". +func agentFromUserAgent(ua string) string { + l := strings.ToLower(ua) + for _, known := range []string{"claude-code", "claude-cli", "codex", "cursor", "cline", "aider", "gemini-cli", "bob"} { + if strings.Contains(l, known) { + return known + } + } + if l == "" { + return "unknown" + } + if i := strings.IndexAny(l, "/ "); i > 0 { + return l[:i] + } + if len(l) > 32 { + return l[:32] + } + return l +} diff --git a/dash/overhead_test.go b/dash/overhead_test.go new file mode 100644 index 0000000..801e5a9 --- /dev/null +++ b/dash/overhead_test.go @@ -0,0 +1,113 @@ +package dash + +import ( + "fmt" + "testing" + "time" +) + +// BenchmarkRecord measures the ENTIRE cost the dashboard adds to a request +// goroutine: stamp the timestamp and one non-blocking channel send. This is the +// number the issue demands be reported — a tool that sells latency awareness +// cannot pay measurable latency for its own dashboard. +// +// The writer goroutine (redaction, gzip, SQLite insert, SSE fan-out) runs +// concurrently and is deliberately NOT in this measurement, because it is not on +// the request path. BenchmarkWriterThroughput covers that instead. +func BenchmarkRecord(b *testing.B) { + r, err := NewRecorder(Options{DBPath: ":memory:", QueueSize: 1 << 16}) + if err != nil { + b.Fatal(err) + } + defer r.Close() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + r.Record(&Event{SessionID: "s", Model: "m", TokensBefore: 1000, TokensAfter: 900}) + } +} + +// BenchmarkRecordFullQueue is the pathological case: the queue is full, so every +// call takes the drop branch. This path MUST stay O(1) — if a full queue could +// block, an overloaded dashboard would become a latency incident. +func BenchmarkRecordFullQueue(b *testing.B) { + r, err := NewRecorder(Options{DBPath: ":memory:", QueueSize: 1}) + if err != nil { + b.Fatal(err) + } + defer r.Close() + for i := 0; i < 64; i++ { + r.Record(&Event{}) // fill it + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + r.Record(&Event{SessionID: "s"}) + } +} + +// BenchmarkObserve measures the cache-attribution bookkeeping, the one other thing +// the request goroutine does (a map lookup under a short mutex). +func BenchmarkObserve(b *testing.B) { + r, err := NewRecorder(Options{DBPath: ":memory:"}) + if err != nil { + b.Fatal(err) + } + defer r.Close() + b.ResetTimer() + for i := 0; i < b.N; i++ { + r.Observe("session", "model", int64(i)) + } +} + +// TestCaptureOverheadIsNegligible asserts the measured per-request cost stays +// inside a budget generous enough not to flake on a loaded CI box but far below +// anything a caller could perceive. It is a REGRESSION guard: if someone later +// moves I/O onto the capture path, this fails. +func TestCaptureOverheadIsNegligible(t *testing.T) { + r, err := NewRecorder(Options{DBPath: ":memory:", QueueSize: 1 << 16}) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + const n = 20000 + evs := make([]*Event, n) + for i := range evs { + evs[i] = &Event{SessionID: "s", Model: "m", TokensBefore: 1000, TokensAfter: 900} + } + start := time.Now() + for _, e := range evs { + r.Record(e) + } + per := time.Since(start) / n + + // 50us is orders of magnitude above what the operation actually costs and a + // rounding error against this workload's multi-second upstream round trip, so the + // assertion catches a design regression rather than scheduler noise. + if per > 50*time.Microsecond { + t.Errorf("Record() costs %v per request; a capture on the hot path must be negligible", per) + } + t.Logf("capture overhead: %v per request (%d events)", per, n) +} + +// BenchmarkWriterThroughput measures how fast the writer drains the queue — +// off-path, but it must outrun realistic traffic or the queue fills and drops. +func BenchmarkWriterThroughput(b *testing.B) { + r, err := NewRecorder(Options{DBPath: ":memory:", QueueSize: 1 << 16, BatchSize: 256, + FlushInterval: 5 * time.Millisecond}) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + e := &Event{SessionID: fmt.Sprintf("s%d", i%64), Model: "m", TokensBefore: 1000, TokensAfter: 900} + e.Components = []CompRow{{Component: "extract", SavedGross: 100, SavedUnique: 100}} + r.Record(e) + } + r.Close() // drains + b.StopTimer() + if d := r.Stats().Dropped; d > 0 { + b.Logf("dropped %d of %d (the queue could not keep up at this rate)", d, b.N) + } +} diff --git a/dash/overview.go b/dash/overview.go new file mode 100644 index 0000000..577eaa1 --- /dev/null +++ b/dash/overview.go @@ -0,0 +1,306 @@ +package dash + +import "database/sql" + +// Denominator is one labelled savings ratio. Shipping a single "savings %" is the +// mistake both reference implementations make in different ways: a whole-request +// ratio recounts the transcript every turn, so a 200-turn session reads ~0% no +// matter how well compaction performed, while a compressible-only ratio flatters +// by excluding everything we chose not to touch. Both are true. Neither is "the" +// number. So each ratio ships with the denominator it divides by, in words. +type Denominator struct { + Key string `json:"key"` + Label string `json:"label"` + Numerator int64 `json:"numerator"` + Denominator int64 `json:"denominator"` + Percent float64 `json:"percent"` + // Description states exactly what this ratio divides by and when to trust it. + Description string `json:"description"` + // Available is false when the inputs were missing; the UI must then show "n/a" + // rather than 0%, and never a ratio computed by dividing savings by themselves. + Available bool `json:"available"` +} + +func denom(key, label string, num, den int64, desc string) Denominator { + d := Denominator{Key: key, Label: label, Numerator: num, Denominator: den, Description: desc} + if den > 0 { + d.Percent = float64(num) / float64(den) * 100 + d.Available = true + } + return d +} + +// WaterfallStep is one bar of the honest-savings waterfall: baseline cost, the +// savings that reduced it, the penalties that gave some back, and the net. +type WaterfallStep struct { + Key string `json:"key"` + Label string `json:"label"` + DeltaUSD float64 `json:"delta_usd"` // signed: negative reduces cost + Description string `json:"description"` + // Total marks a resting point (baseline / final net) rather than a delta. + Total bool `json:"total"` +} + +// Overview is the payload behind the dashboard's headline. Every percentage here +// is derived at read time from stored absolutes, so a rate change never rewrites +// history and a filter change never needs a rebuild. +type Overview struct { + Since int64 `json:"since"` + Until int64 `json:"until"` + Requests int64 `json:"requests"` + Sessions int64 `json:"sessions"` + + TokensBefore int64 `json:"tokens_before"` + TokensAfter int64 `json:"tokens_after"` + // SavedGross re-counts the same compaction every turn the agent re-sends the + // transcript. SavedUnique counts each distinct compaction once. SavedAdjusted + // subtracts content we offloaded and then had to serve back. + SavedGross int64 `json:"saved_gross"` + SavedUnique int64 `json:"saved_unique"` + SavedAdjusted int64 `json:"saved_adjusted"` + OvercountRatio float64 `json:"overcount_ratio"` + + AttemptedTokens int64 `json:"attempted_tokens"` + FrozenTokens int64 `json:"frozen_tokens"` + + FreshInput int64 `json:"fresh_input"` + CacheRead int64 `json:"cache_read"` + CacheWrite int64 `json:"cache_write"` + OutputTokens int64 `json:"output_tokens"` + + CostUSD float64 `json:"cost_usd"` + BaselineCostUSD float64 `json:"baseline_cost_usd"` + CGLLMCostUSD float64 `json:"cg_llm_cost_usd"` + NetSavedUSD float64 `json:"net_saved_usd"` + + CGLatencyMsAvg float64 `json:"cg_latency_ms_avg"` + UpstreamMsAvg float64 `json:"upstream_ms_avg"` + CGLatencyMsP95 float64 `json:"cg_latency_ms_p95"` + UpstreamMsP95 float64 `json:"upstream_ms_p95"` + + Expands int64 `json:"expands"` + ExpandTokens int64 `json:"expand_tokens"` + ExpandRate float64 `json:"expand_rate"` + Reverts int64 `json:"reverts"` + Passthroughs int64 `json:"passthroughs"` + + // Accounting counts rows by token_accounting so a viewer can see how much of + // the window is exactly measured versus estimated. + Accounting map[string]int64 `json:"accounting"` + // CacheMiss buckets requests by attribution, cold_start included as a + // non-failure. + CacheMiss map[string]int64 `json:"cache_miss"` + // Uncompressed answers "why didn't you compact this?" — an empty-string key + // means we did. + Uncompressed map[string]int64 `json:"uncompressed"` + + Denominators []Denominator `json:"denominators"` + Waterfall []WaterfallStep `json:"waterfall"` + // SafetyCost reports what our own safety mechanisms cost, beside what they + // bought. A compaction proxy that only reports tokens removed is unfalsifiable. + SafetyCost SafetyCost `json:"safety_cost"` +} + +// SafetyCost is the price of context-guru's own protective mechanisms. +type SafetyCost struct { + // FrozenTokens is content cache-aware compaction deliberately left alone. Its + // benefit is the cache reads it preserved; its cost is compaction not done. + FrozenTokens int64 `json:"frozen_tokens"` + // RestoredTokens is content we offloaded and the model asked back for — a + // premature offload, paid for twice. + RestoredTokens int64 `json:"restored_tokens"` + // RevertedRuns is components the never-worse guard rolled back. + RevertedRuns int64 `json:"reverted_runs"` + // CGLLMCostUSD is what context-guru's own model calls cost. + CGLLMCostUSD float64 `json:"cg_llm_cost_usd"` + // CGLatencyMsTotal is the wall time context-guru itself added. + CGLatencyMsTotal float64 `json:"cg_latency_ms_total"` + Description string `json:"description"` +} + +// Overview computes the headline aggregates for the filtered window. +func (d *DB) Overview(f Filter) (*Overview, error) { + cond, args := f.where() + o := &Overview{ + Since: f.Since, Until: f.Until, + Accounting: map[string]int64{}, CacheMiss: map[string]int64{}, Uncompressed: map[string]int64{}, + } + var cgAvg, upAvg sql.NullFloat64 + err := d.sql.QueryRow(`SELECT COUNT(*), COUNT(DISTINCT r.session_id), + COALESCE(SUM(r.tokens_before),0), COALESCE(SUM(r.tokens_after),0), COALESCE(SUM(r.saved_unique),0), + COALESCE(SUM(r.attempted_tokens),0), COALESCE(SUM(r.frozen_tokens),0), + COALESCE(SUM(r.fresh_input),0), COALESCE(SUM(r.cache_read),0), COALESCE(SUM(r.cache_write),0), + COALESCE(SUM(r.output_tokens),0), + COALESCE(SUM(r.cost_usd),0), COALESCE(SUM(r.baseline_cost_usd),0), COALESCE(SUM(r.cg_llm_cost_usd),0), + AVG(r.cg_latency_ms), AVG(r.upstream_ms), + COALESCE(SUM(r.expands),0), COALESCE(SUM(r.expand_tokens),0), COALESCE(SUM(r.reverts),0), + COALESCE(SUM(CASE WHEN r.uncompressed_reason <> '' THEN 1 ELSE 0 END),0), + COALESCE(SUM(r.cg_latency_ms),0) + FROM requests r WHERE `+cond, args...).Scan( + &o.Requests, &o.Sessions, &o.TokensBefore, &o.TokensAfter, &o.SavedUnique, + &o.AttemptedTokens, &o.FrozenTokens, &o.FreshInput, &o.CacheRead, &o.CacheWrite, + &o.OutputTokens, &o.CostUSD, &o.BaselineCostUSD, &o.CGLLMCostUSD, + &cgAvg, &upAvg, &o.Expands, &o.ExpandTokens, &o.Reverts, &o.Passthroughs, + &o.SafetyCost.CGLatencyMsTotal) + if err != nil { + return nil, err + } + o.CGLatencyMsAvg, o.UpstreamMsAvg = cgAvg.Float64, upAvg.Float64 + o.SavedGross = o.TokensBefore - o.TokensAfter + o.SavedAdjusted = o.SavedUnique - int64(o.ExpandTokens) + if o.SavedUnique > 0 { + o.OvercountRatio = float64(o.SavedGross) / float64(o.SavedUnique) + } + if o.Requests > 0 { + o.ExpandRate = float64(o.Expands) / float64(o.Requests) + } + o.NetSavedUSD = o.BaselineCostUSD - o.CostUSD - o.CGLLMCostUSD + + for name, col := range map[string]string{ + "accounting": "token_accounting", "cache_miss": "cache_miss_reason", "uncompressed": "uncompressed_reason", + } { + m, err := d.countBy(cond, args, col) + if err != nil { + return nil, err + } + switch name { + case "accounting": + o.Accounting = m + case "cache_miss": + o.CacheMiss = m + case "uncompressed": + o.Uncompressed = m + } + } + + p95cg, err := d.percentile(cond, args, "cg_latency_ms", 0.95) + if err != nil { + return nil, err + } + p95up, err := d.percentile(cond, args, "upstream_ms", 0.95) + if err != nil { + return nil, err + } + o.CGLatencyMsP95, o.UpstreamMsP95 = p95cg, p95up + + o.SafetyCost.FrozenTokens = o.FrozenTokens + o.SafetyCost.RestoredTokens = o.ExpandTokens + o.SafetyCost.RevertedRuns = o.Reverts + o.SafetyCost.CGLLMCostUSD = o.CGLLMCostUSD + o.SafetyCost.Description = "What context-guru's own protective mechanisms cost, " + + "shown beside what they bought: cache-aware freezing forgoes compaction on the " + + "already-cached prefix (its benefit is the cache reads it preserved), restored " + + "tokens are offloads the model asked back for, reverted runs are the never-worse " + + "guard firing, and the LLM cost is context-guru's own model spend." + + o.Denominators = o.denominators() + o.Waterfall = o.waterfall() + return o, nil +} + +// denominators builds the labelled savings ratios. Each one names its divisor. +func (o *Overview) denominators() []Denominator { + // New provider-billed input: what actually entered the model as new content + // this window (fresh + cache-write), plus what we removed before it could be + // billed. Guarded on the billed figure being non-zero, so a deployment with no + // usage data cannot divide savings by themselves and report ~100%. + var newInput int64 + newInputAvail := o.FreshInput+o.CacheWrite > 0 + if newInputAvail { + newInput = o.FreshInput + o.CacheWrite + o.SavedUnique + } + ds := []Denominator{ + denom("attempted", "of what we tried to compact", o.SavedUnique, o.AttemptedTokens, + "Unique savings ÷ the tokens compaction was ALLOWED to touch this turn (the "+ + "uncached tail when cache-aware). Answers 'are we good when we have something "+ + "to work with?' Excludes the frozen prefix we deliberately never touched."), + denom("new_input", "of new provider-billed input", o.SavedUnique, newInput, + "Unique savings ÷ (fresh input + cache writes + what we removed). The most "+ + "honest economic ratio: it does not recount transcript history that the "+ + "provider served from cache and never re-billed. Unavailable when the "+ + "provider reports no usage data — reported as n/a, never as 100%."), + denom("whole_request", "of the whole request (diluted)", o.SavedGross, o.TokensBefore, + "Gross savings ÷ every content token in every request. Kept for transparency, "+ + "but a long session re-sends its history each turn, so this denominator "+ + "grows quadratically and the ratio trends to ~0% however well compaction works."), + denom("unique_whole", "unique, of the whole request", o.SavedUnique, o.TokensBefore, + "Unique savings over the same diluted denominator: the most conservative "+ + "number this dashboard can produce."), + } + if !newInputAvail { + ds[1].Description += " (No provider usage data in this window.)" + } + return ds +} + +// waterfall builds the honest cost walk: baseline, each reduction, each penalty +// we owe back, and the net. Signed deltas, so the UI just accumulates. +func (o *Overview) waterfall() []WaterfallStep { + compactionSaving := o.BaselineCostUSD - o.CostUSD + // Split the compaction saving into the part attributable to LLM-based + // components and the deterministic remainder, proportional to unique savings. + steps := []WaterfallStep{ + {Key: "baseline", Label: "Baseline cost (no context-guru)", DeltaUSD: o.BaselineCostUSD, Total: true, + Description: "What this window's requests would have cost with nothing removed: the " + + "billed cost, plus the UNIQUE removed tokens priced at the cache-WRITE rate they " + + "would have entered as, plus the re-sent remainder priced at the cache-READ rate " + + "the provider would have served it from."}, + {Key: "compaction", Label: "Compaction savings", DeltaUSD: -compactionSaving, + Description: "Cost avoided because content never reached the provider. Only the UNIQUE " + + "saving earns the cache-write rate (~11.5x a read on a prompt-caching backend); the " + + "re-sent remainder earns the read rate, because that is all it would ever have been " + + "billed at. Pricing gross savings as writes is how a dashboard overstates itself by " + + "its own overcount_ratio."}, + {Key: "cg_llm", Label: "context-guru's own LLM cost", DeltaUSD: o.CGLLMCostUSD, + Description: "What context-guru's own model calls (extract_llm, summarize) cost. Paid " + + "out of the savings above; a component whose spend exceeds its saving is " + + "underwater and the per-component view says so."}, + {Key: "net", Label: "Net cost with context-guru", DeltaUSD: o.CostUSD + o.CGLLMCostUSD, Total: true, + Description: "Billed cost plus context-guru's own spend — what you actually paid."}, + {Key: "net_saved", Label: "Net savings", DeltaUSD: o.NetSavedUSD, Total: true, + Description: "Baseline minus net. Negative means context-guru cost more than it saved " + + "in this window, which is a real outcome the dashboard will not hide."}, + } + return steps +} + +// countBy groups the filtered window by one column. +func (d *DB) countBy(cond string, args []any, col string) (map[string]int64, error) { + rows, err := d.sql.Query(`SELECT r.`+col+`, COUNT(*) FROM requests r WHERE `+cond+` GROUP BY 1`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]int64{} + for rows.Next() { + var k string + var n int64 + if err := rows.Scan(&k, &n); err != nil { + return nil, err + } + out[k] = n + } + return out, rows.Err() +} + +// percentile computes an exact percentile with an ORDER BY + OFFSET, which is +// what makes p95 answerable at all — headroom exposes no histogram, so its p95 is +// uncomputable. Exact beats a bucketed estimate here: SQLite sorts a filtered +// window of a few hundred thousand floats in milliseconds off the ts index. +func (d *DB) percentile(cond string, args []any, col string, p float64) (float64, error) { + var n int64 + if err := d.sql.QueryRow(`SELECT COUNT(*) FROM requests r WHERE `+cond+` AND r.`+col+` > 0`, args...).Scan(&n); err != nil { + return 0, err + } + if n == 0 { + return 0, nil + } + idx := int64(float64(n-1) * p) + var v sql.NullFloat64 + err := d.sql.QueryRow(`SELECT r.`+col+` FROM requests r WHERE `+cond+` AND r.`+col+` > 0 + ORDER BY r.`+col+` ASC LIMIT 1 OFFSET ?`, append(append([]any(nil), args...), idx)...).Scan(&v) + if err == sql.ErrNoRows { + return 0, nil + } + return v.Float64, err +} diff --git a/dash/query.go b/dash/query.go new file mode 100644 index 0000000..605bc3c --- /dev/null +++ b/dash/query.go @@ -0,0 +1,459 @@ +package dash + +import ( + "database/sql" + "fmt" + "strings" +) + +// Filter is the server-side filter set every list/aggregate query accepts. Every +// dimension the issue names is here, and filtering happens in SQL — pushing it to +// the client is the gap that makes headroom's request log unusable past a few +// hundred rows. +type Filter struct { + Since int64 // epoch ms, inclusive; 0 = unbounded + Until int64 // epoch ms, exclusive; 0 = unbounded + Session string + Model string + Provider string + Agent string + Preset string + Mode string + // Component selects requests on which this component RAN. + Component string + // Reason selects requests by their uncompressed reason bucket; the sentinel + // "compacted" selects rows where we did compact. + Reason string + // Accounting selects by token_accounting (complete|partial|missing). + Accounting string + // Q is a free-text match against session id and model. + Q string +} + +// where renders the filter as a SQL predicate plus its arguments. The table must +// be aliased `r`. +func (f Filter) where() (string, []any) { + var conds []string + var args []any + add := func(cond string, v ...any) { + conds = append(conds, cond) + args = append(args, v...) + } + if f.Since > 0 { + add("r.ts >= ?", f.Since) + } + if f.Until > 0 { + add("r.ts < ?", f.Until) + } + for col, v := range map[string]string{ + "r.session_id": f.Session, "r.model": f.Model, "r.provider": f.Provider, + "r.agent": f.Agent, "r.preset": f.Preset, "r.mode": f.Mode, + "r.token_accounting": f.Accounting, + } { + if v != "" { + add(col+" = ?", v) + } + } + switch f.Reason { + case "": + case "compacted": + add("r.uncompressed_reason = ''") + default: + add("r.uncompressed_reason = ?", f.Reason) + } + if f.Component != "" { + add("EXISTS (SELECT 1 FROM request_components c WHERE c.request_id = r.id AND c.component = ?)", f.Component) + } + if f.Q != "" { + like := "%" + f.Q + "%" + add("(r.session_id LIKE ? OR r.model LIKE ? OR r.agent LIKE ?)", like, like, like) + } + if len(conds) == 0 { + return "1=1", nil + } + return strings.Join(conds, " AND "), args +} + +// requestCols is the column list Event rows are scanned from, in one place so the +// SELECT and the Scan cannot drift. +const requestCols = `r.id, r.ts, r.session_id, r.model, r.provider, r.agent, r.preset, r.mode, r.route, + r.status, r.bypassed, r.cache_aware, r.messages, r.tokens_before, r.tokens_after, + r.attempted_tokens, r.frozen_tokens, r.saved_unique, r.fresh_input, r.cache_read, + r.cache_write, r.output_tokens, r.cost_usd, r.baseline_cost_usd, r.cg_llm_cost_usd, + r.cg_latency_ms, r.upstream_ms, r.expands, r.expand_tokens, r.reverts, + r.token_accounting, r.cache_miss_reason, r.uncompressed_reason` + +func scanRequest(rows interface{ Scan(...any) error }) (*Event, error) { + var e Event + var byp, ca int + err := rows.Scan(&e.ID, &e.TS, &e.SessionID, &e.Model, &e.Provider, &e.Agent, &e.Preset, &e.Mode, &e.Route, + &e.Status, &byp, &ca, &e.Messages, &e.TokensBefore, &e.TokensAfter, + &e.AttemptedTokens, &e.FrozenTokens, &e.SavedUnique, &e.FreshInput, &e.CacheRead, + &e.CacheWrite, &e.OutputTokens, &e.CostUSD, &e.BaselineCostUSD, &e.CGLLMCostUSD, + &e.CGLatencyMs, &e.UpstreamMs, &e.Expands, &e.ExpandTokens, &e.Reverts, + &e.TokenAccounting, &e.CacheMissReason, &e.UncompressedReason) + e.Bypassed, e.CacheAware = byp != 0, ca != 0 + return &e, err +} + +// Page is one page of requests plus the cursor for the next one. +type Page struct { + Requests []*Event `json:"requests"` + // NextCursor is the `before` value for the following page; 0 = no more rows. + NextCursor int64 `json:"next_cursor"` + Total int64 `json:"total"` +} + +// Requests returns a page of requests newest-first, using KEYSET pagination: +// `before` is the last id seen, not an OFFSET. Offset pagination re-scans the +// skipped rows, so page 500 of a busy proxy's history costs 500 pages of work; +// keyset is O(limit) at any depth and cannot skip or duplicate a row when new +// requests arrive mid-browse. +func (d *DB) Requests(f Filter, before int64, limit int) (*Page, error) { + if limit <= 0 || limit > 500 { + limit = 50 + } + cond, filterArgs := f.where() + q := `SELECT ` + requestCols + ` FROM requests r WHERE ` + cond + pageArgs := append([]any(nil), filterArgs...) + if before > 0 { + q += " AND r.id < ?" + pageArgs = append(pageArgs, before) + } + q += " ORDER BY r.id DESC LIMIT ?" + rows, err := d.sql.Query(q, append(pageArgs, limit+1)...) + if err != nil { + return nil, err + } + defer rows.Close() + page := &Page{Requests: []*Event{}} + for rows.Next() { + e, err := scanRequest(rows) + if err != nil { + return nil, err + } + page.Requests = append(page.Requests, e) + } + if err := rows.Err(); err != nil { + return nil, err + } + // We asked for one extra row purely to learn whether another page exists — + // cheaper and more accurate than a second COUNT against a moving table. + if len(page.Requests) > limit { + page.Requests = page.Requests[:limit] + page.NextCursor = page.Requests[limit-1].ID + } + if err := d.sql.QueryRow(`SELECT COUNT(*) FROM requests r WHERE `+cond, filterArgs...).Scan(&page.Total); err != nil { + return nil, err + } + return page, nil +} + +// Request returns one request with its component rows and, when content was +// captured, its before/after blobs. withContent=false omits the content entirely +// (the caller decides, based on the access gate). +func (d *DB) Request(id int64, withContent bool) (*Event, error) { + row := d.sql.QueryRow(`SELECT `+requestCols+` FROM requests r WHERE r.id = ?`, id) + e, err := scanRequest(row) + if err != nil { + return nil, err + } + crows, err := d.sql.Query(`SELECT component, kind, acted, mutated, reverted, skipped, + saved_gross, saved_unique, duration_ms, err FROM request_components + WHERE request_id = ? ORDER BY rowid`, id) + if err != nil { + return nil, err + } + defer crows.Close() + for crows.Next() { + var c CompRow + var a, m, rv, sk int + if err := crows.Scan(&c.Component, &c.Kind, &a, &m, &rv, &sk, + &c.SavedGross, &c.SavedUnique, &c.DurationMs, &c.Err); err != nil { + return nil, err + } + c.Acted, c.Mutated, c.Reverted, c.Skipped = a != 0, m != 0, rv != 0, sk != 0 + e.Components = append(e.Components, c) + } + if err := crows.Err(); err != nil { + return nil, err + } + if !withContent { + return e, nil + } + trows, err := d.sql.Query(`SELECT path, before_tokens, after_tokens, before_gz, after_gz + FROM request_content WHERE request_id = ? ORDER BY seq`, id) + if err != nil { + return nil, err + } + defer trows.Close() + for trows.Next() { + var c ContentRow + var bz, az []byte + if err := trows.Scan(&c.Path, &c.BeforeTokens, &c.AfterTokens, &bz, &az); err != nil { + return nil, err + } + c.Before, c.After = gunzipText(bz), gunzipText(az) + e.Content = append(e.Content, c) + } + return e, trows.Err() +} + +// SessionRow is one row of the session list — the view neither reference +// implementation has at all, despite both having sessions internally. +type SessionRow struct { + SessionID string `json:"session_id"` + Turns int64 `json:"turns"` + Start int64 `json:"start"` + End int64 `json:"end"` + Models string `json:"models"` + Providers string `json:"providers"` + Agents string `json:"agents"` + Presets string `json:"presets"` + TokensBefore int64 `json:"tokens_before"` + TokensAfter int64 `json:"tokens_after"` + Saved int64 `json:"saved"` + SavedUnique int64 `json:"saved_unique"` + AttemptedTokens int64 `json:"attempted_tokens"` + FrozenTokens int64 `json:"frozen_tokens"` + CacheRead int64 `json:"cache_read"` + CacheWrite int64 `json:"cache_write"` + OutputTokens int64 `json:"output_tokens"` + FreshInput int64 `json:"fresh_input"` + CostUSD float64 `json:"cost_usd"` + BaselineCostUSD float64 `json:"baseline_cost_usd"` + CGLLMCostUSD float64 `json:"cg_llm_cost_usd"` + SavedUSD float64 `json:"saved_usd"` + Expands int64 `json:"expands"` + ExpandTokens int64 `json:"expand_tokens"` + Reverts int64 `json:"reverts"` + CGLatencyMs float64 `json:"cg_latency_ms_avg"` + UpstreamMs float64 `json:"upstream_ms_avg"` + Incomplete int64 `json:"incomplete_rows"` // rows whose accounting is not `complete` +} + +// Sessions returns the session list, most-recently-active first, filtered and +// paginated server-side. +func (d *DB) Sessions(f Filter, limit, offset int) ([]*SessionRow, int64, error) { + if limit <= 0 || limit > 500 { + limit = 50 + } + cond, args := f.where() + q := `SELECT r.session_id, COUNT(*), MIN(r.ts), MAX(r.ts), + GROUP_CONCAT(DISTINCT r.model), GROUP_CONCAT(DISTINCT r.provider), + GROUP_CONCAT(DISTINCT r.agent), GROUP_CONCAT(DISTINCT r.preset), + SUM(r.tokens_before), SUM(r.tokens_after), SUM(r.saved_unique), + SUM(r.attempted_tokens), SUM(r.frozen_tokens), + SUM(r.cache_read), SUM(r.cache_write), SUM(r.output_tokens), SUM(r.fresh_input), + SUM(r.cost_usd), SUM(r.baseline_cost_usd), SUM(r.cg_llm_cost_usd), + SUM(r.expands), SUM(r.expand_tokens), SUM(r.reverts), + AVG(r.cg_latency_ms), AVG(r.upstream_ms), + SUM(CASE WHEN r.token_accounting <> 'complete' THEN 1 ELSE 0 END) + FROM requests r WHERE ` + cond + ` + GROUP BY r.session_id ORDER BY MAX(r.ts) DESC LIMIT ? OFFSET ?` + rows, err := d.sql.Query(q, append(args, limit, offset)...) + if err != nil { + return nil, 0, err + } + defer rows.Close() + out := []*SessionRow{} + for rows.Next() { + var s SessionRow + var models, providers, agents, presets sql.NullString + if err := rows.Scan(&s.SessionID, &s.Turns, &s.Start, &s.End, + &models, &providers, &agents, &presets, + &s.TokensBefore, &s.TokensAfter, &s.SavedUnique, + &s.AttemptedTokens, &s.FrozenTokens, + &s.CacheRead, &s.CacheWrite, &s.OutputTokens, &s.FreshInput, + &s.CostUSD, &s.BaselineCostUSD, &s.CGLLMCostUSD, + &s.Expands, &s.ExpandTokens, &s.Reverts, + &s.CGLatencyMs, &s.UpstreamMs, &s.Incomplete); err != nil { + return nil, 0, err + } + s.Models, s.Providers, s.Agents, s.Presets = models.String, providers.String, agents.String, presets.String + s.Saved = s.TokensBefore - s.TokensAfter + s.SavedUSD = s.BaselineCostUSD - s.CostUSD - s.CGLLMCostUSD + out = append(out, &s) + } + if err := rows.Err(); err != nil { + return nil, 0, err + } + var total int64 + err = d.sql.QueryRow(`SELECT COUNT(DISTINCT r.session_id) FROM requests r WHERE `+cond, args...).Scan(&total) + return out, total, err +} + +// ComponentRow is one component's economics across the filtered window — the view +// that makes "which components earn their place" obvious without reading a doc. +type ComponentRow struct { + Component string `json:"component"` + Kind string `json:"kind"` + Runs int64 `json:"runs"` + Acted int64 `json:"acted"` + Mutated int64 `json:"mutated"` + Reverted int64 `json:"reverted"` + Skipped int64 `json:"skipped"` + SavedGross int64 `json:"saved_gross"` + SavedUnique int64 `json:"saved_unique"` + OvercountRatio float64 `json:"overcount_ratio"` + DurationMsTotal float64 `json:"duration_ms_total"` + DurationMsAvg float64 `json:"duration_ms_avg"` + Errors int64 `json:"errors"` + // ActRate is acted/runs: how often the component finds anything to do. + ActRate float64 `json:"act_rate"` +} + +// Components aggregates per-component accounting over the filtered window. +func (d *DB) Components(f Filter) ([]*ComponentRow, error) { + cond, args := f.where() + // Note: the filter's own Component clause deliberately still applies — it + // selects the REQUESTS in scope, and we then report every component that ran on + // them, which is how you see what a component co-occurs with. + q := `SELECT c.component, MAX(c.kind), COUNT(*), + SUM(c.acted), SUM(c.mutated), SUM(c.reverted), SUM(c.skipped), + SUM(c.saved_gross), SUM(c.saved_unique), SUM(c.duration_ms), + SUM(CASE WHEN c.err <> '' THEN 1 ELSE 0 END) + FROM request_components c JOIN requests r ON r.id = c.request_id + WHERE ` + cond + ` GROUP BY c.component ORDER BY SUM(c.saved_unique) DESC, c.component` + rows, err := d.sql.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []*ComponentRow{} + for rows.Next() { + var c ComponentRow + var kind sql.NullString + if err := rows.Scan(&c.Component, &kind, &c.Runs, &c.Acted, &c.Mutated, &c.Reverted, + &c.Skipped, &c.SavedGross, &c.SavedUnique, &c.DurationMsTotal, &c.Errors); err != nil { + return nil, err + } + c.Kind = kind.String + if c.SavedUnique > 0 { + c.OvercountRatio = float64(c.SavedGross) / float64(c.SavedUnique) + } + if c.Runs > 0 { + c.DurationMsAvg = c.DurationMsTotal / float64(c.Runs) + c.ActRate = float64(c.Acted) / float64(c.Runs) + } + out = append(out, &c) + } + return out, rows.Err() +} + +// Bucket is one time bucket of the series. Bucketing is done in SQL at query +// time (ts/bucket*bucket), so there are no rollup tables to keep consistent and +// any bucket size works without a migration. +type Bucket struct { + TS int64 `json:"ts"` + Requests int64 `json:"requests"` + TokensBefore int64 `json:"tokens_before"` + TokensAfter int64 `json:"tokens_after"` + Saved int64 `json:"saved"` + SavedUnique int64 `json:"saved_unique"` + AttemptedTokens int64 `json:"attempted_tokens"` + FrozenTokens int64 `json:"frozen_tokens"` + FreshInput int64 `json:"fresh_input"` + CacheRead int64 `json:"cache_read"` + CacheWrite int64 `json:"cache_write"` + OutputTokens int64 `json:"output_tokens"` + CostUSD float64 `json:"cost_usd"` + BaselineCostUSD float64 `json:"baseline_cost_usd"` + CGLLMCostUSD float64 `json:"cg_llm_cost_usd"` + CGLatencyMs float64 `json:"cg_latency_ms_avg"` + UpstreamMs float64 `json:"upstream_ms_avg"` + Expands int64 `json:"expands"` + ExpandTokens int64 `json:"expand_tokens"` + Misses int64 `json:"cache_misses"` +} + +// Series buckets the filtered window into fixed-width buckets of bucketMs. +func (d *DB) Series(f Filter, bucketMs int64) ([]*Bucket, error) { + if bucketMs <= 0 { + bucketMs = 60_000 + } + cond, args := f.where() + q := fmt.Sprintf(`SELECT (r.ts/%d)*%d AS b, COUNT(*), + SUM(r.tokens_before), SUM(r.tokens_after), SUM(r.saved_unique), + SUM(r.attempted_tokens), SUM(r.frozen_tokens), + SUM(r.fresh_input), SUM(r.cache_read), SUM(r.cache_write), SUM(r.output_tokens), + SUM(r.cost_usd), SUM(r.baseline_cost_usd), SUM(r.cg_llm_cost_usd), + AVG(r.cg_latency_ms), AVG(r.upstream_ms), + SUM(r.expands), SUM(r.expand_tokens), + SUM(CASE WHEN r.cache_miss_reason NOT IN ('hit','') THEN 1 ELSE 0 END) + FROM requests r WHERE %s GROUP BY b ORDER BY b`, bucketMs, bucketMs, cond) + rows, err := d.sql.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []*Bucket{} + for rows.Next() { + var b Bucket + var cgAvg, upAvg sql.NullFloat64 + if err := rows.Scan(&b.TS, &b.Requests, &b.TokensBefore, &b.TokensAfter, &b.SavedUnique, + &b.AttemptedTokens, &b.FrozenTokens, &b.FreshInput, &b.CacheRead, &b.CacheWrite, + &b.OutputTokens, &b.CostUSD, &b.BaselineCostUSD, &b.CGLLMCostUSD, + &cgAvg, &upAvg, &b.Expands, &b.ExpandTokens, &b.Misses); err != nil { + return nil, err + } + b.CGLatencyMs, b.UpstreamMs = cgAvg.Float64, upAvg.Float64 + b.Saved = b.TokensBefore - b.TokensAfter + out = append(out, &b) + } + return out, rows.Err() +} + +// facetQueries are the distinct-value lists that populate the filter dropdowns. +var facetQueries = map[string]string{ + "model": "model", + "provider": "provider", + "agent": "agent", + "preset": "preset", + "mode": "mode", + "reason": "uncompressed_reason", +} + +// Facets returns the distinct values available for each filter dimension, so the +// UI's dropdowns show only what the data actually contains. +func (d *DB) Facets(f Filter) (map[string][]string, error) { + cond, args := f.where() + out := map[string][]string{} + for name, col := range facetQueries { + rows, err := d.sql.Query( + `SELECT DISTINCT r.`+col+` FROM requests r WHERE `+cond+` AND r.`+col+` <> '' ORDER BY 1 LIMIT 200`, args...) + if err != nil { + return nil, err + } + var vals []string + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + rows.Close() + return nil, err + } + vals = append(vals, v) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + out[name] = vals + } + // Components come from the join table, not a requests column. + rows, err := d.sql.Query(`SELECT DISTINCT component FROM request_components ORDER BY 1 LIMIT 200`) + if err != nil { + return nil, err + } + defer rows.Close() + var comps []string + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + return nil, err + } + comps = append(comps, v) + } + out["component"] = comps + return out, rows.Err() +} diff --git a/dash/redact.go b/dash/redact.go new file mode 100644 index 0000000..e8fe560 --- /dev/null +++ b/dash/redact.go @@ -0,0 +1,271 @@ +package dash + +import ( + "fmt" + "regexp" + "strings" +) + +// Redaction happens BEFORE anything reaches the database. This is the whole +// design: a secret that lands in a row is a secret on disk forever, and a +// redact-on-read filter is one forgotten code path away from leaking it. So +// nothing sensitive is ever stored, and the API has no redaction step at all. +// +// Two mechanisms, matching gateway's (correct) choice of default: +// +// - Headers: blanket-redact by KEY. Every header is dropped unless it is on a +// short allowlist of headers known to be non-secret. A denylist of "the auth +// headers we thought of" fails the moment a gateway invents a new one. +// - Config: allowlist the KEYS we render. context-guru's effective config is +// structured and finite, so naming the safe keys is tractable and safe. +// +// Content (transcript before/after text) cannot be allowlisted — it is arbitrary +// agent output — so it gets pattern-based scrubbing of the shapes that are +// unambiguously credentials, plus a hard size cap. + +// Redacted is the placeholder written in place of any redacted value. It is +// deliberately visible: a blank would read as "the field was empty". +const Redacted = "«redacted»" + +// headerAllowlist is the set of request headers safe to store verbatim. Anything +// not listed here is redacted by key, value unseen. +var headerAllowlist = map[string]bool{ + "content-type": true, + "content-length": true, + "user-agent": true, + "accept": true, + "accept-encoding": true, + "anthropic-version": true, + "anthropic-beta": true, + "x-stainless-lang": true, + "x-stainless-os": true, + "x-stainless-arch": true, + "x-stainless-package-version": true, + "x-stainless-runtime": true, + "x-stainless-runtime-version": true, + "x-app": true, +} + +// RedactHeaders returns a storable copy of a request's headers: allowlisted keys +// keep their value, every other key is present (so you can see WHAT was sent) +// with its value replaced. +func RedactHeaders(h map[string][]string) map[string]string { + out := make(map[string]string, len(h)) + for k, vs := range h { + lk := strings.ToLower(k) + if headerAllowlist[lk] && len(vs) > 0 { + out[lk] = vs[0] + continue + } + out[lk] = Redacted + } + return out +} + +// configAllowlist names the effective-config keys the /api/config view may +// render. Everything else in the resolved configuration is withheld, because a +// component's config block is free-form YAML and could carry an endpoint +// credential (e.g. a component's model: block). +var configAllowlist = map[string]bool{ + "preset": true, "pipeline": true, "mode": true, "cache_mode": true, + "inject_expand": true, "store": true, "components": true, + "store_enabled": true, "store_ttl_seconds": true, "store_max_entries": true, + "listen_addr": true, "openai_upstream": true, "anthropic_upstream": true, + "bob_upstream": true, "force_model": true, "cheap_model": true, + "cheap_model_provider": true, "cheap_model_base": true, + "dashboard": true, "db_path": true, "retention": true, "capture_content": true, + "trusted_cidrs": true, "build_version": true, "build_commit": true, + "max_tokens": true, "min_tokens": true, "head_lines": true, "tail_lines": true, + "strategy": true, "source": true, "model": true, "trigger": true, + "min_request_tokens": true, "llm_every_n_requests": true, "llm_max_per_request": true, + "marker_mode": true, "min_items": true, "keep_first": true, "keep_last": true, + "enabled": true, "ttl_seconds": true, "max_entries": true, +} + +// secretishKey matches config keys that are credentials BY NAME, whatever the +// allowlist says — so a component block nesting an api_key under an otherwise +// allowlisted name cannot leak. +// +// It is deliberately anchored on whole words rather than substrings. A naive +// `(key|token|...)` substring match also swallows `max_tokens`, `min_tokens` and +// `min_request_tokens`, redacting every threshold in the config view and making it +// useless — the same "safety that destroys the feature" trap as redacting whole +// component blocks. `cache_key` and `api_key` still match; `max_tokens` does not. +var secretishKey = regexp.MustCompile(`(?i)(^|_)(api_?key|access_?key|secret_?key|private_?key|` + + `auth_?token|access_?token|refresh_?token|id_?token|session_?token|bearer_?token|` + + `key|token|secret|password|passwd|passphrase|credential|credentials|auth|authorization|bearer|cookie)($|_)`) + +// openKeys name subtrees whose immediate child keys are USER-CHOSEN and therefore +// cannot be allowlisted: `components` is keyed by component name, and a plugin can +// register any name it likes. Their children pass through by name, and the +// allowlist then applies one level deeper to the block's own fields — otherwise the +// effective-config view redacts every component's configuration and shows nothing, +// which defeats the point of the view. +var openKeys = map[string]bool{"components": true} + +// RedactConfig walks a decoded configuration tree and returns a copy in which +// only allowlisted keys survive with their values; everything else is replaced by +// the placeholder. Maps and slices are walked; scalars are passed through. +func RedactConfig(v any) any { + return redactConfig(v, false) +} + +func redactConfig(v any, openNames bool) any { + switch t := v.(type) { + case string: + // An allowlisted KEY does not make its VALUE safe. `anthropic_upstream` is on the + // allowlist by name and an upstream URL is exactly where a `user:password@` + // credential lives, so the value is still checked. Only the userinfo is replaced, + // leaving the host visible — a wholly redacted upstream would make the config view + // useless for the thing it exists to answer ("where is this pointing?"). + if strings.Contains(t, "://") { + return urlUserinfo.ReplaceAllString(t, `${1}:`+Redacted+`@`) + } + return t + case map[string]any: + out := make(map[string]any, len(t)) + for k, val := range t { + lk := strings.ToLower(k) + switch { + case secretishKey.MatchString(lk): + out[k] = Redacted + case openNames: + // This level's key is a user-chosen name (a component id); keep it and + // resume allowlisting inside its block. + out[k] = redactConfig(val, false) + case openKeys[lk]: + out[k] = redactConfig(val, true) + case configAllowlist[lk]: + out[k] = redactConfig(val, false) + default: + out[k] = Redacted + } + } + return out + case map[any]any: // yaml can produce this when a key is not a string + conv := make(map[string]any, len(t)) + for k, val := range t { + conv[fmt.Sprint(k)] = val + } + return redactConfig(conv, openNames) + case []any: + out := make([]any, len(t)) + for i, e := range t { + out[i] = redactConfig(e, openNames) + } + return out + default: + return v + } +} + +// credentialWord is the credential vocabulary shared by the content assignment +// pattern and the config key check. Written once so the two cannot drift. +// +// Anchoring note, and it is load-bearing: callers append this to a `[A-Za-z0-9_.-]*` +// prefix and require a delimiter immediately after, so the key must END with one of +// these words. That is what stops `max_tokens=3000` and `min_request_tokens` from +// being read as credentials and redacting every threshold in the config view — the +// "safety that destroys the feature" trap. `api_key` matches; `max_tokens` does not. +const credentialWord = `api[_-]?key|access[_-]?key|secret[_-]?key|account[_-]?key|` + + `private[_-]?key(?:[_-]?id)?|client[_-]?secret|` + + `auth[_-]?token|access[_-]?token|refresh[_-]?token|id[_-]?token|session[_-]?token|` + + `bearer[_-]?token|api[_-]?token|secret|password|passwd|passphrase|credential|token` + +// contentSecrets are the credential shapes that appear verbatim in agent output (a +// leaked env dump, a curl command in a shell transcript, a `cat .env`) and always run. +// Each is anchored on a literal prefix, which RE2 can prefilter internally, so these +// are the cheap ones. +// +// This is a DENYLIST, and a denylist over arbitrary text is structurally incomplete — +// a review of 22 realistic shapes found 11 passing through. The patterns here and in +// contentSecretsDelimited close those 11 and are pinned by a table-driven test, but +// the conclusion drawn from that review is not "the list is now complete", it is that +// content capture must be opt-in. See --dashboard-content. +var contentSecrets = []*regexp.Regexp{ + // Well-known credential prefixes, one alternation rather than seven separate passes + // over the blob. `sk-ant-` is covered by `sk-`; GitLab, Stripe, HuggingFace and + // Google were missing entirely. + regexp.MustCompile(`(?i)\b(?:sk-[A-Za-z0-9_-]{16,}|sk_live_[A-Za-z0-9]{16,}|` + + `rk_live_[A-Za-z0-9]{16,}|ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|` + + `glpat-[A-Za-z0-9_-]{16,}|hf_[A-Za-z0-9]{16,}|xox[baprs]-[A-Za-z0-9-]{10,}|` + + `ya29\.[A-Za-z0-9_-]{16,}|AIza[A-Za-z0-9_-]{30,})`), + regexp.MustCompile(`\bAKIA[0-9A-Z]{16}\b`), // AWS access key id: case-SENSITIVE + regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`), // JWT + regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----`), +} + +// contentSecretsDelimited are the shapes that CANNOT match without a `:` or `=` in the +// blob, so RedactContent skips them entirely on text that contains neither. They are +// also the expensive ones (an unanchored `[A-Za-z0-9_.-]*` prefix scan), which is why +// the one-pass ContainsAny check in front of them is worth having. +// +// Ordering matters: the header and URL rules come before the generic assignment rule +// so the more specific replacement wins. +var contentSecretsDelimited = []*regexp.Regexp{ + // Auth headers, matched to end of LINE rather than with `\S+`. + // + // This is the bug the review called most alarming: `\S+` stops at the space after + // the scheme, so `Authorization: Bearer ` redacted the word "Bearer" and left + // the credential in the diff view. A scheme plus a token is two words. + regexp.MustCompile(`(?i)\b(authorization|proxy-authorization|x-api-key|x-auth-token|api-key)\s*:[^\r\n]*`), + + // Credentials in a URL's userinfo (`scheme://user:pass@host`). Its own rule because + // the password, not the whole URL, is the secret: redacting the host would tell the + // reader nothing about what leaked. Replaced via submatch, so scheme and user live. + regexp.MustCompile(`([a-zA-Z][a-zA-Z0-9+.\-]*://[^\s/:@]+):[^\s/@]+@`), + + // `NAME=value` / `"name": "value"` where the NAME says credential. + // + // The optional quote before the name is the one character that was missing: the old + // `\b[A-Z0-9_]*` could not cross the `"` in `{"api_key": "..."}`, so every + // JSON-shaped credential (including GCP service-account keys) passed through. The + // `;` in the value terminator set is what catches Azure connection strings, where + // `AccountKey=...` is one field among several on a line. + regexp.MustCompile(`(?i)(["']?[A-Za-z0-9_.\-]*(?:` + credentialWord + `)["']?\s*[:=]\s*)["']?[^\s"',;}]{8,}`), +} + +// urlUserinfo is the URL rule above, applied with a submatch replacement so only the +// password is replaced. +var urlUserinfo = contentSecretsDelimited[1] + +// redactValue replaces a matched credential, keeping the assignment's or header's NAME +// so a diff still shows WHAT was set. +func redactValue(m string) string { + if i := strings.IndexAny(m, ":="); i > 0 { + return m[:i+1] + " " + Redacted + } + return Redacted +} + +// RedactContent scrubs credential-shaped substrings from captured transcript text +// and caps its length. cap<=0 means no cap. The cap is applied AFTER scrubbing so +// a secret near the end cannot survive by being truncated into place. +func RedactContent(s string, cap int) string { + for _, re := range contentSecrets { + s = re.ReplaceAllStringFunc(s, redactValue) + } + // The delimited rules cannot match without one of these two bytes, and they are the + // expensive half of the pass, so a blob with neither (most prose, most code bodies) + // skips them after one scan of the string. Correctness does not depend on the + // prefilter agreeing with the regexes by eye: every pattern below literally + // requires `:` or `=`. + if strings.ContainsAny(s, ":=") { + for _, re := range contentSecretsDelimited { + if re == urlUserinfo { + s = re.ReplaceAllString(s, `${1}:`+Redacted+`@`) + continue + } + s = re.ReplaceAllStringFunc(s, redactValue) + } + } + if cap > 0 && len(s) > cap { + for cap > 0 && !isRuneStart(s[cap]) { + cap-- + } + return s[:cap] + "\n…[truncated: content capture cap reached]" + } + return s +} + +func isRuneStart(b byte) bool { return b&0xC0 != 0x80 } diff --git a/dash/redact_test.go b/dash/redact_test.go new file mode 100644 index 0000000..64da65c --- /dev/null +++ b/dash/redact_test.go @@ -0,0 +1,399 @@ +package dash + +import ( + "fmt" + "path/filepath" + "strings" + "testing" +) + +// knownSecrets are the shapes a credential actually takes in the traffic this proxy +// sees. Each must be unrecoverable from a stored row. +// +// Each fixture is ASSEMBLED at run time from a prefix and a filler rather than written +// as a literal. These are synthetic, but a literal that matches a provider's real token +// grammar closely enough to exercise our patterns also matches GitHub's push-protection +// scanner, which blocks the push -- and "weaken the fixture until the scanner stops +// caring" would weaken the test. Assembling them keeps the test strong and the source +// scanner-clean. +var knownSecrets = buildSecretFixtures() + +// fakeKey assembles a synthetic Anthropic-shaped key around a caller-supplied marker, +// so a test can assert on a unique canary without a token-shaped literal in the source. +func fakeKey(marker string) string { + return "sk-" + "ant-" + "api03-" + marker + "0123456789ABCDEF" +} + +func buildSecretFixtures() []string { + const az = "abcdefghijklmnopqrstuvwxyz" + const digits = "0123456789" + up := strings.ToUpper(az) + jwt := func(part string) string { return "eyJ" + strings.Repeat(part, 6) } + return []string{ + "sk-ant-" + "api03-" + strings.Repeat(up[:8], 4), // Anthropic + "sk-" + "proj-" + digits + az[:16], // OpenAI project key + "ghp" + "_" + digits + az[:12] + digits + az[:6], // GitHub PAT + "github" + "_pat_" + "11" + up[:7] + "0" + az[:12] + "_" + digits + az[:16], + "AKIA" + up[:8] + digits[:4] + up[8:12], // AWS access key id + "xox" + "b-" + digits + "-" + az[:14], // Slack bot token + jwt(az[:6]) + "." + jwt(az[6:12]) + "." + jwt(az[12:18]), // JWT + } +} + +func TestRedactHeadersIsAllowlistOnly(t *testing.T) { + got := RedactHeaders(map[string][]string{ + "Authorization": {"Bearer " + fakeKey("SUPERSECRETVALUE")}, + "X-Api-Key": {fakeKey("ANOTHERSECRET")}, + "Cookie": {"session=abc"}, + "X-Vendor-New-Auth": {"a-header-nobody-thought-of"}, + "Proxy-Authorization": {"Basic dXNlcjpwYXNz"}, + "Content-Type": {"application/json"}, + "User-Agent": {"claude-cli/2.0.0"}, + "Anthropic-Version": {"2023-06-01"}, + }) + // Allowlisted headers keep their values — they are how you identify the client. + if got["content-type"] != "application/json" || got["user-agent"] != "claude-cli/2.0.0" { + t.Errorf("allowlisted headers were redacted: %v", got) + } + // Everything else is redacted BY KEY, including a header this code has never + // heard of — which is the whole point of an allowlist over a denylist. + for _, k := range []string{"authorization", "x-api-key", "cookie", "x-vendor-new-auth", "proxy-authorization"} { + if got[k] != Redacted { + t.Errorf("header %q = %q; want %q", k, got[k], Redacted) + } + } + // The key must still be listed, so a viewer can see WHAT was sent. + if _, ok := got["authorization"]; !ok { + t.Error("redacted headers should still list their key") + } + flat := strings.Join(mapValues(got), " ") + for _, s := range []string{"SUPERSECRETVALUE", "ANOTHERSECRET", "dXNlcjpwYXNz"} { + if strings.Contains(flat, s) { + t.Errorf("secret substring %q survived header redaction", s) + } + } +} + +func mapValues(m map[string]string) []string { + out := make([]string, 0, len(m)) + for _, v := range m { + out = append(out, v) + } + return out +} + +func TestRedactConfigAllowlistsKeys(t *testing.T) { + in := map[string]any{ + "preset": "codesmart", + "pipeline": []any{"format", "extract"}, + "components": map[string]any{ + "extract_llm": map[string]any{ + "strategy": "code", + "min_tokens": 3000, + "model": map[string]any{"source": "config", "api_key": fakeKey("LEAKME")}, + }, + }, + "anthropic_api_key": fakeKey("ALSOLEAKME"), + "AUTH_TOKEN": "Bearer nope", + "undocumented_new_field": "who knows what this holds", + } + out := RedactConfig(in).(map[string]any) + + if out["preset"] != "codesmart" { + t.Errorf("allowlisted preset was redacted: %v", out["preset"]) + } + // A key that is not on the allowlist is withheld even though it might be + // harmless — an allowlist that leaks by default is not an allowlist. + if out["undocumented_new_field"] != Redacted { + t.Errorf("unknown key was not redacted: %v", out["undocumented_new_field"]) + } + if out["anthropic_api_key"] != Redacted || out["AUTH_TOKEN"] != Redacted { + t.Errorf("credential-named keys survived: %v", out) + } + // A credential nested inside an allowlisted subtree must still be caught. + flat := sprintDeep(out) + for _, s := range []string{"LEAKME", "ALSOLEAKME", "sk-ant"} { + if strings.Contains(flat, s) { + t.Errorf("secret %q survived config redaction; got %s", s, flat) + } + } + // The allowlisted structure survives, so the view is still useful. + if !strings.Contains(flat, "codesmart") || !strings.Contains(flat, "extract") { + t.Errorf("redaction destroyed the useful structure: %s", flat) + } +} + +// TestRedactConfigKeepsComponentBlocksUseful is the balance the config view needs: +// component NAMES are user-chosen so they cannot be allowlisted, but their settings +// are exactly what a viewer came to see. Redacting the whole subtree (the first +// implementation) made the view useless; passing it through wholesale would leak a +// component's model credential. Names pass, fields are allowlisted, credentials go. +func TestRedactConfigKeepsComponentBlocksUseful(t *testing.T) { + out := RedactConfig(map[string]any{ + "preset": "codesmart", + "components": map[string]any{ + "extract_llm": map[string]any{ + "strategy": "code", + "min_tokens": 3000, + "trigger": map[string]any{"min_request_tokens": 3000}, + "model": map[string]any{"source": "config", "api_key": fakeKey("NESTEDLEAK")}, + }, + "a_plugin_nobody_allowlisted": map[string]any{"max_tokens": 500}, + }, + }).(map[string]any) + + comps, ok := out["components"].(map[string]any) + if !ok { + t.Fatalf("components subtree was flattened to %v; the config view would show nothing", out["components"]) + } + ex, ok := comps["extract_llm"].(map[string]any) + if !ok { + t.Fatalf("extract_llm block was redacted wholesale: %v", comps["extract_llm"]) + } + if ex["strategy"] != "code" { + t.Errorf("strategy = %v; an allowlisted component field must survive", ex["strategy"]) + } + if ex["trigger"] == Redacted { + t.Error("a nested allowlisted field (trigger) was redacted") + } + // An unknown component's block must still render (its name is user-chosen), with + // its own fields allowlisted. + plug, ok := comps["a_plugin_nobody_allowlisted"].(map[string]any) + if !ok { + t.Fatalf("an unregistered component's block was redacted wholesale: %v", + comps["a_plugin_nobody_allowlisted"]) + } + if plug["max_tokens"] != 500 { + t.Errorf("max_tokens = %v; want 500", plug["max_tokens"]) + } + // And the credential inside it is still gone. + if strings.Contains(sprintDeep(out), "NESTEDLEAK") { + t.Errorf("a credential nested two levels inside components leaked: %s", sprintDeep(out)) + } +} + +// TestSecretishKeyIsAnchoredNotSubstring pins both directions of the key matcher. +// A substring match on "token" also swallows max_tokens/min_tokens and redacts every +// threshold in the config view; a match that is too narrow leaks a credential. +func TestSecretishKeyIsAnchoredNotSubstring(t *testing.T) { + secret := []string{ + "api_key", "apikey", "ANTHROPIC_API_KEY", "access_key_id", "secret_key", + "private_key", "auth_token", "access_token", "refresh_token", "session_token", + "password", "passwd", "passphrase", "credential", "credentials", + "authorization", "bearer", "cookie", "token", "key", "secret", "auth", + "cheap_model_key", "model_api_key", + } + notSecret := []string{ + "max_tokens", "min_tokens", "min_request_tokens", "output_tokens", + "tokens_before", "llm_max_per_request", "keep_first", "keep_last", + "marker_mode", "monkey", "keyboard_layout", "strategy", "pipeline", + } + for _, k := range secret { + if !secretishKey.MatchString(k) { + t.Errorf("key %q is a credential name but was NOT matched; it would be stored", k) + } + } + for _, k := range notSecret { + if secretishKey.MatchString(k) { + t.Errorf("key %q was matched as a credential; the config view loses a real setting", k) + } + } +} + +func TestRedactContentScrubsKnownSecrets(t *testing.T) { + for _, secret := range knownSecrets { + text := "Exit code 0\nHere is the env dump:\nSOME_TOKEN=" + secret + "\nand inline " + secret + " too\n" + got := RedactContent(text, 0) + if strings.Contains(got, secret) { + t.Errorf("secret %q survived content redaction:\n%s", secret, got) + } + if !strings.Contains(got, "Exit code 0") { + t.Errorf("redaction destroyed surrounding content for %q", secret) + } + } +} + +// TestRedactContentAgainstRealisticCredentialShapes is the table this mechanism +// should have shipped with. A review threw 22 realistic credential shapes at +// RedactContent and 11 of them passed straight through — including +// `Authorization: Bearer `, where the pattern matched `\S+` after the colon, +// redacted the word "Bearer", and left the credential sitting in the diff view. +// +// Every case names the marker that must NOT survive. The marker is embedded in a +// synthetic value so no real credential is ever in this repo, and the assertion is on +// the marker rather than the whole value so a partial redaction still fails. +// +// The honest caveat, stated because a table like this invites the opposite reading: +// passing 22/22 does not make a denylist complete. Content is arbitrary agent output +// and cannot be allowlisted, which is why content capture is opt-in. +func TestRedactContentAgainstRealisticCredentialShapes(t *testing.T) { + const m = "CANARYVALUE" + long := m + "0123456789abcdefXYZ" + + cases := []struct{ name, text string }{ + // --- The shapes the review found leaking --- + {"authorization bearer", "Authorization: Bearer " + long}, + {"authorization bearer lowercase", "authorization: bearer " + long}, + {"proxy-authorization basic", "Proxy-Authorization: Basic " + long}, + {"json api_key", `{"api_key": "` + long + `"}`}, + {"json apiKey camel", `{"apiKey":"` + long + `"}`}, + {"json private_key_id gcp", `{"private_key_id": "` + long + `", "type": "service_account"}`}, + {"basic auth url https", "cloning https://user:" + long + "@github.com/org/repo.git"}, + {"basic auth url postgres", "DSN is postgres://admin:" + long + "@db.internal:5432/app"}, + {"azure connection string", "DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=" + long + ";EndpointSuffix=core.windows.net"}, + {"gitlab pat", "token: glpat-" + long}, + {"stripe live key", "using sk_live_" + long}, + {"huggingface token", "export HF_TOKEN=hf_" + long}, + + // --- The shapes that already worked; they must keep working --- + {"anthropic key", "ANTHROPIC_API_KEY=" + fakeKey(m)}, + {"openai project key", "OPENAI_API_KEY=sk-proj-" + long}, + {"github pat", "ghp_" + m + "0123456789abcdefghij"}, + {"github fine-grained pat", "github_pat_11" + m + "_0123456789abcdefghijklmnop"}, + {"aws access key id", "AKIA" + "IOSFODNN7" + m[:7]}, + {"slack bot token", "xoxb-123456789012-" + long}, + {"jwt", "eyJ" + m + "abcdef.eyJhbGciOiJIUzI1NiIs.SflKxwRJSMeKKF2QT4"}, + {"env var named secret", "MY_SERVICE_SECRET=" + long}, + {"password assignment", "password = '" + long + "'"}, + {"pem private key", "-----BEGIN RSA PRIVATE KEY-----\n" + long + "\n-----END RSA PRIVATE KEY-----"}, + } + if len(cases) != 22 { + t.Fatalf("the table has %d cases; the reviewed set is 22", len(cases)) + } + + leaked := 0 + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := RedactContent(tc.text, 0) + if strings.Contains(got, m) { + leaked++ + t.Errorf("credential survived redaction\n in: %s\n out: %s", tc.text, got) + } + }) + } + if leaked > 0 { + t.Logf("%d of %d shapes leaked", leaked, len(cases)) + } +} + +// TestRedactConfigChecksAllowlistedValuesForEmbeddedCredentials closes the other half +// of the reviewed gap: `anthropic_upstream` is allowlisted BY NAME, so its value was +// passed through verbatim — and an upstream URL is exactly where a `user:password@` +// credential lives. Allowlisting the key must not mean trusting the value. +func TestRedactConfigChecksAllowlistedValuesForEmbeddedCredentials(t *testing.T) { + const m = "URLCANARY" + out := RedactConfig(map[string]any{ + "anthropic_upstream": "https://svc:" + m + "0123456789@api.anthropic.com", + "openai_upstream": "https://api.openai.com", // no credential: must survive intact + "components": map[string]any{ + "summarize": map[string]any{ + "source": "postgres://admin:" + m + "0123456789@db.internal/app", + }, + }, + }).(map[string]any) + + if flat := sprintDeep(out); strings.Contains(flat, m) { + t.Errorf("a credential embedded in an allowlisted VALUE survived: %s", flat) + } + if out["openai_upstream"] != "https://api.openai.com" { + t.Errorf("a credential-free allowlisted URL was redacted: %v; the config view must stay useful", + out["openai_upstream"]) + } +} + +func TestRedactContentKeepsOrdinaryCode(t *testing.T) { + code := `func Fib(n int) int { + if n < 2 { return n } + a, b := 0, 1 + for i := 2; i <= n; i++ { a, b = b, a+b } + return b +} +// a comment mentioning the word key and token, harmlessly +skeleton := "skip-this-not-a-secret" +` + got := RedactContent(code, 0) + if strings.Contains(got, Redacted) { + t.Errorf("ordinary code was shredded by the secret patterns:\n%s", got) + } +} + +func TestRedactContentCapAppliesAfterScrubbing(t *testing.T) { + secret := fakeKey("TRAILINGSECRET") + // Put the secret right at the cap boundary: a cap applied FIRST could truncate + // the pattern into an unmatchable prefix and store it. + text := strings.Repeat("x", 200) + secret + strings.Repeat("y", 200) + got := RedactContent(text, 210) + if strings.Contains(got, "TRAILINGSECRET") { + t.Errorf("a secret at the truncation boundary survived:\n%s", got) + } + if len(got) > 260 { // cap + the truncation notice + t.Errorf("cap not applied: %d bytes", len(got)) + } + if !strings.Contains(got, "truncated") { + t.Error("truncation should be visible, not silent") + } +} + +func TestRedactContentCapIsRuneSafe(t *testing.T) { + // Cap lands mid-rune; the result must still be valid UTF-8. + got := RedactContent(strings.Repeat("é", 100), 51) + for i, r := range got { + if r == '�' { + t.Fatalf("cap produced an invalid rune at %d: %q", i, got) + } + } +} + +// TestNoSecretReachesTheDatabase is the end-to-end version of the guarantee: run a +// captured event carrying a secret through the real capture path, then read every +// byte of every stored column back and assert the secret is not there. +func TestNoSecretReachesTheDatabase(t *testing.T) { + path := filepath.Join(t.TempDir(), "d.db") + rec, err := NewRecorder(Options{DBPath: path, CaptureContent: true, ContentCap: 1 << 20, + ContentMaxPerRequest: 10}) + if err != nil { + t.Fatal(err) + } + secret := fakeKey("DBLEAKCANARY") + e := mkEvent(1000, "s", "m", 500, 100) + e.Content = []ContentRow{{ + Path: "messages.2", + Before: "ANTHROPIC_API_KEY=" + secret + "\nsome more tool output\n", + After: "ANTHROPIC_API_KEY=" + secret, + }} + // Record the event RAW — no pre-redaction. That is the contract now: the writer + // goroutine redacts immediately before the INSERT, so the secret is handed to the + // pipeline in the clear and must still never reach a column. This is the stronger + // version of the test: it fails if redaction is skipped anywhere on the write path, + // whereas scrubbing here first would only have proved RedactContent works. + e.ContentCap = 1 << 20 + rec.Record(e) + if err := rec.Close(); err != nil { + t.Fatal(err) + } + + db, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + got, err := db.Request(1, true) + if err != nil { + t.Fatal(err) + } + if len(got.Content) == 0 { + t.Fatal("no content stored; the test would pass vacuously") + } + for _, c := range got.Content { + if strings.Contains(c.Before, secret) || strings.Contains(c.After, secret) { + t.Fatalf("the secret reached disk: %q / %q", c.Before, c.After) + } + if !strings.Contains(c.Before, "ANTHROPIC_API_KEY") { + t.Error("redaction removed the variable NAME too; the diff loses its meaning") + } + } +} + +// sprintDeep flattens a redacted config tree to one string, so a test can assert +// no secret substring appears anywhere in it. +func sprintDeep(v any) string { return fmt.Sprintf("%v", v) } diff --git a/dash/schema.go b/dash/schema.go new file mode 100644 index 0000000..9c2d326 --- /dev/null +++ b/dash/schema.go @@ -0,0 +1,183 @@ +// Package dash is context-guru's persistent observability layer: the durable +// per-request store behind the dashboard, the off-hot-path capture pipeline that +// fills it, the JSON/SSE API the UI reads, and the embedded single-file UI. +// +// Layering, and why it is boring on purpose: +// +// - metrics.Aggregator stays the fast in-process counter behind /stats. dash +// never replaces it and never changes its shape — the benchmark harnesses +// (deploy/harbor/*.py) parse that payload. +// - Capture is strictly out of band. A request handler builds one Event and +// hands it to a buffered channel; when the channel is full the event is +// DROPPED and counted. Observability can never add latency to, or fail, a +// request — the property gateway gets right and the one worth keeping. +// - One writer goroutine owns the database. It batches inserts in a +// transaction and fans a summary row out to SSE clients. Nothing else writes. +// - Percentages are derived at read time; COST is computed at write time, so +// history does not silently reprice when a model's published rate changes. +// - No rollup tables. Time series are bucketed in SQL at query time +// (ts/bucket*bucket GROUP BY 1). SQLite handles millions of rows; a +// pre-aggregation layer is the speculative complexity to skip until a query +// is measurably slow. +// +// The driver is modernc.org/sqlite (pure Go), so a dashboard build needs no C +// toolchain beyond the one tree-sitter already forces. +package dash + +import ( + "database/sql" + "fmt" + "strings" +) + +// schemaVersion is bumped whenever the DDL below changes incompatibly. On a +// mismatch Open PRESERVES the old file (renamed with its version suffix) and +// starts a fresh database: a dashboard is a derived view, so discarding history +// beats refusing to boot, and keeping the file beats deleting a user's data. +const schemaVersion = 1 + +// ddl is the whole schema. Timestamps are epoch MILLISECONDS everywhere — never +// a formatted locale string, which cannot be range-queried, sorted portably, or +// bucketed (gateway's mistake). +const ddl = ` +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +-- One row per proxied request. Cost columns are USD, priced at write time from +-- the model's rates; NULL-equivalent (0) with token_accounting<>'complete' means +-- "we could not price this", which the UI must render as unknown, not as free. +CREATE TABLE IF NOT EXISTS requests ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, -- epoch ms + session_id TEXT NOT NULL DEFAULT '', + model TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT '', + agent TEXT NOT NULL DEFAULT '', -- client user-agent family (claude-code, codex, …) + preset TEXT NOT NULL DEFAULT '', + mode TEXT NOT NULL DEFAULT '', -- operating mode (active|bypass|observe) + route TEXT NOT NULL DEFAULT '', + status INTEGER NOT NULL DEFAULT 0, -- upstream HTTP status (0 = no upstream) + bypassed INTEGER NOT NULL DEFAULT 0, + cache_aware INTEGER NOT NULL DEFAULT 0, + messages INTEGER NOT NULL DEFAULT 0, + tokens_before INTEGER NOT NULL DEFAULT 0, + tokens_after INTEGER NOT NULL DEFAULT 0, + attempted_tokens INTEGER NOT NULL DEFAULT 0, -- denominator: what we were allowed to compact + frozen_tokens INTEGER NOT NULL DEFAULT 0, -- cost of cache safety: what we deliberately did not touch + saved_unique INTEGER NOT NULL DEFAULT 0, -- this request's NEW (not re-sent) savings + fresh_input INTEGER NOT NULL DEFAULT 0, + cache_read INTEGER NOT NULL DEFAULT 0, + cache_write INTEGER NOT NULL DEFAULT 0, + output_tokens INTEGER NOT NULL DEFAULT 0, + cost_usd REAL NOT NULL DEFAULT 0, + baseline_cost_usd REAL NOT NULL DEFAULT 0, -- what the same request would have cost uncompacted + cg_llm_cost_usd REAL NOT NULL DEFAULT 0, -- context-guru's OWN model spend attributable here + cg_latency_ms REAL NOT NULL DEFAULT 0, + upstream_ms REAL NOT NULL DEFAULT 0, + expands INTEGER NOT NULL DEFAULT 0, + expand_tokens INTEGER NOT NULL DEFAULT 0, -- restoration: content we offloaded and had to serve back + reverts INTEGER NOT NULL DEFAULT 0, + token_accounting TEXT NOT NULL DEFAULT 'missing', -- complete|partial|missing + cache_miss_reason TEXT NOT NULL DEFAULT '', -- cold_start|ttl_expiry|prefix_change|unknown|hit + uncompressed_reason TEXT NOT NULL DEFAULT '' -- why we did not compact: '' = we did +); +CREATE INDEX IF NOT EXISTS idx_requests_ts ON requests(ts DESC); +CREATE INDEX IF NOT EXISTS idx_requests_session ON requests(session_id, ts); +CREATE INDEX IF NOT EXISTS idx_requests_model ON requests(model, ts); + +-- One row per component per request: the answer to "which components earn their +-- place". saved_gross is what the component removed THIS turn (re-counted every +-- turn the agent re-sends the transcript); saved_unique counts each distinct +-- compaction once. +CREATE TABLE IF NOT EXISTS request_components ( + request_id INTEGER NOT NULL REFERENCES requests(id) ON DELETE CASCADE, + component TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT '', + acted INTEGER NOT NULL DEFAULT 0, + mutated INTEGER NOT NULL DEFAULT 0, + reverted INTEGER NOT NULL DEFAULT 0, + skipped INTEGER NOT NULL DEFAULT 0, + saved_gross INTEGER NOT NULL DEFAULT 0, + saved_unique INTEGER NOT NULL DEFAULT 0, + duration_ms REAL NOT NULL DEFAULT 0, + err TEXT NOT NULL DEFAULT '' +); +CREATE INDEX IF NOT EXISTS idx_rc_request ON request_components(request_id); +CREATE INDEX IF NOT EXISTS idx_rc_comp ON request_components(component); + +-- Before/after text of each rewritten message — the diff view's data. Stored +-- gzip-compressed and size-capped, and skippable entirely (content capture is +-- opt-out). Redaction happens BEFORE the insert, never on read. +CREATE TABLE IF NOT EXISTS request_content ( + request_id INTEGER NOT NULL REFERENCES requests(id) ON DELETE CASCADE, + seq INTEGER NOT NULL, + path TEXT NOT NULL DEFAULT '', + before_tokens INTEGER NOT NULL DEFAULT 0, + after_tokens INTEGER NOT NULL DEFAULT 0, + before_gz BLOB, + after_gz BLOB +); +CREATE INDEX IF NOT EXISTS idx_content_request ON request_content(request_id); + +-- Ingested benchmark runs (deploy/harbor's summary.json + rows-*.json). +CREATE TABLE IF NOT EXISTS bench_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, -- run directory name, so re-ingesting replaces + ts INTEGER NOT NULL, + dataset TEXT NOT NULL DEFAULT '', + model TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '{}' +); +CREATE TABLE IF NOT EXISTS bench_tasks ( + run_id INTEGER NOT NULL REFERENCES bench_runs(id) ON DELETE CASCADE, + arm TEXT NOT NULL, -- config name: off|codesmart|headroom|rtk|… + task TEXT NOT NULL, + reward REAL NOT NULL DEFAULT 0, + steps INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cache_read INTEGER NOT NULL DEFAULT 0, + cache_write INTEGER NOT NULL DEFAULT 0, + fresh_input INTEGER NOT NULL DEFAULT 0, + cost_usd REAL NOT NULL DEFAULT 0, + norm_cost_usd REAL NOT NULL DEFAULT 0, + wall_s REAL NOT NULL DEFAULT 0, + exception INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS idx_bt_run ON bench_tasks(run_id, arm); +` + +// migrate creates the schema and validates its version. A version mismatch is +// reported to the caller, which renames the old file aside and retries — see Open. +func migrate(db *sql.DB) error { + var have string + err := db.QueryRow(`SELECT value FROM meta WHERE key='schema_version'`).Scan(&have) + switch { + case err == sql.ErrNoRows || isMissingTable(err): + // Fresh (or pre-meta) database: create everything and stamp the version. + case err != nil: + return err + case have != fmt.Sprint(schemaVersion): + return &versionMismatch{have: have} + default: + return nil // already at this version + } + if _, err := db.Exec(ddl); err != nil { + return err + } + _, err = db.Exec(`INSERT OR REPLACE INTO meta(key,value) VALUES('schema_version',?)`, fmt.Sprint(schemaVersion)) + return err +} + +// versionMismatch signals that the file on disk was written by another schema. +type versionMismatch struct{ have string } + +func (e *versionMismatch) Error() string { + return fmt.Sprintf("dash: database schema version %s, want %d", e.have, schemaVersion) +} + +func isMissingTable(err error) bool { + return err != nil && strings.Contains(err.Error(), "no such table") +} diff --git a/dash/sse.go b/dash/sse.go new file mode 100644 index 0000000..2456852 --- /dev/null +++ b/dash/sse.go @@ -0,0 +1,221 @@ +package dash + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "sync" + "time" +) + +// sseWriteTimeout is how long a client has to accept an event before it is +// considered dead and evicted. A browser tab that is suspended, or a curl that +// stopped reading, must not be able to stall the writer goroutine — which would +// stall persistence, which would stall nothing on the request path but would still +// silently stop the dashboard. Bounded per-client buffering plus this timeout +// makes a hung client the hung client's own problem. +const sseWriteTimeout = 2 * time.Second + +// sseClientBuffer is how many events a slow-but-alive client may fall behind by +// before it is dropped. +const sseClientBuffer = 64 + +// Hub fans captured events out to SSE clients. Published events carry SUMMARY +// rows only — no before/after content, ever. The live feed is a monitoring +// surface; content is fetched deliberately, per request, through the access-gated +// detail route. +type Hub struct { + mu sync.Mutex + clients map[*client]struct{} + nextID uint64 + closed bool + // ring is a small backlog so a reconnecting client with Last-Event-ID gets the + // gap replayed instead of silently losing it. Bounded on purpose: a client that + // was away longer than the ring reloads from /api/requests, which is the + // authoritative history. + ring []*Event + ringCap int +} + +type client struct { + ch chan *Event + closed chan struct{} + once sync.Once +} + +func (c *client) stop() { c.once.Do(func() { close(c.closed) }) } + +// NewHub returns an empty hub. +func NewHub() *Hub { + return &Hub{clients: map[*client]struct{}{}, ringCap: 256} +} + +// Clients reports the current subscriber count. +func (h *Hub) Clients() int { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.clients) +} + +// Publish sends one event to every live client, dropping (and evicting) any +// client whose buffer is full. Called only from the writer goroutine. +func (h *Hub) Publish(e *Event) { + // Summary only: strip content before it can reach a live-feed client. + sum := *e + sum.Content = nil + h.mu.Lock() + if h.closed { + h.mu.Unlock() + return + } + h.ring = append(h.ring, &sum) + if len(h.ring) > h.ringCap { + h.ring = h.ring[len(h.ring)-h.ringCap:] + } + var evict []*client + for c := range h.clients { + select { + case c.ch <- &sum: + default: + evict = append(evict, c) // buffer full: the client is not keeping up + } + } + for _, c := range evict { + delete(h.clients, c) + } + h.mu.Unlock() + for _, c := range evict { + c.stop() + } +} + +// Close disconnects every client. +func (h *Hub) Close() { + h.mu.Lock() + h.closed = true + cs := make([]*client, 0, len(h.clients)) + for c := range h.clients { + cs = append(cs, c) + } + h.clients = map[*client]struct{}{} + h.mu.Unlock() + for _, c := range cs { + c.stop() + } +} + +// backlogSince returns ring events with an id greater than lastID (the +// Last-Event-ID a reconnecting browser sends), so a reconnect backfills the gap +// instead of pretending nothing happened while it was away. +func (h *Hub) backlogSince(lastID int64) []*Event { + h.mu.Lock() + defer h.mu.Unlock() + var out []*Event + for _, e := range h.ring { + if e.ID > lastID { + out = append(out, e) + } + } + return out +} + +func (h *Hub) subscribe() (*client, bool) { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return nil, false + } + c := &client{ch: make(chan *Event, sseClientBuffer), closed: make(chan struct{})} + h.clients[c] = struct{}{} + return c, true +} + +func (h *Hub) unsubscribe(c *client) { + h.mu.Lock() + delete(h.clients, c) + h.mu.Unlock() + c.stop() +} + +// ServeHTTP streams events as SSE. It honors Last-Event-ID (header or the +// ?last_event_id= query param, for clients that cannot set headers), enforces a +// per-write timeout, and evicts itself on a stalled write. +func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", http.StatusInternalServerError) + return + } + lastID := int64(0) + if v := r.Header.Get("Last-Event-ID"); v != "" { + lastID, _ = strconv.ParseInt(v, 10, 64) + } else if v := r.URL.Query().Get("last_event_id"); v != "" { + lastID, _ = strconv.ParseInt(v, 10, 64) + } + + c, ok := h.subscribe() + if !ok { + http.Error(w, "shutting down", http.StatusServiceUnavailable) + return + } + defer h.unsubscribe(c) + + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.Header().Set("X-Accel-Buffering", "no") // don't let a reverse proxy buffer the stream + w.WriteHeader(http.StatusOK) + flusher.Flush() + + // Backfill the gap first, in order, so the client's view is continuous. + for _, e := range h.backlogSince(lastID) { + if !writeEvent(w, flusher, e) { + return + } + } + + keepalive := time.NewTicker(20 * time.Second) + defer keepalive.Stop() + for { + select { + case <-r.Context().Done(): + return + case <-c.closed: + return + case e := <-c.ch: + if !writeEvent(w, flusher, e) { + return + } + case <-keepalive.C: + // A comment frame keeps intermediaries from reaping an idle stream. + if _, err := io.WriteString(w, ": keepalive\n\n"); err != nil { + return + } + flusher.Flush() + } + } +} + +// writeEvent writes one SSE frame under a deadline. A write that does not +// complete in sseWriteTimeout means the client is not reading; we give up on it +// rather than block. Uses ResponseController so the deadline applies to the +// underlying connection, not just to our own bookkeeping. +func writeEvent(w http.ResponseWriter, flusher http.Flusher, e *Event) bool { + rc := http.NewResponseController(w) + // SetWriteDeadline is unsupported on some ResponseWriters (e.g. httptest's); + // that is fine — the timeout is a safety net, not a correctness requirement. + _ = rc.SetWriteDeadline(time.Now().Add(sseWriteTimeout)) + defer func() { _ = rc.SetWriteDeadline(time.Time{}) }() + + b, err := json.Marshal(e) + if err != nil { + return true // skip a malformed event; do not kill the stream + } + if _, err := fmt.Fprintf(w, "id: %d\nevent: request\ndata: %s\n\n", e.ID, b); err != nil { + return false + } + flusher.Flush() + return true +} diff --git a/dash/sse_test.go b/dash/sse_test.go new file mode 100644 index 0000000..0f726fd --- /dev/null +++ b/dash/sse_test.go @@ -0,0 +1,235 @@ +package dash + +import ( + "bufio" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +func TestHubFansOutAndStripsContent(t *testing.T) { + h := NewHub() + defer h.Close() + c1, _ := h.subscribe() + c2, _ := h.subscribe() + if h.Clients() != 2 { + t.Fatalf("clients = %d; want 2", h.Clients()) + } + + e := &Event{ID: 7, SessionID: "s", TokensBefore: 100, TokensAfter: 50} + e.Content = []ContentRow{{Path: "messages.1", Before: "a customer's source code", After: "x"}} + h.Publish(e) + + for i, c := range []*client{c1, c2} { + select { + case got := <-c.ch: + if got.ID != 7 { + t.Errorf("client %d got id %d", i, got.ID) + } + // The live feed is a monitoring surface; content is fetched deliberately + // through the access-gated detail route, never pushed. + if len(got.Content) != 0 { + t.Errorf("client %d received request CONTENT over SSE: %+v", i, got.Content) + } + case <-time.After(time.Second): + t.Errorf("client %d received nothing", i) + } + } + // The caller's event must not have been mutated by the strip. + if len(e.Content) != 1 { + t.Error("Publish mutated the caller's event") + } +} + +// TestHubEvictsHungClient is the test the issue names: a subscriber that stops +// reading must be dropped, not allowed to wedge the writer goroutine. +func TestHubEvictsHungClient(t *testing.T) { + h := NewHub() + defer h.Close() + hung, _ := h.subscribe() + live, _ := h.subscribe() + + // Overflow the hung client's buffer while draining the live one. + drained := 0 + done := make(chan struct{}) + go func() { + defer close(done) + for range live.ch { + drained++ + if drained > sseClientBuffer*3 { + return + } + } + }() + + start := time.Now() + for i := 0; i < sseClientBuffer*4; i++ { + h.Publish(&Event{ID: int64(i + 1)}) + time.Sleep(time.Millisecond) // let the live reader keep up + } + elapsed := time.Since(start) + + // Publishing must never have blocked on the hung client. + if elapsed > 10*time.Second { + t.Fatalf("Publish stalled behind a hung client (%v)", elapsed) + } + select { + case <-hung.closed: + // evicted, as required + case <-time.After(time.Second): + t.Error("a client that never read was not evicted") + } + if h.Clients() > 1 { + t.Errorf("clients = %d after eviction; want at most 1", h.Clients()) + } + <-done +} + +func TestHubBacklogHonorsLastEventID(t *testing.T) { + h := NewHub() + defer h.Close() + for i := 1; i <= 5; i++ { + h.Publish(&Event{ID: int64(i)}) + } + got := h.backlogSince(3) + if len(got) != 2 || got[0].ID != 4 || got[1].ID != 5 { + t.Fatalf("backlog after id 3 = %v; want ids 4,5", ids(got)) + } + if all := h.backlogSince(0); len(all) != 5 { + t.Errorf("backlog from 0 = %d events; want 5", len(all)) + } + if none := h.backlogSince(99); len(none) != 0 { + t.Errorf("backlog past the head = %d; want 0", len(none)) + } +} + +func ids(evs []*Event) []int64 { + out := make([]int64, len(evs)) + for i, e := range evs { + out[i] = e.ID + } + return out +} + +func TestHubRingIsBounded(t *testing.T) { + h := NewHub() + defer h.Close() + for i := 1; i <= h.ringCap*3; i++ { + h.Publish(&Event{ID: int64(i)}) + } + if len(h.ring) > h.ringCap { + t.Errorf("ring grew to %d; cap is %d", len(h.ring), h.ringCap) + } + // It must retain the NEWEST events, not the oldest. + if h.ring[len(h.ring)-1].ID != int64(h.ringCap*3) { + t.Errorf("ring head = %d; want the newest event", h.ring[len(h.ring)-1].ID) + } +} + +func TestSSEEndpointStreamsAndBackfills(t *testing.T) { + h := NewHub() + defer h.Close() + for i := 1; i <= 3; i++ { + h.Publish(&Event{ID: int64(i), SessionID: "s"}) + } + srv := httptest.NewServer(http.HandlerFunc(h.ServeHTTP)) + defer srv.Close() + + // Reconnect claiming to have seen id 1: ids 2 and 3 must be backfilled. + req, _ := http.NewRequest(http.MethodGet, srv.URL+"?last_event_id=1", nil) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if ct := resp.Header.Get("Content-Type"); ct != "text/event-stream" { + t.Errorf("content type = %q", ct) + } + + // Publish one more so the stream carries a live event after the backfill. + go func() { + time.Sleep(50 * time.Millisecond) + h.Publish(&Event{ID: 4, SessionID: "s"}) + }() + + sc := bufio.NewScanner(resp.Body) + var seen []string + deadline := time.Now().Add(5 * time.Second) + for sc.Scan() && time.Now().Before(deadline) { + line := sc.Text() + if strings.HasPrefix(line, "id: ") { + seen = append(seen, strings.TrimPrefix(line, "id: ")) + } + if len(seen) >= 3 { + break + } + } + want := []string{"2", "3", "4"} + if len(seen) != 3 || seen[0] != want[0] || seen[1] != want[1] || seen[2] != want[2] { + t.Errorf("event ids = %v; want %v (backfill then live)", seen, want) + } +} + +func TestHubCloseDisconnectsEveryone(t *testing.T) { + h := NewHub() + c, _ := h.subscribe() + h.Close() + select { + case <-c.closed: + case <-time.After(time.Second): + t.Error("Close did not disconnect the client") + } + if _, ok := h.subscribe(); ok { + t.Error("subscribe succeeded after Close") + } + // Publishing after Close must be a harmless no-op, not a panic. + h.Publish(&Event{ID: 1}) +} + +// TestHubConcurrent drives publish, subscribe and unsubscribe at once; run under +// -race this is the mandatory SSE-hub check. +func TestHubConcurrent(t *testing.T) { + h := NewHub() + var wg sync.WaitGroup + stop := make(chan struct{}) + + wg.Add(1) + go func() { + defer wg.Done() + for i := 1; ; i++ { + select { + case <-stop: + return + default: + } + h.Publish(&Event{ID: int64(i), SessionID: "s"}) + } + }() + for g := 0; g < 6; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 50; i++ { + c, ok := h.subscribe() + if !ok { + return + } + // Read a little, then leave — the churn a browser tab actually produces. + select { + case <-c.ch: + case <-time.After(time.Millisecond): + } + h.unsubscribe(c) + _ = h.Clients() + _ = h.backlogSince(int64(i)) + } + }() + } + time.Sleep(150 * time.Millisecond) + close(stop) + wg.Wait() + h.Close() +} diff --git a/dash/store.go b/dash/store.go new file mode 100644 index 0000000..da0ce06 --- /dev/null +++ b/dash/store.go @@ -0,0 +1,307 @@ +package dash + +import ( + "bytes" + "compress/gzip" + "database/sql" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "sync/atomic" + "time" + + _ "modernc.org/sqlite" // pure-Go driver: no extra C toolchain for the dashboard +) + +// memSeq names in-memory databases uniquely. See Open. +var memSeq atomic.Uint64 + +// DB is the dashboard's durable store. All writes go through one goroutine (see +// capture.go), so the only concurrency here is reads racing that writer, which +// SQLite in WAL mode handles. +type DB struct { + sql *sql.DB + path string // "" for the in-memory store +} + +// Open opens (creating if needed) the dashboard database at path. path ":memory:" +// or "" yields an ephemeral in-memory database, which is also the fallback the +// proxy uses when the configured path is unwritable — the proxy must keep serving +// traffic whatever the disk says. +// +// On a schema-version mismatch the existing file is renamed aside +// (.v.bak) and a fresh database is created: the dashboard is a derived +// view, so discarding history beats refusing to boot, and renaming beats deleting. +func Open(path string) (*DB, error) { + if path == "" || path == ":memory:" { + // A UNIQUE name per Open, not a bare `file::memory:`. + // + // `cache=shared` is required: database/sql keeps a connection POOL and a private + // in-memory database exists per connection, so every pooled connection would + // otherwise see its own empty database. But under `cache=shared` the NAME + // identifies the database, and `file::memory:` is a single name — so every + // in-memory dashboard in the process WAS the same database. Two proxies falling + // back to :memory: silently merged their history, and :memory: tests leaked rows + // into each other (the flakiest possible failure). A per-instance name keeps the + // pooling behaviour and removes the collision. + return openDSN(fmt.Sprintf("file:dashmem%d?mode=memory&cache=shared", memSeq.Add(1)), "") + } + if dir := filepath.Dir(path); dir != "" { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + } + db, err := openDSN(dsn(path), path) + var mismatch *versionMismatch + if errors.As(err, &mismatch) { + aside := fmt.Sprintf("%s.v%s.bak", path, sanitizeVersion(mismatch.have)) + slog.Warn("dash: schema version changed; preserving the old database and starting fresh", + "old_version", mismatch.have, "new_version", schemaVersion, "preserved_at", aside) + if rerr := os.Rename(path, aside); rerr != nil { + return nil, fmt.Errorf("dash: %w (and could not preserve it: %v)", err, rerr) + } + return openDSN(dsn(path), path) + } + return db, err +} + +// dsn builds the driver DSN: WAL for concurrent reads while the writer commits, +// NORMAL synchronous (a lost tail of observability rows on power loss is +// acceptable; halving write cost is not), a busy timeout so a read never errors +// out under a concurrent commit, and foreign keys on for the ON DELETE CASCADEs +// retention relies on. +func dsn(path string) string { + return "file:" + path + "?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)" + + "&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)" +} + +func openDSN(d, path string) (*DB, error) { + sdb, err := sql.Open("sqlite", d) + if err != nil { + return nil, err + } + if err := sdb.Ping(); err != nil { + sdb.Close() + return nil, err + } + if err := migrate(sdb); err != nil { + sdb.Close() + return nil, err + } + return &DB{sql: sdb, path: path}, nil +} + +// sanitizeVersion keeps a version string safe to embed in a filename. +func sanitizeVersion(v string) string { + v = strings.Map(func(r rune) rune { + if (r >= '0' && r <= '9') || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') { + return r + } + return '-' + }, v) + if v == "" { + return "unknown" + } + if len(v) > 16 { + v = v[:16] + } + return v +} + +// Close releases the database. +func (d *DB) Close() error { + if d == nil || d.sql == nil { + return nil + } + return d.sql.Close() +} + +// Path returns the on-disk path ("" when in-memory). +func (d *DB) Path() string { return d.path } + +// insertBatch writes a batch of captured events in ONE transaction — the whole +// point of batching: a per-request fsync would make the writer the bottleneck +// under agent traffic. A failed batch is logged and dropped; observability never +// retries into a growing backlog. +func (d *DB) insertBatch(evs []*Event) error { + tx, err := d.sql.Begin() + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit + + reqStmt, err := tx.Prepare(`INSERT INTO requests( + ts, session_id, model, provider, agent, preset, mode, route, status, bypassed, cache_aware, + messages, tokens_before, tokens_after, attempted_tokens, frozen_tokens, saved_unique, + fresh_input, cache_read, cache_write, output_tokens, + cost_usd, baseline_cost_usd, cg_llm_cost_usd, cg_latency_ms, upstream_ms, + expands, expand_tokens, reverts, token_accounting, cache_miss_reason, uncompressed_reason + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) + if err != nil { + return err + } + defer reqStmt.Close() + compStmt, err := tx.Prepare(`INSERT INTO request_components( + request_id, component, kind, acted, mutated, reverted, skipped, saved_gross, saved_unique, duration_ms, err + ) VALUES (?,?,?,?,?,?,?,?,?,?,?)`) + if err != nil { + return err + } + defer compStmt.Close() + contentStmt, err := tx.Prepare(`INSERT INTO request_content( + request_id, seq, path, before_tokens, after_tokens, before_gz, after_gz + ) VALUES (?,?,?,?,?,?,?)`) + if err != nil { + return err + } + defer contentStmt.Close() + + for _, e := range evs { + res, err := reqStmt.Exec( + e.TS, e.SessionID, e.Model, e.Provider, e.Agent, e.Preset, e.Mode, e.Route, e.Status, + boolInt(e.Bypassed), boolInt(e.CacheAware), + e.Messages, e.TokensBefore, e.TokensAfter, e.AttemptedTokens, e.FrozenTokens, e.SavedUnique, + e.FreshInput, e.CacheRead, e.CacheWrite, e.OutputTokens, + e.CostUSD, e.BaselineCostUSD, e.CGLLMCostUSD, e.CGLatencyMs, e.UpstreamMs, + e.Expands, e.ExpandTokens, e.Reverts, e.TokenAccounting, e.CacheMissReason, e.UncompressedReason, + ) + if err != nil { + return err + } + id, err := res.LastInsertId() + if err != nil { + return err + } + e.ID = id + for _, c := range e.Components { + if _, err := compStmt.Exec(id, c.Component, c.Kind, + boolInt(c.Acted), boolInt(c.Mutated), boolInt(c.Reverted), boolInt(c.Skipped), + c.SavedGross, c.SavedUnique, c.DurationMs, c.Err); err != nil { + return err + } + } + for i, c := range e.Content { + if _, err := contentStmt.Exec(id, i, c.Path, c.BeforeTokens, c.AfterTokens, + gzipText(c.Before), gzipText(c.After)); err != nil { + return err + } + } + } + return tx.Commit() +} + +func boolInt(b bool) int { + if b { + return 1 + } + return 0 +} + +// gzipText compresses one captured before/after blob. Content is the bulk of the +// database, it is highly repetitive agent transcript text, and it is only ever +// read one request at a time by the diff view — so paying CPU on the writer +// goroutine to keep the file small is the right trade. +func gzipText(s string) []byte { + if s == "" { + return nil + } + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := io.WriteString(zw, s); err != nil { + return nil + } + if err := zw.Close(); err != nil { + return nil + } + return buf.Bytes() +} + +func gunzipText(b []byte) string { + if len(b) == 0 { + return "" + } + zr, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + return "" + } + defer zr.Close() + out, err := io.ReadAll(zr) + if err != nil { + return "" + } + return string(out) +} + +// Prune enforces retention by BOTH age and size, in that order: drop everything +// older than maxAge, then — if the file is still over maxBytes — drop the oldest +// requests until it fits. Age alone cannot bound a burst; size alone silently +// erases a quiet week. Content rows and component rows go with their request via +// ON DELETE CASCADE. Returns how many request rows were deleted. +// +// maxAge <= 0 disables the age rule; maxBytes <= 0 disables the size rule. +func (d *DB) Prune(now time.Time, maxAge time.Duration, maxBytes int64) (int64, error) { + var deleted int64 + if maxAge > 0 { + cutoff := now.Add(-maxAge).UnixMilli() + res, err := d.sql.Exec(`DELETE FROM requests WHERE ts < ?`, cutoff) + if err != nil { + return deleted, err + } + n, _ := res.RowsAffected() + deleted += n + } + if maxBytes <= 0 { + return deleted, nil + } + // Size rule. Loop because deleting a slice does not immediately shrink the + // file (SQLite reuses freed pages), so we bound the work: at most a few + // rounds, each dropping the oldest 10% of rows, and stop as soon as the + // estimated payload fits. + for round := 0; round < 8; round++ { + size, err := d.sizeBytes() + if err != nil || size <= maxBytes { + return deleted, err + } + var total int64 + if err := d.sql.QueryRow(`SELECT COUNT(*) FROM requests`).Scan(&total); err != nil { + return deleted, err + } + if total == 0 { + return deleted, nil + } + drop := total / 10 + if drop < 1 { + drop = 1 + } + res, err := d.sql.Exec( + `DELETE FROM requests WHERE id IN (SELECT id FROM requests ORDER BY ts ASC LIMIT ?)`, drop) + if err != nil { + return deleted, err + } + n, _ := res.RowsAffected() + deleted += n + // Reclaim the pages so the next sizeBytes reflects the deletion. + if _, err := d.sql.Exec(`VACUUM`); err != nil { + return deleted, err + } + } + return deleted, nil +} + +// sizeBytes reports the database's payload size. page_count*page_size is exact +// for the main file and works for an in-memory database too (where a stat would +// have nothing to look at). +func (d *DB) sizeBytes() (int64, error) { + var pages, pageSize int64 + if err := d.sql.QueryRow(`PRAGMA page_count`).Scan(&pages); err != nil { + return 0, err + } + if err := d.sql.QueryRow(`PRAGMA page_size`).Scan(&pageSize); err != nil { + return 0, err + } + return pages * pageSize, nil +} diff --git a/dash/store_test.go b/dash/store_test.go new file mode 100644 index 0000000..097d35e --- /dev/null +++ b/dash/store_test.go @@ -0,0 +1,627 @@ +package dash + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// mkEvent builds a plausible captured request for the query tests. +func mkEvent(ts int64, session, model string, before, after int) *Event { + return &Event{ + TS: ts, SessionID: session, Model: model, Provider: "anthropic", + Agent: "claude-code", Preset: "codesmart", Mode: ModeActive, + TokensBefore: before, TokensAfter: after, AttemptedTokens: before, + SavedUnique: before - after, FreshInput: 10, CacheRead: 1000, CacheWrite: 100, + OutputTokens: 50, CostUSD: 0.01, BaselineCostUSD: 0.02, + CGLatencyMs: 5, UpstreamMs: 500, TokenAccounting: AccountingComplete, + CacheMissReason: CacheHit, + Components: []CompRow{ + {Component: "extract", Kind: "offload", Acted: before > after, Mutated: before > after, + SavedGross: before - after, SavedUnique: before - after, DurationMs: 1.5}, + {Component: "cacheinject", Kind: "reformat", Mutated: true, DurationMs: 0.1}, + }, + } +} + +func openTestDB(t *testing.T) *DB { + t.Helper() + db, err := Open(filepath.Join(t.TempDir(), "d.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +// TestInMemoryDatabasesAreIsolated pins the fix for a latent collision: Open(":memory:") +// used the DSN `file::memory:?cache=shared`, and under cache=shared the NAME identifies +// the database — so every in-memory dashboard in the process was the SAME database. +// Production opens one, so it was not a live bug, but it silently merged the history of +// two proxies both falling back to :memory:, and it leaked rows between :memory: tests, +// which is the flakiest class of failure there is. +func TestInMemoryDatabasesAreIsolated(t *testing.T) { + a, err := Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer a.Close() + b, err := Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer b.Close() + + if err := a.insertBatch([]*Event{mkEvent(1000, "only-in-a", "m", 100, 90)}); err != nil { + t.Fatal(err) + } + page, err := b.Requests(Filter{}, 0, 10) + if err != nil { + t.Fatal(err) + } + if page.Total != 0 { + t.Errorf("the second in-memory DB sees %d rows from the first; they are the same database", page.Total) + } + // And the first must still hold its own row — a per-connection private database + // would isolate them by losing the data instead. + pa, err := a.Requests(Filter{}, 0, 10) + if err != nil { + t.Fatal(err) + } + if pa.Total != 1 { + t.Errorf("the first in-memory DB holds %d rows; want the 1 just inserted "+ + "(a pooled connection must still see the write)", pa.Total) + } +} + +func TestSchemaMigrationIsIdempotent(t *testing.T) { + path := filepath.Join(t.TempDir(), "d.db") + db, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := db.insertBatch([]*Event{mkEvent(1000, "s1", "m", 100, 90)}); err != nil { + t.Fatal(err) + } + db.Close() + + // Re-opening the same file must find the schema already at version and keep data. + db2, err := Open(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer db2.Close() + page, err := db2.Requests(Filter{}, 0, 10) + if err != nil { + t.Fatal(err) + } + if page.Total != 1 { + t.Fatalf("reopen lost data: total=%d, want 1", page.Total) + } +} + +func TestSchemaVersionMismatchPreservesOldFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "d.db") + db, err := Open(path) + if err != nil { + t.Fatal(err) + } + if err := db.insertBatch([]*Event{mkEvent(1000, "s1", "m", 100, 90)}); err != nil { + t.Fatal(err) + } + // Pretend the file was written by a future version. + if _, err := db.sql.Exec(`UPDATE meta SET value='9999' WHERE key='schema_version'`); err != nil { + t.Fatal(err) + } + db.Close() + + db2, err := Open(path) + if err != nil { + t.Fatalf("a version mismatch must start fresh, not fail: %v", err) + } + defer db2.Close() + page, err := db2.Requests(Filter{}, 0, 10) + if err != nil { + t.Fatal(err) + } + if page.Total != 0 { + t.Errorf("mismatch should start a FRESH database; found %d rows", page.Total) + } + // The user's old data must be preserved, not deleted. + entries, _ := os.ReadDir(dir) + found := false + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".bak") { + found = true + } + } + if !found { + t.Errorf("old database was not preserved alongside; files: %v", entries) + } +} + +func TestRetentionPrunesByAgeAndCascades(t *testing.T) { + db := openTestDB(t) + now := time.Now() + old := now.Add(-48 * time.Hour).UnixMilli() + fresh := now.Add(-1 * time.Hour).UnixMilli() + ev := mkEvent(old, "old", "m", 100, 90) + ev.Content = []ContentRow{{Path: "messages.0", Before: "aaa", After: "a"}} + if err := db.insertBatch([]*Event{ev, mkEvent(fresh, "new", "m", 100, 90)}); err != nil { + t.Fatal(err) + } + oldID := ev.ID + + n, err := db.Prune(now, 24*time.Hour, 0) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatalf("pruned %d rows; want 1", n) + } + page, _ := db.Requests(Filter{}, 0, 10) + if page.Total != 1 || page.Requests[0].SessionID != "new" { + t.Fatalf("wrong row survived: %+v", page.Requests) + } + // Component and content rows must have gone with the request (ON DELETE CASCADE), + // or the database grows without bound behind a retention policy that looks fine. + var comps, content int + db.sql.QueryRow(`SELECT COUNT(*) FROM request_components WHERE request_id=?`, oldID).Scan(&comps) + db.sql.QueryRow(`SELECT COUNT(*) FROM request_content WHERE request_id=?`, oldID).Scan(&content) + if comps != 0 || content != 0 { + t.Errorf("orphaned rows after prune: %d component, %d content", comps, content) + } +} + +func TestRetentionPrunesBySize(t *testing.T) { + db := openTestDB(t) + now := time.Now() + // Write enough content to exceed a small size cap. + big := strings.Repeat("some agent transcript text that compresses but not to nothing ", 400) + var evs []*Event + for i := 0; i < 60; i++ { + e := mkEvent(now.Add(-time.Duration(60-i)*time.Minute).UnixMilli(), "s", "m", 1000, 900) + e.Content = []ContentRow{{Path: "messages.0", Before: big, After: big[:200]}} + evs = append(evs, e) + } + if err := db.insertBatch(evs); err != nil { + t.Fatal(err) + } + before, _ := db.sizeBytes() + if before < 100<<10 { + t.Skipf("test data only produced %d bytes; cannot exercise the size rule", before) + } + + // Age rule off, size rule at a fraction of the current size. + if _, err := db.Prune(now, 0, before/3); err != nil { + t.Fatal(err) + } + after, _ := db.sizeBytes() + if after >= before { + t.Errorf("size prune did not shrink the database: %d -> %d", before, after) + } + page, _ := db.Requests(Filter{}, 0, 200) + if page.Total == 0 { + t.Error("size prune deleted everything; it should drop only the oldest rows") + } + // The survivors must be the NEWEST rows. + if page.Total > 0 { + var minTS int64 + db.sql.QueryRow(`SELECT MIN(ts) FROM requests`).Scan(&minTS) + if minTS <= evs[0].TS { + t.Errorf("oldest row survived a size prune (min ts %d)", minTS) + } + } +} + +func TestKeysetPaginationCoversEveryRowExactlyOnce(t *testing.T) { + db := openTestDB(t) + const n = 37 + var evs []*Event + for i := 0; i < n; i++ { + evs = append(evs, mkEvent(int64(1000+i), "s", "m", 100, 90)) + } + if err := db.insertBatch(evs); err != nil { + t.Fatal(err) + } + + seen := map[int64]int{} + cursor := int64(0) + pages := 0 + for { + page, err := db.Requests(Filter{}, cursor, 10) + if err != nil { + t.Fatal(err) + } + pages++ + if page.Total != n { + t.Errorf("page %d reported total %d; want %d", pages, page.Total, n) + } + for _, r := range page.Requests { + seen[r.ID]++ + } + // Newest first, strictly descending. + for i := 1; i < len(page.Requests); i++ { + if page.Requests[i].ID >= page.Requests[i-1].ID { + t.Fatalf("page not strictly newest-first: %d then %d", + page.Requests[i-1].ID, page.Requests[i].ID) + } + } + if page.NextCursor == 0 { + break + } + cursor = page.NextCursor + if pages > 20 { + t.Fatal("pagination did not terminate") + } + } + if len(seen) != n { + t.Errorf("paged over %d distinct rows; want %d", len(seen), n) + } + for id, c := range seen { + if c != 1 { + t.Errorf("row %d returned %d times; keyset pagination must not duplicate", id, c) + } + } +} + +func TestFilterEveryDimension(t *testing.T) { + db := openTestDB(t) + a := mkEvent(1000, "sess-a", "model-a", 100, 50) + a.Provider, a.Agent, a.Preset, a.Mode = "anthropic", "claude-code", "codesmart", ModeActive + a.TokenAccounting = AccountingComplete + b := mkEvent(5000, "sess-b", "model-b", 200, 200) + b.Provider, b.Agent, b.Preset, b.Mode = "openai", "codex", "codesafe", ModeBypass + b.TokenAccounting = AccountingPartial + b.UncompressedReason = ReasonBypassed + b.Components = []CompRow{{Component: "dedup", Kind: "offload"}} + if err := db.insertBatch([]*Event{a, b}); err != nil { + t.Fatal(err) + } + + cases := []struct { + name string + f Filter + want int64 + }{ + {"session", Filter{Session: "sess-a"}, 1}, + {"model", Filter{Model: "model-b"}, 1}, + {"provider", Filter{Provider: "anthropic"}, 1}, + {"agent", Filter{Agent: "codex"}, 1}, + {"preset", Filter{Preset: "codesmart"}, 1}, + {"mode", Filter{Mode: ModeBypass}, 1}, + {"accounting", Filter{Accounting: AccountingPartial}, 1}, + {"component", Filter{Component: "dedup"}, 1}, + {"component-shared", Filter{Component: "extract"}, 1}, + {"reason-bucket", Filter{Reason: ReasonBypassed}, 1}, + {"reason-compacted", Filter{Reason: "compacted"}, 1}, + {"since", Filter{Since: 2000}, 1}, + {"until", Filter{Until: 2000}, 1}, + {"since+until", Filter{Since: 900, Until: 6000}, 2}, + {"q-session", Filter{Q: "sess-b"}, 1}, + {"q-model", Filter{Q: "model-a"}, 1}, + {"q-agent", Filter{Q: "claude"}, 1}, + {"q-nomatch", Filter{Q: "nothing-here"}, 0}, + {"combined", Filter{Provider: "anthropic", Model: "model-b"}, 0}, + {"unset", Filter{}, 2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + page, err := db.Requests(tc.f, 0, 50) + if err != nil { + t.Fatal(err) + } + if page.Total != tc.want { + t.Errorf("total=%d, want %d", page.Total, tc.want) + } + if int64(len(page.Requests)) != tc.want { + t.Errorf("rows=%d, want %d", len(page.Requests), tc.want) + } + // Every aggregate must accept the same filter without erroring. + if _, err := db.Overview(tc.f); err != nil { + t.Errorf("Overview: %v", err) + } + if _, _, err := db.Sessions(tc.f, 10, 0); err != nil { + t.Errorf("Sessions: %v", err) + } + if _, err := db.Components(tc.f); err != nil { + t.Errorf("Components: %v", err) + } + if _, err := db.Series(tc.f, 60000); err != nil { + t.Errorf("Series: %v", err) + } + }) + } +} + +func TestQueryTimeBucketing(t *testing.T) { + db := openTestDB(t) + // Three requests inside one minute, two in the next. + base := int64(1_700_000_000_000) + base -= base % 60000 // align so the assertion is about bucketing, not phase + var evs []*Event + for _, off := range []int64{0, 10_000, 59_999, 60_000, 119_000} { + evs = append(evs, mkEvent(base+off, "s", "m", 100, 90)) + } + if err := db.insertBatch(evs); err != nil { + t.Fatal(err) + } + + minute, err := db.Series(Filter{}, 60_000) + if err != nil { + t.Fatal(err) + } + if len(minute) != 2 { + t.Fatalf("60s bucketing produced %d buckets; want 2", len(minute)) + } + if minute[0].Requests != 3 || minute[1].Requests != 2 { + t.Errorf("bucket counts = %d,%d; want 3,2", minute[0].Requests, minute[1].Requests) + } + if minute[0].TS != base { + t.Errorf("first bucket ts = %d; want the floor %d", minute[0].TS, base) + } + if minute[0].Saved != 30 { + t.Errorf("bucket saved = %d; want 30 (3 requests x 10)", minute[0].Saved) + } + + // A wider bucket must merge them without any schema change (no rollup tables). + hour, err := db.Series(Filter{}, 3_600_000) + if err != nil { + t.Fatal(err) + } + if len(hour) != 1 || hour[0].Requests != 5 { + t.Errorf("hour bucketing = %d buckets, first has %d requests; want 1 bucket of 5", len(hour), hour[0].Requests) + } +} + +func TestContentRoundTripsCompressed(t *testing.T) { + db := openTestDB(t) + before := strings.Repeat("line of tool output\n", 200) + e := mkEvent(1000, "s", "m", 500, 100) + e.Content = []ContentRow{{Path: "messages.3", BeforeTokens: 500, AfterTokens: 100, + Before: before, After: "line of tool output\n<>"}} + if err := db.insertBatch([]*Event{e}); err != nil { + t.Fatal(err) + } + + // Without permission, no content at all — not empty strings that read as "nothing + // was changed", but no rows. + got, err := db.Request(e.ID, false) + if err != nil { + t.Fatal(err) + } + if len(got.Content) != 0 { + t.Errorf("withContent=false returned %d content rows", len(got.Content)) + } + if len(got.Components) != 2 { + t.Errorf("component rows = %d; want 2 (components are not gated)", len(got.Components)) + } + + got, err = db.Request(e.ID, true) + if err != nil { + t.Fatal(err) + } + if len(got.Content) != 1 { + t.Fatalf("content rows = %d; want 1", len(got.Content)) + } + if got.Content[0].Before != before { + t.Error("before text did not survive the gzip round trip") + } + if got.Content[0].Path != "messages.3" { + t.Errorf("path = %q", got.Content[0].Path) + } +} + +func TestOverviewDenominatorsAndSafety(t *testing.T) { + db := openTestDB(t) + e := mkEvent(1000, "s", "m", 1000, 800) + e.AttemptedTokens, e.FrozenTokens = 400, 600 + e.SavedUnique = 200 + e.FreshInput, e.CacheWrite = 100, 300 + e.ExpandTokens, e.Expands = 50, 1 + if err := db.insertBatch([]*Event{e}); err != nil { + t.Fatal(err) + } + o, err := db.Overview(Filter{}) + if err != nil { + t.Fatal(err) + } + if o.SavedGross != 200 || o.SavedUnique != 200 { + t.Errorf("saved gross/unique = %d/%d", o.SavedGross, o.SavedUnique) + } + if o.SavedAdjusted != 150 { + t.Errorf("adjusted saved = %d; want 150 (200 unique − 50 restored)", o.SavedAdjusted) + } + byKey := map[string]Denominator{} + for _, d := range o.Denominators { + byKey[d.Key] = d + if d.Description == "" { + t.Errorf("denominator %q has no description; every ratio must name its divisor", d.Key) + } + } + if got := byKey["attempted"]; got.Denominator != 400 || got.Percent != 50 { + t.Errorf("attempted denominator = %d, %.1f%%; want 400, 50%%", got.Denominator, got.Percent) + } + // new_input = 200 saved / (100 fresh + 300 cache-write + 200 saved) = 33.33%. + if got := byKey["new_input"]; got.Denominator != 600 || !got.Available || + got.Percent < 33.3 || got.Percent > 33.4 { + t.Errorf("new_input = %d denominator, %.2f%%, available=%v; want 600, ~33.33%%, true", + got.Denominator, got.Percent, got.Available) + } + if got := byKey["whole_request"]; got.Denominator != 1000 { + t.Errorf("whole_request denominator = %d; want 1000", got.Denominator) + } + if o.SafetyCost.FrozenTokens != 600 || o.SafetyCost.RestoredTokens != 50 { + t.Errorf("safety cost = %+v", o.SafetyCost) + } + if len(o.Waterfall) < 4 { + t.Errorf("waterfall has %d steps; want the full walk", len(o.Waterfall)) + } +} + +// TestNewInputRatioNeverDividesSavingsByThemselves is the guard the issue calls +// non-negotiable: with no provider usage data the denominator would be `saved` +// alone and the ratio would read ~100%. It must read n/a. +func TestNewInputRatioNeverDividesSavingsByThemselves(t *testing.T) { + db := openTestDB(t) + e := mkEvent(1000, "s", "m", 1000, 500) + e.FreshInput, e.CacheRead, e.CacheWrite, e.OutputTokens = 0, 0, 0, 0 + e.SavedUnique = 500 + e.TokenAccounting = AccountingPartial + if err := db.insertBatch([]*Event{e}); err != nil { + t.Fatal(err) + } + o, err := db.Overview(Filter{}) + if err != nil { + t.Fatal(err) + } + for _, d := range o.Denominators { + if d.Key != "new_input" { + continue + } + if d.Available { + t.Fatalf("new_input reported as available with no usage data (%.1f%%)", d.Percent) + } + if d.Percent != 0 { + t.Errorf("new_input percent = %.1f; want 0 with a clear unavailable flag", d.Percent) + } + return + } + t.Fatal("new_input denominator missing") +} + +func TestComponentsAggregateAndOvercount(t *testing.T) { + db := openTestDB(t) + // The same compaction re-sent three turns: gross triples, unique stays put. + var evs []*Event + for i := 0; i < 3; i++ { + e := mkEvent(int64(1000+i), "s", "m", 1000, 700) + e.Components = []CompRow{{Component: "extract", Kind: "offload", Acted: true, Mutated: true, + SavedGross: 300, SavedUnique: map[bool]int{true: 300, false: 0}[i == 0], DurationMs: 2}} + evs = append(evs, e) + } + if err := db.insertBatch(evs); err != nil { + t.Fatal(err) + } + rows, err := db.Components(Filter{}) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("component rows = %d; want 1", len(rows)) + } + c := rows[0] + if c.Runs != 3 || c.Acted != 3 || c.SavedGross != 900 || c.SavedUnique != 300 { + t.Errorf("aggregation wrong: %+v", c) + } + if c.OvercountRatio != 3 { + t.Errorf("overcount ratio = %v; want 3 (900 gross ÷ 300 unique)", c.OvercountRatio) + } + if c.DurationMsTotal != 6 || c.DurationMsAvg != 2 { + t.Errorf("latency = %v total, %v avg", c.DurationMsTotal, c.DurationMsAvg) + } + if c.ActRate != 1 { + t.Errorf("act rate = %v; want 1", c.ActRate) + } +} + +func TestSessionsAggregate(t *testing.T) { + db := openTestDB(t) + if err := db.insertBatch([]*Event{ + mkEvent(1000, "sess-1", "m", 1000, 900), + mkEvent(2000, "sess-1", "m", 1000, 800), + mkEvent(3000, "sess-2", "m", 500, 500), + }); err != nil { + t.Fatal(err) + } + rows, total, err := db.Sessions(Filter{}, 10, 0) + if err != nil { + t.Fatal(err) + } + if total != 2 || len(rows) != 2 { + t.Fatalf("sessions = %d rows, total %d; want 2/2", len(rows), total) + } + // Most recently active first. + if rows[0].SessionID != "sess-2" { + t.Errorf("first session = %q; want sess-2 (most recent)", rows[0].SessionID) + } + var s1 *SessionRow + for _, r := range rows { + if r.SessionID == "sess-1" { + s1 = r + } + } + if s1 == nil { + t.Fatal("sess-1 missing") + } + if s1.Turns != 2 || s1.Saved != 300 || s1.Start != 1000 || s1.End != 2000 { + t.Errorf("sess-1 = %+v", s1) + } + // Pagination. + page2, _, err := db.Sessions(Filter{}, 1, 1) + if err != nil { + t.Fatal(err) + } + if len(page2) != 1 || page2[0].SessionID != "sess-1" { + t.Errorf("offset paging wrong: %+v", page2) + } +} + +func TestPercentileExact(t *testing.T) { + db := openTestDB(t) + var evs []*Event + for i := 1; i <= 100; i++ { + e := mkEvent(int64(1000+i), "s", "m", 100, 90) + e.CGLatencyMs = float64(i) + evs = append(evs, e) + } + if err := db.insertBatch(evs); err != nil { + t.Fatal(err) + } + o, err := db.Overview(Filter{}) + if err != nil { + t.Fatal(err) + } + // index = floor(99 * 0.95) = 94 -> the 95th smallest value, which is 95. + if o.CGLatencyMsP95 != 95 { + t.Errorf("p95 = %v; want 95", o.CGLatencyMsP95) + } + if o.CGLatencyMsAvg != 50.5 { + t.Errorf("avg = %v; want 50.5", o.CGLatencyMsAvg) + } +} + +func TestInMemoryModeWorks(t *testing.T) { + for _, path := range []string{"", ":memory:"} { + db, err := Open(path) + if err != nil { + t.Fatalf("Open(%q): %v", path, err) + } + if err := db.insertBatch([]*Event{mkEvent(1000, "s", "m", 100, 90)}); err != nil { + t.Errorf("insert into in-memory db: %v", err) + } + if db.Path() != "" { + t.Errorf("in-memory Path() = %q; want empty", db.Path()) + } + db.Close() + } +} + +func TestUnwritablePathDegradesToMemory(t *testing.T) { + // A path under a file (not a directory) can never be created. + f := filepath.Join(t.TempDir(), "a-file") + if err := os.WriteFile(f, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + rec, err := NewRecorder(Options{DBPath: filepath.Join(f, "nested", "d.db")}) + if err != nil { + t.Fatalf("an unwritable dashboard path must NOT stop the proxy: %v", err) + } + defer rec.Close() + if rec.DB().Path() != "" { + t.Errorf("expected the in-memory fallback; got path %q", rec.DB().Path()) + } +} diff --git a/dash/ui.go b/dash/ui.go new file mode 100644 index 0000000..f373303 --- /dev/null +++ b/dash/ui.go @@ -0,0 +1,45 @@ +package dash + +import ( + "embed" + "io/fs" + "net/http" +) + +// The UI is ONE embedded directory: an HTML file, a stylesheet, and a script. +// No npm, no bundler, no build step, and — the part that matters for a tool that +// ships into VPCs and air-gapped clusters — no CDN. Every byte the page needs is +// in the binary, so the dashboard works with the network unplugged. headroom's +// Alpine/Tailwind/htmx script tags are exactly what we are not doing. +// +// Charts are hand-drawn SVG rather than a vendored chart library: the ladder's +// "native platform feature covers it" rung. SVG path/rect/text is a native +// browser feature, the series here are small, and 45 KB of vendored library would +// buy tooltips we can write in fifteen lines. +// +//go:embed ui +var uiFS embed.FS + +// uiHandler serves the embedded UI. Assets are immutable per build, so they are +// cacheable; the HTML is not, so a redeploy is picked up on reload. +func uiHandler() http.Handler { + sub, err := fs.Sub(uiFS, "ui") + if err != nil { + return http.NotFoundHandler() + } + files := http.FileServer(http.FS(sub)) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "", "/", "index.html": + w.Header().Set("Cache-Control", "no-cache") + default: + w.Header().Set("Cache-Control", "public, max-age=3600") + } + // The page loads nothing from the network beyond its own origin; say so, so a + // stray CDN tag added later fails loudly in the browser instead of silently + // breaking the air-gapped install. + w.Header().Set("Content-Security-Policy", + "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'") + files.ServeHTTP(w, r) + }) +} diff --git a/dash/ui/app.js b/dash/ui/app.js new file mode 100644 index 0000000..eaed01d --- /dev/null +++ b/dash/ui/app.js @@ -0,0 +1,1208 @@ +// context-guru dashboard. No framework, no build step, no CDN — the page is three +// files in a Go binary. State is a plain object, rendering is direct DOM writes, +// and charts are hand-drawn SVG. Everything that reads provider- or agent-supplied +// text goes through textContent or el(), never innerHTML, because a tool output in +// a transcript is attacker-influenced content (gateway interpolates it; we do not). +'use strict'; + +// ── tiny DOM helpers ─────────────────────────────────────────────────────── +const $ = (s, r = document) => r.querySelector(s); +const $$ = (s, r = document) => Array.from(r.querySelectorAll(s)); + +/** el(tag, props, ...children) — children are appended as text unless they are Nodes. */ +function el(tag, props, ...kids) { + const n = document.createElement(tag); + if (props) for (const [k, v] of Object.entries(props)) { + if (v === null || v === undefined || v === false) continue; + if (k === 'class') n.className = v; + else if (k === 'style') setStyle(n, v); + else if (k === 'text') n.textContent = String(v); + else if (k === 'html') throw new Error('el(): raw html is not allowed'); + else if (k.startsWith('on')) n.addEventListener(k.slice(2), v); + else n.setAttribute(k, String(v)); + } + for (const kid of kids.flat()) { + if (kid === null || kid === undefined || kid === false) continue; + n.appendChild(kid instanceof Node ? kid : document.createTextNode(String(kid))); + } + return n; +} +const clear = (n) => { while (n.firstChild) n.removeChild(n.firstChild); return n; }; + +/** + * setStyle applies "prop:value;prop:value" via the CSSOM. + * + * Not as a style ATTRIBUTE: the page ships a strict `style-src 'self'` CSP, which + * blocks inline style attributes — and that CSP is worth keeping, because the diff + * view renders tool output the model was fed, i.e. attacker-influenced text. Going + * through el.style.setProperty is exempt from style-src and equally expressive. + */ +function setStyle(node, decls) { + for (const part of String(decls).split(';')) { + const i = part.indexOf(':'); + if (i < 0) continue; + const prop = part.slice(0, i).trim(); + const val = part.slice(i + 1).trim(); + if (prop) node.style.setProperty(prop, val); + } +} +const svgEl = (tag, attrs) => { + const n = document.createElementNS('http://www.w3.org/2000/svg', tag); + for (const [k, v] of Object.entries(attrs || {})) n.setAttribute(k, String(v)); + return n; +}; + +// ── formatting ───────────────────────────────────────────────────────────── +const nf = new Intl.NumberFormat(); +function num(v) { return v === null || v === undefined ? '—' : nf.format(Math.round(v)); } +function compact(v) { + if (v === null || v === undefined) return '—'; + const a = Math.abs(v); + if (a >= 1e9) return (v / 1e9).toFixed(a >= 1e10 ? 0 : 1) + 'B'; + if (a >= 1e6) return (v / 1e6).toFixed(a >= 1e7 ? 0 : 1) + 'M'; + if (a >= 1e3) return (v / 1e3).toFixed(a >= 1e4 ? 0 : 1) + 'k'; + return nf.format(Math.round(v)); +} +function usd(v) { + if (v === null || v === undefined) return '—'; + const a = Math.abs(v); + if (a === 0) return '$0'; + if (a < 0.01) return (v < 0 ? '-' : '') + '$' + a.toFixed(4); + if (a < 1000) return (v < 0 ? '-' : '') + '$' + a.toFixed(2); + return (v < 0 ? '-' : '') + '$' + nf.format(Math.round(a)); +} +function pct(v, digits = 1) { return v === null || v === undefined ? '—' : v.toFixed(digits) + '%'; } +function ms(v) { + if (!v) return '0 ms'; + return v >= 1000 ? (v / 1000).toFixed(2) + ' s' : v.toFixed(v < 10 ? 1 : 0) + ' ms'; +} +// Timestamps are epoch ms on the wire and formatted here, in the VIEWER's locale. +// The server never stores or sends a formatted date — a locale string cannot be +// range-queried, sorted, or bucketed. +function when(tsMs) { + if (!tsMs) return '—'; + const d = new Date(tsMs); + const today = new Date(); + const sameDay = d.toDateString() === today.toDateString(); + return sameDay ? d.toLocaleTimeString() : d.toLocaleString(); +} +function dur(msv) { + if (!msv || msv < 0) return '—'; + // Below a second, show the actual milliseconds: rounding a component's 300 ms of + // total hot-path time to "0s" hides exactly the cost this view exists to expose. + if (msv < 1000) return ms(msv); + const s = Math.round(msv / 1000); + if (s < 60) return s + 's'; + const m = Math.floor(s / 60); + if (m < 60) return m + 'm ' + (s % 60) + 's'; + return Math.floor(m / 60) + 'h ' + (m % 60) + 'm'; +} +function firstOf(csv) { return (csv || '').split(',')[0] || '—'; } + +// ── state ────────────────────────────────────────────────────────────────── +const state = { + view: 'overview', + filter: {}, + range: 0, + reqCursor: 0, + reqStack: [], + sessOffset: 0, + live: [], + overview: null, +}; + +function qs(extra) { + const p = new URLSearchParams(); + for (const [k, v] of Object.entries(state.filter)) if (v) p.set(k, v); + if (state.range > 0) p.set('since', String(Date.now() - state.range)); + for (const [k, v] of Object.entries(extra || {})) if (v !== '' && v !== 0 && v !== undefined) p.set(k, String(v)); + const s = p.toString(); + return s ? '?' + s : ''; +} + +async function api(path, extra) { + const res = await fetch('/api/' + path + qs(extra), { headers: { accept: 'application/json' } }); + if (!res.ok) { + let msg = res.status + ' ' + res.statusText; + try { const j = await res.json(); if (j.error) msg = j.error; } catch (_) { /* not json */ } + const e = new Error(msg); e.status = res.status; throw e; + } + return res.json(); +} + +function emptyState(host, title, detail) { + clear(host).appendChild(el('div', { class: 'empty' }, el('strong', { text: title }), detail || '')); +} +function loadingState(host, rows = 3) { + clear(host); + for (let i = 0; i < rows; i++) { + host.appendChild(el('div', { class: 'skel', style: 'margin:8px 0;width:' + (100 - i * 12) + '%' })); + } +} + +// ── charts ───────────────────────────────────────────────────────────────── +// A tiny SVG line/area/bar renderer. Native SVG covers everything the issue asks +// for; a vendored chart library would add 45 KB to buy the tooltip below. +const CH = { w: 900, h: 220, pad: { t: 12, r: 14, b: 26, l: 56 } }; + +function ticks(min, max, n = 4) { + if (!isFinite(min) || !isFinite(max) || max === min) return [min || 0]; + const step = (max - min) / n, out = []; + for (let i = 0; i <= n; i++) out.push(min + step * i); + return out; +} + +/** + * lineChart(host, series, opts) + * series: [{name, color, points:[[x,y],…], area?:bool, dashed?:bool}] + * opts: {yFmt, xFmt, tipFmt, stacked?} + */ +function lineChart(host, series, opts = {}) { + clear(host); + const live = series.filter((s) => s.points && s.points.length); + if (!live.length) { emptyState(host, 'No data in this window', 'Send traffic through the proxy, or widen the time range.'); return; } + const yFmt = opts.yFmt || compact, xFmt = opts.xFmt || when; + + const xs = live.flatMap((s) => s.points.map((p) => p[0])); + const ys = live.flatMap((s) => s.points.map((p) => p[1])); + const xMin = Math.min(...xs), xMax = Math.max(...xs); + const yMin = Math.min(0, ...ys), yMax = Math.max(...ys) || 1; + const { w, h, pad } = CH; + const px = (x) => pad.l + (xMax === xMin ? 0 : ((x - xMin) / (xMax - xMin)) * (w - pad.l - pad.r)); + const py = (y) => h - pad.b - ((y - yMin) / (yMax - yMin || 1)) * (h - pad.t - pad.b); + + const svg = svgEl('svg', { viewBox: `0 0 ${w} ${h}`, role: 'img', preserveAspectRatio: 'none' }); + svg.setAttribute('aria-label', opts.label || 'time series chart'); + + for (const t of ticks(yMin, yMax)) { + svg.appendChild(svgEl('line', { class: 'gridline', x1: pad.l, x2: w - pad.r, y1: py(t), y2: py(t) })); + const lab = svgEl('text', { class: 'axis-text', x: pad.l - 6, y: py(t) + 3, 'text-anchor': 'end' }); + lab.textContent = yFmt(t); + svg.appendChild(lab); + } + svg.appendChild(svgEl('line', { class: 'axis', x1: pad.l, x2: w - pad.r, y1: h - pad.b, y2: h - pad.b })); + for (const t of [xMin, (xMin + xMax) / 2, xMax]) { + const lab = svgEl('text', { + class: 'axis-text', x: px(t), y: h - pad.b + 14, + 'text-anchor': t === xMin ? 'start' : t === xMax ? 'end' : 'middle', + }); + lab.textContent = xFmt(t); + svg.appendChild(lab); + } + + // Shaded band between the first two series (the "money saved" area). + if (opts.band && live.length >= 2) { + const a = live[0].points, b = live[1].points; + const dPath = a.map((p, i) => `${i ? 'L' : 'M'}${px(p[0])},${py(p[1])}`).join('') + + b.slice().reverse().map((p) => `L${px(p[0])},${py(p[1])}`).join('') + 'Z'; + svg.appendChild(svgEl('path', { d: dPath, fill: live[0].color, opacity: '0.14' })); + } + + for (const s of live) { + const d = s.points.map((p, i) => `${i ? 'L' : 'M'}${px(p[0])},${py(p[1])}`).join(''); + if (s.area) { + svg.appendChild(svgEl('path', { + d: d + `L${px(s.points[s.points.length - 1][0])},${py(yMin)}L${px(s.points[0][0])},${py(yMin)}Z`, + fill: s.color, opacity: '0.13', + })); + } + svg.appendChild(svgEl('path', { + d, fill: 'none', stroke: s.color, 'stroke-width': 2, + 'stroke-linejoin': 'round', 'stroke-linecap': 'round', + 'stroke-dasharray': s.dashed ? '5 4' : null, + })); + // A path through a single point renders nothing, so one bucket of traffic would + // look identical to no traffic. Draw explicit markers on short series. + if (s.points.length <= 12) { + for (const pt of s.points) { + svg.appendChild(svgEl('circle', { cx: px(pt[0]), cy: py(pt[1]), r: 3.5, fill: s.color })); + } + } + } + + const hover = svgEl('line', { class: 'axis', x1: 0, x2: 0, y1: pad.t, y2: h - pad.b, opacity: '0' }); + svg.appendChild(hover); + // A transparent capture rect over the plot area. Without it, pointer events only + // land on the rendered strokes — an SVG's own box is not a hit target — so the + // tooltip fires on a 2px line and nowhere else. Added last so it sits on top. + svg.appendChild(svgEl('rect', { + x: pad.l, y: pad.t, width: Math.max(0, w - pad.l - pad.r), height: Math.max(0, h - pad.t - pad.b), + fill: 'transparent', + })); + host.appendChild(svg); + + const tip = el('div', { class: 'tooltip' }); + host.appendChild(tip); + svg.addEventListener('pointerleave', () => { tip.classList.remove('show'); hover.setAttribute('opacity', '0'); }); + svg.addEventListener('pointermove', (ev) => { + const rect = svg.getBoundingClientRect(); + const relX = ((ev.clientX - rect.left) / rect.width) * w; + const dataX = xMin + ((relX - pad.l) / (w - pad.l - pad.r)) * (xMax - xMin); + let best = null; + for (const p of live[0].points) if (!best || Math.abs(p[0] - dataX) < Math.abs(best[0] - dataX)) best = p; + if (!best) return; + hover.setAttribute('x1', px(best[0])); hover.setAttribute('x2', px(best[0])); + hover.setAttribute('opacity', '0.5'); + const lines = [xFmt(best[0])]; + for (const s of live) { + const p = s.points.find((q) => q[0] === best[0]); + if (p) lines.push(s.name + ': ' + (opts.tipFmt || yFmt)(p[1])); + } + tip.textContent = lines.join('\n'); + tip.classList.add('show'); + const hostRect = host.getBoundingClientRect(); + tip.style.left = Math.min(hostRect.width - 190, Math.max(0, ev.clientX - hostRect.left + 12)) + 'px'; + tip.style.top = Math.max(0, ev.clientY - hostRect.top - 10) + 'px'; + }); + + host.appendChild(el('div', { class: 'legend' }, ...live.map((s) => + el('span', {}, el('i', { style: 'background:' + s.color }), s.name)))); +} + +/** barRows(host, rows) — rows: [{label, value, display, max, negative, desc}] */ +function barRows(host, rows, opts = {}) { + clear(host); + if (!rows.length) { emptyState(host, 'Nothing to show yet', opts.emptyDetail || ''); return; } + const max = Math.max(...rows.map((r) => Math.abs(r.max !== undefined ? r.max : r.value)), 1); + const wrap = el('div', { class: 'bars' }); + for (const r of rows) { + const width = r.available === false ? 0 : Math.min(100, (Math.abs(r.value) / max) * 100); + const row = el('div', { class: 'bar-row' }, + el('div', { class: 'bar-label', text: r.label }), + el('div', { class: 'bar-track' }, el('div', { + class: 'bar-fill' + (r.value < 0 ? ' neg' : ''), + style: 'width:' + width + '%' + (r.color ? ';background:' + r.color : ''), + })), + el('div', { class: 'bar-val' + (r.available === false ? ' na' : ''), text: r.display })); + wrap.appendChild(row); + if (r.desc) wrap.appendChild(el('div', { class: 'bar-desc', text: r.desc })); + } + host.appendChild(wrap); +} + +// ── overview ─────────────────────────────────────────────────────────────── +function tile(key, label, value, sub, cls) { + return el('div', { class: 'tile ' + (cls || ''), 'data-testid': 'tile-' + key }, + el('div', { class: 'k', text: label }), + el('div', { class: 'v', 'data-testid': 'tile-' + key + '-value', text: value }), + sub ? el('div', { class: 's', text: sub }) : null); +} + +function renderTiles(o) { + const host = clear($('#tiles')); + const exact = (o.accounting && o.accounting.complete) || 0; + const costKnown = exact > 0; + const tiles = [ + tile('requests', 'Requests', num(o.requests), num(o.sessions) + ' sessions'), + tile('tokens-before', 'Tokens before', compact(o.tokens_before), 'content tokens in'), + tile('tokens-after', 'Tokens after', compact(o.tokens_after), 'content tokens out'), + tile('saved-gross', 'Saved (gross)', compact(o.saved_gross), 'recounts re-sent history', 'accent'), + tile('saved-unique', 'Saved (unique)', compact(o.saved_unique), 'each compaction once', 'good'), + tile('saved-adjusted', 'Saved (net of restores)', compact(o.saved_adjusted), + compact(o.expand_tokens) + ' restored back', o.saved_adjusted < 0 ? 'bad' : ''), + tile('overcount', 'Overcount ratio', o.overcount_ratio ? o.overcount_ratio.toFixed(1) + '×' : '—', + 'gross ÷ unique'), + tile('cost-baseline', 'Baseline cost', costKnown ? usd(o.baseline_cost_usd) : 'unknown', + costKnown ? 'without context-guru' : 'no priced requests'), + tile('cost-actual', 'Actual cost', costKnown ? usd(o.cost_usd) : 'unknown', + costKnown ? 'as billed' : 'no priced requests'), + tile('cost-cg', "context-guru's own LLM", costKnown ? usd(o.cg_llm_cost_usd) : 'unknown', + 'our components’ model spend'), + tile('saved-usd', 'Net dollars saved', costKnown ? usd(o.net_saved_usd) : 'unknown', + 'baseline − actual − our spend', o.net_saved_usd < 0 ? 'bad' : 'good'), + tile('cache-read', 'Cache reads', compact(o.cache_read), 'billed at the read rate'), + tile('cache-write', 'Cache writes', compact(o.cache_write), '~11.5× a read'), + tile('fresh-input', 'Fresh input', compact(o.fresh_input), 'uncached new tokens'), + tile('output', 'Output tokens', compact(o.output_tokens), 'completions'), + tile('cg-latency', 'context-guru latency', ms(o.cg_latency_ms_avg), 'p95 ' + ms(o.cg_latency_ms_p95)), + tile('upstream-latency', 'Upstream latency', ms(o.upstream_ms_avg), 'p95 ' + ms(o.upstream_ms_p95)), + tile('expands', 'Restorations', num(o.expands), + pct(o.expand_rate * 100) + ' of requests · ' + compact(o.expand_tokens) + ' tok', + o.expands > 0 ? 'bad' : ''), + tile('reverts', 'Reverts', num(o.reverts), 'never-worse guard fired'), + tile('passthroughs', 'Not compacted', num(o.passthroughs), 'see reason buckets below'), + ]; + tiles.forEach((t) => host.appendChild(t)); +} + +function renderDenominators(o) { + barRows($('#denominators'), (o.denominators || []).map((d) => ({ + label: d.label, + value: d.available ? d.percent : 0, + max: 100, + display: d.available ? pct(d.percent, 2) : 'n/a', + available: d.available, + desc: d.description + (d.available ? ` (${compact(d.numerator)} ÷ ${compact(d.denominator)} tokens)` : ''), + })), { emptyDetail: 'No requests match the filter.' }); +} + +function renderWaterfall(o) { + const host = clear($('#waterfall')); + const steps = o.waterfall || []; + if (!steps.length || !o.baseline_cost_usd) { + emptyState(host, 'No priced requests yet', + 'The waterfall needs provider usage data (all four token tiers) and a known model price.'); + return; + } + const max = Math.max(...steps.map((s) => Math.abs(s.delta_usd)), 0.0001); + const wrap = el('div', { class: 'bars' }); + for (const s of steps) { + const color = s.total ? 'var(--s2)' : s.delta_usd < 0 ? 'var(--good)' : 'var(--bad)'; + wrap.appendChild(el('div', { class: 'bar-row' }, + el('div', { class: 'bar-label', text: s.label }), + el('div', { class: 'bar-track' }, el('div', { + class: 'bar-fill', style: `width:${(Math.abs(s.delta_usd) / max) * 100}%;background:${color}`, + })), + el('div', { class: 'bar-val', text: (s.delta_usd < 0 ? '−' : s.total ? '' : '+') + usd(Math.abs(s.delta_usd)) }))); + wrap.appendChild(el('div', { class: 'bar-desc', text: s.description })); + } + host.appendChild(wrap); +} + +function renderDistribution(hostSel, map, labels, testid) { + const host = clear($(hostSel)); + const entries = Object.entries(map || {}).filter(([, v]) => v > 0); + if (!entries.length) { emptyState(host, 'No requests in this window', ''); return; } + entries.sort((a, b) => b[1] - a[1]); + const total = entries.reduce((n, [, v]) => n + v, 0); + barRows(host, entries.map(([k, v]) => ({ + label: (labels && labels[k]) || (k === '' ? 'compacted' : k), + value: v, max: total, + display: num(v) + ' (' + pct((v / total) * 100, 0) + ')', + }))); +} + +function renderSafety(o) { + const s = o.safety_cost || {}; + $('#safety-note').textContent = s.description || ''; + barRows($('#safety'), [ + { label: 'Frozen for cache safety', value: s.frozen_tokens || 0, display: compact(s.frozen_tokens) + ' tok', + desc: 'Compaction we deliberately did NOT do on the already-cached prefix. The benefit ' + + 'is the ' + compact(o.cache_read) + ' cache-read tokens that stayed cheap; the cost is this.' }, + { label: 'Restored after offload', value: s.restored_tokens || 0, display: compact(s.restored_tokens) + ' tok', + color: 'var(--bad)', + desc: 'Content we removed and the model asked back for — a premature offload, paid for twice.' }, + { label: 'Reverted component runs', value: s.reverted_runs || 0, display: num(s.reverted_runs) + ' runs', + color: 'var(--s3)', + desc: 'The never-worse guard rolling a component back. Safety working, and its cost is the ' + + 'latency of the attempt.' }, + { label: "context-guru's own latency", value: s.cg_latency_ms_total || 0, display: dur(s.cg_latency_ms_total), + color: 'var(--s4)', desc: 'Total wall time context-guru itself added across the window.' }, + { label: "context-guru's own LLM spend", value: (s.cg_llm_cost_usd || 0) * 1000, display: usd(s.cg_llm_cost_usd), + color: 'var(--s5)', desc: 'Paid out of the savings above.' }, + ]); +} + +function renderLive() { + const body = clear($('#live-body')); + if (!state.live.length) { + body.appendChild(el('tr', {}, el('td', { colspan: '8' }, + el('div', { class: 'empty' }, el('strong', { text: 'Waiting for traffic' }), + 'Requests appear here the moment they are captured.')))); + return; + } + for (const e of state.live.slice(0, 25)) { + body.appendChild(el('tr', { class: 'click', onclick: () => openRequest(e.id) }, + el('td', { text: when(e.ts) }), + el('td', {}, el('span', { class: 'trunc', title: e.session_id, text: e.session_id || '—' })), + el('td', { text: e.model || '—' }), + el('td', { class: 'num', text: compact(e.tokens_before) }), + el('td', { class: 'num', text: compact(e.tokens_after) }), + el('td', { class: 'num', text: compact(e.tokens_before - e.tokens_after) }), + el('td', { class: 'num', text: ms(e.cg_latency_ms) }), + el('td', {}, el('span', { class: 'pill ' + e.token_accounting, text: e.token_accounting })))); + } +} + +async function loadOverview() { + loadingState($('#tiles'), 4); + try { + const [o, s] = await Promise.all([api('stats'), api('series', { bucket: bucketFor() })]); + state.overview = o; + renderTiles(o); + renderDenominators(o); + renderWaterfall(o); + renderSafety(o); + renderDistribution('#cachemiss', o.cache_miss, { + hit: 'cache hit', cold_start: 'cold start (not a failure)', ttl_expiry: 'TTL expiry', + prefix_change: 'prefix change', unknown: 'unknown', '': 'no cache data', + }); + renderDistribution('#reasons', o.uncompressed, { + '': 'compacted', bypassed: 'bypassed by header', below_trigger: 'below every trigger', + cache_frozen: 'frozen for cache safety', found_nothing: 'nothing to remove', + reverted: 'all components reverted', no_messages: 'no messages', + }); + renderDistribution('#accounting', o.accounting, { + complete: 'exact (all four tiers)', partial: 'estimated', missing: 'unmeasured', + }); + renderSeries(s.buckets || []); + } catch (err) { + emptyState($('#tiles'), 'Could not load statistics', String(err.message || err)); + } +} + +function bucketFor() { + if (state.range === 0) return 3600000; + if (state.range <= 3600000) return 60000; + if (state.range <= 86400000) return 300000; + return 3600000; +} + +function renderSeries(buckets) { + if (!buckets.length) { + for (const id of ['#chart-cost', '#chart-tokens', '#chart-cache', '#chart-latency', '#chart-volume']) { + emptyState($(id), 'No data in this window', 'Send traffic through the proxy, or widen the time range.'); + } + return; + } + // Cumulative cost: the headline chart. The area between the lines is the money. + let cumBase = 0, cumAct = 0; + const base = [], act = []; + for (const b of buckets) { + cumBase += b.baseline_cost_usd; + cumAct += b.cost_usd + b.cg_llm_cost_usd; + base.push([b.ts, cumBase]); + act.push([b.ts, cumAct]); + } + const anyCost = cumBase > 0 || cumAct > 0; + if (anyCost) { + lineChart($('#chart-cost'), [ + { name: 'Without context-guru (cumulative)', color: 'var(--s5)', points: base }, + { name: 'With context-guru (incl. our own spend)', color: 'var(--s1)', points: act, area: true }, + ], { band: true, yFmt: usd, tipFmt: usd, label: 'cumulative cost with and without context-guru' }); + } else { + emptyState($('#chart-cost'), 'No priced requests yet', + 'Cost needs provider usage data (all four token tiers) and a known model price. Token charts below still work.'); + } + + lineChart($('#chart-tokens'), [ + { name: 'Tokens before', color: 'var(--s2)', points: buckets.map((b) => [b.ts, b.tokens_before]) }, + { name: 'Tokens after', color: 'var(--s1)', points: buckets.map((b) => [b.ts, b.tokens_after]), area: true }, + { name: 'Saved (unique)', color: 'var(--s3)', points: buckets.map((b) => [b.ts, b.saved_unique]) }, + ], { label: 'content tokens over time' }); + + lineChart($('#chart-cache'), [ + { name: 'Cache reads', color: 'var(--s1)', points: buckets.map((b) => [b.ts, b.cache_read]), area: true }, + { name: 'Cache writes', color: 'var(--s3)', points: buckets.map((b) => [b.ts, b.cache_write]) }, + { name: 'Fresh input', color: 'var(--s2)', points: buckets.map((b) => [b.ts, b.fresh_input]) }, + ], { label: 'cache reads versus writes over time' }); + + lineChart($('#chart-latency'), [ + { name: 'context-guru added (avg)', color: 'var(--s1)', points: buckets.map((b) => [b.ts, b.cg_latency_ms_avg]) }, + { name: 'Upstream round-trip (avg)', color: 'var(--s2)', points: buckets.map((b) => [b.ts, b.upstream_ms_avg]), dashed: true }, + ], { yFmt: ms, tipFmt: ms, label: 'latency over time' }); + + lineChart($('#chart-volume'), [ + { name: 'Requests', color: 'var(--s2)', points: buckets.map((b) => [b.ts, b.requests]), area: true }, + { name: 'Restorations (expands)', color: 'var(--s5)', points: buckets.map((b) => [b.ts, b.expands]) }, + { name: 'Cache misses', color: 'var(--s3)', points: buckets.map((b) => [b.ts, b.cache_misses]) }, + ], { yFmt: num, label: 'request volume and restorations' }); +} + +// ── components ───────────────────────────────────────────────────────────── +/** + * verdict summarises whether a component earns its place, from what it saved + * against what it cost. Order matters: a component that burned real wall time for + * nothing is a worse finding than one that simply never fired, so the cost test + * comes FIRST — otherwise extract_llm's 15 s of model calls for zero savings reads + * as a bland "inert here". + */ +function verdict(c) { + if (c.runs === 0) return ['—', 'neutral']; + if (c.errors > 0) return ['errors', 'missing']; + // Spent >1s of hot-path time and returned nothing: paid for, unused. + if (c.saved_unique === 0 && c.duration_ms_total > 1000) return ['costly and inert', 'missing']; + if (c.mutated === 0) return ['inert here', 'partial']; + if (c.saved_unique === 0) return ['mutates, saves no content', 'neutral']; + // More than a millisecond of latency per 100 tokens saved. + if (c.duration_ms_total > 1000 && c.duration_ms_total / c.saved_unique > 0.01) { + return ['expensive for its yield', 'partial']; + } + if (c.act_rate < 0.02) return ['rarely fires', 'partial']; + return ['earning its place', 'complete']; +} + +async function loadComponents() { + const body = clear($('#components-body')); + body.appendChild(el('tr', {}, el('td', { colspan: '13' }, el('div', { class: 'skel' })))); + try { + const { components } = await api('components'); + clear(body); + if (!components.length) { + body.appendChild(el('tr', {}, el('td', { colspan: '13' }, + el('div', { class: 'empty' }, el('strong', { text: 'No component runs captured' }), + 'Run some traffic through the proxy with a non-empty pipeline.')))); + emptyState($('#chart-comp'), 'No component data', ''); + return; + } + for (const c of components) { + const [vtext, vcls] = verdict(c); + body.appendChild(el('tr', { class: 'click', onclick: () => { setFilter('component', c.component); go('requests'); } }, + el('td', {}, el('code', { text: c.component })), + el('td', { text: c.kind || '—' }), + el('td', { class: 'num', text: num(c.runs) }), + el('td', { class: 'num', text: num(c.acted) }), + el('td', { class: 'num', text: pct(c.act_rate * 100, 1) }), + el('td', { class: 'num', text: num(c.reverted) }), + el('td', { class: 'num', text: compact(c.saved_unique) }), + el('td', { class: 'num', text: compact(c.saved_gross) }), + el('td', { class: 'num', text: c.overcount_ratio ? c.overcount_ratio.toFixed(1) + '×' : '—' }), + el('td', { class: 'num', text: dur(c.duration_ms_total) }), + el('td', { class: 'num', text: ms(c.duration_ms_avg) }), + el('td', { class: 'num', text: num(c.errors) }), + el('td', {}, el('span', { class: 'pill ' + vcls, text: vtext })))); + } + const top = components.filter((c) => c.saved_unique > 0).slice(0, 12); + barRows($('#chart-comp'), top.map((c, i) => ({ + label: c.component, value: c.saved_unique, display: compact(c.saved_unique) + ' tok', + color: `var(--s${(i % 5) + 1})`, + desc: `${num(c.runs)} runs, acted on ${pct(c.act_rate * 100, 1)}, own latency ${dur(c.duration_ms_total)}, ` + + `overcount ${c.overcount_ratio ? c.overcount_ratio.toFixed(1) + '×' : 'n/a'}`, + })), { emptyDetail: 'No component saved any content tokens in this window.' }); + } catch (err) { + clear(body).appendChild(el('tr', {}, el('td', { colspan: '13' }, + el('div', { class: 'empty' }, el('strong', { text: 'Could not load components' }), String(err.message || err))))); + } +} + +// ── sessions ─────────────────────────────────────────────────────────────── +async function loadSessions() { + const body = clear($('#sessions-body')); + body.appendChild(el('tr', {}, el('td', { colspan: '12' }, el('div', { class: 'skel' })))); + try { + const { sessions, total } = await api('sessions', { limit: 25, offset: state.sessOffset }); + clear(body); + if (!sessions.length) { + body.appendChild(el('tr', {}, el('td', { colspan: '12' }, + el('div', { class: 'empty' }, el('strong', { text: 'No sessions yet' }), + 'A session appears as soon as its first request is captured.')))); + } + for (const s of sessions) { + body.appendChild(el('tr', { class: 'click', onclick: () => { setFilter('session', s.session_id); go('requests'); } }, + el('td', {}, el('span', { class: 'trunc', title: s.session_id, text: s.session_id || '(none)' })), + el('td', { text: firstOf(s.models) }), + el('td', { text: firstOf(s.agents) }), + el('td', { text: firstOf(s.presets) }), + el('td', { class: 'num', text: num(s.turns) }), + el('td', { class: 'num', text: compact(s.tokens_before) }), + el('td', { class: 'num', text: compact(s.saved) }), + el('td', { class: 'num', text: s.baseline_cost_usd ? usd(s.saved_usd) : '—' }), + el('td', { class: 'num', text: compact(s.cache_read) + ' / ' + compact(s.cache_write) }), + el('td', { class: 'num', text: num(s.expands) }), + el('td', { class: 'num', text: ms(s.cg_latency_ms_avg) }), + el('td', { text: when(s.start) }))); + } + const from = total ? state.sessOffset + 1 : 0; + $('#sess-page').textContent = `${from}–${Math.min(state.sessOffset + 25, total)} of ${num(total)}`; + $('#sess-prev').disabled = state.sessOffset === 0; + $('#sess-next').disabled = state.sessOffset + 25 >= total; + } catch (err) { + clear(body).appendChild(el('tr', {}, el('td', { colspan: '12' }, + el('div', { class: 'empty' }, el('strong', { text: 'Could not load sessions' }), String(err.message || err))))); + } +} + +// ── requests ─────────────────────────────────────────────────────────────── +async function loadRequests() { + const body = clear($('#requests-body')); + body.appendChild(el('tr', {}, el('td', { colspan: '13' }, el('div', { class: 'skel' })))); + try { + const page = await api('requests', { limit: 50, before: state.reqCursor }); + clear(body); + if (!page.requests.length) { + body.appendChild(el('tr', {}, el('td', { colspan: '13' }, + el('div', { class: 'empty' }, el('strong', { text: 'No requests match' }), + 'Clear a filter, widen the range, or send traffic through the proxy.')))); + } + for (const e of page.requests) { + body.appendChild(el('tr', { class: 'click', 'data-testid': 'request-row', onclick: () => openRequest(e.id) }, + el('td', { text: e.id }), + el('td', { text: when(e.ts) }), + el('td', {}, el('span', { class: 'trunc', title: e.session_id, text: e.session_id || '—' })), + el('td', { text: e.model || '—' }), + el('td', {}, el('span', { class: 'pill neutral', text: e.mode || '—' })), + el('td', { class: 'num', text: compact(e.tokens_before) }), + el('td', { class: 'num', text: compact(e.tokens_after) }), + el('td', { class: 'num', text: compact(e.tokens_before - e.tokens_after) }), + el('td', { class: 'num', text: compact(e.cache_read) + ' / ' + compact(e.cache_write) }), + el('td', { class: 'num', text: e.token_accounting === 'complete' ? usd(e.cost_usd) : '—' }), + el('td', { class: 'num', text: ms(e.cg_latency_ms) }), + el('td', {}, el('span', { class: 'pill ' + (e.cache_miss_reason || 'neutral'), text: e.cache_miss_reason || '—' })), + el('td', {}, el('span', { class: 'pill ' + e.token_accounting, text: e.token_accounting })))); + } + $('#req-page').textContent = `${num(page.requests.length)} shown of ${num(page.total)} matching`; + $('#req-next').disabled = !page.next_cursor; + $('#req-prev').disabled = state.reqStack.length === 0; + state.nextCursor = page.next_cursor; + } catch (err) { + clear(body).appendChild(el('tr', {}, el('td', { colspan: '13' }, + el('div', { class: 'empty' }, el('strong', { text: 'Could not load requests' }), String(err.message || err))))); + } +} + +// ── request detail + diff ────────────────────────────────────────────────── +/** + * Myers-style LCS diff over lines, then rendered Git-style. This is the view both + * reference implementations carry the data for and neither built: it answers + * "what did context-guru actually remove or rewrite?" instead of asserting a + * token count. + */ +function diffLines(a, b) { + const n = a.length, m = b.length; + // Trim the common head/tail first: agent transcripts share long identical + // stretches, so this cuts the DP table to the part that actually differs. + let head = 0; + while (head < n && head < m && a[head] === b[head]) head++; + let tail = 0; + while (tail < n - head && tail < m - head && a[n - 1 - tail] === b[m - 1 - tail]) tail++; + const as = a.slice(head, n - tail), bs = b.slice(head, m - tail); + + const out = []; + for (let i = 0; i < head; i++) out.push({ op: ' ', text: a[i], ai: i + 1, bi: i + 1 }); + + // Guard the quadratic table: a huge rewrite renders as a whole-block replace + // rather than hanging the tab. + // ponytail: LCS is O(n·m); the cap below is the ceiling. Switch to a real Myers + // O(nd) if multi-megabyte single-message diffs ever matter. + const LIMIT = 1500; + if (as.length > LIMIT || bs.length > LIMIT) { + if (as.length) out.push({ op: 'gap', text: `… ${as.length} lines replaced (too large to line-diff) …` }); + for (let i = 0; i < as.length; i++) out.push({ op: '-', text: as[i], ai: head + i + 1 }); + for (let j = 0; j < bs.length; j++) out.push({ op: '+', text: bs[j], bi: head + j + 1 }); + } else { + const dp = Array.from({ length: as.length + 1 }, () => new Uint32Array(bs.length + 1)); + for (let i = as.length - 1; i >= 0; i--) { + for (let j = bs.length - 1; j >= 0; j--) { + dp[i][j] = as[i] === bs[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]); + } + } + let i = 0, j = 0; + while (i < as.length && j < bs.length) { + if (as[i] === bs[j]) { out.push({ op: ' ', text: as[i], ai: head + i + 1, bi: head + j + 1 }); i++; j++; } + else if (dp[i + 1][j] >= dp[i][j + 1]) { out.push({ op: '-', text: as[i], ai: head + i + 1 }); i++; } + else { out.push({ op: '+', text: bs[j], bi: head + j + 1 }); j++; } + } + while (i < as.length) { out.push({ op: '-', text: as[i], ai: head + i + 1 }); i++; } + while (j < bs.length) { out.push({ op: '+', text: bs[j], bi: head + j + 1 }); j++; } + } + for (let k = 0; k < tail; k++) { + out.push({ op: ' ', text: a[n - tail + k], ai: n - tail + k + 1, bi: m - tail + k + 1 }); + } + return out; +} + +/** Collapse runs of unchanged lines to CTX lines of context, Git-style. */ +function withHunks(rows, ctx = 3) { + const keep = new Array(rows.length).fill(false); + rows.forEach((r, i) => { + if (r.op === ' ') return; + for (let k = Math.max(0, i - ctx); k <= Math.min(rows.length - 1, i + ctx); k++) keep[k] = true; + }); + const out = []; + let skipped = 0; + rows.forEach((r, i) => { + if (keep[i]) { + if (skipped) { out.push({ op: 'gap', text: `… ${skipped} unchanged lines …` }); skipped = 0; } + out.push(r); + } else skipped++; + }); + if (skipped) out.push({ op: 'gap', text: `… ${skipped} unchanged lines …` }); + return out; +} + +function renderDiff(host, before, after, mode) { + clear(host); + if (mode === 'side') { + host.appendChild(el('div', { class: 'side' }, + el('pre', { text: before || '(empty)' }), el('pre', { text: after || '(empty)' }))); + return; + } + if (mode === 'raw') { + host.appendChild(el('pre', { style: 'margin:0;padding:8px 10px;white-space:pre-wrap', text: after || '(empty)' })); + return; + } + const rows = withHunks(diffLines((before || '').split('\n'), (after || '').split('\n'))); + if (!rows.length) { host.appendChild(el('div', { class: 'empty', text: 'Identical.' })); return; } + const frag = document.createDocumentFragment(); + for (const r of rows) { + if (r.op === 'gap') { + frag.appendChild(el('div', { class: 'dl gap' }, + el('span', { class: 'ln' }), el('span', { class: 'ln' }), el('span', { class: 'tx', text: r.text }))); + continue; + } + const cls = r.op === '+' ? 'add' : r.op === '-' ? 'del' : 'ctx'; + frag.appendChild(el('div', { class: 'dl ' + cls }, + el('span', { class: 'ln', text: r.ai || '' }), + el('span', { class: 'ln', text: r.bi || '' }), + el('span', { class: 'tx', text: r.text }))); + } + host.appendChild(frag); +} + +function kv(k, v) { return el('div', {}, el('div', { class: 'k', text: k }), el('div', { class: 'v', text: v })); } + +async function openRequest(id) { + $('#drawer').hidden = false; + $('#scrim').hidden = false; + $('#drawer-title').textContent = 'Request #' + id; + const body = clear($('#drawer-body')); + loadingState(body, 5); + try { + const res = await fetch('/api/requests/' + id); + if (!res.ok) throw new Error(res.status + ' ' + res.statusText); + const { request: e, content_visible: visible, content_captured: captured } = await res.json(); + clear(body); + + body.appendChild(el('div', { class: 'kv', 'data-testid': 'detail-summary' }, + kv('Session', e.session_id || '—'), + kv('When', when(e.ts)), + kv('Model', e.model || '—'), + kv('Provider', e.provider || '—'), + kv('Agent', e.agent || '—'), + kv('Preset', e.preset || '—'), + kv('Mode', e.mode || '—'), + kv('Upstream status', e.status || '—'), + kv('Messages', num(e.messages)), + kv('Tokens before → after', compact(e.tokens_before) + ' → ' + compact(e.tokens_after)), + kv('Saved (gross / unique)', compact(e.tokens_before - e.tokens_after) + ' / ' + compact(e.saved_unique)), + kv('Attempted (eligible)', compact(e.attempted_tokens)), + kv('Frozen for cache safety', compact(e.frozen_tokens)), + kv('Fresh / read / write / out', + [e.fresh_input, e.cache_read, e.cache_write, e.output_tokens].map(compact).join(' / ')), + kv('Cost (actual / baseline)', e.token_accounting === 'complete' + ? usd(e.cost_usd) + ' / ' + usd(e.baseline_cost_usd) : 'not priced'), + kv("context-guru's own LLM", e.token_accounting === 'complete' ? usd(e.cg_llm_cost_usd) : '—'), + kv('context-guru latency', ms(e.cg_latency_ms)), + kv('Upstream latency', ms(e.upstream_ms)), + kv('Restorations', num(e.expands) + ' (' + compact(e.expand_tokens) + ' tok)'), + kv('Reverts', num(e.reverts)), + kv('Cache attribution', e.cache_miss_reason || '—'), + kv('Token accounting', e.token_accounting), + kv('Compaction outcome', e.uncompressed_reason || 'compacted'))); + + body.appendChild(el('h2', { text: 'Components, in the order they ran' })); + if (!e.components || !e.components.length) { + body.appendChild(el('div', { class: 'empty', text: 'No components ran on this request.' })); + } else { + const tbl = el('table', { class: 'tbl compact', 'data-testid': 'detail-components' }, + el('thead', {}, el('tr', {}, + el('th', { text: '#' }), el('th', { text: 'Component' }), el('th', { text: 'Kind' }), + el('th', { class: 'num', text: 'Saved' }), el('th', { class: 'num', text: 'Unique' }), + el('th', { class: 'num', text: 'Latency' }), el('th', { text: 'Outcome' })))); + const tb = el('tbody'); + e.components.forEach((c, i) => { + const outcome = c.reverted ? ['reverted', 'missing'] : c.skipped ? ['skipped', 'neutral'] + : c.acted ? ['acted', 'complete'] : ['mutated only', 'partial']; + tb.appendChild(el('tr', {}, + el('td', { text: i + 1 }), + el('td', {}, el('code', { text: c.component })), + el('td', { text: c.kind || '—' }), + el('td', { class: 'num', text: compact(c.saved_gross) }), + el('td', { class: 'num', text: compact(c.saved_unique) }), + el('td', { class: 'num', text: ms(c.duration_ms) }), + el('td', {}, el('span', { class: 'pill ' + outcome[1], text: outcome[0] }), + c.err ? el('div', { class: 's', text: c.err }) : null))); + }); + tbl.appendChild(tb); + body.appendChild(el('div', { class: 'tblwrap' }, tbl)); + } + + body.appendChild(el('h2', { style: 'margin-top:18px', text: 'What context-guru changed' })); + if (!visible) { + body.appendChild(el('div', { class: 'empty' }, + el('strong', { text: 'Content is not visible from this address' }), + 'Per-request content is served to loopback or a configured trusted CIDR only, because a ' + + 'transcript can carry your source code. Aggregates are open.')); + } else if (!captured) { + body.appendChild(el('div', { class: 'empty' }, + el('strong', { text: 'Content capture is disabled' }), + 'Start the proxy with content capture on to record before/after text for the diff view.')); + } else if (!e.content || !e.content.length) { + body.appendChild(el('div', { class: 'empty' }, + el('strong', { text: 'Nothing was rewritten' }), + 'This request passed through unchanged' + (e.uncompressed_reason ? ' (' + e.uncompressed_reason + ')' : '') + '.')); + } else { + // Biggest saving first, and open that one: the point of the view is "what did + // context-guru actually remove?", so leading with an unchanged block (and + // collapsing the 2k-token rewrite below it) buries the answer. + const blocks = e.content.slice().sort( + (a, b) => (b.before_tokens - b.after_tokens) - (a.before_tokens - a.after_tokens)); + blocks.forEach((c, idx) => { + const saved = c.before_tokens - c.after_tokens; + const det = el('details', { class: 'diff', 'data-testid': 'diff-block' }, el('summary', { + text: `${c.path} — ${compact(c.before_tokens)} → ${compact(c.after_tokens)} tokens ` + + (saved > 0 ? `(saved ${compact(saved)})` : '(rewritten, no token saving)'), + })); + if (idx === 0 && c.before_tokens > c.after_tokens) det.open = true; + const bodyHost = el('div', { class: 'diffbody' }); + const bar = el('div', { class: 'difftoolbar' }, 'View:'); + // testids spelled out in full so a grep (and the Go test that guards them) + // finds them literally rather than reconstructing a concatenation. + for (const [mode, label, testid] of [ + ['git', 'Git diff', 'diff-mode-git'], + ['side', 'Side by side', 'diff-mode-side'], + ['raw', 'After only', 'diff-mode-raw'], + ]) { + bar.appendChild(el('button', { + class: 'ghost', 'data-testid': testid, + onclick: () => renderDiff(bodyHost, c.before, c.after, mode), + }, label)); + } + det.appendChild(bar); + det.appendChild(bodyHost); + renderDiff(bodyHost, c.before, c.after, 'git'); + body.appendChild(det); + }); + } + } catch (err) { + emptyState(clear(body), 'Could not load this request', String(err.message || err)); + } +} + +function closeDrawer() { $('#drawer').hidden = true; $('#scrim').hidden = true; } + +// ── benchmarks ───────────────────────────────────────────────────────────── +async function loadBenchmarks() { + const host = clear($('#bench-list')); + loadingState(host, 3); + try { + const { runs } = await api('benchmarks'); + clear(host); + if (!runs || !runs.length) { + emptyState(host, 'No benchmark runs ingested', + 'Point --dash-bench-dirs at a harness jobs root (a directory of runs, each with summary.json and rows-*.json) and re-scan.'); + return; + } + // 42 ingested runs rendered flat is 40k pixels of table. Collapse each run and + // open only the newest, so the view opens on the run you just finished. + runs.forEach((run, runIdx) => { + const sec = el('details', { class: 'panel diff', 'data-testid': 'bench-run' }); + if (runIdx === 0) sec.open = true; + const armNames = (run.arms || []).map((a) => a.arm).join(', '); + sec.appendChild(el('summary', {}, + el('strong', { text: run.name }), + ' ' + [run.dataset, run.model, armNames && 'arms: ' + armNames, + when(run.ts)].filter(Boolean).join(' · '))); + const inner = el('div', { style: 'padding:12px 14px' }); + const tbl = el('table', { class: 'tbl' }, el('thead', {}, el('tr', {}, + el('th', { text: 'Arm' }), el('th', { class: 'num', text: 'Tasks' }), + el('th', { class: 'num', text: 'Solved' }), el('th', { class: 'num', text: 'Solve rate' }), + el('th', { class: 'num', text: 'Mean reward' }), el('th', { class: 'num', text: 'Mean steps' }), + el('th', { class: 'num', text: 'Total cost' }), el('th', { class: 'num', text: 'Cost / task' }), + el('th', { class: 'num', text: '$ per solve' }), + el('th', { class: 'num', text: 'Cache hit' }), el('th', { class: 'num', text: 'Mean wall' }), + el('th', { class: 'num', text: 'Exceptions' })))); + const tb = el('tbody'); + for (const a of run.arms || []) { + const perSolve = a.solved > 0 ? a.total_cost_usd / a.solved : null; + tb.appendChild(el('tr', { class: 'click', onclick: () => toggleBenchTasks(inner, run.id, a.arm) }, + el('td', {}, el('code', { text: a.arm })), + el('td', { class: 'num', text: num(a.tasks) }), + el('td', { class: 'num', text: num(a.solved) }), + el('td', { class: 'num', text: pct(a.solve_rate * 100) }), + el('td', { class: 'num', text: a.mean_reward.toFixed(3) }), + el('td', { class: 'num', text: a.mean_steps.toFixed(1) }), + el('td', { class: 'num', text: usd(a.total_cost_usd) }), + el('td', { class: 'num', text: usd(a.mean_cost_usd) }), + el('td', { class: 'num', text: perSolve === null ? '—' : usd(perSolve) }), + el('td', { class: 'num', text: pct(a.cache_hit_rate * 100, 2) }), + el('td', { class: 'num', text: dur(a.mean_wall_s * 1000) }), + el('td', { class: 'num', text: num(a.exceptions) }))); + } + tbl.appendChild(tb); + inner.appendChild(el('div', { class: 'tblwrap' }, tbl)); + inner.appendChild(el('p', { class: 'note', text: 'Cost per solve is the number that matters: an arm that spends less by solving fewer tasks has not saved anything. Click an arm for its per-task rows.' })); + // Cost-vs-reward scatter: the visualization the issue asks for. + inner.appendChild(el('h2', { text: 'Cost vs reward, by arm' })); + const scatter = el('div', { class: 'chart', 'data-testid': 'bench-scatter' }); + inner.appendChild(scatter); + sec.appendChild(inner); + host.appendChild(sec); + renderScatter(scatter, run.arms || []); + }); + } catch (err) { + emptyState(host, 'Could not load benchmarks', String(err.message || err)); + } +} + +function renderScatter(host, arms) { + clear(host); + const pts = arms.filter((a) => a.tasks > 0); + if (!pts.length) { emptyState(host, 'No arms to plot', ''); return; } + const { w, h, pad } = CH; + const xMax = Math.max(...pts.map((a) => a.mean_cost_usd)) * 1.15 || 1; + const svg = svgEl('svg', { viewBox: `0 0 ${w} ${h}`, role: 'img' }); + svg.setAttribute('aria-label', 'mean cost per task versus solve rate, by arm'); + const px = (v) => pad.l + (v / xMax) * (w - pad.l - pad.r); + const py = (v) => h - pad.b - v * (h - pad.t - pad.b); + for (const t of [0, 0.25, 0.5, 0.75, 1]) { + svg.appendChild(svgEl('line', { class: 'gridline', x1: pad.l, x2: w - pad.r, y1: py(t), y2: py(t) })); + const lab = svgEl('text', { class: 'axis-text', x: pad.l - 6, y: py(t) + 3, 'text-anchor': 'end' }); + lab.textContent = (t * 100).toFixed(0) + '%'; + svg.appendChild(lab); + } + svg.appendChild(svgEl('line', { class: 'axis', x1: pad.l, x2: w - pad.r, y1: h - pad.b, y2: h - pad.b })); + for (const t of [0, xMax / 2, xMax]) { + const lab = svgEl('text', { class: 'axis-text', x: px(t), y: h - pad.b + 14, 'text-anchor': 'middle' }); + lab.textContent = usd(t); + svg.appendChild(lab); + } + pts.forEach((a, i) => { + const cx = px(a.mean_cost_usd), cy = py(a.solve_rate); + svg.appendChild(svgEl('circle', { cx, cy, r: 7, fill: `var(--s${(i % 5) + 1})`, opacity: '0.85' })); + const lab = svgEl('text', { class: 'axis-text', x: cx + 11, y: cy + 4 }); + lab.textContent = a.arm; + svg.appendChild(lab); + }); + host.appendChild(svg); + host.appendChild(el('div', { class: 'legend' }, + el('span', { text: 'x: mean billed cost per task · y: solve rate · up and to the left is better' }))); +} + +async function toggleBenchTasks(sec, runID, arm) { + const existing = sec.querySelector('[data-tasks="' + arm + '"]'); + if (existing) { existing.remove(); return; } + const host = el('div', { class: 'tblwrap', 'data-tasks': arm, 'data-testid': 'bench-tasks' }); + sec.appendChild(host); + loadingState(host, 2); + try { + const { tasks } = await api('benchmarks/' + runID + '/tasks', { arm }); + clear(host); + const tbl = el('table', { class: 'tbl compact' }, el('thead', {}, el('tr', {}, + el('th', { text: 'Task' }), el('th', { class: 'num', text: 'Reward' }), + el('th', { class: 'num', text: 'Steps' }), el('th', { class: 'num', text: 'Cache r/w' }), + el('th', { class: 'num', text: 'Fresh' }), el('th', { class: 'num', text: 'Out' }), + el('th', { class: 'num', text: 'Cost' }), el('th', { class: 'num', text: 'Wall' }), el('th', { text: '' })))); + const tb = el('tbody'); + for (const t of tasks) { + tb.appendChild(el('tr', {}, + el('td', {}, el('span', { class: 'trunc', title: t.task, text: t.task })), + el('td', { class: 'num', text: t.reward.toFixed(2) }), + el('td', { class: 'num', text: num(t.steps) }), + el('td', { class: 'num', text: compact(t.cache_read) + ' / ' + compact(t.cache_write) }), + el('td', { class: 'num', text: compact(t.fresh_input) }), + el('td', { class: 'num', text: compact(t.completion_tokens) }), + el('td', { class: 'num', text: usd(t.cost_usd) }), + el('td', { class: 'num', text: dur(t.wall_s * 1000) }), + el('td', {}, t.exception ? el('span', { class: 'pill missing', text: 'exception' }) + : t.reward >= 1 ? el('span', { class: 'pill complete', text: 'solved' }) + : el('span', { class: 'pill neutral', text: 'unsolved' })))); + } + tbl.appendChild(tb); + host.appendChild(tbl); + } catch (err) { + emptyState(host, 'Could not load tasks', String(err.message || err)); + } +} + +// ── config ───────────────────────────────────────────────────────────────── +function renderTree(v, key) { + if (v === null || v === undefined) return el('div', { class: 'v', text: '—' }); + if (Array.isArray(v)) { + return el('div', {}, el('div', { class: 'k', text: key }), + el('div', { class: 'v', text: v.map((x) => (typeof x === 'object' ? JSON.stringify(x) : String(x))).join(', ') || '(empty)' })); + } + if (typeof v === 'object') { + const box = el('details', { class: 'diff', open: key === undefined ? 'open' : null }, + el('summary', { text: key === undefined ? 'effective configuration' : key })); + const inner = el('div', { style: 'padding:10px 12px' }); + const grid = el('div', { class: 'kv' }); + for (const [k, val] of Object.entries(v)) { + if (val !== null && typeof val === 'object' && !Array.isArray(val)) inner.appendChild(renderTree(val, k)); + else grid.appendChild(kv(k, Array.isArray(val) ? (val.join(', ') || '(empty)') : String(val))); + } + inner.insertBefore(grid, inner.firstChild); + box.appendChild(inner); + return box; + } + return el('div', {}, el('div', { class: 'k', text: key }), el('div', { class: 'v', text: String(v) })); +} + +async function loadConfig() { + const host = clear($('#config-body')); + loadingState(host, 3); + try { + const cfg = await api('config'); + clear(host).appendChild(renderTree(cfg)); + } catch (err) { + emptyState(host, err.status === 403 ? 'Configuration is not visible from this address' + : 'Could not load configuration', String(err.message || err)); + } + const chost = clear($('#capture-body')); + try { + const { capture: c, description } = await api('capture'); + chost.appendChild(el('div', { class: 'kv' }, + kv('Captured', num(c.captured)), kv('Written', num(c.written)), + kv('Dropped', num(c.dropped)), kv('Insert errors', num(c.errors)), + kv('Queue', c.queued + ' / ' + c.queue_cap), kv('SSE clients', num(c.sse_clients)), + kv('Database', c.db_path || '(in memory — history is lost on restart)'), + kv('Database size', compact(c.db_bytes) + ' B'))); + chost.appendChild(el('p', { class: 'note', text: description })); + } catch (err) { + emptyState(chost, 'Could not load capture health', String(err.message || err)); + } +} + +// ── capture-drop + observe-mode banners ──────────────────────────────────── +async function checkCapture() { + try { + const { capture: c } = await api('capture'); + const b = $('#capture-warning'); + if (c.dropped > 0) { + b.textContent = `${num(c.dropped)} captured request(s) were dropped because the capture queue was full — ` + + 'the figures below under-report. Requests were never delayed; observability was. Raise the queue size.'; + b.hidden = false; + } else b.hidden = true; + + // Observe mode has to be unmissable. Every request was forwarded UNTOUCHED, so + // reading these figures as achieved savings is exactly the wrong conclusion — and it + // is the conclusion a dashboard invites unless it says otherwise. + const o = $('#observe-banner'); + if (c.mode === 'observe') { + const q = c.observe_queue; + let text = 'You are currently in OBSERVE mode. context-guru did not modify any request: ' + + 'every request above was forwarded to the provider untouched, and the pipeline ran ' + + 'off-path on a copy. Savings shown here are what compaction WOULD have achieved, ' + + 'not what it did.'; + if (q) { + text += ` Off-path queue: ${num(q.processed)} measured, ${num(q.pending)} in flight`; + // Drops matter more than depth: a dropped observation never happened, so the + // projection understates. Say which direction the error runs. + text += q.dropped > 0 + ? `, ${num(q.dropped)} DROPPED — the projection under-reports by whatever those would have saved.` + : ', 0 dropped.'; + } + o.textContent = text; + o.hidden = false; + } else o.hidden = true; + } catch (_) { /* the banners are best-effort */ } +} + +// ── views + filters ──────────────────────────────────────────────────────── +const loaders = { + overview: loadOverview, components: loadComponents, sessions: loadSessions, + requests: loadRequests, benchmarks: loadBenchmarks, config: loadConfig, +}; + +function go(view) { + if (!Object.prototype.hasOwnProperty.call(loaders, view)) view = 'overview'; + state.view = view; + for (const t of $$('.tab')) t.setAttribute('aria-selected', String(t.dataset.view === view)); + for (const s of $$('.view')) s.hidden = s.id !== 'view-' + view; + location.hash = view; + loaders[view](); +} + +function setFilter(key, value) { + state.filter[key] = value; + const ctl = $('#f-' + key); + if (ctl) ctl.value = value; + resetPaging(); +} + +function resetPaging() { state.reqCursor = 0; state.reqStack = []; state.sessOffset = 0; } + +function readFilters() { + state.filter = { + q: $('#f-q').value.trim(), model: $('#f-model').value, provider: $('#f-provider').value, + agent: $('#f-agent').value, preset: $('#f-preset').value, mode: $('#f-mode').value, + component: $('#f-component').value, reason: $('#f-reason').value, + accounting: $('#f-accounting').value, session: state.filter.session || '', + }; + state.range = Number($('#f-range').value) || 0; + resetPaging(); + loaders[state.view](); +} + +async function loadFacets() { + try { + const f = await api('facets'); + for (const dim of ['model', 'provider', 'agent', 'preset', 'mode', 'component']) { + const sel = $('#f-' + dim); + const keep = sel.value; + while (sel.options.length > 1) sel.remove(1); + for (const v of f[dim] || []) sel.appendChild(el('option', { value: v }, v)); + sel.value = keep; + } + } catch (_) { /* dropdowns degrade to "All" */ } +} + +// ── SSE ──────────────────────────────────────────────────────────────────── +let lastEventID = 0; +function connectLive() { + const src = new EventSource('/api/events' + (lastEventID ? '?last_event_id=' + lastEventID : '')); + const label = $('#live-label'), box = $('.live'); + src.onopen = () => { box.className = 'live on'; label.textContent = 'live'; }; + src.onerror = () => { box.className = 'live off'; label.textContent = 'reconnecting…'; }; + src.addEventListener('request', (ev) => { + let e; + try { e = JSON.parse(ev.data); } catch (_) { return; } + lastEventID = Math.max(lastEventID, e.id || 0); + state.live.unshift(e); + if (state.live.length > 60) state.live.length = 60; + if (state.view === 'overview') renderLive(); + }); +} + +// ── boot ─────────────────────────────────────────────────────────────────── +function initTheme() { + const saved = localStorage.getItem('cg-theme'); + if (saved) document.documentElement.setAttribute('data-theme', saved); + $('#theme').addEventListener('click', () => { + const cur = document.documentElement.getAttribute('data-theme'); + const next = cur === 'dark' ? 'light' : cur === 'light' ? 'auto' : 'dark'; + document.documentElement.setAttribute('data-theme', next); + if (next === 'auto') localStorage.removeItem('cg-theme'); + else localStorage.setItem('cg-theme', next); + }); +} + +function init() { + initTheme(); + for (const t of $$('.tab')) t.addEventListener('click', () => go(t.dataset.view)); + for (const id of ['f-model', 'f-provider', 'f-agent', 'f-preset', 'f-mode', 'f-component', + 'f-reason', 'f-accounting', 'f-range']) { + $('#' + id).addEventListener('change', readFilters); + } + let deb; + $('#f-q').addEventListener('input', () => { clearTimeout(deb); deb = setTimeout(readFilters, 250); }); + $('#f-clear').addEventListener('click', () => { + for (const s of $$('.filters select')) s.value = s.id === 'f-range' ? '0' : ''; + $('#f-q').value = ''; + state.filter = {}; + readFilters(); + }); + $('#req-next').addEventListener('click', () => { + if (!state.nextCursor) return; + state.reqStack.push(state.reqCursor); + state.reqCursor = state.nextCursor; + loadRequests(); + }); + $('#req-prev').addEventListener('click', () => { + state.reqCursor = state.reqStack.pop() || 0; + loadRequests(); + }); + $('#sess-prev').addEventListener('click', () => { state.sessOffset = Math.max(0, state.sessOffset - 25); loadSessions(); }); + $('#sess-next').addEventListener('click', () => { state.sessOffset += 25; loadSessions(); }); + $('#bench-refresh').addEventListener('click', async () => { + await fetch('/api/benchmarks?refresh=1'); + loadBenchmarks(); + }); + $('#drawer-close').addEventListener('click', closeDrawer); + $('#scrim').addEventListener('click', closeDrawer); + document.addEventListener('keydown', (ev) => { if (ev.key === 'Escape') closeDrawer(); }); + + // An unknown or absent hash must land on Overview, not call loaders[undefined]. + const wanted = (location.hash || '').replace(/^#/, ''); + renderLive(); // show the feed's empty state immediately, not only on the first event + go(Object.prototype.hasOwnProperty.call(loaders, wanted) ? wanted : 'overview'); + loadFacets(); + checkCapture(); + connectLive(); + // Poll the aggregates: SSE carries individual rows, but a rollup must be + // recomputed server-side, and 10 s is well under a human's patience. + setInterval(() => { if (state.view === 'overview') loadOverview(); }, 10000); + setInterval(() => { loadFacets(); checkCapture(); }, 30000); +} + +document.addEventListener('DOMContentLoaded', init); diff --git a/dash/ui/index.html b/dash/ui/index.html new file mode 100644 index 0000000..0a8ea92 --- /dev/null +++ b/dash/ui/index.html @@ -0,0 +1,269 @@ + + + + + +context-guru dashboard + + + + + +
+
+ + context-guru +
+ +
+ connecting… + +
+
+ +
+ + + + + + + + + + + +
+ + + + + +
+ + +
+
+ +
+

Savings percentage — four denominators, four questions

+

A single savings figure would be a lie of omission. Each bar below divides + the same savings by a different denominator, and says which. Where the inputs are + missing the bar reads n/a rather than inventing a number.

+
+
+ +
+

Cumulative cost: without context-guru vs with

+

The shaded area between the lines is money saved. Baseline prices the + unique tokens we removed at the cache-write rate they would have entered + as (~11.5× a cache read on a prompt-caching backend), and the re-sent remainder at + the cache-read rate the provider would have served it from. Pricing gross savings as + writes is how a dashboard overstates itself by its own overcount ratio — which is why + dollar savings and token savings diverge so sharply on this workload.

+
+
+ +
+

Honest savings waterfall

+

Baseline, every reduction, every penalty we owe back, and the net. If + context-guru cost more than it saved, this chart says so.

+
+
+ +
+
+

Tokens over time

+
+
+
+

Cache reads vs writes

+
+
+
+

Latency: context-guru vs upstream

+
+
+
+

Request volume & restorations

+
+
+
+ +
+
+

What our own safety mechanisms cost

+

+
+
+
+

Cache-miss attribution

+

A cold start is not a failure — the first request of a session, or the + first for a model, has nothing to hit. TTL expiry wins ties over a changed prefix.

+
+
+
+

Why we didn't compact

+

A first-class reason bucket, not an absence of data.

+
+
+
+

Token-accounting confidence

+

Only complete rows have all four billed token tiers. Anything + else is an estimate and is never rendered as exact.

+
+
+
+ +
+

Live feed

+

Summary rows only, streamed over SSE. Content is never pushed to a live + client; open a request to inspect it.

+
+ + + +
TimeSessionModelBeforeAfterSavedCG msAccounting
+
+
+
+ + + + + + + + + + + + + + + + +
+ + + + + + + + diff --git a/dash/ui/style.css b/dash/ui/style.css new file mode 100644 index 0000000..ac8de7a --- /dev/null +++ b/dash/ui/style.css @@ -0,0 +1,272 @@ +/* Design tokens first, hex never inline. headroom had to bolt ~25 light-mode + overrides onto hardcoded colours; defining the palette as custom properties on + line one means dark mode is a nine-line block instead of a retrofit. */ +:root { + color-scheme: light dark; + + --bg: #f7f8fa; + --bg-raised: #ffffff; + --bg-sunken: #eef0f4; + --border: #d7dbe2; + --border-soft:#e6e9ef; + --fg: #14181f; + --fg-muted: #5b6472; + --fg-faint: #838c9a; + + --accent: #0f7d78; + --accent-soft: #d7efee; + --good: #157f4a; + --good-soft: #d6f0e1; + --warn: #9a6100; + --warn-soft: #fbeed2; + --bad: #b3271f; + --bad-soft: #fadedd; + --info: #245ea8; + --info-soft: #dbe7f8; + + /* Chart series. Sequential-neutral, distinguishable in both themes and for the + common colour-vision deficiencies (no red/green-only pairing). */ + --s1: #0f7d78; + --s2: #245ea8; + --s3: #9a6100; + --s4: #7a4fa3; + --s5: #b3271f; + + --radius: 8px; + --radius-sm: 5px; + --gap: 14px; + --mono: ui-monospace, "JetBrains Mono", "SF Mono", Menlo, Consolas, monospace; + --sans: system-ui, -apple-system, "Segoe UI", Inter, Roboto, sans-serif; + --shadow: 0 1px 2px rgb(20 24 31 / 6%), 0 4px 12px rgb(20 24 31 / 5%); +} + +/* Dark mode: only the tokens change. Applies both to an explicit choice and to a + system preference, so the toggle wins in both directions. */ +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + --bg: #0f1216; --bg-raised: #171b21; --bg-sunken: #10141a; + --border: #2b323c; --border-soft: #232a33; + --fg: #e7eaef; --fg-muted: #a2acba; --fg-faint: #78828f; + --accent: #4ecdc7; --accent-soft: #123833; + --good: #4bcc8a; --good-soft: #12331f; + --warn: #e0a83c; --warn-soft: #33280f; + --bad: #f2726a; --bad-soft: #3a1a18; + --info: #6fa8f0; --info-soft: #14243c; + --s1: #4ecdc7; --s2: #6fa8f0; --s3: #e0a83c; --s4: #b48ae0; --s5: #f2726a; + --shadow: 0 1px 2px rgb(0 0 0 / 40%), 0 6px 18px rgb(0 0 0 / 30%); + } +} +:root[data-theme="dark"] { + --bg: #0f1216; --bg-raised: #171b21; --bg-sunken: #10141a; + --border: #2b323c; --border-soft: #232a33; + --fg: #e7eaef; --fg-muted: #a2acba; --fg-faint: #78828f; + --accent: #4ecdc7; --accent-soft: #123833; + --good: #4bcc8a; --good-soft: #12331f; + --warn: #e0a83c; --warn-soft: #33280f; + --bad: #f2726a; --bad-soft: #3a1a18; + --info: #6fa8f0; --info-soft: #14243c; + --s1: #4ecdc7; --s2: #6fa8f0; --s3: #e0a83c; --s4: #b48ae0; --s5: #f2726a; + --shadow: 0 1px 2px rgb(0 0 0 / 40%), 0 6px 18px rgb(0 0 0 / 30%); +} + +* { box-sizing: border-box; } +html, body { margin: 0; padding: 0; } +body { + background: var(--bg); color: var(--fg); + font: 14px/1.5 var(--sans); + -webkit-font-smoothing: antialiased; +} +h1, h2, h3 { margin: 0 0 6px; font-weight: 600; letter-spacing: -0.01em; } +h2 { font-size: 15px; } +a { color: var(--accent); } +code { font-family: var(--mono); font-size: 0.92em; } + +.skip { position: absolute; left: -9999px; } +.skip:focus { left: 8px; top: 8px; background: var(--bg-raised); padding: 8px; z-index: 100; } + +/* ── top bar ───────────────────────────────────────────────────────────── */ +.topbar { + display: flex; align-items: center; gap: 18px; flex-wrap: wrap; + padding: 10px 18px; background: var(--bg-raised); + border-bottom: 1px solid var(--border); + position: sticky; top: 0; z-index: 20; +} +.brand { display: flex; align-items: center; gap: 8px; font-weight: 650; letter-spacing: -0.02em; } +.logo { width: 20px; height: 20px; color: var(--accent); } +.tabs { display: flex; gap: 2px; flex-wrap: wrap; } +.tab { + background: none; border: 0; padding: 7px 12px; border-radius: var(--radius-sm); + color: var(--fg-muted); font: inherit; font-weight: 500; cursor: pointer; +} +.tab:hover { background: var(--bg-sunken); color: var(--fg); } +.tab[aria-selected="true"] { background: var(--accent-soft); color: var(--accent); font-weight: 600; } +.topbar-right { margin-left: auto; display: flex; align-items: center; gap: 12px; } +.live { display: inline-flex; align-items: center; gap: 6px; font-size: 12px; color: var(--fg-muted); } +.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--fg-faint); display: inline-block; } +.live.on .dot { background: var(--good); box-shadow: 0 0 0 3px var(--good-soft); } +.live.off .dot { background: var(--bad); } +button.ghost { + background: var(--bg-raised); color: var(--fg-muted); font: inherit; + border: 1px solid var(--border); border-radius: var(--radius-sm); + padding: 5px 10px; cursor: pointer; +} +button.ghost:hover { color: var(--fg); border-color: var(--fg-faint); } +button.ghost:disabled { opacity: 0.45; cursor: default; } + +/* ── filters ───────────────────────────────────────────────────────────── */ +.filters { + display: flex; gap: 8px; flex-wrap: wrap; align-items: center; + padding: 10px 18px; background: var(--bg-sunken); + border-bottom: 1px solid var(--border-soft); +} +.filters input, .filters select { + background: var(--bg-raised); color: var(--fg); font: inherit; + border: 1px solid var(--border); border-radius: var(--radius-sm); padding: 5px 8px; + max-width: 220px; +} +.filters input[type="search"] { min-width: 220px; flex: 1 1 220px; } +:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } + +/* ── layout ────────────────────────────────────────────────────────────── */ +main { padding: var(--gap) 18px 60px; max-width: 1600px; margin: 0 auto; } +.panel { + background: var(--bg-raised); border: 1px solid var(--border-soft); + border-radius: var(--radius); padding: 14px 16px; margin-bottom: var(--gap); + box-shadow: var(--shadow); +} +.grid-2 { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 520px), 1fr)); gap: var(--gap); } +.grid-2 .panel { margin-bottom: 0; } +.note { color: var(--fg-muted); font-size: 12.5px; margin: 0 0 10px; max-width: 78ch; } +.banner { padding: 10px 14px; margin: var(--gap) 18px 0; border-radius: var(--radius); font-size: 13px; } +.banner.warn { background: var(--warn-soft); color: var(--warn); border: 1px solid currentColor; } + +/* ── stat tiles ────────────────────────────────────────────────────────── */ +.tiles { display: grid; grid-template-columns: repeat(auto-fill, minmax(178px, 1fr)); gap: 10px; margin-bottom: var(--gap); } +.tile { + background: var(--bg-raised); border: 1px solid var(--border-soft); + border-radius: var(--radius); padding: 11px 13px; box-shadow: var(--shadow); +} +.tile .k { color: var(--fg-muted); font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.04em; } +.tile .v { font-size: 21px; font-weight: 620; font-variant-numeric: tabular-nums; margin-top: 3px; letter-spacing: -0.02em; } +.tile .s { color: var(--fg-faint); font-size: 11.5px; margin-top: 2px; } +.tile.good .v { color: var(--good); } +.tile.bad .v { color: var(--bad); } +.tile.accent .v { color: var(--accent); } + +/* ── bars (denominators, distributions, waterfall) ─────────────────────── */ +.bars { display: grid; gap: 8px; } +.bar-row { display: grid; grid-template-columns: minmax(150px, 34%) 1fr auto; gap: 10px; align-items: center; } +.bar-label { font-size: 12.5px; color: var(--fg-muted); } +.bar-track { background: var(--bg-sunken); border-radius: 99px; height: 12px; overflow: hidden; position: relative; } +.bar-fill { height: 100%; background: var(--s1); border-radius: 99px; } +.bar-fill.neg { background: var(--bad); } +.bar-val { font-variant-numeric: tabular-nums; font-size: 12.5px; font-weight: 600; min-width: 84px; text-align: right; } +.bar-desc { grid-column: 1 / -1; color: var(--fg-faint); font-size: 11.5px; margin: -3px 0 6px; max-width: 90ch; } +.na { color: var(--fg-faint); font-style: italic; } + +/* ── tables ────────────────────────────────────────────────────────────── */ +/* Wide content scrolls INSIDE its own container; the page body must never scroll + horizontally. min-width:0 is what actually lets a grid/flex child shrink below + its content's intrinsic width — without it the table pushes the panel wider than + the viewport and the whole page scrolls sideways on a phone. */ +.tblwrap { overflow-x: auto; max-width: 100%; } +.panel, .grid-2, .tiles, main { min-width: 0; } +.panel > * { min-width: 0; } +.tbl { width: 100%; border-collapse: collapse; font-size: 13px; } +.tbl th, .tbl td { text-align: left; padding: 7px 9px; border-bottom: 1px solid var(--border-soft); white-space: nowrap; } +.tbl th { color: var(--fg-muted); font-weight: 600; font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.03em; position: sticky; top: 0; background: var(--bg-raised); } +.tbl td.num, .tbl th.num { text-align: right; font-variant-numeric: tabular-nums; } +.tbl tbody tr:hover { background: var(--bg-sunken); } +.tbl tbody tr.click { cursor: pointer; } +.tbl.compact th, .tbl.compact td { padding: 5px 8px; } +/* Long session ids / long content must never blow out the layout. */ +.trunc { max-width: 190px; overflow: hidden; text-overflow: ellipsis; display: inline-block; vertical-align: bottom; font-family: var(--mono); font-size: 12px; } +.pager { display: flex; align-items: center; gap: 12px; margin-top: 10px; font-size: 12.5px; color: var(--fg-muted); } + +.pill { display: inline-block; padding: 1px 7px; border-radius: 99px; font-size: 11px; font-weight: 600; } +.pill.complete { background: var(--good-soft); color: var(--good); } +.pill.partial { background: var(--warn-soft); color: var(--warn); } +.pill.missing { background: var(--bad-soft); color: var(--bad); } +.pill.hit { background: var(--good-soft); color: var(--good); } +.pill.cold_start { background: var(--info-soft); color: var(--info); } +.pill.ttl_expiry, .pill.prefix_change, .pill.unknown { background: var(--warn-soft); color: var(--warn); } +.pill.neutral { background: var(--bg-sunken); color: var(--fg-muted); } + +/* ── charts ────────────────────────────────────────────────────────────── */ +.chart { width: 100%; min-height: 200px; position: relative; } +.chart svg { width: 100%; height: auto; display: block; overflow: visible; } +.axis { stroke: var(--border); stroke-width: 1; } +.gridline { stroke: var(--border-soft); stroke-width: 1; } +.axis-text { fill: var(--fg-faint); font-size: 10px; font-family: var(--sans); } +.legend { display: flex; gap: 14px; flex-wrap: wrap; font-size: 12px; color: var(--fg-muted); margin-top: 6px; } +.legend i { width: 10px; height: 10px; border-radius: 2px; display: inline-block; margin-right: 5px; vertical-align: -1px; } +.tooltip { + position: absolute; pointer-events: none; opacity: 0; transition: opacity .1s; + background: var(--bg-raised); border: 1px solid var(--border); border-radius: var(--radius-sm); + padding: 7px 9px; font-size: 12px; box-shadow: var(--shadow); z-index: 30; white-space: pre-line; + font-variant-numeric: tabular-nums; +} +.tooltip.show { opacity: 1; } + +/* ── empty / loading ───────────────────────────────────────────────────── */ +.empty { color: var(--fg-muted); text-align: center; padding: 34px 16px; font-size: 13px; } +.empty strong { display: block; color: var(--fg); font-size: 14.5px; margin-bottom: 4px; } +.skel { background: linear-gradient(90deg, var(--bg-sunken) 25%, var(--border-soft) 50%, var(--bg-sunken) 75%); + background-size: 200% 100%; animation: shimmer 1.2s infinite; border-radius: var(--radius-sm); height: 14px; } +@keyframes shimmer { to { background-position: -200% 0; } } +@media (prefers-reduced-motion: reduce) { .skel { animation: none; } } + +/* ── drawer ────────────────────────────────────────────────────────────── */ +/* `hidden` must actually hide: a `display: flex`/`block` rule overrides the + attribute's UA `display: none`, which left the invisible drawer swallowing every + click on the page underneath it. Restate it explicitly. */ +[hidden] { display: none !important; } +.scrim { position: fixed; inset: 0; background: rgb(0 0 0 / 42%); z-index: 40; } +.drawer { + position: fixed; top: 0; right: 0; bottom: 0; width: min(920px, 100%); + background: var(--bg); border-left: 1px solid var(--border); z-index: 41; + display: flex; flex-direction: column; +} +.drawer-head { display: flex; align-items: center; gap: 12px; padding: 12px 16px; border-bottom: 1px solid var(--border); background: var(--bg-raised); } +.drawer-head h2 { margin: 0; } +.drawer-body { overflow-y: auto; padding: 14px 16px 60px; } +.kv { display: grid; grid-template-columns: repeat(auto-fill, minmax(190px, 1fr)); gap: 8px 16px; margin-bottom: 14px; } +.kv div { font-size: 12.5px; } +.kv .k { color: var(--fg-muted); } +.kv .v { font-weight: 600; font-variant-numeric: tabular-nums; overflow-wrap: anywhere; } + +/* ── diff view ─────────────────────────────────────────────────────────── */ +.diff { border: 1px solid var(--border-soft); border-radius: var(--radius); margin-bottom: 12px; overflow: hidden; background: var(--bg-raised); } +.diff summary { padding: 8px 12px; cursor: pointer; font-size: 12.5px; background: var(--bg-sunken); font-family: var(--mono); } +/* A collapsed benchmark run: name in the app font, metadata muted beside it. */ +.panel.diff { padding: 0; } +.panel.diff > summary { font-family: var(--sans); color: var(--fg-muted); } +.panel.diff > summary strong { color: var(--fg); font-size: 14px; margin-right: 4px; } +.diff summary::marker { color: var(--fg-faint); } +.difftoolbar { display: flex; gap: 8px; padding: 7px 12px; border-bottom: 1px solid var(--border-soft); align-items: center; font-size: 12px; color: var(--fg-muted); } +.diffbody { font-family: var(--mono); font-size: 12px; line-height: 1.45; overflow-x: auto; max-height: 460px; overflow-y: auto; } +.dl { display: grid; grid-template-columns: 46px 46px 1fr; } +.dl > span { padding: 0 8px; white-space: pre-wrap; overflow-wrap: anywhere; } +.dl .ln { color: var(--fg-faint); text-align: right; user-select: none; background: var(--bg-sunken); font-size: 11px; } +.dl.add { background: var(--good-soft); } +.dl.del { background: var(--bad-soft); } +.dl.add .tx::before { content: "+ "; color: var(--good); } +.dl.del .tx::before { content: "- "; color: var(--bad); } +.dl.ctx .tx::before { content: " "; } +.dl.gap { background: var(--bg-sunken); color: var(--fg-faint); font-style: italic; } +.side { display: grid; grid-template-columns: 1fr 1fr; gap: 1px; background: var(--border-soft); } +.side > pre { margin: 0; padding: 8px 10px; background: var(--bg-raised); overflow-x: auto; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 12px; max-height: 420px; } + +/* ── small viewport ────────────────────────────────────────────────────── */ +@media (max-width: 720px) { + .topbar { gap: 10px; padding: 8px 12px; } + .tabs { order: 3; width: 100%; overflow-x: auto; } + .filters { padding: 8px 12px; } + .filters input, .filters select { max-width: none; flex: 1 1 130px; } + main { padding: 10px 12px 50px; } + .tiles { grid-template-columns: repeat(auto-fill, minmax(140px, 1fr)); } + .grid-2 { grid-template-columns: 1fr; } + .side { grid-template-columns: 1fr; } + .bar-row { grid-template-columns: 1fr; gap: 3px; } + .bar-val { text-align: left; } +} diff --git a/docs/dashboard.md b/docs/dashboard.md new file mode 100644 index 0000000..abfbd51 --- /dev/null +++ b/docs/dashboard.md @@ -0,0 +1,344 @@ +# Dashboard + +context-guru ships a **persistent observability dashboard**: an embedded single-page UI +plus a JSON/SSE API, backed by a durable per-request store. It exists to answer one +question honestly — **what value is context-guru providing?** — and to make the answer +falsifiable, including when the answer is "less than you hoped". + +```sh +context-guru-proxy --preset codesmart --dashboard +# open http://localhost:4000/dashboard/ +``` + +It is **off by default**. Turning it on adds `/dashboard/` and `/api/*`; every existing +route, including [`/stats`](reference/routes.md), is byte-for-byte unchanged. + +![The dashboard's Overview](img/dashboard/01-overview.jpg) + +!!! info "No CDN, no build step, no network" + The whole UI is three files (`index.html`, `style.css`, `app.js`) embedded in the + binary with `go:embed`, served under a `default-src 'self'` CSP. Charts are + hand-drawn SVG — no chart library, no framework, no npm. The page fetches nothing + off-origin, so it works in a VPC or fully air-gapped, and a test asserts that. + +## What it shows + +### Overview + +Every headline number, with the honesty machinery visible rather than hidden: + +| Group | Fields | +|---|---| +| Volume | requests · sessions · tokens before/after | +| Savings | gross · unique · net-of-restores · overcount ratio | +| Money | baseline cost · actual cost · context-guru's own LLM spend · net dollars saved | +| Tokens billed | fresh input · cache reads · cache writes · output | +| Latency | context-guru added (mean + **p95**) · upstream (mean + p95) | +| Quality | restorations + rate · reverts · not-compacted count | + +Three of those deserve their own explanation. + +#### Four savings denominators, not one + +A single "savings %" is a lie of omission. The dashboard ships four ratios, each +labelled with **what it divides by**, and each carrying that explanation in the payload +itself (`GET /api/stats` → `denominators[].description`): + +| Denominator | Question it answers | +|---|---| +| **of what we tried to compact** | Are we good *when we have something to work with?* Divides by the tokens compaction was allowed to touch — the uncached tail when cache-aware. | +| **of new provider-billed input** | The honest economic ratio. Divides by fresh input + cache writes + what we removed, so transcript history the provider served from cache is not recounted. | +| **of the whole request (diluted)** | Kept for transparency. A long session re-sends its history every turn, so this denominator grows quadratically and trends to ~0% however well compaction works. | +| **unique, of the whole request** | The most conservative figure the dashboard can produce. | + +The new-input ratio is guarded: with no provider usage data the denominator would be +`saved` alone and the ratio would read ~100%. It reports **n/a** instead. + +#### The cost of our own safety mechanisms + +Reported next to their benefit, because a compaction proxy that shows only tokens +removed is unfalsifiable: + +- **Frozen for cache safety** — compaction deliberately *not* done on the already-cached + prefix. Its benefit is the cache reads that stayed cheap; its cost is this. +- **Restored after offload** — content we removed and the model asked back for. A + premature offload, paid for twice. +- **Reverted component runs** — the never-worse guard firing. Safety working, and its + cost is the latency of the attempt. +- **context-guru's own latency and LLM spend** — paid out of the savings above. + +#### Cumulative cost, with and without + +![Cost chart with a tooltip](img/dashboard/02-savings-cost-graph-tooltip.jpg) + +The shaded band between the lines is money saved. Baseline prices the **unique** tokens we +removed at the **cache-write** rate they would have entered as — on a prompt-caching +backend that is ~11.5× a cache read — and the re-sent remainder at the **cache-read** rate +the provider would have served it from. That split is the whole reason token savings and +dollar savings diverge so sharply on this workload (see +[the SWE-bench comparison](results/comparison.md)). + +Both halves matter, and getting either wrong inflates the headline. `saved_tokens` is +gross: the agent re-sends its transcript every turn, so one compaction is re-counted once +per remaining turn — a 4.7× overcount on the 63-request replay above, and 13.1× on a +longer one. Only `saved_unique` is content that genuinely never reached the provider, and +only that part can be priced as a cache write; the re-sent remainder would have come from +the provider's cache at 1/11.5 the rate. The dashboard shows the correction factor as +`overcount_ratio` right beside the dollar figure — an earlier version of this page +described pricing gross savings as writes, which overstated net savings by ~9× on the +same data. + +Beneath it, the **honest savings waterfall**: baseline → compaction savings → +context-guru's own LLM cost → net cost → net savings. If context-guru cost more than it +saved, the waterfall says so. + +### Components — which ones earn their place + +![Per-component economics](img/dashboard/03-component-metrics.jpg) + +Runs · acted · act rate · reverted · **unique** saved · **gross** saved · **overcount +ratio** · own latency · avg ms · errors, plus a plain-language verdict. On the real +traffic in that screenshot the verdicts are doing their job: `extract` is *earning its +place*, `cacheinject` *mutates, saves no content* (its win is provider-side and invisible +to content-token counts), and a component that burned wall time for nothing reads +*costly and inert*. + +`overcount_ratio` is the number that keeps the rest honest: a ratio of 7× means the +gross figure counted the same compaction seven times as the agent re-sent its transcript. + +### Sessions + +![Session list](img/dashboard/04-sessions.jpg) + +Searchable, filterable, paginated. Per session: model · agent · preset · turns · tokens +before/saved · dollars saved · cache reads/writes · restorations · context-guru latency · +start time. Clicking a session filters the request list to it. + +### Requests, and the diff + +![Request list](img/dashboard/05-requests.jpg) + +Server-side filters across every dimension, with **keyset** pagination (a cursor, not an +`OFFSET`, so page 500 costs the same as page 1). Click any row for the detail drawer: + +![Request detail](img/dashboard/08-request-detail.jpg) + +…and, the headline feature, **what context-guru actually did to the wire**: + +![Git-style content diff](img/dashboard/09-content-git-diff.jpg) + +Git-style hunks with line numbers and collapsed unchanged runs, plus side-by-side and +after-only views. Both reference implementations carry this data and neither renders it. + +![Side-by-side diff](img/dashboard/10-diff-side-by-side.jpg) + +### Benchmarks + +![Benchmark comparison](img/dashboard/11-benchmark-comparison.jpg) + +Point `--dashboard-bench-dirs` at a harness jobs root and every run's `summary.json` + +`rows-.json` is ingested — no new export format. Per arm: tasks · solved · solve +rate · mean reward · mean steps · total cost · cost per task · **cost per solve** · cache +hit rate · mean wall · exceptions, with a cost-vs-reward scatter and per-task drill-down. + +Cost per solve is the number that matters: an arm that spends less by solving fewer tasks +has not saved anything. + +Re-ingesting replaces a run rather than duplicating it, so restarting the proxy against a +jobs root is idempotent. + +### Configuration + +![Effective configuration](img/dashboard/13-configuration.jpg) + +The **resolved** configuration — preset expanded, defaults filled, overrides applied — +not what was typed. Alongside it, the capture pipeline's own health, drop count included. + +### Dark mode, small screens, empty states + +Dark mode is not a retrofit: the palette is CSS custom properties from line one, and dark +mode redefines only the tokens. + +![Dark mode](img/dashboard/14-overview-dark.jpg) + +Small viewports reflow; wide tables scroll inside their own container so the page body +never scrolls sideways. + +
+ ![Small viewport](img/dashboard/16-overview-small-viewport.jpg){ width="320" } +
390 px viewport
+
+ +An empty dashboard reads zero and explains why each panel is blank, rather than +rendering nothing: + +![Empty state](img/dashboard/18-empty-dashboard.jpg) + +## Honest-metrics rules + +The dashboard follows five rules. They are the difference between a dashboard and a +marketing surface. + +1. **Every ratio names its denominator.** See above. +2. **Gross, unique and adjusted are visibly distinct.** `overcount_ratio` is surfaced, + not hidden. +3. **The cost of our own safety mechanisms sits beside their benefit.** +4. **`token_accounting` per row: `complete | partial | missing`.** Only `complete` rows + have all four billed token tiers. A row we cannot price is rendered as *unknown* — + never as free, and never as exact. +5. **Cache misses are attributed, and a cold start is not a failure.** Buckets: + `hit · cold_start · ttl_expiry · prefix_change · unknown`. The first request of a + session, or the first for a given model, has nothing to hit. TTL expiry wins ties + against a changed prefix — a prefix that changed after the entry had already expired + was not the cause. + +Plus a sixth, which is really rule 0: **"why didn't you compact this?" is a first-class +answer**, not an absence of data. Buckets: `bypassed · no_messages · below_trigger · +cache_frozen · found_nothing · reverted`, and an empty reason means we did compact. + +## Architecture + +```mermaid +flowchart LR + R[chat request] --> P[pipeline
apply.BodyTrace] + P --> U[upstream] + U --> C[client] + P -. one struct, one
non-blocking send .-> Q[[capture channel
buffered, drops + counts]] + Q --> W[writer goroutine
batched tx] + W --> DB[(SQLite
requests · components · content)] + W --> H[SSE hub] + DB --> API[/api/*] + H --> API + API --> UI[embedded UI] +``` + +Four properties, in order of importance: + +**Capture is off the hot path.** The handler builds one struct from values the request +path already computed and hands it to a buffered channel with a `default:` branch. When +the channel is full the event is **dropped and counted** — never queued into a growing +backlog, never blocking. Observability cannot add latency to, or fail, a request. + +**Measured overhead: no detectable per-request cost, content capture included.** Driving +the real handler over one keep-alive connection with a 24-tool-result transcript and +content capture ON, median of 40 paired requests against the same fake upstream: the +dashboard-on figure lands within noise of dashboard-off (repeatedly a shade *below* it). +The channel send itself is ~175 ns (`go test -bench BenchmarkRecord ./dash`). + +That second number used to be the only one published, and on its own it was misleading. +`finish` is called from the handler's `defer`, which runs **before the handler returns** — +so work placed there is paid by the next request on a keep-alive connection, i.e. by +every real agent. Content redaction sat there and cost **~53 ms/request**, ~25% of a +request, while the documented figure was "0.000002%". A benchmark that calls `Record` +directly cannot see that, so the guard is now an end-to-end handler-latency test with +content capture on (`TestDashboardAddsNoRequestLatencyWithContentCapture`, budget 5 ms); +putting redaction back on the request goroutine measures +87 ms and fails it. + +**Redaction happens before the database, never on read.** Headers are blanket-redacted by +key against a short allowlist; config keys are allowlisted, and an allowlisted key's +*value* is still checked for an embedded `user:password@` credential; captured content is +scrubbed of credential-shaped strings and size-capped. All of it runs on the writer +goroutine, immediately before the INSERT — off the request path, but still before anything +touches disk. A secret that reaches disk is a secret forever, and a redact-on-read filter +is one forgotten code path from leaking it. + +Content is the one surface that **cannot** be allowlisted, because it is arbitrary agent +output. It gets pattern scrubbing, and a pattern denylist is structurally always behind +reality: a review of 22 realistic credential shapes found 11 passing through, the worst +being `Authorization: Bearer `, where the pattern matched the scheme and left the +token in the diff view. Those are fixed and pinned by a table-driven test — but 22/22 +passing does not prove completeness, which is why **content capture is opt-in** +(`--dashboard-content`, default off) rather than opt-out. + +**Percentages at read time, cost at write time.** Every ratio is derived when queried, so +a filter change needs no rebuild; every cost is computed when the row is written, so +history does not silently reprice when a model's published rate changes. + +One more, worth stating because it is what we chose *not* to build: **no rollup tables.** +Time series are bucketed in SQL at query time (`ts/bucket*bucket GROUP BY 1`). SQLite +handles millions of rows, and any bucket width works without a migration. + +## Storage + +SQLite via `modernc.org/sqlite` (pure Go — no C toolchain beyond the one tree-sitter +already requires), in WAL mode. + +| Table | Holds | +|---|---| +| `requests` | one row per proxied request: identity, all four token tiers, costs, latencies, attribution | +| `request_components` | one row per component per request — the "which components earn their place" data | +| `request_content` | before/after text, gzip-compressed and size-capped; skippable entirely | +| `bench_runs` / `bench_tasks` | ingested harness runs and their per-task rows | + +Timestamps are **epoch milliseconds** everywhere. A formatted locale string cannot be +range-queried, sorted portably, or bucketed; the UI formats in the viewer's locale at +render time. + +Retention is bounded by **age and size**: rows older than `--dashboard-retention` are +dropped, then — if the file is still over `--dashboard-max-bytes` — the oldest requests +go until it fits. Age alone cannot bound a burst; size alone silently erases a quiet week. + +The schema carries a version. On a mismatch the existing file is **renamed aside** +(`.v.bak`) and a fresh database is created: a dashboard is a derived view, so +discarding history beats refusing to boot, and renaming beats deleting a user's data. + +**No-persistence mode:** `--dashboard-db :memory:` keeps everything in RAM. It is also +the automatic fallback when the configured path cannot be opened — the proxy's job is to +proxy, so an unwritable dashboard path logs a warning and degrades rather than failing to +start. + +## Access + +| Surface | Who can see it | +|---|---| +| Aggregates, series, component and session rollups, request metrics | anyone who can reach the port | +| Per-request **content** (the diff view) | loopback, or an explicit `--dashboard-trusted-cidrs` entry | +| Effective **configuration** | loopback, or a trusted CIDR | + +Aggregates are deliberately open: a proxy bound to `0.0.0.0` should still show its own +numbers, and the point of this tool is observability. Content is gated because a +transcript can carry a user's source code. There is **no** "disable observability in +production" switch — for a tool whose value *is* observability, that would be backwards. + +## Configuration + +See [Config & environment](reference/config.md) for the full flag table. The short +version: + +```sh +context-guru-proxy --preset codesmart \ + --dashboard \ + --dashboard-db /var/lib/context-guru/dashboard.db \ + --dashboard-retention 168h \ + --dashboard-max-bytes 1073741824 \ + --dashboard-bench-dirs /var/lib/context-guru/benchruns \ + --dashboard-trusted-cidrs 10.0.0.0/8 +``` + +Every flag has an environment equivalent (`DASHBOARD`, `DASHBOARD_DB`, …) for container +deployments. + +## API + +See [Routes & headers](reference/routes.md) for the full list. All of it is plain JSON +plus one SSE stream, so the dashboard is not the only possible consumer: + +```sh +curl -s localhost:4000/api/stats | jq '.denominators[] | {label, percent, available}' +curl -s 'localhost:4000/api/requests?component=extract&reason=compacted&limit=20' +curl -s 'localhost:4000/api/series?bucket=300000&since=1786300000000' +curl -N localhost:4000/api/events # live summary rows over SSE +``` + +## Verifying it yourself + +```sh +CGO_ENABLED=1 go test ./dash/ ./proxy/ # unit + integration +CGO_ENABLED=1 go test -race ./dash/ ./proxy/ # capture path + SSE hub +CGO_ENABLED=1 go test -bench BenchmarkRecord ./dash/ # the overhead number +``` + +The UI itself is regression-tested two ways: a Go test asserts every stat tile's +`data-testid` exists (and that `app.js` parses — a dropped paren renders a blank page +that no Go test would otherwise catch), and a browser check drives the rendered app +end to end. See [Measure savings](how-to/measure-savings.md) for the workflow. diff --git a/docs/design.md b/docs/design.md index 41c139e..64c7ac1 100644 --- a/docs/design.md +++ b/docs/design.md @@ -22,6 +22,7 @@ infrastructure the components sit on. | `session/` | resolve the session key (explicit id, else content hash) | | `modes/` | per-session cached-prefix boundary (`Tracker`) + the bounded off-path worker pool (`Pool`) | | `metrics/` | `Emitter` implementations: `Slog`, `Aggregator` (for `/stats`), `Tee` | +| `dash/` | the persistent observability layer: SQLite store, off-hot-path capture, SSE hub, JSON API, embedded UI | | `config/` | strict YAML loader, presets, pipeline builder | | `proxy/` | the standalone/gateway HTTP proxy | | `adapters/bifrost/` | `LLMPlugin` adapter to embed the pipeline in a bifrost deployment | @@ -195,30 +196,6 @@ provider requires one `tool_result` per `tool_call_id`). A miss silently turns a lossy — the known TTL edge, much narrower now the TTL slides on every read (see [Freeze lifetime](#freeze-lifetime-and-which-way-to-fail)). -### The loop on a streaming response - -The loop needs a whole assistant message; SSE delivers events. So the host decides per request, -from the request bytes, whether it can afford to look: - -- **no marker in `messages`/`system`** → nothing to expand → stream through untouched; -- **marker present** → buffer the stream, rebuild the message with `expand.AggregateSSE` - (Anthropic dialect only — other dialects return `ok=false` and are replayed raw), inspect, and - either continue the loop or replay the buffered bytes verbatim. - -Buffering is the one thing that turns a stream into a non-stream, so the marker test must be tight -in *both* directions: a false negative loses a real expand call, a false positive silently costs -every request its time-to-first-byte. It scans only model-visible content (`messages`, `system`) — -scanning the whole body also matched the expand tool description the host injects itself, which made -it unconditionally true (issue #26). `/stats` exposes `sse_streamed` / `sse_buffered` / -`sse_buffered_pct` and the two TTFB averages so the fast path is measured, not assumed. - -**Markers on the wire are usually HTML-escaped.** `encoding/json` escapes `<` by default — a caller -can opt out with `Encoder.SetEscapeHTML(false)`, and some non-Go clients never escape it — and `sjson` -escapes it whenever the value contains a newline, which is how every marker is appended. So `<>` -in the model's view is normally `<>` in the bytes. Marker matching on *decoded* content -(`expand.HasPlaceholder`, used by the components) sees the plain form; matching on *raw request -bytes* (`expand.rawMarkerRe`, used by the host's streaming decision) must accept both, and does. - ## State: the Store One `Store` interface, in-memory TTL+LRU default (both hosts share it). Defaults: **10000s sliding @@ -321,10 +298,16 @@ of per-request percentages. It also reports: and individual filters pay off, and which output shapes matched nothing; - `saved_tokens` vs `saved_tokens_unique` and `overcount_ratio` — cumulative vs distinct. The agent re-sends history verbatim every turn, so the cumulative figure double-counts. Quote the unique one; -- `mode` / `sync_enforced`, and the `potential_*` / `projected_*` observe namespace. +- `mode` / `sync_enforced`, and the `potential_*` / `projected_*` observe namespace; +- the four provider-billed token tiers (`fresh_input_tokens`, `cache_read_tokens`, + `cache_write_tokens`, `output_tokens`), plus `attempted_tokens` / `frozen_tokens` and the + two extra ratios derived from them (`savings_pct_attempted`, `savings_pct_new_input`). -Fields are only ever **added** to `/stats`; the harbor harnesses parse it, so no field is renamed -or removed. The full field list is in [Routes](reference/routes.md#get-stats). +`/stats` is **append-only by contract**: `deploy/harbor/*.py` parses it to produce every +published benchmark result, so a rename would invalidate the reproduction path *silently* +(the harness would keep running and report zeros). A golden test asserts the exact key set +of both the top-level object and each per-component object. The full field list is in +[Routes](reference/routes.md#get-stats). ## Operating modes @@ -393,6 +376,82 @@ or cached content falls back into the mutable tail. - `observe`: the forwarded body *is* the input, so there is nothing for a failure to damage; a panicking observation is contained by the pool and counted. +## Observability: the dashboard store (D11) + +`Aggregator` answers "what is happening now" and forgets everything on restart. The +[dashboard](dashboard.md) answers "what happened, and was it worth it" — which needs +durability, filtering and per-request detail. It is a **separate, additive layer**: the +aggregator is untouched and stays the fast in-process counter. + +```mermaid +flowchart LR + H[chat handler] -->|apply.BodyOpts| P[pipeline] + P --> U[upstream] + U --> C[client] + H -. one struct,
one non-blocking send .-> Q[[capture channel
buffered · drops + counts]] + Q --> W[writer goroutine
batched transaction] + W --> DB[(SQLite · WAL
requests
request_components
request_content
bench_runs/tasks)] + W --> S[SSE hub
write timeout + eviction] + DB --> A[/api/*
filters · keyset paging · query-time buckets/] + S --> A + A --> UI[go:embed single-page UI] +``` + +Five decisions, and what each one refuses: + +**Capture is off the hot path, and drops rather than blocks.** The handler builds one +`dash.Event` from values the request path already computed and does a channel send with a +`default:` branch. A full queue increments a drop counter that the dashboard itself +displays. *Refuses:* observability that can add latency to, or fail, a request. + +"Off the hot path" has to mean the whole pipeline, not just the send. The first version of +this layer redacted captured content inside the handler's `defer` — which runs *before the +handler returns*, so a keep-alive client's next request queued behind nine regexes over up +to 48 blobs. The channel send was genuinely ~175 ns and the request still got ~53 ms +slower. Everything expensive (redaction, gzip, the insert, the SSE fan-out) now happens on +the **writer goroutine**, and the regression test drives a real handler with content +capture ON rather than calling `Record` directly — a benchmark of the cheap half of a +two-part path will report the cheap half. + +**`apply.BodyOpts` is the capture point.** `BodyFull` delegates to it, so the rewrite is +byte-identical whether or not anyone is looking; the `Trace` embedded in its `Result` +carries the resolved session, the `RunReport`, the cache-awareness facts +(`AttemptedTokens` / `FrozenTokens`) and each rewritten message's before/after text — the +same material `CONTEXT_GURU_DUMP` writes to a file. *Refuses:* a parallel accounting path +that could disagree with the pipeline's own. + +**Redaction before the database, never on read.** Headers are blanket-redacted by key +against a short allowlist (a denylist fails the moment a gateway invents a new auth +header); config keys are allowlisted, with credential-named keys always withheld, and an +allowlisted key's *value* is still checked for an embedded `user:password@` credential. + +Captured message **content** is the one surface that cannot be allowlisted — it is +arbitrary agent output — so it gets pattern scrubbing, and a pattern denylist is +structurally always behind reality: a review of 22 realistic credential shapes found 11 +passing through, including `Authorization: Bearer `, where the pattern matched the +scheme and left the token. The patterns are fixed and the shapes are now a table-driven +test, but the honest conclusion is that this mechanism cannot be *proved* complete, so +content capture is **opt-in** (`--dashboard-content`, default off) rather than opt-out. +*Refuses:* a secret on disk, a redact-on-read filter one forgotten code path from leaking +it, and a default that writes arbitrary transcripts to disk behind a denylist. + +**Percentages at read time, cost at write time.** Ratios are derived per query, so a +filter change needs no rebuild. Costs are computed when the row is written, so history does +not reprice when a model's published rate changes. *Refuses:* a "savings" figure that +silently changes retroactively. + +**No rollup tables.** Time series are bucketed in SQL (`ts/bucket*bucket GROUP BY 1`). +*Refuses:* a pre-aggregation layer to keep consistent before any query is measurably slow. + +Timestamps are epoch **milliseconds** throughout — a formatted locale string cannot be +range-queried, sorted portably or bucketed. Retention is bounded by age **and** size. The +schema carries a version; a mismatch renames the old file aside and starts fresh, because a +dashboard is a derived view and discarding it beats refusing to boot. + +Per-request **content** and the effective **configuration** are served to loopback or an +explicit trusted CIDR only; aggregates are open, because a proxy bound to `0.0.0.0` should +still report its own numbers. + ## Config & registry One strict YAML struct serves both hosts. `pipeline:` is an ordered name-list (order + diff --git a/docs/how-to/measure-savings.md b/docs/how-to/measure-savings.md index 5e26660..a67aaaa 100644 --- a/docs/how-to/measure-savings.md +++ b/docs/how-to/measure-savings.md @@ -1,24 +1,162 @@ # Measure savings -context-guru reports what it actually saved through the proxy's `GET /stats` endpoint, backed by -an in-process metrics aggregator. Savings are measured on **message content text** — what the model -reads — not the JSON envelope, so a control directive like a `cache_control` breakpoint never looks -"worse". +The honest answer to "what did context-guru save me?" is **several numbers, each with its +denominator named**. This guide walks the fastest path to those numbers, then explains +which one to quote and why the obvious one is usually the wrong one. -## `GET /stats` +Savings are measured on **message content text** — what the model reads — not the JSON +envelope, so a control directive like `cacheinject` never looks "worse". -The proxy exposes `GET /stats` with in-process savings rollups. Savings are **token-weighted** -(Σ saved / Σ before) — the honest aggregate, not a mean of per-request percentages. It also reports: +## The fast path: turn on the dashboard + +```sh +context-guru-proxy --preset codesmart --dashboard +# point your agent at http://localhost:4000/anthropic (or /openai), then open: +# http://localhost:4000/dashboard/ +``` + +Within a few turns the [Overview](../dashboard.md) shows tokens before/after, gross vs +unique vs net-of-restores savings, baseline vs actual dollars, the cumulative-cost chart +with the saved area shaded, and the honest savings waterfall. + +![The dashboard's Overview](../img/dashboard/01-overview.jpg) + +Everything below is about reading those numbers correctly. + +## Which savings number to quote + +context-guru reports **four** savings ratios because there is no single honest one. Pick +by the question you are actually asking: + +| If you want to know… | Use | Why | +|---|---|---| +| Is compaction working when there is something to compact? | **of what we tried to compact** | Divides by the tokens compaction was *allowed* to touch — the uncached tail on a caching backend. Excludes the frozen prefix we deliberately never touched. | +| What is the economic effect? | **of new provider-billed input** | Divides by fresh input + cache writes + what we removed. Does not recount transcript history the provider served from cache and never re-billed. | +| What is the most conservative claim I can make? | **unique, of the whole request** | Each distinct compaction counted once, over every content token in every request. | +| Why does my long session read ~0%? | **of the whole request (diluted)** | This one. A 200-turn session re-sends its history every turn, so the denominator grows quadratically. It is not a bug; it is what a whole-request ratio *means*. | + +!!! warning "The trap" + On this workload a whole-request ratio is dominated by re-sent history, so it trends + toward zero however well compaction performs. If a tool quotes one savings percentage + and does not say what it divided by, you cannot tell which of these you are looking at. + +## Gross vs unique vs adjusted + +Three savings figures, all true, all different: + +| Figure | Meaning | +|---|---| +| **gross** | Every token removed, re-counted each turn the agent re-sends the same transcript. | +| **unique** | Each distinct compaction counted once (deduped by the content key the offloader stashed). | +| **net of restores** | Unique, minus content the model asked back for via `context_guru_expand`. | + +`overcount_ratio = gross ÷ unique` is how inflated the gross figure is. **7×** means the +gross number counted the same compaction seven times. Quote unique or adjusted; use gross +only when you say it is cumulative. + +## Read the cost, not just the tokens + +The counterintuitive result from [the SWE-bench study](../results/comparison.md): on a +prompt-caching backend the request is ~99.95% cached and a **cache write bills ~11.5× a +cache read**, so removing unique tokens moves 0.02–0.13% of the billed total while cost +tracks agent **steps** at r = 0.95. + +So the dashboard prices every request at write time and shows: + +- **baseline cost** — what the same requests would have cost with nothing removed (the + tokens we removed priced at the cache-write rate they would have entered as); +- **actual cost** — as billed; +- **context-guru's own LLM spend** — what `extract_llm` and friends cost us; +- **net dollars saved** — baseline − actual − our own spend. + +The waterfall walks exactly that, and will show a negative net if we spent more than we +saved. + +!!! note "`token_accounting` gates all of it" + A request is priced only when the provider reported all four token tiers **and** the + model's rates are known. Otherwise the row is marked `partial` or `missing` and its + cost reads *unknown* — never zero. Filter to `complete` before quoting a dollar figure. + +## Check what it cost you + +Savings without their costs is not a measurement. The dashboard's **"What our own safety +mechanisms cost"** panel reports, beside the benefit: + +- **frozen for cache safety** — compaction we did not do on the already-cached prefix; +- **restored after offload** — premature offloads the model asked back for; +- **reverted component runs** — the never-worse guard firing; +- **context-guru's own latency and LLM spend**. + +And the **"why didn't you compact this?"** panel turns non-events into data: +`bypassed · below_trigger · cache_frozen · found_nothing · reverted · no_messages`. + +## Find the components that earn their place + +![Per-component economics](../img/dashboard/03-component-metrics.jpg) + +Per component: runs · acted · act rate · reverted · unique/gross saved · overcount · own +latency · errors · verdict. This is how you find: + +- a component that never fires on your traffic (**inert here**) — drop it from the pipeline; +- one that spent real wall time and returned nothing (**costly and inert**) — the most + expensive kind of dead weight; +- one whose latency dwarfs its yield (**expensive for its yield**); +- `cacheinject`, which always reads **mutates, saves no content** because its win is a + provider-side KV-cache hit, invisible to content-token counts. + +Click a component to filter the request list to the requests it ran on, then open one and +read the diff. + +## See exactly what changed + +![Git-style content diff](../img/dashboard/09-content-git-diff.jpg) + +The request drawer shows every rewritten message as a Git-style diff (plus side-by-side +and after-only views), ordered biggest-saving first. This is the check that a savings +number cannot give you: *did it remove the right thing?* + +Content capture is on by default, redacted and size-capped before storage, and visible +from loopback or a trusted CIDR only. Disable it with `--dashboard-content=false`. + +## A real session end to end + +`scripts/cc-demo.sh` routes a real `claude` CLI session through the proxy: it builds a +tiny repo, starts the proxy, points Claude Code's `ANTHROPIC_BASE_URL` at it, and runs one +`claude -p` task. Add `--dashboard` to the proxy it starts and you get the whole picture +rather than a stats delta: + +```sh +export ANTHROPIC_BASE_URL=... # upstream Anthropic-compatible endpoint +export ANTHROPIC_AUTH_TOKEN=... +scripts/cc-demo.sh +# then open http://localhost:4000/dashboard/#sessions and click the session +``` + +It's the shortest way to see real savings on your own model without a full benchmark +harness. + +## `GET /stats` — the scriptable snapshot + +`/stats` remains the in-process snapshot the benchmark harnesses parse. Its shape is +**stable**: fields are only ever added, guarded by a golden test, because +`deploy/harbor/*.py` reads it by name and a rename would invalidate the published +reproduction path silently. | Field | Meaning | |---|---| -| token-weighted savings | Σ saved / Σ before across all requests | +| `savings_pct` | Token-weighted Σ saved / Σ before — the whole-request (diluted) ratio | +| `savings_pct_attempted` | Σ saved / Σ attempted — the "of what we tried to compact" ratio | +| `savings_pct_new_input` | Σ saved / (fresh + cache-write + saved); **0** when the provider reported no usage, never ~100% | +| `attempted_tokens` / `frozen_tokens` | What compaction was allowed to touch, and what cache safety made us leave | +| `fresh_input_tokens` / `cache_read_tokens` / `cache_write_tokens` / `output_tokens` | The four billed tiers | | `wasted_tokens` | content offloaded then re-served via expand (a premature offload) | | `bounces` | how many offloads were re-served (the count behind `wasted_tokens`) | | `adjusted_saved` | `saved − wasted` — bounce-adjusted, may be negative | | `top_passthrough` | components that ran but never changed a request: dead weight to drop | | `top_discarded` | components whose changes the **writeback layer threw away** — they mutated but never reached the wire. Always worth investigating. | | `saved_tokens_unique` / `overcount_ratio` | distinct compactions, and how many times each was re-counted. Prefer the unique figure: the agent re-sends history verbatim every turn, so the cumulative `saved_tokens` is inflated. | +| `components..saved_tokens_unique` / `.overcount_ratio` | The same split, per component | +| `cg_added_ms_avg` / `upstream_ms_avg` / `upstream_ms_avg_bypassed` | Latency, split by whether the request bypassed us | | `mode` | the operating mode these numbers came from: `sync` \| `observe` | | `sync_enforced` | requests whose forwarded body context-guru actually shaped. **0 in observe mode by construction.** | @@ -28,6 +166,9 @@ The proxy exposes `GET /stats` with in-process savings rollups. Savings are **to component itself deliberately always skips (the rewrite is body-level). But a content-offloader that never fires is a candidate to drop from your pipeline. +`/stats` is in-memory and resets with the process. For history, retention, filtering, +sessions and diffs, use the dashboard. + !!! warning "`top_discarded` is never expected" An entry in `top_discarded` means a component ran, mutated the request, and the writeback layer threw the change away before it reached the wire. Unlike `top_passthrough` this is @@ -47,8 +188,9 @@ The proxy exposes `GET /stats` with in-process savings rollups. Savings are **to ## The Emitter interface -The pipeline depends only on the `Emitter` interface (`Component(Report)` + `Run(RunReport)`), so it -carries no telemetry-backend dependency. Swap implementations to route metrics where you need: +The pipeline depends only on the `Emitter` interface (`Component(Report)` + +`Run(RunReport)`), so it carries no telemetry-backend dependency. Swap implementations to +route metrics where you need: | Emitter | Role | |---|---| @@ -57,25 +199,18 @@ carries no telemetry-backend dependency. Swap implementations to route metrics w | `Tee` | fan-out to several emitters | | `NopEmitter` | discard | -## A real session: `scripts/cc-demo.sh` - -`scripts/cc-demo.sh` routes a real `claude` CLI session through the proxy and reads `/stats` before -and after. It builds a tiny repo, starts the proxy with `--preset balanced`, points Claude Code's -`ANTHROPIC_BASE_URL` at it, runs one `claude -p` task, and prints the stats delta: +The dashboard does not replace any of these — it captures out of band from +[`apply.BodyTrace`](../design.md), so the aggregator stays the fast in-process counter. -```sh -export ANTHROPIC_BASE_URL=... # upstream Anthropic-compatible endpoint -export ANTHROPIC_AUTH_TOKEN=... -scripts/cc-demo.sh -# == stats before == {...} -# (claude reads main.go + README.md through the proxy) -# == stats after == {...} ← token-weighted savings for the session -``` +## Benchmarks -It's the shortest way to see real savings on your own model without a full benchmark harness. +For the full per-component SWE-bench evaluation — where `mask` delivers ~27% +content-token savings with no reward loss, and how the `/stats` within-run metric is +derived — see [Benchmarks](../RESULTS.md). To view a harness run in the dashboard, point +`--dashboard-bench-dirs` at its jobs root: each run's `summary.json` + `rows-.json` +is ingested, with cost-vs-reward per arm and per-task drill-down. -## Benchmarks +![Benchmark comparison](../img/dashboard/11-benchmark-comparison.jpg) -For the full per-component SWE-bench evaluation — where `mask` delivers ~27% content-token savings -with no reward loss, and how the `/stats` within-run metric is derived — see -[Benchmarks](../RESULTS.md). +Quote **cost per solve**, not cost: an arm that spends less by solving fewer tasks has not +saved anything. diff --git a/docs/img/dashboard/01-overview.jpg b/docs/img/dashboard/01-overview.jpg new file mode 100644 index 0000000..82e714f Binary files /dev/null and b/docs/img/dashboard/01-overview.jpg differ diff --git a/docs/img/dashboard/02-savings-cost-graph-tooltip.jpg b/docs/img/dashboard/02-savings-cost-graph-tooltip.jpg new file mode 100644 index 0000000..4a9344c Binary files /dev/null and b/docs/img/dashboard/02-savings-cost-graph-tooltip.jpg differ diff --git a/docs/img/dashboard/03-component-metrics.jpg b/docs/img/dashboard/03-component-metrics.jpg new file mode 100644 index 0000000..bf32619 Binary files /dev/null and b/docs/img/dashboard/03-component-metrics.jpg differ diff --git a/docs/img/dashboard/04-sessions.jpg b/docs/img/dashboard/04-sessions.jpg new file mode 100644 index 0000000..49139d8 Binary files /dev/null and b/docs/img/dashboard/04-sessions.jpg differ diff --git a/docs/img/dashboard/05-requests.jpg b/docs/img/dashboard/05-requests.jpg new file mode 100644 index 0000000..d15fc9f Binary files /dev/null and b/docs/img/dashboard/05-requests.jpg differ diff --git a/docs/img/dashboard/08-request-detail.jpg b/docs/img/dashboard/08-request-detail.jpg new file mode 100644 index 0000000..820cb59 Binary files /dev/null and b/docs/img/dashboard/08-request-detail.jpg differ diff --git a/docs/img/dashboard/09-content-git-diff.jpg b/docs/img/dashboard/09-content-git-diff.jpg new file mode 100644 index 0000000..d660a5b Binary files /dev/null and b/docs/img/dashboard/09-content-git-diff.jpg differ diff --git a/docs/img/dashboard/10-diff-side-by-side.jpg b/docs/img/dashboard/10-diff-side-by-side.jpg new file mode 100644 index 0000000..a4a69f1 Binary files /dev/null and b/docs/img/dashboard/10-diff-side-by-side.jpg differ diff --git a/docs/img/dashboard/11-benchmark-comparison.jpg b/docs/img/dashboard/11-benchmark-comparison.jpg new file mode 100644 index 0000000..85ae082 Binary files /dev/null and b/docs/img/dashboard/11-benchmark-comparison.jpg differ diff --git a/docs/img/dashboard/13-configuration.jpg b/docs/img/dashboard/13-configuration.jpg new file mode 100644 index 0000000..f7af4fb Binary files /dev/null and b/docs/img/dashboard/13-configuration.jpg differ diff --git a/docs/img/dashboard/14-overview-dark.jpg b/docs/img/dashboard/14-overview-dark.jpg new file mode 100644 index 0000000..60c5458 Binary files /dev/null and b/docs/img/dashboard/14-overview-dark.jpg differ diff --git a/docs/img/dashboard/16-overview-small-viewport.jpg b/docs/img/dashboard/16-overview-small-viewport.jpg new file mode 100644 index 0000000..1092041 Binary files /dev/null and b/docs/img/dashboard/16-overview-small-viewport.jpg differ diff --git a/docs/img/dashboard/18-empty-dashboard.jpg b/docs/img/dashboard/18-empty-dashboard.jpg new file mode 100644 index 0000000..1b97fa9 Binary files /dev/null and b/docs/img/dashboard/18-empty-dashboard.jpg differ diff --git a/docs/reference/config.md b/docs/reference/config.md index 6aa1557..59538a9 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -110,11 +110,47 @@ An unparseable or absent value silently keeps the default — pricing must never Independently of pricing, the component declines to run on prompt-caching traffic unless `allow_on_caching_backend: true` is set — measured net-negative there. See [extract_llm](../components/extract_llm.md#the-honest-verdict). +## Dashboard + +The [dashboard](../dashboard.md) is **off by default**. Enabling it adds `/dashboard/` and +`/api/*`; nothing else about the proxy changes. + +| Flag / env | Default | Purpose | +|---|---|---| +| `--dashboard` / `DASHBOARD` | off | Enable the persistent dashboard (embedded UI + JSON/SSE API). | +| `--dashboard-db` / `DASHBOARD_DB` | `./context-guru-dashboard.db` | SQLite path. `:memory:` keeps history in RAM only (the no-persistence mode). An unwritable path falls back to in-memory with a warning rather than failing to start. | +| `--dashboard-retention` / `DASHBOARD_RETENTION` | `168h` (7 days) | Drop rows older than this. `0` disables the age rule. | +| `--dashboard-max-bytes` / `DASHBOARD_MAX_BYTES` | `536870912` (512 MiB) | Cap the database size, dropping the oldest requests first. `0` disables the size rule. | +| `--dashboard-content` / `DASHBOARD_CONTENT` | `false` | Capture before/after message text for the diff view. **Opt-in**: it stores arbitrary agent output on disk, scrubbed of known credential shapes and size-capped **before** storage — but content cannot be allowlisted the way headers and config keys are, so the safe default is off. | +| `--dashboard-content-cap` / `DASHBOARD_CONTENT_CAP` | `16384` | Maximum bytes stored per captured before/after blob. | +| `--dashboard-queue` / `DASHBOARD_QUEUE` | `4096` | Capture-channel depth. A full channel **drops** events (counted, and shown in the UI) rather than delaying a request. | +| `--dashboard-trusted-cidrs` / `DASHBOARD_TRUSTED_CIDRS` | — | Comma-separated CIDRs allowed to view per-request **content** and the effective config. Loopback always is; aggregates are open to everyone. | +| `--dashboard-bench-dirs` / `DASHBOARD_BENCH_DIRS` | — | Comma-separated directories of benchmark runs (each with `summary.json` + `rows-*.json`) to ingest at startup. Re-ingesting replaces a run rather than duplicating it. | + +!!! note "Retention is bounded by age AND size" + Age alone cannot bound a burst of traffic; size alone silently erases a quiet week. + The age rule runs first, then the size rule drops the oldest remaining requests until + the file fits. + +!!! warning "There is deliberately no 'disable observability in production' switch" + For a tool whose value *is* observability, that would be backwards. What is gated is + per-request **content** and the effective **configuration** — not the metrics. + +### Example (container) + +```sh +DASHBOARD=true \ +DASHBOARD_DB=/var/lib/context-guru/dashboard.db \ +DASHBOARD_RETENTION=720h \ +DASHBOARD_MAX_BYTES=2147483648 \ +DASHBOARD_TRUSTED_CIDRS=10.0.0.0/8,192.168.0.0/16 \ +context-guru-proxy --preset codesmart +``` ## Diagnostics | Env | Effect | |---|---| | `CONTEXT_GURU_DEBUG=1` | Logs each tool output's token count + first line. | -| `CONTEXT_GURU_DUMP=` | Appends a before → after JSON record per rewritten message. | -| `CONTEXT_GURU_CAPTURE=` | Appends the pristine inbound request body to a JSONL file — the input for offline replay. | +| `CONTEXT_GURU_DUMP=` | Appends a before → after JSON record per rewritten message. The [dashboard](../dashboard.md) captures the same material into a queryable store with a diff view. | +| `CONTEXT_GURU_CAPTURE=` | Appends each pristine inbound request as one JSONL record, for offline replay through `/compact`. | diff --git a/docs/reference/routes.md b/docs/reference/routes.md index 39786b0..a27eb00 100644 --- a/docs/reference/routes.md +++ b/docs/reference/routes.md @@ -137,6 +137,60 @@ outside observe mode. `cg_added_ms_avg` and the `llm_*` fields deliberately **do** accumulate in observe mode: they are real measurements and real spend. Zeroing them would hide a true number rather than protect anyone. See [Operating modes](../how-to/operating-modes.md). +## Dashboard routes (`--dashboard`) + +Present only when the [dashboard](../dashboard.md) is enabled. Without the flag the route +table above is unchanged and every path below returns 404. + +| Route | Purpose | +|---|---| +| `GET /dashboard/` | The embedded single-page UI (HTML + CSS + JS from `go:embed`; no CDN, no build step). `/dashboard` redirects here. | +| `GET /api/stats` | Overview aggregates: token tiers, costs, the four labelled savings denominators, the honest-savings waterfall, safety-mechanism costs, and the accounting / cache-miss / uncompressed-reason distributions. Accepts every filter parameter below. | +| `GET /api/series?bucket=` | Time series, bucketed **at query time** (no rollup tables). One object per bucket with tokens, the four billed tiers, costs, mean latencies, restorations and cache misses. | +| `GET /api/requests` | Paginated request list. Server-side filters + **keyset** pagination (`before=`, not an offset). Returns `{requests, next_cursor, total}`. | +| `GET /api/requests/{id}` | One request with its per-component rows and, for a permitted caller, the before/after content the diff view renders. | +| `GET /api/sessions` | Session list with per-session rollups (`limit` / `offset`). | +| `GET /api/components` | Per-component economics: runs, acted, reverted, unique/gross savings, `overcount_ratio`, total and mean own-latency, errors. | +| `GET /api/facets` | The distinct values present for each filter dimension, so a UI shows only what the data contains. | +| `GET /api/config` | The **effective** (resolved, key-allowlisted) configuration. Access-gated. | +| `GET /api/benchmarks` | Ingested harness runs with per-arm aggregates. `?refresh=1` re-scans the configured run directories. | +| `GET /api/benchmarks/{id}/tasks` | Per-task rows for a run (`?arm=` to restrict). | +| `GET /api/capture` | The capture pipeline's own health, **including its drop count**. | +| `GET /api/events` | SSE stream of captured requests (summary rows only — never content). Honors `Last-Event-ID` (or `?last_event_id=`) so a reconnect backfills the gap. | + +### Filter parameters + +Accepted by `/api/stats`, `/api/series`, `/api/requests`, `/api/sessions`, +`/api/components` and `/api/facets`. All filtering happens in SQL, server-side. + +| Parameter | Matches | +|---|---| +| `since` / `until` | Epoch-**millisecond** bounds (`since` inclusive, `until` exclusive). | +| `session` · `model` · `provider` · `agent` · `preset` · `mode` | Exact match. | +| `component` | Requests on which that component ran. | +| `reason` | The uncompressed-reason bucket (`bypassed`, `below_trigger`, `cache_frozen`, `found_nothing`, `reverted`, `no_messages`), or `compacted` for requests we did compact. | +| `accounting` | `complete` \| `partial` \| `missing`. | +| `q` | Free-text match against session id, model and agent. | +| `limit` · `before` · `offset` | Page size; keyset cursor (`/api/requests`); offset (`/api/sessions`). | + +### Access gating + +| Surface | Who can see it | +|---|---| +| Aggregates, series, session/component rollups, per-request **metrics** | anyone who can reach the port | +| Per-request **content** (`/api/requests/{id}` content, the diff view) | loopback, or a `--dashboard-trusted-cidrs` entry | +| Effective **configuration** (`/api/config`) | loopback, or a trusted CIDR | + +Aggregates stay open on purpose: a proxy bound to `0.0.0.0` should still report its own +numbers. Content is gated because a transcript can carry a user's source code. An +untrusted caller still gets the metrics row, plus `content_visible: false` so the UI can +say *why* the panel is empty rather than implying nothing changed. + +!!! note "`POST /compact` (compaction-service mode)" + The [llm-d compaction service example](../examples/llm-d-service.md) adds a + stateless `POST /compact` route: it runs the pipeline and returns the + rewritten body directly (`200` + JSON) with no upstream call, no store, and + no markers. See [Quickstart: Compaction service](../get-started/quickstart-compaction.md). ## Per-request headers diff --git a/go.mod b/go.mod index 3d320e4..208f082 100644 --- a/go.mod +++ b/go.mod @@ -39,13 +39,17 @@ require ( github.com/bytedance/sonic/loader v0.5.1 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/mailru/easyjson v0.9.1 // indirect github.com/mark3labs/mcp-go v0.43.2 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-pointer v0.0.1 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spf13/cast v1.10.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -55,5 +59,9 @@ require ( github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect golang.org/x/arch v0.23.0 // indirect - golang.org/x/sys v0.45.0 // indirect + golang.org/x/sys v0.47.0 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.56.0 // indirect ) diff --git a/go.sum b/go.sum index 82fd711..3675f55 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -68,13 +70,19 @@ github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8 github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mark3labs/mcp-go v0.43.2 h1:21PUSlWWiSbUPQwXIJ5WKlETixpFpq+WBpbMGDSVy/I= github.com/mark3labs/mcp-go v0.43.2/go.mod h1:YnJfOL382MIWDx1kMY+2zsRHU/q78dBg9aFb8W6Thdw= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/maximhq/bifrost/core v1.7.0 h1:iUmSfqXwVdDbyJXWIO+S8AYz6JOTWoyhGON4Z4a/p7Q= github.com/maximhq/bifrost/core v1.7.0/go.mod h1:jjdqJc0+fCNl3irgUGfSDzgZupMSRLNm4E/2Q7KZKks= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= @@ -145,6 +153,8 @@ golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= @@ -152,3 +162,11 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= diff --git a/internal/modelinfo/modelinfo.go b/internal/modelinfo/modelinfo.go index d7c6079..b139873 100644 --- a/internal/modelinfo/modelinfo.go +++ b/internal/modelinfo/modelinfo.go @@ -27,6 +27,33 @@ type Resolver interface { Window(ctx context.Context, model string) (tokens int, ok bool) } +// Price is a model's per-token USD rates, in the four tiers a prompt-caching +// provider bills. Zero rates mean "unknown" — a caller must treat a Price it did +// not get an ok=true for as "no pricing", never as free. +type Price struct { + Input float64 `json:"input"` // fresh (uncached) input per token + Output float64 `json:"output"` // completion per token + CacheRead float64 `json:"cache_read"` // cache-hit input per token + CacheWrite float64 `json:"cache_write"` // cache-creation input per token +} + +// Cost prices one request's four token tiers in USD. +func (p Price) Cost(fresh, cacheRead, cacheWrite, output int64) float64 { + return float64(fresh)*p.Input + float64(cacheRead)*p.CacheRead + + float64(cacheWrite)*p.CacheWrite + float64(output)*p.Output +} + +// Zero reports whether no rate at all is known (so a cost figure would be a lie). +func (p Price) Zero() bool { + return p.Input == 0 && p.Output == 0 && p.CacheRead == 0 && p.CacheWrite == 0 +} + +// Pricer resolves a model's per-token rates. ok=false means "unknown"; callers +// must then report cost as unavailable rather than zero. +type Pricer interface { + Price(ctx context.Context, model string) (Price, bool) +} + // LiteLLMPricesURL is the community-maintained map of model -> {max_input_tokens,…}. // Overridable (air-gapped mirrors) via NewLiteLLM. const LiteLLMPricesURL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" @@ -50,9 +77,10 @@ type LiteLLM struct { TTL time.Duration mu sync.Mutex - byKey map[string]int // normalized key -> max_input_tokens - fetched time.Time // last fetch ATTEMPT (success or failure) - fetching bool // a background fetch is in flight (single-flight guard) + byKey map[string]int // normalized key -> max_input_tokens + priceBy map[string]Price // normalized key -> per-token USD rates + fetched time.Time // last fetch ATTEMPT (success or failure) + fetching bool // a background fetch is in flight (single-flight guard) } // negTTL is how long to wait before retrying after a failed/empty fetch when no map @@ -96,76 +124,90 @@ func (l *LiteLLM) refreshIfStale(context.Context) { go func() { // Detached context: the fetch outlives the triggering request; its own Client // timeout bounds it. Record the attempt time regardless of outcome (negative cache). - m, err := l.fetch(context.Background()) + m, pm, err := l.fetch(context.Background()) l.mu.Lock() l.fetching = false l.fetched = time.Now() if err == nil && len(m) > 0 { - l.byKey = m // on failure keep any prior map (fail open) + l.byKey, l.priceBy = m, pm // on failure keep any prior map (fail open) } l.mu.Unlock() }() } -func (l *LiteLLM) fetch(ctx context.Context) (map[string]int, error) { +func (l *LiteLLM) fetch(ctx context.Context) (map[string]int, map[string]Price, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.URL, nil) if err != nil { - return nil, err + return nil, nil, err } resp, err := l.Client.Do(req) if err != nil { - return nil, err + return nil, nil, err } defer resp.Body.Close() b, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) if err != nil { - return nil, err - } - // Decode PER ENTRY, not into one typed map[string]struct{…}. The upstream - // document is community-maintained and not schema-clean: it carries a - // `sample_spec` documentation entry whose numeric fields hold prose - // ("deprecation_date": "date when the model becomes deprecated…"), and a - // handful of models spell an integer field as a float. encoding/json aborts the - // WHOLE map on the first type error, so a strict decode returned (nil, err) - // after successfully parsing ~2,900 good rows — which is why every context - // window resolved to "unknown" in production and no fraction-based trigger has - // ever fired. Skip the bad entries; keep the good ones; say so out loud. + return nil, nil, err + } + // Per-entry decode (see the package note): the upstream document is not + // schema-clean, so one typed whole-map unmarshal yields nothing. This loop also + // collects the per-token PRICES the dashboard needs, from the same pass. var raw map[string]json.RawMessage if err := json.Unmarshal(b, &raw); err != nil { - return nil, err + return nil, nil, err } m := make(map[string]int, len(raw)*2) + pm := make(map[string]Price, len(raw)*2) var skipped []string for k, rv := range raw { if k == sampleSpecKey { continue // the document's own schema documentation, not a model } var v struct { - // float64, not int: a few entries spell an integer window as 128000.0, + // float64, not int: a few entries spell an integer field as 128000.0, // which an int field rejects. - MaxInputTokens float64 `json:"max_input_tokens"` - MaxTokens float64 `json:"max_tokens"` + MaxInputTokens float64 `json:"max_input_tokens"` + MaxTokens float64 `json:"max_tokens"` + InputCost float64 `json:"input_cost_per_token"` + OutputCost float64 `json:"output_cost_per_token"` + CacheReadCost float64 `json:"cache_read_input_token_cost"` + CacheCreateCost float64 `json:"cache_creation_input_token_cost"` } if err := json.Unmarshal(rv, &v); err != nil { skipped = append(skipped, k) continue } - w := int(v.MaxInputTokens) - if w == 0 { - w = int(v.MaxTokens) + full, tail := normalize(k) + if w := int(v.MaxInputTokens); w != 0 || v.MaxTokens != 0 { + if w == 0 { + w = int(v.MaxTokens) + } + m[full] = w + if _, ok := m[tail]; !ok { // don't clobber a more-specific full key + m[tail] = w + } } - if w == 0 { + p := Price{Input: v.InputCost, Output: v.OutputCost, CacheRead: v.CacheReadCost, CacheWrite: v.CacheCreateCost} + if p.Zero() { continue } - full, tail := normalize(k) - m[full] = w - if _, ok := m[tail]; !ok { // don't clobber a more-specific full key - m[tail] = w + // LiteLLM omits cache rates for models that do not cache; fall back to the + // provider-standard Anthropic multiples ONLY when a cache tier is missing but + // input pricing is known, so a cached request is never priced as free. + if p.CacheRead == 0 { + p.CacheRead = p.Input * 0.1 + } + if p.CacheWrite == 0 { + p.CacheWrite = p.Input * 1.25 + } + pm[full] = p + if _, ok := pm[tail]; !ok { + pm[tail] = p } } - // Degrading silently here is what hid this bug for the life of the package: an - // empty map is indistinguishable from "no model has a window" at every call - // site, because every lookup fails open. Both outcomes get a log line. + // Degrading silently here is what hid the whole-map decode bug for the life of + // the package: an empty map is indistinguishable from "no model has a window" at + // every call site, because every lookup fails open. Both outcomes get a log line. if len(m) == 0 { slog.Warn("modelinfo: the model-window document decoded to nothing; every context window will read as unknown and fraction-based triggers will not fire", "url", l.URL, "entries", len(raw), "skipped", len(skipped)) @@ -173,7 +215,31 @@ func (l *LiteLLM) fetch(ctx context.Context) (map[string]int, error) { slog.Info("modelinfo: skipped malformed model entries", "skipped", len(skipped), "kept", len(m), "examples", skipped[:min(3, len(skipped))]) } - return m, nil + return m, pm, nil +} + +// Price returns the model's per-token rates from the cached LiteLLM map, matched +// the same way Window matches (full id, then bare tail, then a contains scan). +func (l *LiteLLM) Price(ctx context.Context, model string) (Price, bool) { + l.refreshIfStale(ctx) + l.mu.Lock() + defer l.mu.Unlock() + if l.priceBy == nil { + return Price{}, false + } + full, tail := normalize(model) + if p, ok := l.priceBy[full]; ok { + return p, true + } + if p, ok := l.priceBy[tail]; ok { + return p, true + } + for k, p := range l.priceBy { + if strings.Contains(k, tail) { + return p, true + } + } + return Price{}, false } // sampleSpecKey is the LiteLLM document's self-documenting entry: its fields are @@ -246,3 +312,17 @@ func (c Chain) Window(ctx context.Context, model string) (int, bool) { } return 0, false } + +// Price tries each element that can price a model; the first ok wins. Elements +// that only resolve windows are skipped, so a Chain{LiteLLM, Static} prices from +// LiteLLM and reports unknown when it has not loaded. +func (c Chain) Price(ctx context.Context, model string) (Price, bool) { + for _, r := range c { + if p, ok := r.(Pricer); ok { + if pr, found := p.Price(ctx, model); found { + return pr, true + } + } + } + return Price{}, false +} diff --git a/internal/modelinfo/modelinfo_test.go b/internal/modelinfo/modelinfo_test.go index 94cc693..5acf51c 100644 --- a/internal/modelinfo/modelinfo_test.go +++ b/internal/modelinfo/modelinfo_test.go @@ -109,3 +109,91 @@ func TestChainAndStaticFallback(t *testing.T) { t.Fatalf("chain should fall back to static: %d,%v", w, ok) } } + +// priceSample carries the four billed tiers plus a model with no published cache +// rates, on top of the malformed-entry shapes litellm_decode_test.go pins. +const priceSample = `{ + "sample_spec": {"max_input_tokens": "max input tokens, if the provider specifies it", + "input_cost_per_token": "the cost"}, + "aws/claude-sonnet-5": {"max_input_tokens": 1000000, "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 2e-07, + "cache_creation_input_token_cost": 2.5e-06}, + "no-cache-model": {"max_input_tokens": 8192, "input_cost_per_token": 4e-06, "output_cost_per_token": 8e-06} +}` + +func TestLiteLLMPrices(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte(priceSample)) + })) + defer srv.Close() + l := NewLiteLLM(srv.URL, srv.Client(), time.Hour) + ctx := context.Background() + for i := 0; i < 400; i++ { + if _, ok := l.Price(ctx, "aws/claude-sonnet-5"); ok { + break + } + time.Sleep(5 * time.Millisecond) + } + + // Prices must resolve, with the four tiers intact. + p, ok := l.Price(ctx, "aws/claude-sonnet-5") + if !ok { + t.Fatal("Price(aws/claude-sonnet-5) not resolved") + } + want := Price{Input: 2e-06, Output: 1e-05, CacheRead: 2e-07, CacheWrite: 2.5e-06} + if p != want { + t.Errorf("price = %+v; want %+v", p, want) + } + // A model with no published cache rates must not be priced as if a cached + // request were FREE: the provider-standard multiples fill in. + np, ok := l.Price(ctx, "no-cache-model") + if !ok { + t.Fatal("Price(no-cache-model) not resolved") + } + if np.CacheRead == 0 || np.CacheWrite == 0 { + t.Errorf("missing cache tiers left at zero (%+v): a cached request would price as free", np) + } + if np.CacheRead >= np.Input || np.CacheWrite <= np.Input { + t.Errorf("filled cache tiers are not read 1e-12 || diff < -1e-12 { + t.Errorf("Cost = %v; want %v", got, want) + } +} + +func TestChainPriceSkipsNonPricers(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte(priceSample)) + })) + defer srv.Close() + l := NewLiteLLM(srv.URL, srv.Client(), time.Hour) + ctx := context.Background() + // Static resolves windows but cannot price; the Chain must fall through to the + // LiteLLM element rather than reporting unknown. + c := Chain{DefaultStatic(), l} + for i := 0; i < 400; i++ { + if _, ok := c.Price(ctx, "aws/claude-sonnet-5"); ok { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("Chain.Price never resolved through a non-Pricer element") +} diff --git a/internal/modelinfo/zz_price_live_test.go b/internal/modelinfo/zz_price_live_test.go new file mode 100644 index 0000000..ceba5aa --- /dev/null +++ b/internal/modelinfo/zz_price_live_test.go @@ -0,0 +1,29 @@ +package modelinfo + +import ( + "context" + "os" + "testing" + "time" +) + +// TestLivePriceLookup is a diagnostic against the real LiteLLM prices map. It is +// skipped unless CG_LIVE_PRICE=1, so CI never depends on the network. +func TestLivePriceLookup(t *testing.T) { + if os.Getenv("CG_LIVE_PRICE") == "" { + t.Skip("set CG_LIVE_PRICE=1 to probe the live LiteLLM prices map") + } + l := NewLiteLLM("", nil, 0) + ctx := context.Background() + for i := 0; i < 60; i++ { + if _, ok := l.Price(ctx, "aws/claude-sonnet-5"); ok { + break + } + time.Sleep(250 * time.Millisecond) + } + for _, m := range []string{"aws/claude-sonnet-5", "claude-sonnet-4-5", "aws/claude-haiku-4-5", "gpt-4o"} { + p, ok := l.Price(ctx, m) + w, wok := l.Window(ctx, m) + t.Logf("%-24s price_ok=%v %+v window=%d %v", m, ok, p, w, wok) + } +} diff --git a/metrics/metrics.go b/metrics/metrics.go index bfdd62c..4aa7eb8 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -127,6 +127,18 @@ type Aggregator struct { filterFam map[string]*filterStat filterName map[string]*filterStat filterMiss map[string]int64 + // Provider-billed usage, summed from response bodies (W8). These are the tiers + // that actually cost money — on a prompt-caching backend a cache write bills + // ~11.5x a read, so content-token savings alone cannot express the economics. + freshInput int64 + cacheRead int64 + cacheWrite int64 + outputTok int64 + // attempted is the tokens compaction was ALLOWED to touch (the uncached tail + // when cache-aware); frozen is what cache safety made us leave alone. Together + // they give /stats an honest denominator and the cost of its own safety. + attempted int64 + frozen int64 } // filterStat is one cmdfilter family's or filter's ledger. SavedUnique dedups by @@ -333,6 +345,28 @@ func (a *Aggregator) RecordExpand(tokens int) { a.mu.Unlock() } +// RecordUsage adds one response's provider-billed token tiers. Called with what +// the provider reported; a response that reports nothing contributes nothing +// (never a zero-filled row that would read as "free"). +func (a *Aggregator) RecordUsage(fresh, cacheRead, cacheWrite, output int64) { + a.mu.Lock() + a.freshInput += fresh + a.cacheRead += cacheRead + a.cacheWrite += cacheWrite + a.outputTok += output + a.mu.Unlock() +} + +// RecordEligibility notes how many tokens this request's offloaders were allowed +// to touch, and how many cache-awareness froze — the numerator's honest +// denominator, and the cost of our own safety mechanism. +func (a *Aggregator) RecordEligibility(attempted, frozen int) { + a.mu.Lock() + a.attempted += int64(attempted) + a.frozen += int64(frozen) + a.mu.Unlock() +} + // RecordAddedLatency notes the wall time (ms) context-guru added to one request // (normalize + pipeline + writeback). Only meaningful on the active path. func (a *Aggregator) RecordAddedLatency(ms float64) { @@ -506,6 +540,26 @@ type Snapshot struct { // exactly the gap noted in headroom's dashboard. Omitted when no pool is running, so // a sync-only deployment shows no phantom queue. ObserveQueue *QueueStats `json:"observe_queue,omitempty"` + + // Provider-billed token tiers (W8), summed from response usage. ADDITIVE: the + // benchmark harnesses parse this payload, so fields are only ever added here, + // never renamed or removed (see the golden shape test). + FreshInputTokens int64 `json:"fresh_input_tokens"` + CacheReadTokens int64 `json:"cache_read_tokens"` + CacheWriteTokens int64 `json:"cache_write_tokens"` + OutputTokens int64 `json:"output_tokens"` + // AttemptedTokens is what compaction was ALLOWED to touch; FrozenTokens is what + // cache-awareness deliberately left alone. SavingsPctAttempted divides savings + // by the former — the honest ratio, since SavingsPct's whole-request denominator + // recounts the transcript every turn and trends to ~0% on a long session. + AttemptedTokens int64 `json:"attempted_tokens"` + FrozenTokens int64 `json:"frozen_tokens"` + // SavingsPctAttempted = saved / attempted. 0 when nothing was attempted. + SavingsPctAttempted float64 `json:"savings_pct_attempted"` + // SavingsPctNewInput = saved / (fresh + cache_write + saved): savings as a + // fraction of what would have newly entered the provider. 0 (not 100) when the + // provider reported no usage — savings must never be divided by themselves. + SavingsPctNewInput float64 `json:"savings_pct_new_input"` } // QueueStats mirrors modes.Stats. Declared here as a plain struct rather than importing @@ -590,6 +644,16 @@ func (a *Aggregator) Snapshot() Snapshot { if mode == "" { mode = components.ModeSync } + attemptedPct := 0.0 + if a.attempted > 0 { + attemptedPct = float64(saved) / float64(a.attempted) * 100 + } + // Guard on the BILLED figure, not the sum: with no usage data the denominator + // would be `saved` alone and the ratio would read ~100%. No data => report 0. + newInputPct := 0.0 + if a.freshInput+a.cacheWrite > 0 { + newInputPct = float64(saved) / float64(a.freshInput+a.cacheWrite+saved) * 100 + } snap := Snapshot{ Requests: a.requests, TokensBefore: a.before, TokensAfter: a.after, SavedTokens: saved, SavingsPct: pct, @@ -603,6 +667,10 @@ func (a *Aggregator) Snapshot() Snapshot { CmdfilterMisses: topMisses(a.filterMiss, 20), Mode: string(mode), SyncEnforced: a.syncRequests, + FreshInputTokens: a.freshInput, CacheReadTokens: a.cacheRead, + CacheWriteTokens: a.cacheWrite, OutputTokens: a.outputTok, + AttemptedTokens: a.attempted, FrozenTokens: a.frozen, + SavingsPctAttempted: attemptedPct, SavingsPctNewInput: newInputPct, } if a.potentialRuns > 0 || mode == components.ModeObserve { snap.ObserveNotice = observeNotice diff --git a/mkdocs.yml b/mkdocs.yml index 87914c4..3128fc3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -112,6 +112,7 @@ nav: - Setup & evaluation: setup.md - Concepts: - Architecture (deep dive): design.md + - Dashboard: dashboard.md - Components: - Overview: components.md - Reformat: diff --git a/proxy/dash_test.go b/proxy/dash_test.go new file mode 100644 index 0000000..5ea5385 --- /dev/null +++ b/proxy/dash_test.go @@ -0,0 +1,562 @@ +package proxy + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + _ "github.com/rossoctl/context-guru/components/all" + "github.com/rossoctl/context-guru/config" + "github.com/rossoctl/context-guru/dash" + "github.com/rossoctl/context-guru/internal/modelinfo" + "github.com/rossoctl/context-guru/metrics" + "github.com/rossoctl/context-guru/store" +) + +// bigToolOutput is a tool result large and repetitive enough that the pipeline's +// offloaders actually fire, so the end-to-end test exercises a real compaction +// rather than a no-op. +func bigToolOutput() string { + var b strings.Builder + b.WriteString("Exit code 1\nTraceback (most recent call last):\n") + for i := 0; i < 400; i++ { + b.WriteString(" File \"/repo/pkg/mod/thing.py\", line 42, in handler\n result = compute(x, y)\n") + } + b.WriteString("ValueError: bad input\n") + return b.String() +} + +// anthropicRequest builds a realistic Anthropic-dialect body with a big tool result. +func anthropicRequest(model string) []byte { + body := map[string]any{ + "model": model, + "max_tokens": 64, + "messages": []any{ + map[string]any{"role": "user", "content": "please fix the failing test"}, + map[string]any{"role": "assistant", "content": []any{ + map[string]any{"type": "tool_use", "id": "tu_1", "name": "bash", "input": map[string]any{"command": "pytest"}}, + }}, + map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "tool_result", "tool_use_id": "tu_1", "content": bigToolOutput()}, + }}, + }, + } + b, _ := json.Marshal(body) + return b +} + +// fakeUpstream returns an Anthropic-shaped response with real usage tiers. +func fakeUpstream(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"id":"msg_1","type":"message","role":"assistant", + "content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn", + "usage":{"input_tokens":12,"output_tokens":34, + "cache_read_input_tokens":9000,"cache_creation_input_tokens":1500}}`)) + })) +} + +// dashHandler wires a real pipeline + recorder in front of a fake upstream. +func dashHandler(t *testing.T, up string, opts dash.Options) (*Handler, *dash.Recorder) { + t.Helper() + cfg, err := config.LoadBytes([]byte("preset: codesafe\n")) + if err != nil { + t.Fatal(err) + } + agg := metrics.NewAggregator() + pipe, err := cfg.Build(agg) + if err != nil { + t.Fatal(err) + } + if opts.DBPath == "" { + opts.DBPath = filepath.Join(t.TempDir(), "d.db") + } + opts.BatchSize, opts.FlushInterval = 1, time.Millisecond + rec, err := dash.NewRecorder(opts) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { rec.Close() }) + h := New(pipe, store.NewMemory(store.Options{}), agg, Options{ + AnthropicUpstream: up, + Preset: cfg.Preset, + Dashboard: rec, + Prices: fixedPricer{}, + }) + return h, rec +} + +// fixedPricer prices at the real aws/claude-sonnet-5 rates so the cost assertions +// are about the arithmetic, not about network access to a prices map. +type fixedPricer struct{} + +func (fixedPricer) Price(context.Context, string) (modelinfo.Price, bool) { + return modelinfo.Price{Input: 2e-06, Output: 1e-05, CacheRead: 2e-07, CacheWrite: 2.5e-06}, true +} + +// waitForRows polls until the writer goroutine has persisted n rows. +func waitForRows(t *testing.T, rec *dash.Recorder, n int64) { + t.Helper() + for i := 0; i < 400; i++ { + if rec.Stats().Written >= n { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("only %d of %d rows persisted (dropped %d, errors %d)", + rec.Stats().Written, n, rec.Stats().Dropped, rec.Stats().Errors) +} + +func TestDashboardCapturesARealRequestEndToEnd(t *testing.T) { + up := fakeUpstream(t) + defer up.Close() + h, rec := dashHandler(t, up.URL, dash.Options{CaptureContent: true, ContentCap: 1 << 16, + ContentMaxPerRequest: 10}) + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/anthropic/v1/messages", + strings.NewReader(string(anthropicRequest("aws/claude-sonnet-5")))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "claude-cli/2.0.14 (external, cli)") + req.Header.Set("x-context-guru-session", "sess-e2e") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("proxy returned %d", resp.StatusCode) + } + waitForRows(t, rec, 1) + + page, err := rec.DB().Requests(dash.Filter{}, 0, 10) + if err != nil { + t.Fatal(err) + } + if page.Total != 1 { + t.Fatalf("captured %d rows; want 1", page.Total) + } + e := page.Requests[0] + + if e.SessionID != "sess-e2e" { + t.Errorf("session = %q; want the header-supplied sess-e2e", e.SessionID) + } + if e.Model != "aws/claude-sonnet-5" { + t.Errorf("model = %q", e.Model) + } + if e.Provider != "anthropic" || e.Agent != "claude-cli" || e.Preset != "codesafe" { + t.Errorf("labels wrong: provider=%q agent=%q preset=%q", e.Provider, e.Agent, e.Preset) + } + if e.Mode != dash.ModeActive { + t.Errorf("mode = %q; want active", e.Mode) + } + if e.Status != 200 { + t.Errorf("upstream status = %d", e.Status) + } + // Usage came off the real response. + if e.FreshInput != 12 || e.CacheRead != 9000 || e.CacheWrite != 1500 || e.OutputTokens != 34 { + t.Errorf("usage tiers not captured: %+v", e) + } + if e.TokenAccounting != dash.AccountingComplete { + t.Errorf("accounting = %q; want complete (usage + a known price)", e.TokenAccounting) + } + if e.CostUSD <= 0 { + t.Error("cost not priced despite complete accounting") + } + // The pipeline actually compacted, so baseline must exceed actual. + if e.TokensBefore <= e.TokensAfter { + t.Fatalf("the pipeline did not compact (%d -> %d); the assertions below would be vacuous", + e.TokensBefore, e.TokensAfter) + } + if e.BaselineCostUSD <= e.CostUSD { + t.Errorf("baseline %v is not above actual %v despite %d tokens removed", + e.BaselineCostUSD, e.CostUSD, e.TokensBefore-e.TokensAfter) + } + if e.CGLatencyMs <= 0 || e.UpstreamMs <= 0 { + t.Errorf("latency not captured: cg=%v upstream=%v", e.CGLatencyMs, e.UpstreamMs) + } + if e.CacheMissReason != dash.CacheHit { + t.Errorf("cache attribution = %q; a response with cache reads is a hit", e.CacheMissReason) + } + if e.UncompressedReason != "" { + t.Errorf("uncompressed reason = %q on a request that compacted", e.UncompressedReason) + } + + // Components and the diff content are both there. + full, err := rec.DB().Request(e.ID, true) + if err != nil { + t.Fatal(err) + } + if len(full.Components) == 0 { + t.Error("no component rows captured") + } + acted := false + for _, c := range full.Components { + if c.Acted { + acted = true + } + } + if !acted { + t.Error("no component recorded as having acted despite a net saving") + } + if len(full.Content) == 0 { + t.Fatal("no before/after content captured; the diff view would be empty") + } + found := false + for _, c := range full.Content { + if strings.Contains(c.Before, "Traceback") && len(c.After) < len(c.Before) { + found = true + } + } + if !found { + t.Errorf("the captured content does not show the compaction: %+v", full.Content) + } + + // /stats must have been updated in lockstep and still parse for the harness. + w := httptest.NewRecorder() + h.stats(w, httptest.NewRequest("GET", "/stats", nil)) + var snap map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &snap); err != nil { + t.Fatal(err) + } + if snap["cache_read_tokens"].(float64) != 9000 { + t.Errorf("/stats cache_read_tokens = %v; want 9000", snap["cache_read_tokens"]) + } + if snap["attempted_tokens"].(float64) <= 0 { + t.Error("/stats attempted_tokens not recorded") + } +} + +func TestDashboardCapturesABypassedRequest(t *testing.T) { + up := fakeUpstream(t) + defer up.Close() + h, rec := dashHandler(t, up.URL, dash.Options{}) + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/anthropic/v1/messages", + strings.NewReader(string(anthropicRequest("aws/claude-sonnet-5")))) + req.Header.Set("x-context-guru-bypass", "true") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + waitForRows(t, rec, 1) + + page, _ := rec.DB().Requests(dash.Filter{}, 0, 10) + e := page.Requests[0] + if !e.Bypassed || e.Mode != dash.ModeBypass { + t.Errorf("bypass not recorded: bypassed=%v mode=%q", e.Bypassed, e.Mode) + } + if e.UncompressedReason != dash.ReasonBypassed { + t.Errorf("reason = %q; want %q", e.UncompressedReason, dash.ReasonBypassed) + } + if e.TokensBefore != e.TokensAfter { + t.Errorf("a bypassed request must not report a saving: %d -> %d", e.TokensBefore, e.TokensAfter) + } +} + +func TestDashboardCapturesCompactRoute(t *testing.T) { + h, rec := dashHandler(t, "", dash.Options{CaptureContent: true, ContentCap: 1 << 16, + ContentMaxPerRequest: 10}) + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + + resp, err := srv.Client().Post(srv.URL+"/compact?provider=anthropic", "application/json", + strings.NewReader(string(anthropicRequest("aws/claude-sonnet-5")))) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + waitForRows(t, rec, 1) + + page, _ := rec.DB().Requests(dash.Filter{}, 0, 10) + e := page.Requests[0] + if e.Route != "/compact" { + t.Errorf("route = %q; want /compact", e.Route) + } + // /compact never calls a provider, so the row must be honestly marked as + // partially accounted rather than priced as free. + if e.TokenAccounting == dash.AccountingComplete { + t.Error("/compact has no provider usage; the row must not claim complete accounting") + } + if e.CostUSD != 0 { + t.Errorf("/compact priced a request with no billed usage: %v", e.CostUSD) + } +} + +func TestDisabledDashboardChangesNothing(t *testing.T) { + up := fakeUpstream(t) + defer up.Close() + cfg, _ := config.LoadBytes([]byte("preset: codesafe\n")) + agg := metrics.NewAggregator() + pipe, _ := cfg.Build(agg) + h := New(pipe, store.NewMemory(store.Options{}), agg, Options{AnthropicUpstream: up.URL}) + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + + // The request still works. + resp, err := srv.Client().Post(srv.URL+"/anthropic/v1/messages", "application/json", + strings.NewReader(string(anthropicRequest("m")))) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("proxy returned %d with the dashboard off", resp.StatusCode) + } + // And the dashboard routes are absent, so an unconfigured proxy's surface is + // byte-identical to before this feature existed. + for _, path := range []string{"/dashboard/", "/api/stats", "/api/events"} { + r, err := srv.Client().Get(srv.URL + path) + if err != nil { + t.Fatal(err) + } + r.Body.Close() + if r.StatusCode != http.StatusNotFound { + t.Errorf("%s -> %d with the dashboard disabled; want 404", path, r.StatusCode) + } + } + // /stats still works and keeps its shape. + r, err := srv.Client().Get(srv.URL + "/stats") + if err != nil { + t.Fatal(err) + } + defer r.Body.Close() + var snap map[string]any + if err := json.NewDecoder(r.Body).Decode(&snap); err != nil { + t.Fatal(err) + } + if snap["requests"] == nil { + t.Error("/stats broken with the dashboard off") + } +} + +// manyToolResultsRequest builds a transcript shaped like a real agent's mid-session +// turn: many large tool results, not one. The shape is the point — content capture +// redacts up to ContentMaxPerRequest x (before+after) blobs, so a body with a single +// tool result exercises 1/24th of the work and would let a regression through. The +// blobs are log-shaped (KEY=value lines, paths, a URL) because that is the text the +// credential regexes are slowest over. +func manyToolResultsRequest(model string, results int) []byte { + msgs := []any{map[string]any{"role": "user", "content": "please fix the failing test"}} + var logish strings.Builder + for i := 0; i < 120; i++ { + logish.WriteString("2026-08-09T12:00:00Z INFO worker=7 path=/repo/pkg/mod/thing.py status=ok\n") + logish.WriteString(" DATABASE_URL=postgres://localhost:5432/app RETRIES=3 TIMEOUT_MS=2500\n") + logish.WriteString(" File \"/repo/pkg/mod/thing.py\", line 42, in handler\n result = compute(x, y)\n") + } + blob := logish.String() + for i := 0; i < results; i++ { + id := "tu_" + strconv.Itoa(i) + msgs = append(msgs, + map[string]any{"role": "assistant", "content": []any{ + map[string]any{"type": "tool_use", "id": id, "name": "bash", + "input": map[string]any{"command": "pytest -k case" + strconv.Itoa(i)}}, + }}, + map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "tool_result", "tool_use_id": id, "content": blob}, + }}) + } + b, _ := json.Marshal(map[string]any{ + "model": model, "max_tokens": 64, "messages": msgs, + }) + return b +} + +// TestDashboardAddsNoRequestLatencyWithContentCapture is the regression test the +// original overhead claim was missing. +// +// dash.BenchmarkRecord measures a channel send (~175 ns) and is honest about that, +// but it is not the dashboard's per-request cost: `finish` is called from serve's +// `defer`, which runs BEFORE the handler returns, so anything expensive there is paid +// by the next request on a keep-alive connection — every real agent. Content +// redaction (nine regexes over up to ContentMaxPerRequest x 2 blobs) sat there and +// cost ~53 ms/request, ~25% of a real request, while the documented figure was +// ~0.000002%. A benchmark that calls Record directly can never catch that. +// +// So this drives the REAL handler over ONE keep-alive connection with content capture +// ON — the path a client actually pays for — and compares against the same handler +// with the dashboard off. It fails if the dashboard adds a perceptible cost. +func TestDashboardAddsNoRequestLatencyWithContentCapture(t *testing.T) { + up := fakeUpstream(t) + defer up.Close() + + cfg, err := config.LoadBytes([]byte("preset: codesafe\n")) + if err != nil { + t.Fatal(err) + } + offAgg := metrics.NewAggregator() + offPipe, err := cfg.Build(offAgg) + if err != nil { + t.Fatal(err) + } + offH := New(offPipe, store.NewMemory(store.Options{}), offAgg, + Options{AnthropicUpstream: up.URL, Prices: fixedPricer{}}) + // Content capture ON: the configuration with the most work to do per request is + // the one worth guarding. + onH, _ := dashHandler(t, up.URL, dash.Options{ + CaptureContent: true, ContentCap: 16 << 10, ContentMaxPerRequest: 24, + }) + + // 24 tool results = ContentMaxPerRequest, so the capture path does the full amount + // of work it is ever allowed to do on one request. A body with a single tool result + // exercises 1/24th of it and lets a regression through. + body := string(manyToolResultsRequest("m", 24)) + + // One connection per handler, reused, so a cost paid in the handler's defer shows + // up as latency on the NEXT request rather than being hidden by a fresh dial. + newClient := func(h *Handler) (*http.Client, string, func()) { + srv := httptest.NewServer(h.Mux()) + c := &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 1}} + return c, srv.URL + "/anthropic/v1/messages", func() { + c.CloseIdleConnections() + srv.Close() + } + } + one := func(c *http.Client, url string) time.Duration { + start := time.Now() + resp, err := c.Post(url, "application/json", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, resp.Body) // drain, or the connection is not reusable + resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("upstream returned %d", resp.StatusCode) + } + return time.Since(start) + } + + offClient, offURL, closeOff := newClient(offH) + defer closeOff() + onClient, onURL, closeOn := newClient(onH) + defer closeOn() + + // PAIRED and INTERLEAVED, then compared on the MEDIAN. Measuring all the off + // requests and then all the on requests attributes any drift in machine load to the + // dashboard: consecutive runs of that shape disagreed by 4x (+3 ms to +12 ms) on an + // idle box, which is not a usable gate. Alternating and taking medians cancels drift + // and discards the outliers a shared CI box produces. + const warmup, iters = 5, 40 + for i := 0; i < warmup; i++ { + one(offClient, offURL) + one(onClient, onURL) + } + offs := make([]time.Duration, 0, iters) + ons := make([]time.Duration, 0, iters) + for i := 0; i < iters; i++ { + if i%2 == 0 { // alternate which one goes first, so ordering cannot bias either + offs = append(offs, one(offClient, offURL)) + ons = append(ons, one(onClient, onURL)) + continue + } + ons = append(ons, one(onClient, onURL)) + offs = append(offs, one(offClient, offURL)) + } + median := func(ds []time.Duration) time.Duration { + slices.Sort(ds) + return ds[len(ds)/2] + } + off, on := median(offs), median(ons) + + added := on - off + t.Logf("median per-request latency over %d paired requests: dashboard off %v, "+ + "on with content ON %v (added %v)", iters, off, on, added) + + // The budget is loose in absolute terms (a fake-upstream request still moves by a + // millisecond or two under load) but an order of magnitude below the ~53 ms the + // regression cost. Anything that puts redaction, gzip or an insert back on the + // request goroutine blows straight through it. + if added > 5*time.Millisecond { + t.Errorf("the dashboard added %v per request with content capture on; "+ + "something expensive is back on the request goroutine (budget 5ms)", added) + } +} + +// TestDashboardUnderConcurrentTraffic drives the whole path (pipeline, capture, +// writer, SSE, reads) at once. Run under -race this is the mandatory check. +func TestDashboardUnderConcurrentTraffic(t *testing.T) { + up := fakeUpstream(t) + defer up.Close() + h, rec := dashHandler(t, up.URL, dash.Options{CaptureContent: true, ContentCap: 4096, + ContentMaxPerRequest: 4, QueueSize: 64}) + srv := httptest.NewServer(h.Mux()) + defer srv.Close() + + // An SSE subscriber that reads, and one that goes away mid-stream. + sseDone := make(chan struct{}) + go func() { + defer close(sseDone) + r, err := srv.Client().Get(srv.URL + "/api/events") + if err != nil { + return + } + defer r.Body.Close() + io.CopyN(io.Discard, r.Body, 512) //nolint:errcheck // a short read then leave, on purpose + }() + + var wg sync.WaitGroup + for g := 0; g < 6; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < 8; i++ { + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/anthropic/v1/messages", + strings.NewReader(string(anthropicRequest("aws/claude-sonnet-5")))) + req.Header.Set("x-context-guru-session", "sess-"+string(rune('a'+g))) + resp, err := srv.Client().Do(req) + if err != nil { + t.Errorf("request failed: %v", err) + return + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + } + }(g) + } + // Concurrent dashboard readers. + for g := 0; g < 3; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 20; i++ { + for _, path := range []string{"/api/stats", "/api/requests?limit=5", "/api/sessions", + "/api/components", "/api/capture"} { + r, err := srv.Client().Get(srv.URL + path) + if err != nil { + t.Errorf("%s: %v", path, err) + return + } + io.Copy(io.Discard, r.Body) + r.Body.Close() + } + } + }() + } + wg.Wait() + <-sseDone + + // Whatever the interleaving, nothing may be silently lost: written + dropped + // must account for everything captured. + s := rec.Stats() + if s.Captured != 48 { + t.Errorf("captured %d of 48 requests", s.Captured) + } + if s.Errors > 0 { + t.Errorf("%d insert errors", s.Errors) + } +} diff --git a/proxy/dashcapture.go b/proxy/dashcapture.go new file mode 100644 index 0000000..c3690c9 --- /dev/null +++ b/proxy/dashcapture.go @@ -0,0 +1,171 @@ +package proxy + +import ( + "context" + "net/http" + "time" + + "github.com/rossoctl/context-guru/apply" + "github.com/rossoctl/context-guru/dash" + "github.com/rossoctl/context-guru/internal/cheapmodel" + "github.com/rossoctl/context-guru/internal/modelinfo" +) + +// The dashboard's capture point. It is a plain struct built from values the +// request path has already computed, handed to a channel with a `default:` branch. +// No I/O, no lock held across a call, no token re-counting: the entire cost on the +// request goroutine is a few field copies and one non-blocking send, which is why +// enabling the dashboard does not show up in request latency (see docs/dashboard.md +// for the measurement). +// +// Everything expensive — redaction of captured content, gzip, the insert, the SSE +// fan-out — happens on the writer goroutine, after the response has been sent. + +// capture is the per-request scratchpad the chat handler fills as it goes. +type capture struct { + rec *dash.Recorder + pricer modelinfo.Pricer + preset string + route string + provider string + model string + agent string + start time.Time + cgMs float64 + upstream float64 + status int + expands int + expandTok int + trace apply.Trace + // llmCallsAtStart / llmCostAtStart snapshot context-guru's own cheap-model usage + // before the pipeline runs, so this request is charged only its own share of it. + llmInAtStart int64 + llmOutAtStart int64 + unique map[string]int +} + +// newCapture starts a capture for one request, or returns nil when the dashboard +// is off — every call site is nil-safe, so the disabled path costs one nil check. +func (h *Handler) newCapture(r *http.Request, provider, route string) *capture { + if h.rec == nil { + return nil + } + _, in, out := cheapmodel.Usage() + return &capture{ + rec: h.rec, pricer: h.opts.Prices, preset: h.opts.Preset, + route: route, provider: provider, + agent: r.UserAgent(), start: time.Now(), + llmInAtStart: in, llmOutAtStart: out, + unique: map[string]int{}, + } +} + +// noteTrace records the pipeline's outcome and computes each component's +// unique-savings share. The dedup map lives in the recorder (process-wide), which +// is what makes "unique" mean the same thing here and in /stats. +func (c *capture) noteTrace(tr apply.Trace) { + if c == nil { + return + } + c.trace = tr + if tr.Run == nil { + return + } + for _, rep := range tr.Run.Components { + if saved := rep.Saved(); saved > 0 && !rep.Reverted && !rep.Skipped { + c.unique[rep.Component] = c.rec.MarkUnique(rep.Component, rep.CacheKeys, saved) + } + } +} + +func (c *capture) noteCG(ms float64) { + if c != nil { + c.cgMs = ms + } +} +func (c *capture) noteUpstream(ms float64, status int) { + if c != nil { + c.upstream, c.status = ms, status + } +} +func (c *capture) noteModel(model string) { + if c != nil { + c.model = model + } +} +func (c *capture) noteExpand(tokens int) { + if c != nil { + c.expands++ + c.expandTok += tokens + } +} + +// finish builds the event and hands it off. Called once, after the response has +// been written, so nothing here is on the client's critical path. usage may be +// zero-valued with ok=false, in which case the row is marked partial and left +// unpriced rather than priced as free. +func (c *capture) finish(usage Usage, usageOK bool, captureContent bool, contentCap, contentMax int) { + if c == nil { + return + } + e := &dash.Event{ + TS: c.start.UnixMilli(), + Model: c.model, + Provider: c.provider, + Route: c.route, + Preset: c.preset, + Status: c.status, + } + e.Agent = dash.AgentFor(c.agent) + e.FromTrace(c.trace, c.unique) + e.CGLatencyMs, e.UpstreamMs = c.cgMs, c.upstream + e.Expands, e.ExpandTokens = c.expands, c.expandTok + e.FreshInput, e.CacheRead = usage.FreshInput, usage.CacheRead + e.CacheWrite, e.OutputTokens = usage.CacheWrite, usage.Output + + // context-guru's own model spend attributable to THIS request: the delta of the + // process-wide cheap-model counters across the request. Priced with the same + // model rates; a cheap model configured to a different id is close enough here + // that over-reporting our own cost is the safe direction. + _, inNow, outNow := cheapmodel.Usage() + cgIn, cgOut := inNow-c.llmInAtStart, outNow-c.llmOutAtStart + + var price modelinfo.Price + priced := false + if c.pricer != nil && c.model != "" { + price, priced = c.pricer.Price(context.Background(), c.model) + } + e.Price(price, usageOK && priced) + if priced && (cgIn > 0 || cgOut > 0) { + e.CGLLMCostUSD = price.Cost(cgIn, 0, 0, cgOut) + } + + // Cache attribution, with a cold start treated as the non-failure it is. + seenSession, seenModel, sinceMs := c.rec.Observe(e.SessionID, e.Model, e.TS) + // Anthropic's prompt cache has a 5-minute TTL; a gap wider than that explains a + // miss without blaming a prefix change (TTL wins ties). + e.AttributeCache(seenSession, seenModel, sinceMs, 5*60*1000, e.CacheWrite > 0) + + if !captureContent { + e.Content = nil + } else { + // Truncate here (a slice reslice, free), but do NOT redact here. + // + // finish is called from serve's defer, which runs before the handler RETURNS — + // not merely after the response body is written. So anything expensive on this + // line is paid by the next request on a keep-alive connection, i.e. by every real + // agent. Redaction is nine regexes over up to contentMax x 2 blobs, measured at + // ~53 ms/request (~25% of a request), against the ~175 ns the channel send costs. + // It therefore belongs on the writer goroutine, which already owns the event and + // is already off the hot path. + // + // Secrets still never reach disk: the writer redacts BEFORE the INSERT (see + // Event.Redact, called from Recorder.run). What changed is which goroutine pays, + // not whether redaction happens. + if len(e.Content) > contentMax { + e.Content = e.Content[:contentMax] + } + e.ContentCap = contentCap + } + c.rec.Record(e) +} diff --git a/proxy/modes.go b/proxy/modes.go index 1fd63db..41b6ee7 100644 --- a/proxy/modes.go +++ b/proxy/modes.go @@ -22,9 +22,14 @@ import ( // is nothing for a failure to damage, because the forwarded body is the input. // applyMode rewrites body for forwarding according to the handler's mode, and returns the -// body to forward plus the wall time to charge to the request path. Never returns a nil -// body. -func (h *Handler) applyMode(r *reqInfo) ([]byte, time.Duration) { +// body to forward, the wall time to charge to the request path, and the observational +// trace the dashboard's capture reads. Never returns a nil body. +// +// In observe mode the returned trace is the ZERO value, and deliberately so: the enforced +// path ran nothing, so there is nothing about this request to observe. Reporting the +// off-path projection here would credit a hypothetical saving to a request that was +// forwarded untouched — the exact confusion the potential_* namespace exists to prevent. +func (h *Handler) applyMode(r *reqInfo) ([]byte, time.Duration, apply.Trace) { mode := h.mode() start := time.Now() @@ -33,7 +38,7 @@ func (h *Handler) applyMode(r *reqInfo) ([]byte, time.Duration) { // happens off-path, on a copy, and the request pays only the enqueue. if mode == components.ModeObserve && !r.bypassed { h.observe(r) - return r.body, time.Since(start) + return r.body, time.Since(start), apply.Trace{} } res := apply.BodyOpts(r.ctx, h.pipe, h.store, apply.Opts{ @@ -43,9 +48,9 @@ func (h *Handler) applyMode(r *reqInfo) ([]byte, time.Duration) { }) added := time.Since(start) if res.Body == nil { - return r.body, added + return r.body, added, res.Trace } - return res.Body, added + return res.Body, added, res.Trace } // reqInfo is the per-request input both the inline pass and an off-path observation need. diff --git a/proxy/proxy.go b/proxy/proxy.go index 1f48a72..525e02a 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -26,8 +26,10 @@ import ( "github.com/rossoctl/context-guru/apply" "github.com/rossoctl/context-guru/components" "github.com/rossoctl/context-guru/components/offload" + "github.com/rossoctl/context-guru/dash" "github.com/rossoctl/context-guru/expand" "github.com/rossoctl/context-guru/internal/cheapmodel" + "github.com/rossoctl/context-guru/internal/modelinfo" "github.com/rossoctl/context-guru/metrics" "github.com/rossoctl/context-guru/modes" "github.com/rossoctl/context-guru/schema" @@ -76,6 +78,19 @@ type Options struct { // auto/on keep offloaders from mutating already-cached content on prompt-caching // backends; off restores legacy compact-everything (for confirmed non-caching backends). CacheMode string + // Prices resolves a model's per-token rates so the dashboard can price each + // request AT WRITE TIME (so history does not reprice when a rate changes). nil = + // no pricing, and every captured row is marked partially accounted rather than + // being priced as free. Built in main from internal/modelinfo. + Prices modelinfo.Pricer + // Preset labels captured rows with the configuration in effect, so the dashboard + // can filter and compare by preset. + Preset string + // Dashboard, when non-nil, enables the persistent observability layer: each + // request is captured off the hot path into a durable store and the dashboard UI + // + API are mounted. nil = the proxy behaves exactly as before. + Dashboard *dash.Recorder + // PipelineFor builds a pipeline for a per-request override on /compact // (?preset=… or x-context-guru-pipeline: a,b,c). nil = overrides ignored, the // handler always uses the configured pipeline. Supplied by main (which holds @@ -133,6 +148,10 @@ type Handler struct { // So observe gets a store of its own: as persistent as the live one, and completely // disjoint from it. shadow store.Store + // rec is the dashboard's capture pipeline (nil when the dashboard is off). Every + // use is nil-guarded, so the disabled path costs one nil check per request. + rec *dash.Recorder + api *dash.API } // New builds the proxy handler. agg may be nil (no /stats rollups). @@ -141,7 +160,8 @@ func New(pipe *components.Pipeline, st store.Store, agg *metrics.Aggregator, opt if c == nil { c = &http.Client{Timeout: 5 * time.Minute} } - h := &Handler{pipe: pipe, store: st, agg: agg, opts: opts, client: c, tracker: modes.NewTracker(0)} + h := &Handler{pipe: pipe, store: st, agg: agg, opts: opts, client: c, + tracker: modes.NewTracker(0), rec: opts.Dashboard} if h.mode() == components.ModeObserve { h.pool = modes.NewPool(opts.Observe.MaxQueue, opts.Observe.Workers) h.shadow = store.NewMemory(store.Options{}) @@ -149,6 +169,21 @@ func New(pipe *components.Pipeline, st store.Store, agg *metrics.Aggregator, opt if agg != nil { agg.SetMode(h.mode()) } + if h.rec != nil { + h.api = dash.NewAPI(h.rec) + // Publish the off-path pool's counters to the dashboard, the same layering /stats + // uses: the pool lives in `modes`, which sits above both `metrics` and `dash`, so + // the host is the only place that can join them. Read at serve time, not captured + // now, and left unset in sync mode so the UI shows no phantom queue. + if h.pool != nil { + pool := h.pool + h.rec.SetObserveQueue(func() dash.QueueStats { + q := pool.Stats() + return dash.QueueStats{Queued: q.Queued, Pending: q.Pending, + Processed: q.Processed, Dropped: q.Dropped, Errors: q.Errors} + }) + } + } return h } @@ -174,6 +209,11 @@ func (h *Handler) Mux() *http.ServeMux { m.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("ok")) }) m.HandleFunc("GET /stats", h.stats) m.HandleFunc("GET /expand", h.expand) + // The dashboard mounts /dashboard/ (embedded UI) and /api/* (JSON + SSE) only + // when enabled, so an unconfigured proxy's route table is byte-identical to before. + if h.api != nil { + h.api.Mount(m) + } // Bob (BobShell) gateway. Bob is OpenAI-compatible but calls Bob-specific // paths: its model call is POST /inference/v1/chat/completions (reduced like // any OpenAI chat), and its control-plane calls (/admin/v1/profile, @@ -288,6 +328,9 @@ func (h *Handler) compact(w http.ResponseWriter, r *http.Request) { // eval measures a different component than the one that ships. That is the same class of // divergence as the window this handler used to hard-code as unknown, a few lines above — // and it went unnoticed for the same reason, because both are silent. + cp := h.newCapture(r, string(provider), "/compact") + cp.noteModel(gjson.GetBytes(body, "model").String()) + start := time.Now() res := apply.BodyOpts(r.Context(), pipe, h.store, apply.Opts{ Provider: provider, Body: body, @@ -298,9 +341,13 @@ func (h *Handler) compact(w http.ResponseWriter, r *http.Request) { CacheMode: cacheMode, Tracker: h.tracker, }) - out := res.Body + cp.noteCG(float64(time.Since(start).Microseconds()) / 1000.0) + cp.noteTrace(res.Trace) w.Header().Set("Content-Type", "application/json") - w.Write(out) + w.Write(res.Body) + // After the response: /compact never calls a provider, so there is no usage to + // report and the row is honestly marked partially accounted. + cp.finish(Usage{}, false, h.captureContent(), h.contentCap(), h.contentMax()) } // splitComma splits a comma-separated header value into trimmed, non-empty names. @@ -426,6 +473,10 @@ func (h *Handler) chat(provider bschemas.ModelProvider, up upstream) http.Handle } } bypassed := strings.EqualFold(r.Header.Get("x-context-guru-bypass"), "true") + // Start the dashboard capture (nil when the dashboard is off). It only holds + // values the request path already computed; nothing here does I/O. + cp := h.newCapture(r, string(provider), up.path) + cp.noteModel(gjson.GetBytes(body, "model").String()) // Fail open around the whole pre-forward rewrite (pipeline + expand injection): a // panic anywhere here must forward the PRISTINE inbound body, never 500 the client. // apply.BodyFull has its own recover; this backstops expand.Inject and anything else. @@ -438,7 +489,8 @@ func (h *Handler) chat(provider bschemas.ModelProvider, up upstream) http.Handle } }() var added time.Duration - body, added = h.applyMode(&reqInfo{ + var tr apply.Trace + body, added, tr = h.applyMode(&reqInfo{ ctx: r.Context(), provider: provider, body: body, @@ -447,8 +499,12 @@ func (h *Handler) chat(provider bschemas.ModelProvider, up upstream) http.Handle models: models, window: window, }) + addedMs := float64(added.Microseconds()) / 1000.0 + cp.noteCG(addedMs) + cp.noteTrace(tr) if h.agg != nil && !bypassed { - h.agg.RecordAddedLatency(float64(added.Microseconds()) / 1000.0) + h.agg.RecordAddedLatency(addedMs) + h.agg.RecordEligibility(tr.AttemptedTokens, tr.FrozenTokens) } // Advertise the expand tool so the model can recover any offloaded content // (closes the reversibility loop h.serve drives). Sticky/idempotent + appended @@ -464,7 +520,7 @@ func (h *Handler) chat(provider bschemas.ModelProvider, up upstream) http.Handle body, _ = expand.Inject(string(provider), im, body, h.store.Persists()) } }() - h.serve(w, r, provider, up, body, bypassed) + h.serve(w, r, provider, up, body, bypassed, cp) } } @@ -500,7 +556,7 @@ var errNoUpstream = errors.New("no upstream configured") // → /stats sse_streamed / sse_buffered). It previously matched the expand tool // description this proxy injects itself, so it was unconditionally true and the // zero-added-latency promise above never held for any request (issue #26). -func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschemas.ModelProvider, up upstream, body []byte, bypassed bool) { +func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschemas.ModelProvider, up upstream, body []byte, bypassed bool, cp *capture) { injectOn := h.opts.InjectExpand != expand.InjectNever // For SSE we must buffer to inspect (a latency cost), so only do it when the request // actually carries expandable markers (offload happened → the model might expand). @@ -518,10 +574,19 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschema reqStart := time.Now() sse, sseBuffered := false, false var sseFirstByte time.Time // zero on buffered paths: the client's first byte is the write itself + // The response's billed token tiers, harvested out of band (see sniffer) and + // handed to the dashboard AFTER the client's response is complete, so capture + // can never delay or fail a request. + var usage Usage + var usageOK bool defer func() { if sse && h.agg != nil { h.agg.RecordSSE(msSince(reqStart, sseFirstByte), sseBuffered) } + if h.agg != nil && usageOK { + h.agg.RecordUsage(usage.FreshInput, usage.CacheRead, usage.CacheWrite, usage.Output) + } + cp.finish(usage, usageOK, h.captureContent(), h.contentCap(), h.contentMax()) }() for round := 0; ; round++ { upStart := time.Now() @@ -530,8 +595,10 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschema http.Error(w, "upstream: "+err.Error(), http.StatusBadGateway) return } + upMs := float64(time.Since(upStart).Microseconds()) / 1000.0 + cp.noteUpstream(upMs, resp.StatusCode) if h.agg != nil { - h.agg.RecordUpstreamLatency(float64(time.Since(upStart).Microseconds())/1000.0, bypassed) + h.agg.RecordUpstreamLatency(upMs, bypassed) } isSSE := strings.Contains(resp.Header.Get("Content-Type"), "event-stream") // Inspect for a lone expand call when injection is on and we haven't hit the round @@ -542,9 +609,15 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschema // sseBuffered is sticky: if an earlier round was buffered the client already // lost its stream, so this request counts as buffered however it ends. sse = sse || isSSE - if first := h.stream(w, resp); !sseBuffered { + // Stream straight through, sniffing usage from a bounded head+tail window as + // the bytes go by (no buffering of the whole response, no added latency). + first, u, ok := h.stream(w, resp) + if !sseBuffered { sseFirstByte = first } + if ok { + usage, usageOK = u, true + } return } respBody, _ := io.ReadAll(resp.Body) @@ -554,6 +627,9 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, provider bschema // first byte lands no earlier than the write on whichever path we return from. sse, sseBuffered, sseFirstByte = true, true, time.Time{} } + if u, ok := responseUsage(resp.Header.Get("Content-Type"), respBody); ok { + usage, usageOK = u, true + } // Reconstruct the message the loop reasons over. For SSE, aggregate the events; // if that fails, replay the raw stream unchanged (fail-open). @@ -629,13 +705,24 @@ func (h *Handler) doUpstream(r *http.Request, up upstream, body []byte) (*http.R return h.client.Do(req) } -// stream copies an upstream response through with flushing (SSE-friendly) and -// returns the instant the client got its first byte (zero if the body was empty). -func (h *Handler) stream(w http.ResponseWriter, resp *http.Response) (firstByte time.Time) { +// stream copies an upstream response through with flushing (SSE-friendly), while +// keeping a BOUNDED head+tail window of the bytes so the response's billed token +// tiers can be read afterwards. The window is why observability costs nothing +// here: no whole-response buffering, no extra pass, and the client sees each chunk +// as soon as it arrives. +// +// head+tail rather than tail alone because Anthropic reports the input tiers in +// the FIRST SSE event (message_start) and the output count in the last, while +// OpenAI reports everything in a final chunk. +// +// It also returns the instant the client got its first byte (zero if the body was +// empty), which is the SSE TTFB accounting in serve. +func (h *Handler) stream(w http.ResponseWriter, resp *http.Response) (firstByte time.Time, u Usage, ok bool) { defer resp.Body.Close() copyHeaders(w.Header(), resp.Header) w.WriteHeader(resp.StatusCode) flush, _ := w.(http.Flusher) + sn := newSniffer(h.rec != nil || h.agg != nil) buf := make([]byte, 16*1024) for { n, rerr := resp.Body.Read(buf) @@ -644,6 +731,7 @@ func (h *Handler) stream(w http.ResponseWriter, resp *http.Response) (firstByte firstByte = time.Now() } w.Write(buf[:n]) + sn.write(buf[:n]) if flush != nil { flush.Flush() } @@ -652,7 +740,8 @@ func (h *Handler) stream(w http.ResponseWriter, resp *http.Response) (firstByte break } } - return firstByte + u, ok = responseUsage(resp.Header.Get("Content-Type"), sn.bytes()) + return firstByte, u, ok } // msSince returns milliseconds from start to at (falling back to now if the @@ -664,6 +753,24 @@ func msSince(start, at time.Time) float64 { return float64(at.Sub(start).Microseconds()) / 1000.0 } +// captureContent / contentCap / contentMax read the dashboard's content-capture +// settings, with safe zero values when the dashboard is off. +func (h *Handler) captureContent() bool { + return h.rec != nil && h.rec.Opts().CaptureContent +} +func (h *Handler) contentCap() int { + if h.rec == nil { + return 0 + } + return h.rec.Opts().ContentCap +} +func (h *Handler) contentMax() int { + if h.rec == nil { + return 0 + } + return h.rec.Opts().ContentMaxPerRequest +} + func (h *Handler) stats(w http.ResponseWriter, _ *http.Request) { if h.agg == nil { w.Write([]byte("{}")) diff --git a/proxy/stats_golden_test.go b/proxy/stats_golden_test.go new file mode 100644 index 0000000..4fe63cd --- /dev/null +++ b/proxy/stats_golden_test.go @@ -0,0 +1,246 @@ +package proxy + +import ( + "encoding/json" + "net/http/httptest" + "sort" + "testing" + + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/metrics" +) + +// statsGoldenTopLevel is the /stats contract. deploy/harbor/*.py parses this +// payload to produce every published benchmark result, so a rename or a removal +// silently invalidates the reproduction path — a far worse failure than a build +// break, because the harness would keep running and report zeros. +// +// The rule this test enforces: fields may be ADDED, never renamed or removed. +// Adding a field here alongside the new key is the intended way to change it. +var statsGoldenTopLevel = []string{ + "actual_baseline_tokens", + "adjusted_saved", + "attempted_tokens", + "bounces", + "cache_read_tokens", + "cache_write_tokens", + "cg_added_ms_avg", + "components", + "extract", + "fresh_input_tokens", + "frozen_dropped", + "frozen_flips", + "frozen_hits", + "frozen_misses", + "frozen_repaired", + "frozen_tokens", + "llm_calls", + "llm_input_tokens", + "llm_output_tokens", + + "mode", + "observe_hypothetical_requests", + "output_tokens", + "potential_overhead_ms_avg", + "potential_saved_tokens", + "potential_savings_pct", + "projected_optimized_tokens", + "requests", + "saved_tokens", + "savings_pct", + "savings_pct_attempted", + "savings_pct_new_input", + "sse_buffered", + "sse_buffered_pct", + "sse_streamed", + "sse_ttfb_ms_avg", + "sse_ttfb_ms_avg_buffered", + "sync_enforced", + "tokens_after", + "tokens_before", + "top_discarded", + "top_passthrough", + "upstream_ms_avg", + "upstream_ms_avg_bypassed", + "wasted_tokens", +} + +// statsGoldenComponent is the per-component object's contract. swebench.py reads +// saved_tokens, saved_tokens_unique, overcount_ratio, runs, acted and duration_ms +// by name. +var statsGoldenComponent = []string{ + "acted", + "discarded_changes", + "duration_ms", + "mutated", + "overcount_ratio", + "reverted", + "runs", + "saved_tokens", + "saved_tokens_unique", +} + +// harnessRequiredFields are the exact keys deploy/harbor reads. Listed separately +// and explicitly so the coupling is documented at the point of enforcement. +var harnessRequiredFields = []string{ + "requests", "tokens_before", "tokens_after", "saved_tokens", "savings_pct", + "wasted_tokens", "bounces", "adjusted_saved", "components", "top_passthrough", + "llm_calls", "llm_input_tokens", "llm_output_tokens", + "cg_added_ms_avg", "upstream_ms_avg", "upstream_ms_avg_bypassed", +} + +func TestStatsShapeIsUnchanged(t *testing.T) { + agg := metrics.NewAggregator() + // Populate enough that the component map is present and non-empty. + agg.RecordAddedLatency(3) + agg.RecordUpstreamLatency(100, false) + agg.RecordUpstreamLatency(120, true) + agg.RecordExpand(50) + agg.RecordUsage(10, 1000, 100, 20) + agg.RecordEligibility(400, 600) + h := New(nil, nil, agg, Options{}) + + w := httptest.NewRecorder() + h.stats(w, httptest.NewRequest("GET", "/stats", nil)) + if w.Code != 200 { + t.Fatalf("/stats -> %d", w.Code) + } + if ct := w.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("content type = %q; want application/json", ct) + } + + var got map[string]json.RawMessage + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("/stats is not a JSON object: %v\n%s", err, w.Body.String()) + } + keys := make([]string, 0, len(got)) + for k := range got { + keys = append(keys, k) + } + sort.Strings(keys) + + // Every golden key must still be present. This is the half that protects the + // harness. + have := map[string]bool{} + for _, k := range keys { + have[k] = true + } + for _, want := range statsGoldenTopLevel { + if !have[want] { + t.Errorf("/stats lost field %q — deploy/harbor/*.py parses it; fields may be added, never renamed or removed", want) + } + } + // And the explicitly documented harness dependencies, spelled out again so the + // failure message names the consumer. + for _, want := range harnessRequiredFields { + if !have[want] { + t.Errorf("/stats lost %q, which deploy/harbor reads by name; the published benchmark reproduction breaks silently", want) + } + } + // New keys are fine, but they must be recorded in the golden list so a reviewer + // sees the payload growing on purpose. + golden := map[string]bool{} + for _, k := range statsGoldenTopLevel { + golden[k] = true + } + for _, k := range keys { + if !golden[k] { + t.Errorf("/stats gained field %q; add it to statsGoldenTopLevel so the contract stays reviewed", k) + } + } +} + +func TestStatsComponentShapeIsUnchanged(t *testing.T) { + agg := metrics.NewAggregator() + agg.Component(mkReport("extract", 1000, 800)) + h := New(nil, nil, agg, Options{}) + w := httptest.NewRecorder() + h.stats(w, httptest.NewRequest("GET", "/stats", nil)) + + var got struct { + Components map[string]map[string]json.RawMessage `json:"components"` + } + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + comp, ok := got.Components["extract"] + if !ok { + t.Fatalf("component missing from /stats: %s", w.Body.String()) + } + have := map[string]bool{} + for k := range comp { + have[k] = true + } + for _, want := range statsGoldenComponent { + if !have[want] { + t.Errorf("component object lost %q — deploy/harbor/swebench.py reads it by name", want) + } + } + golden := map[string]bool{} + for _, k := range statsGoldenComponent { + golden[k] = true + } + for k := range comp { + if !golden[k] { + t.Errorf("component object gained %q; record it in statsGoldenComponent", k) + } + } + // The internal dedup working set must never be serialized — it is unbounded and + // meaningless to a consumer. + if have["seenKeys"] || have["seen_keys"] { + t.Error("the unique-savings working set leaked into /stats") + } +} + +func TestStatsWithNoAggregatorStaysAnEmptyObject(t *testing.T) { + h := New(nil, nil, nil, Options{}) + w := httptest.NewRecorder() + h.stats(w, httptest.NewRequest("GET", "/stats", nil)) + if got := w.Body.String(); got != "{}" { + t.Errorf("/stats with no aggregator = %q; want {} (the harness treats {} as 'not ready')", got) + } +} + +// TestStatsNewFieldsCarryHonestValues checks the ADDED fields actually report the +// semantics they claim, not just that they exist. +func TestStatsNewFieldsCarryHonestValues(t *testing.T) { + agg := metrics.NewAggregator() + agg.Run(mkRunReport(1000, 800)) + agg.RecordEligibility(400, 600) + // No usage recorded: the new-input ratio must be 0, NOT 100% from dividing + // savings by themselves. + snap := agg.Snapshot() + if snap.SavedTokens != 200 { + t.Fatalf("saved = %d", snap.SavedTokens) + } + if snap.SavingsPctAttempted != 50 { + t.Errorf("savings_pct_attempted = %v; want 50 (200 saved of 400 attempted)", snap.SavingsPctAttempted) + } + if snap.SavingsPctNewInput != 0 { + t.Errorf("savings_pct_new_input = %v with no usage data; must be 0, never ~100", snap.SavingsPctNewInput) + } + if snap.FrozenTokens != 600 { + t.Errorf("frozen_tokens = %d; want 600", snap.FrozenTokens) + } + + // With usage data it becomes computable: 200 / (100 fresh + 300 write + 200) = 33.3%. + agg.RecordUsage(100, 5000, 300, 50) + snap = agg.Snapshot() + if snap.CacheReadTokens != 5000 || snap.CacheWriteTokens != 300 || snap.OutputTokens != 50 { + t.Errorf("usage tiers wrong: %+v", snap) + } + if p := snap.SavingsPctNewInput; p < 33.3 || p > 33.4 { + t.Errorf("savings_pct_new_input = %v; want ~33.33", p) + } +} + +// mkReport / mkRunReport build minimal component/run reports for the shape tests. +func mkReport(name string, before, after int) components.Report { + return components.Report{Component: name, Kind: "offload", + TokensBefore: before, TokensAfter: after, DurationMs: 1.5, + CacheKeys: []string{"k1"}} +} + +func mkRunReport(before, after int) components.RunReport { + return components.RunReport{Session: "s", TokensBefore: before, TokensAfter: after} +} diff --git a/proxy/usage.go b/proxy/usage.go new file mode 100644 index 0000000..8db9c0d --- /dev/null +++ b/proxy/usage.go @@ -0,0 +1,172 @@ +package proxy + +import ( + "strings" + + "github.com/tidwall/gjson" +) + +// Usage is one response's provider-billed token tiers. ok=false means the +// provider told us nothing, in which case a caller must report the request as +// partially accounted rather than pricing it as free. +type Usage struct { + FreshInput int64 + CacheRead int64 + CacheWrite int64 + Output int64 +} + +// parseUsage extracts the four billed token tiers from a buffered response body, +// in whichever dialect it is written. This is the number that actually matters on +// this workload — the request is ~99.95% cached and a cache write bills ~11.5x a +// read, so content-token savings alone cannot express the economics. +// +// Dialects handled: +// +// Anthropic usage.{input_tokens, output_tokens, +// cache_read_input_tokens, cache_creation_input_tokens} +// OpenAI usage.{prompt_tokens, completion_tokens, +// prompt_tokens_details.cached_tokens} +// +// Anthropic's `input_tokens` already EXCLUDES the cached tiers, so it is the fresh +// figure directly. OpenAI's `prompt_tokens` INCLUDES its cached_tokens, so fresh is +// the difference — getting this backwards double-counts the whole transcript on +// every turn, which is exactly the kind of error a "savings" number hides. +func parseUsage(body []byte) (Usage, bool) { + u := gjson.GetBytes(body, "usage") + if !u.Exists() { + return Usage{}, false + } + var out Usage + switch { + case u.Get("input_tokens").Exists(): // Anthropic + out.FreshInput = u.Get("input_tokens").Int() + out.Output = u.Get("output_tokens").Int() + out.CacheRead = u.Get("cache_read_input_tokens").Int() + out.CacheWrite = u.Get("cache_creation_input_tokens").Int() + case u.Get("prompt_tokens").Exists(): // OpenAI + prompt := u.Get("prompt_tokens").Int() + out.Output = u.Get("completion_tokens").Int() + out.CacheRead = u.Get("prompt_tokens_details.cached_tokens").Int() + out.FreshInput = prompt - out.CacheRead + if out.FreshInput < 0 { + out.FreshInput = 0 + } + case u.Get("output_tokens").Exists(): + // An OUTPUT-ONLY block. Anthropic's streaming message_delta carries exactly + // this, and it holds the FINAL completion count — treating it as "no usage" + // under-reports every streamed response's output tokens. + out.Output = u.Get("output_tokens").Int() + default: + return Usage{}, false + } + if out.FreshInput|out.CacheRead|out.CacheWrite|out.Output == 0 { + return Usage{}, false + } + return out, true +} + +// parseSSEUsage pulls usage out of a streamed response. Both dialects report it in +// terminal events (Anthropic: message_start carries the input tiers and +// message_delta the output; OpenAI: a final chunk with `usage`), so the tiers are +// merged across events, taking the maximum of each — a later event repeating a +// value must not double it. +func parseSSEUsage(raw []byte) (Usage, bool) { + var out Usage + found := false + for _, line := range strings.Split(string(raw), "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "data:") { + continue + } + payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if payload == "" || payload == "[DONE]" { + continue + } + // Anthropic nests the first usage under message.usage; OpenAI puts it at the top. + for _, path := range []string{"usage", "message.usage"} { + u := gjson.Get(payload, path) + if !u.Exists() { + continue + } + one, ok := parseUsage([]byte(`{"usage":` + u.Raw + `}`)) + if !ok { + continue + } + found = true + out.FreshInput = max64(out.FreshInput, one.FreshInput) + out.CacheRead = max64(out.CacheRead, one.CacheRead) + out.CacheWrite = max64(out.CacheWrite, one.CacheWrite) + out.Output = max64(out.Output, one.Output) + } + } + return out, found +} + +// responseUsage picks the right parser for a response's content type. +func responseUsage(contentType string, body []byte) (Usage, bool) { + if len(body) == 0 { + return Usage{}, false + } + if strings.Contains(contentType, "event-stream") { + return parseSSEUsage(body) + } + return parseUsage(body) +} + +// sniffMax bounds each half of the sniffer's window. Usage blocks are a few +// hundred bytes; 64 KiB each way is generous and hard-caps the memory an +// adversarially long response can make us hold per in-flight request. +const sniffMax = 64 << 10 + +// sniffer keeps a bounded head+tail window of a streamed response so its usage +// block can be read after the stream completes, without buffering the response. +// A disabled sniffer allocates nothing and does no work. +type sniffer struct { + on bool + head []byte + tail []byte + total int // bytes written, so bytes() knows whether the head alone is the whole body +} + +func newSniffer(on bool) *sniffer { return &sniffer{on: on} } + +func (s *sniffer) write(p []byte) { + if !s.on { + return + } + s.total += len(p) + if len(s.head) < sniffMax { + n := min(len(p), sniffMax-len(s.head)) + s.head = append(s.head, p[:n]...) + } + s.tail = append(s.tail, p...) + if len(s.tail) > sniffMax { + // Keep the last sniffMax bytes, re-slicing into a fresh buffer so the old one + // can be collected rather than growing forever behind the slice header. + keep := append(make([]byte, 0, sniffMax), s.tail[len(s.tail)-sniffMax:]...) + s.tail = keep + } +} + +// bytes returns the retained window: head, then tail, with a newline between so a +// truncated SSE line in the middle cannot glue two events into one bogus line. +func (s *sniffer) bytes() []byte { + if !s.on { + return nil + } + if s.total <= len(s.head) { + return s.head // the whole response fit in the head window; head IS the body + } + out := make([]byte, 0, len(s.head)+len(s.tail)+1) + out = append(out, s.head...) + out = append(out, '\n') + return append(out, s.tail...) +} + +func max64(a, b int64) int64 { + if a > b { + return a + } + return b +} diff --git a/proxy/usage_test.go b/proxy/usage_test.go new file mode 100644 index 0000000..c7cb379 --- /dev/null +++ b/proxy/usage_test.go @@ -0,0 +1,188 @@ +package proxy + +import ( + "strings" + "testing" +) + +func TestParseUsageAnthropic(t *testing.T) { + // Anthropic's input_tokens EXCLUDES the cached tiers, so it is fresh input as-is. + body := `{"id":"msg_1","usage":{"input_tokens":14,"output_tokens":11, + "cache_read_input_tokens":8612,"cache_creation_input_tokens":2780}}` + u, ok := parseUsage([]byte(body)) + if !ok { + t.Fatal("usage not parsed") + } + if u.FreshInput != 14 || u.Output != 11 || u.CacheRead != 8612 || u.CacheWrite != 2780 { + t.Errorf("got %+v", u) + } +} + +func TestParseUsageOpenAISubtractsCachedFromPrompt(t *testing.T) { + // OpenAI's prompt_tokens INCLUDES cached_tokens. Getting this backwards + // double-counts the whole transcript on every turn — the kind of error a + // "savings" figure conceals. + body := `{"usage":{"prompt_tokens":10000,"completion_tokens":120, + "prompt_tokens_details":{"cached_tokens":9500}}}` + u, ok := parseUsage([]byte(body)) + if !ok { + t.Fatal("usage not parsed") + } + if u.FreshInput != 500 { + t.Errorf("fresh = %d; want 500 (10000 prompt − 9500 cached)", u.FreshInput) + } + if u.CacheRead != 9500 || u.Output != 120 { + t.Errorf("got %+v", u) + } + // A cached count larger than the prompt (never seen, but arithmetic must not go + // negative and turn into a credit). + u, _ = parseUsage([]byte(`{"usage":{"prompt_tokens":10,"completion_tokens":1, + "prompt_tokens_details":{"cached_tokens":50}}}`)) + if u.FreshInput < 0 { + t.Errorf("fresh went negative: %+v", u) + } +} + +func TestParseUsageAbsentOrEmpty(t *testing.T) { + for _, body := range []string{ + `{}`, + `{"id":"msg_1"}`, + `{"usage":{}}`, + `{"usage":{"input_tokens":0,"output_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}}`, + `not json at all`, + } { + if _, ok := parseUsage([]byte(body)); ok { + t.Errorf("reported usage for %q; a response that tells us nothing must report ok=false, "+ + "so the row is flagged partial rather than priced as free", body) + } + } +} + +func TestParseSSEUsageMergesAcrossEvents(t *testing.T) { + // Anthropic reports the input tiers in message_start and the output count in + // message_delta, so the tiers must be merged, not taken from one event. + stream := `event: message_start +data: {"type":"message_start","message":{"id":"m","usage":{"input_tokens":7,"output_tokens":1,"cache_read_input_tokens":12345,"cache_creation_input_tokens":89}}} + +event: content_block_delta +data: {"type":"content_block_delta","delta":{"text":"hello"}} + +event: message_delta +data: {"type":"message_delta","usage":{"output_tokens":64}} + +event: message_stop +data: {"type":"message_stop"} +` + u, ok := parseSSEUsage([]byte(stream)) + if !ok { + t.Fatal("SSE usage not parsed") + } + if u.FreshInput != 7 || u.CacheRead != 12345 || u.CacheWrite != 89 { + t.Errorf("input tiers lost across events: %+v", u) + } + if u.Output != 64 { + t.Errorf("output = %d; want the final 64, not the initial 1", u.Output) + } +} + +func TestParseSSEUsageOpenAIFinalChunk(t *testing.T) { + stream := `data: {"choices":[{"delta":{"content":"hi"}}]} + +data: {"choices":[],"usage":{"prompt_tokens":900,"completion_tokens":40,"prompt_tokens_details":{"cached_tokens":850}}} + +data: [DONE] +` + u, ok := parseSSEUsage([]byte(stream)) + if !ok { + t.Fatal("SSE usage not parsed") + } + if u.FreshInput != 50 || u.CacheRead != 850 || u.Output != 40 { + t.Errorf("got %+v", u) + } +} + +func TestParseSSEUsageNoneReported(t *testing.T) { + stream := "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\ndata: [DONE]\n" + if _, ok := parseSSEUsage([]byte(stream)); ok { + t.Error("reported usage for a stream that carried none") + } +} + +func TestSnifferKeepsHeadAndTailBounded(t *testing.T) { + s := newSniffer(true) + // A response far larger than the window, with the usage block at the very end. + filler := strings.Repeat("x", sniffMax*3) + s.write([]byte("HEAD-MARKER")) + s.write([]byte(filler)) + s.write([]byte("TAIL-MARKER")) + + got := string(s.bytes()) + if !strings.Contains(got, "HEAD-MARKER") { + t.Error("head window lost") + } + if !strings.Contains(got, "TAIL-MARKER") { + t.Error("tail window lost") + } + // Bounded: at most the two windows plus the separator. + if len(got) > 2*sniffMax+1 { + t.Errorf("sniffer retained %d bytes; the window must be bounded at %d", len(got), 2*sniffMax+1) + } +} + +func TestSnifferDisabledCostsNothing(t *testing.T) { + s := newSniffer(false) + s.write([]byte(strings.Repeat("x", 1<<20))) + if got := s.bytes(); got != nil { + t.Errorf("a disabled sniffer retained %d bytes", len(got)) + } +} + +func TestSnifferSmallResponseReturnsItOnce(t *testing.T) { + s := newSniffer(true) + body := `{"usage":{"input_tokens":5,"output_tokens":2}}` + s.write([]byte(body)) + got := string(s.bytes()) + if got != body { + t.Errorf("small response mangled: %q", got) + } + // And it must still parse (no duplicated head+tail confusing the decoder). + if u, ok := parseUsage(s.bytes()); !ok || u.FreshInput != 5 { + t.Errorf("parse from sniffer = %+v ok=%v", u, ok) + } +} + +// TestSnifferSeparatorPreventsGluedSSELines guards a subtle failure: joining a +// truncated head to a tail without a newline could fuse two SSE `data:` lines into +// one unparseable line, silently losing usage for every large stream. +func TestSnifferSeparatorPreventsGluedSSELines(t *testing.T) { + s := newSniffer(true) + head := "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"usage\":{\"input_tokens\":9,\"output_tokens\":1,\"cache_read_input_tokens\":100,\"cache_creation_input_tokens\":2}}}\n\n" + s.write([]byte(head)) + s.write([]byte("data: " + strings.Repeat("{\"filler\":1}", sniffMax/6) + "\n\n")) + s.write([]byte("event: message_delta\ndata: {\"type\":\"message_delta\",\"usage\":{\"output_tokens\":77}}\n\n")) + + u, ok := parseSSEUsage(s.bytes()) + if !ok { + t.Fatal("usage lost across a windowed SSE stream") + } + if u.FreshInput != 9 || u.CacheRead != 100 { + t.Errorf("head tiers lost: %+v", u) + } + if u.Output != 77 { + t.Errorf("tail output lost: %+v", u) + } +} + +func TestResponseUsagePicksTheParserByContentType(t *testing.T) { + json := []byte(`{"usage":{"input_tokens":3,"output_tokens":4}}`) + if u, ok := responseUsage("application/json", json); !ok || u.FreshInput != 3 { + t.Errorf("json path: %+v ok=%v", u, ok) + } + sse := []byte("data: {\"usage\":{\"input_tokens\":3,\"output_tokens\":4}}\n\n") + if u, ok := responseUsage("text/event-stream; charset=utf-8", sse); !ok || u.FreshInput != 3 { + t.Errorf("sse path: %+v ok=%v", u, ok) + } + if _, ok := responseUsage("application/json", nil); ok { + t.Error("empty body reported usage") + } +}