diff --git a/CHANGELOG.md b/CHANGELOG.md index fbfd9cc6..d1d264aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed + +- **OpenTelemetry-aligned rename of the storage entities and HTTP API.** The + `agent_turns` table is now `traces`, `llm_calls` is now `spans`, and + `traces.call_ids` is now `span_ids`; a new forward-looking `spans.kind` + column (always `'llm'` today) leaves room for wire-visible tool spans. The + Rust domain/query types (`AgentTurn`→`Trace`, `Turn*`/`Call*` DTOs → + `Trace*`/`Span*`) and `StorageBackend` methods (`write_calls`→`write_spans`, + `query_turns`→`query_traces`, …) follow suit. New canonical routes + `/api/traces*` and `/api/spans*`; the pre-rename `/api/agent-turns*` and + `/api/llm-calls*` keep working as deprecated aliases (RFC 8594 `Deprecation` + header). Retention config keys `calls`/`turns` are accepted as serde aliases + for `spans`/`traces`. Existing DuckDB/ClickHouse databases auto-migrate in + place on init() with no data loss (idempotent detect-then-rename). + ## [0.6.0] — 2026-06-16 ### Changed diff --git a/CLAUDE.md b/CLAUDE.md index 984be3c4..9de2bf0e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,9 +82,9 @@ See [docs/design/04-turn.md](docs/design/04-turn.md) for Turn (agent interaction ## Storage -Three entities: `agent_turns` (agent turn), `llm_calls` (per-call detail + full body), `llm_metrics` (pre-aggregated time-series). Relation: `agent_turns 1─N llm_calls`. Pluggable backends: +Three entities: `traces` (agent turn), `spans` (per-call detail + full body), `llm_metrics` (pre-aggregated time-series). Relation: `traces 1─N spans`. Pluggable backends: -**Query rule — no JOIN.** Read-path SQL MUST NOT use any `JOIN`. Cross-entity reads are split into multiple point lookups: e.g. fetch `call_ids` from `agent_turns` by PK, then `SELECT ... FROM llm_calls WHERE id IN (?, ?, ...)`. See `query_turn_calls` in `h-storage-duckdb/src/turns.rs` for the canonical pattern. This keeps queries uniformly cheap across DuckDB/PG/ClickHouse and avoids planner surprises at scale. +**Query rule — no JOIN.** Read-path SQL MUST NOT use any `JOIN`. Cross-entity reads are split into multiple point lookups: e.g. fetch `span_ids` from `traces` by PK, then `SELECT ... FROM spans WHERE id IN (?, ?, ...)`. See `query_trace_spans` in `h-storage-duckdb/src/turns.rs` for the canonical pattern. This keeps queries uniformly cheap across DuckDB/PG/ClickHouse and avoids planner surprises at scale. | Backend | Use case | |---------|----------| diff --git a/console/src/hooks/use-agent-overview.ts b/console/src/hooks/use-agent-overview.ts index 0dcffe89..f1c6d9b9 100644 --- a/console/src/hooks/use-agent-overview.ts +++ b/console/src/hooks/use-agent-overview.ts @@ -9,7 +9,7 @@ export function useAgentSummary() { return useQuery({ queryKey: ["agent-summary", { start, end }], queryFn: () => - apiFetch("/api/agent-turns/summary", { start, end }), + apiFetch("/api/traces/summary", { start, end }), placeholderData: (prev) => prev, }) } @@ -20,7 +20,7 @@ export function useAgentActivity() { return useQuery({ queryKey: ["agent-activity", { start, end }], queryFn: () => - apiFetch("/api/agent-turns/activity", { start, end }), + apiFetch("/api/traces/activity", { start, end }), placeholderData: (prev) => prev, }) } diff --git a/console/src/hooks/use-agent-turns.ts b/console/src/hooks/use-agent-turns.ts index d6fef3da..38dd60a5 100644 --- a/console/src/hooks/use-agent-turns.ts +++ b/console/src/hooks/use-agent-turns.ts @@ -34,7 +34,7 @@ export function useAgentTurns({ page, pageSize, sortBy, sortOrder, status, agent status, agentKind, clientIp, serverPort, includeProxyHops, }], queryFn: () => - apiFetch("/api/agent-turns", { + apiFetch("/api/traces", { start, end, page, @@ -55,7 +55,7 @@ export function useAgentTurns({ page, pageSize, sortBy, sortOrder, status, agent export function useAgentTurnDetail(id: string | null) { return useQuery({ queryKey: ["agent-turn-detail", id], - queryFn: () => apiFetch(`/api/agent-turns/${id}`), + queryFn: () => apiFetch(`/api/traces/${id}`), enabled: id != null, }) } @@ -75,7 +75,7 @@ export function useAgentTurnCalls(id: string | null, lite = false) { return useQuery({ queryKey: ["agent-turn-calls", id, lite], queryFn: () => - apiFetch(`/api/agent-turns/${id}/calls`, lite ? { lite: 1 } : {}), + apiFetch(`/api/traces/${id}/spans`, lite ? { lite: 1 } : {}), enabled: id != null, }) } @@ -87,7 +87,7 @@ export function useAgentTurnCalls(id: string | null, lite = false) { export function useAgentTurnProxyView(id: string | null, enabled = true) { return useQuery({ queryKey: ["agent-turn-proxy-view", id], - queryFn: () => apiFetch(`/api/agent-turns/${id}/proxy-view`), + queryFn: () => apiFetch(`/api/traces/${id}/proxy-view`), enabled: id != null && enabled, }) } diff --git a/console/src/hooks/use-llm-call-detail.ts b/console/src/hooks/use-llm-call-detail.ts index 99cfd562..22ffd611 100644 --- a/console/src/hooks/use-llm-call-detail.ts +++ b/console/src/hooks/use-llm-call-detail.ts @@ -5,7 +5,7 @@ import type { LlmCallDetail } from "@/types/api" export function useLlmCallDetail(id: string | null) { return useQuery({ queryKey: ["llm-call-detail", id], - queryFn: () => apiFetch(`/api/llm-calls/${id}`), + queryFn: () => apiFetch(`/api/spans/${id}`), enabled: id != null, }) } diff --git a/console/src/hooks/use-llm-calls.ts b/console/src/hooks/use-llm-calls.ts index e2c904e8..3b98f6ad 100644 --- a/console/src/hooks/use-llm-calls.ts +++ b/console/src/hooks/use-llm-calls.ts @@ -43,7 +43,7 @@ export function useLlmCalls({ statusCode, finishReason, clientIp, serverPort, requestPath, isStream, }], queryFn: () => - apiFetch("/api/llm-calls", { + apiFetch("/api/spans", { start, end, page, diff --git a/docs/configure.md b/docs/configure.md index dc162342..2196dfe7 100644 --- a/docs/configure.md +++ b/docs/configure.md @@ -220,7 +220,7 @@ optimize_on_sweep = false # OPTIMIZE TABLE ... FINAL after ``` The backend talks to ClickHouse over its HTTP interface and creates the schema -(MergeTree fact tables + a ReplacingMergeTree `agent_turns`) on first run. +(MergeTree fact tables + a ReplacingMergeTree `traces`) on first run. Retention runs through the same `[storage.retention]` schedule as DuckDB, using lightweight `DELETE`. `optimize_on_sweep` eagerly reclaims space after each sweep at the cost of a full-table merge; off by default (TTL-style background @@ -249,8 +249,8 @@ defaults** — you only need to add it to override: [storage.retention] enabled = true # set to false to opt out entirely check_interval_secs = 3600 # how often to run the cleanup sweep -calls = 30 # keep llm_calls for N days; caps `turns` -turns = 30 # keep agent_turns for N days; must be <= calls +spans = 30 # keep spans (per-call detail) for N days; caps `traces` +traces = 30 # keep traces (agent turns) for N days; must be <= spans http_exchanges = 7 # keep http_exchanges (bulkiest table) for N days # Per-granularity retention for llm_metrics. Missing keys fall back to @@ -274,11 +274,13 @@ Behavior: - `http_exchanges` stores raw HTTP transport records (headers + bodies) per request/response and is by far the bulkiest table; keep its TTL short unless you specifically need a longer forensics window. -- `turns` must not outlive `calls`. `agent_turns.call_ids` references - rows in `llm_calls`, and the no-JOIN turn-detail read returns - empty/partial call lists once those references go stale. `heron - config validate` enforces `turns <= calls` (with `calls = 0` treated - as infinite, which satisfies any finite `turns`). +- `traces` must not outlive `spans`. `traces.span_ids` references + rows in `spans`, and the no-JOIN trace-detail read returns + empty/partial span lists once those references go stale. `heron + config validate` enforces `traces <= spans` (with `spans = 0` treated + as infinite, which satisfies any finite `traces`). +- The pre-rename keys `calls` / `turns` are still accepted as deprecated + aliases for `spans` / `traces`, so existing config files keep working. ## `[api]` — REST server diff --git a/docs/design/07-schema.md b/docs/design/07-schema.md index 0a099f44..c9ac634a 100644 --- a/docs/design/07-schema.md +++ b/docs/design/07-schema.md @@ -5,21 +5,21 @@ Four data entities, described in a storage-agnostic format (no SQL DDL). Each entity maps to a table/collection in the chosen storage backend (DuckDB / PostgreSQL / ClickHouse). ``` -agent_turns ─── 1:N ─── llm_calls ─── aggregated into ─── llm_metrics +traces ─── 1:N ─── spans ─── aggregated into ─── llm_metrics ╲ ─── llm_finish_metrics ``` -Per the project-wide read-path rule, cross-entity reads never use `JOIN` — `agent_turns` carries its child `call_ids` inline and detail reads issue a follow-up `IN (?, ?, ...)` lookup against `llm_calls`. Finish-reason counts live in their own long-format table (`llm_finish_metrics`) so the wide pre-aggregation table can keep a fixed column set. +Per the project-wide read-path rule, cross-entity reads never use `JOIN` — `traces` carries its child `span_ids` inline and detail reads issue a follow-up `IN (?, ?, ...)` lookup against `spans`. Finish-reason counts live in their own long-format table (`llm_finish_metrics`) so the wide pre-aggregation table can keep a fixed column set. --- -## 1. `llm_calls` — Per-Request Detail +## 1. `spans` — Per-Request Detail One record per LLM API call. The core fact table. Includes full request/response body content. ``` -llm_calls +spans ├── Primary Key │ └── id: string (UUID v7, time-ordered) │ @@ -96,12 +96,12 @@ Future provider values flow through verbatim; no normalization is performed. The --- -## 2. `agent_turns` — One Agent Interaction (1:N over `llm_calls`) +## 2. `traces` — One Agent Interaction (1:N over `spans`) -One record per agent turn (a contiguous sequence of `llm_calls` for a single agent run). Inline `call_ids` keeps cross-entity reads JOIN-free per the project-wide rule. +One record per agent turn (a contiguous sequence of `spans` for a single agent run). Inline `span_ids` keeps cross-entity reads JOIN-free per the project-wide rule. ``` -agent_turns +traces ├── Primary Key │ └── id: string (UUID v7, time-ordered, minted at finalize) │ @@ -109,7 +109,7 @@ agent_turns │ ├── agent_kind: string # e.g. anthropic / openai-codex / generic │ ├── session_id: string? # client-supplied session marker, when available │ ├── source_id: u32 # capture source (matches llm_metrics.source_id) -│ └── call_ids: string[] # ordered list of llm_calls.id in this turn +│ └── span_ids: string[] # ordered list of spans.id in this turn │ ├── Timestamps │ ├── start_time: timestamp # first call's request_time @@ -131,8 +131,8 @@ Indexes: ### Field notes - **`status`** (`VARCHAR NOT NULL`) — Two values only: `complete` (a wire-level terminal landed before finalize) or `incomplete` (idle timeout, pcap EOF, server shutdown, or connection RST mid-stream). The wire-level reason — `end_turn`, `max_tokens`, `refusal`, etc. — lives in `final_finish_reason`. Older databases written when the column carried `length` / `failed` / `cancelled` are upgraded in place on init: `length → complete`, `failed`/`cancelled → incomplete`. The wire reason for those legacy rows is unrecoverable. -- **`final_finish_reason`** (`VARCHAR`, nullable) — Raw `finish_reason` of the call that closed this turn. Same vocabulary as `llm_calls.finish_reason` (per `wire_api`); same migration caveat for rows pre-dating the raw-string refactor. -- **`call_ids`** carries the ordered child UUIDs inline so detail reads can issue a single `SELECT ... FROM llm_calls WHERE id IN (?, ?, ...)` follow-up. See `query_turn_calls` in `ts-storage/src/duckdb.rs` for the canonical pattern. Storing this list inline is the project's chosen alternative to a relational JOIN. +- **`final_finish_reason`** (`VARCHAR`, nullable) — Raw `finish_reason` of the call that closed this turn. Same vocabulary as `spans.finish_reason` (per `wire_api`); same migration caveat for rows pre-dating the raw-string refactor. +- **`span_ids`** carries the ordered child UUIDs inline so detail reads can issue a single `SELECT ... FROM spans WHERE id IN (?, ?, ...)` follow-up. See `query_trace_spans` in `h-storage-duckdb/src/turns.rs` for the canonical pattern. Storing this list inline is the project's chosen alternative to a relational JOIN. --- @@ -213,7 +213,7 @@ Indexes: - `active_calls_max` → `MAX()`. - Percentiles → `SUM(*_p* * *_count) / SUM(*_count)` (approximation — weighting by the matching `*_count` keeps slow-response rows with `call_count=0` from collapsing the result to zero, but it is not equivalent to merging the underlying t-digests. Serialized t-digest bytes is the planned long-term fix.) - **Aggregation levels**: finest `(wire_api, model, server_ip)` for drilldown, global `(*, *, *)` for overview. Additional dimensions will be added as they are validated with real traffic. -- **Other dimension analysis**: query `llm_calls` detail table with GROUP BY for dimensions not yet in pre-aggregation. +- **Other dimension analysis**: query `spans` detail table with GROUP BY for dimensions not yet in pre-aggregation. - **`*_sum / *_count` instead of `*_avg`**: averages are not additive across rows; storing the exact sum and count lets the query layer SUM over any set of rows (multi-source, multi-drain-slice) and divide to get a correct average. The per-row percentiles (`*_p*`) are t-digest estimates over that row's slice only — single-row views can read them directly. - **Multi-granularity**: Fine-grained (10s) for realtime dashboards, coarse (1h) for historical trends. Each granularity has its own drain cadence equal to its window size, so steady-state row count per granularity matches the number of windows covered. - **Active Calls**: Per-`DimensionKey` counter (+1 on `Start`, -1 on `Complete`); every Start writes the current value as a sample into `active_calls_sum / active_calls_sample_count` and updates `active_calls_max`. Cross-row avg via the `sum / count` pair; peak via `MAX(active_calls_max)`. @@ -257,12 +257,12 @@ Indexes: Retention is **enabled by default** with sane TTLs (`calls = turns = 30` days, `http_exchanges = 7` days). Operators tune via `[storage.retention]`; set `enabled = false` to opt out, or set any field to `0` to make that table never expire. A background sweeper (spawned at startup, cancelled on Ctrl+C) runs every `check_interval_secs` (default 3600) and deletes rows older than the per-table / per-granularity cutoff. **Cutoff columns** (what "old" means): -- `llm_calls.request_time` -- `agent_turns.end_time` (NOT NULL; turn completion — safer than start_time) +- `spans.request_time` +- `traces.end_time` (NOT NULL; turn completion — safer than start_time) - `llm_metrics.timestamp`, further keyed by `granularity` - `llm_finish_metrics.timestamp`, further keyed by `granularity` (sweeper reuses the `llm_metrics` per-granularity cutoffs) -**Cross-table constraint:** `turns` must not outlive `calls`. `agent_turns.call_ids` references `llm_calls.id`, and the no-JOIN turn-detail read trusts those references — turns surviving past their calls would render with empty/partial call lists. `validate()` enforces `turns <= calls` (with `calls = 0` treated as infinite, satisfying any finite `turns`). +**Cross-table constraint:** `traces` must not outlive `spans`. `traces.span_ids` references `spans.id`, and the no-JOIN trace-detail read trusts those references — traces surviving past their spans would render with empty/partial span lists. `validate()` enforces `traces <= spans` (with `spans = 0` treated as infinite, satisfying any finite `traces`). The pre-rename keys `calls`/`turns` are still accepted as deprecated serde aliases. **Defaults** (active out of the box): @@ -270,8 +270,8 @@ Retention is **enabled by default** with sane TTLs (`calls = turns = 30` days, ` [storage.retention] enabled = true check_interval_secs = 3600 -calls = 30 # llm_calls max age in days; caps `turns` -turns = 30 # agent_turns max age in days; must be <= calls (or set calls = 0) +spans = 30 # per-call detail (spans) max age in days; caps `traces` +traces = 30 # agent-trace summaries max age; must be <= spans (or set spans = 0) http_exchanges = 7 [storage.retention.metrics] @@ -284,7 +284,7 @@ http_exchanges = 7 Each backend implements `StorageBackend::apply_retention` with a dialect-appropriate strategy: - **DuckDB** (current): per-table DELETE + `CHECKPOINT` once per sweep to reclaim on-disk space (DuckDB DELETEs are MVCC tombstones until checkpoint). - **PostgreSQL** (planned): simple DELETE, or declarative partitioning + `DROP TABLE partition`; with TimescaleDB, `drop_chunks`. -- **ClickHouse** (implemented): per-table lightweight `DELETE FROM ... WHERE < cutoff` (counts gathered via a `count()` immediately before each delete, since CH `DELETE` returns no affected-row count); `agent_turns`/`llm_metrics` use the same per-table / per-granularity cutoffs as DuckDB. Optional `OPTIMIZE TABLE ... FINAL` per swept table when `optimize_on_sweep = true` (off by default — background merges reclaim lazily). No declarative `TTL` is set, so a single `apply_retention` path drives both backends identically. +- **ClickHouse** (implemented): per-table lightweight `DELETE FROM ... WHERE < cutoff` (counts gathered via a `count()` immediately before each delete, since CH `DELETE` returns no affected-row count); `traces`/`llm_metrics` use the same per-table / per-granularity cutoffs as DuckDB. Optional `OPTIMIZE TABLE ... FINAL` per swept table when `optimize_on_sweep = true` (off by default — background merges reclaim lazily). No declarative `TTL` is set, so a single `apply_retention` path drives both backends identically. --- @@ -295,7 +295,7 @@ Each backend implements `StorageBackend::apply_retention` with a dialect-appropr | `id` (UUID v7) | VARCHAR | `uuid` native type | String | | `timestamp` | TIMESTAMP | `timestamptz` | `DateTime64(6)` | | `request_body` / `response_body` | VARCHAR or JSON | TEXT | String | -| `llm_calls` ordering | B-tree on `request_time` | B-tree on `request_time` | `ORDER BY (request_time, id)` MergeTree | +| `spans` ordering | B-tree on `request_time` | B-tree on `request_time` | `ORDER BY (request_time, id)` MergeTree | | `llm_metrics` optimization | plain table | TimescaleDB hypertable on `timestamp` (optional) | `ORDER BY (granularity, timestamp, model)` | | `llm_finish_metrics` optimization | plain table | TimescaleDB hypertable on `timestamp` (optional) | `ORDER BY (granularity, timestamp, finish_reason)` | | Percentile storage | plain DOUBLE | plain f64 | plain f64, or `AggregateFunction(quantilesTDigest, Float64)` for re-aggregation | @@ -306,18 +306,41 @@ Each backend implements `StorageBackend::apply_retention` with a dialect-appropr ## Upgrade Notes +### OpenTelemetry-aligned rename (`agent_turns`→`traces`, `llm_calls`→`spans`) + +Aligns the storage + API vocabulary with the industry-standard Session → Trace → +Span hierarchy (a session is the existing `(source_id, session_id)` view over +traces). + +- Tables `agent_turns` → `traces`, `llm_calls` → `spans`; column + `traces.call_ids` → `span_ids`. +- New forward-looking `spans.kind` column (always `'llm'` today) so future + wire-visible tool spans can carry `kind='tool'`. +- **In-place auto-migration** on init() (unlike the older rename below): an + idempotent detect-then-rename runs before `CREATE TABLE IF NOT EXISTS`, so + existing DuckDB/ClickHouse databases migrate without data loss — no need to + delete the file. `RENAME` has no `IF EXISTS`, so each is guarded by a + table/column existence check. +- Back-compat: HTTP routes `/api/agent-turns*` and `/api/llm-calls*` keep + working as deprecated aliases (RFC 8594 `Deprecation` header) for the + canonical `/api/traces*` and `/api/spans*`. Retention config keys + `calls`/`turns` remain accepted as serde aliases for `spans`/`traces`. + ### `AgentTurn` rename (`LlmTurn` → `AgentTurn`) -- Table `llm_turns` → `agent_turns` +- Table `llm_turns` → `agent_turns` (now further renamed to `traces` — see above) - Column `client_kind` → `agent_kind` -No online migration is performed. Existing `server/data/heron.duckdb` files from before the rename should be deleted before restart — the backend will recreate the new schema on first run via `CREATE TABLE IF NOT EXISTS`. +No online migration was performed for this older rename. Existing +`server/data/heron.duckdb` files from before *this* rename should be deleted +before restart — the backend recreates the schema on first run via +`CREATE TABLE IF NOT EXISTS`. ### `finish_reason` raw-string refactor (see `CHANGELOG`) Idempotent on-init migrations on the DuckDB backend: - `llm_metrics`: `ALTER TABLE ... DROP COLUMN IF EXISTS finish_complete_count` (and the four sibling columns: `finish_length_count`, `finish_tool_use_count`, `finish_error_count`, `finish_cancelled_count`). -- `agent_turns`: one-time `UPDATE agent_turns SET status='complete' WHERE status='length'` and `UPDATE agent_turns SET status='incomplete' WHERE status IN ('failed','cancelled')`. After this update the legacy `length` / `failed` / `cancelled` values cannot reappear; the wire reason for those rows is unrecoverable. -- `llm_calls.finish_reason` is **not** rewritten. Pre-refactor rows keep their normalized labels (`complete`, `length`, `tool_use`, `error`, `cancelled`); post-refactor rows carry raw provider values. The two are distinguishable by row date. Application code that filters or groups across the boundary must handle both vocabularies. +- `traces`: one-time `UPDATE traces SET status='complete' WHERE status='length'` and `UPDATE traces SET status='incomplete' WHERE status IN ('failed','cancelled')`. After this update the legacy `length` / `failed` / `cancelled` values cannot reappear; the wire reason for those rows is unrecoverable. +- `spans.finish_reason` is **not** rewritten. Pre-refactor rows keep their normalized labels (`complete`, `length`, `tool_use`, `error`, `cancelled`); post-refactor rows carry raw provider values. The two are distinguishable by row date. Application code that filters or groups across the boundary must handle both vocabularies. - `llm_finish_metrics` is created via `CREATE TABLE IF NOT EXISTS` on first run; no historical backfill from the old `finish_*_count` columns is performed. diff --git a/server/app/heron/src/bin/storage_bench.rs b/server/app/heron/src/bin/storage_bench.rs index 72c79406..33382a8a 100644 --- a/server/app/heron/src/bin/storage_bench.rs +++ b/server/app/heron/src/bin/storage_bench.rs @@ -25,7 +25,7 @@ use h_llm::wire_apis as wa; use h_metrics::model::LlmMetric; use h_storage::query::*; use h_storage::StorageBackend; -use h_turn::{AgentTurn, TurnStatus}; +use h_turn::{Trace, TraceStatus}; use heron::create_backend; #[derive(Parser, Debug)] @@ -114,9 +114,9 @@ fn make_call(i: usize, body: &str) -> LlmCall { } } -fn make_turn(i: usize) -> AgentTurn { +fn make_turn(i: usize) -> Trace { let ts = BASE_US + (i as i64 * WINDOW_US / 20_000).min(WINDOW_US); - AgentTurn { + Trace { source_id: "bench".into(), turn_id: format!("turn-{i:012}"), session_id: format!("sess-{}", i % 2000), @@ -135,13 +135,13 @@ fn make_turn(i: usize) -> AgentTurn { total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, total_cost_usd: Some(0.05), - status: TurnStatus::Complete, + status: TraceStatus::Complete, final_finish_reason: Some("stop".into()), user_input_preview: Some("hello".into()), user_call_id: Some(format!("call-{:012}", i * 3)), final_answer_preview: Some("done".into()), final_call_id: Some(format!("call-{:012}", i * 3 + 2)), - call_ids: vec![format!("call-{:012}", i * 3)], + span_ids: vec![format!("call-{:012}", i * 3)], metadata: serde_json::json!({}), tool_surfaces: vec![], tool_call_total: 0, @@ -247,7 +247,7 @@ async fn main() -> Result<(), Box> { for chunk_start in (0..args.calls).step_by(args.batch) { let end = (chunk_start + args.batch).min(args.calls); let batch: Vec = (chunk_start..end).map(|i| make_call(i, &body)).collect(); - backend.write_calls(batch).await?; + backend.write_spans(batch).await?; } let calls_secs = t.elapsed().as_secs_f64(); let calls_per_sec = args.calls as f64 / calls_secs; @@ -263,8 +263,8 @@ async fn main() -> Result<(), Box> { let t = Instant::now(); for chunk_start in (0..args.turns).step_by(args.batch) { let end = (chunk_start + args.batch).min(args.turns); - let batch: Vec = (chunk_start..end).map(make_turn).collect(); - backend.write_turns(batch).await?; + let batch: Vec = (chunk_start..end).map(make_turn).collect(); + backend.write_traces(batch).await?; } let turns_per_sec = args.turns as f64 / t.elapsed().as_secs_f64(); @@ -276,7 +276,7 @@ async fn main() -> Result<(), Box> { let r = range.clone(); async move { let _ = b - .query_calls(&CallsQuery { + .query_spans(&SpansQuery { time_range: r, filter: DimensionFilter::default(), status_codes: vec![], @@ -334,7 +334,7 @@ async fn main() -> Result<(), Box> { let r = range.clone(); async move { let _ = b - .query_turns(&TurnsQuery { + .query_traces(&TracesQuery { time_range: r, filter: DimensionFilter::default(), client_ips: vec![], @@ -382,10 +382,10 @@ async fn main() -> Result<(), Box> { "turns": turns_per_sec.round(), }, "query_ms": { - "query_calls": { "p50": qc_p50, "p95": qc_p95, "mean": qc_mean }, + "query_spans": { "p50": qc_p50, "p95": qc_p95, "mean": qc_mean }, "query_summary": { "p50": qs_p50, "p95": qs_p95, "mean": qs_mean }, "query_timeseries": { "p50": qt_p50, "p95": qt_p95, "mean": qt_mean }, - "query_turns": { "p50": qn_p50, "p95": qn_p95, "mean": qn_mean }, + "query_traces": { "p50": qn_p50, "p95": qn_p95, "mean": qn_mean }, "query_services": { "p50": qv_p50, "p95": qv_p95, "mean": qv_mean }, }, }); diff --git a/server/app/heron/src/cmd/backfill_tokens.rs b/server/app/heron/src/cmd/backfill_tokens.rs index d08724e3..f975fd0e 100644 --- a/server/app/heron/src/cmd/backfill_tokens.rs +++ b/server/app/heron/src/cmd/backfill_tokens.rs @@ -96,7 +96,7 @@ pub fn run(args: &BackfillTokensArgs) -> i32 { }; let select_sql = format!( "SELECT id, wire_api, status_code, request_body, response_body \ - FROM llm_calls \ + FROM spans \ WHERE COALESCE(input_tokens, 0) = 0 \ AND COALESCE(output_tokens, 0) = 0 \ AND response_body IS NOT NULL \ @@ -140,7 +140,7 @@ pub fn run(args: &BackfillTokensArgs) -> i32 { let mut skipped_unknown_wire: u64 = 0; let mut update_stmt = match conn.prepare( - "UPDATE llm_calls SET input_tokens = ?, output_tokens = ?, total_tokens = ? WHERE id = ?", + "UPDATE spans SET input_tokens = ?, output_tokens = ?, total_tokens = ? WHERE id = ?", ) { Ok(s) => s, Err(e) => { @@ -273,14 +273,14 @@ mod tests { use duckdb::Connection; use tempfile::tempdir; - /// Bootstrap a temp DuckDB matching the production llm_calls schema + /// Bootstrap a temp DuckDB matching the production spans schema /// (only the columns we actually read/write — minimal but matching /// types). Returns (path, conn). fn make_temp_db(dir: &std::path::Path) -> (PathBuf, Connection) { let db = dir.join("test.duckdb"); let c = Connection::open(&db).expect("open"); c.execute_batch( - "CREATE TABLE llm_calls ( + "CREATE TABLE spans ( id VARCHAR PRIMARY KEY, source_id VARCHAR DEFAULT '', client_ip VARCHAR DEFAULT '', @@ -316,7 +316,7 @@ mod tests { out_t: Option, ) { c.execute( - "INSERT INTO llm_calls (id, wire_api, status_code, request_body, response_body, input_tokens, output_tokens) \ + "INSERT INTO spans (id, wire_api, status_code, request_body, response_body, input_tokens, output_tokens) \ VALUES (?, ?, ?, ?, ?, ?, ?)", duckdb::params![id, wire_api, status, req, resp, in_t, out_t], ) @@ -325,7 +325,7 @@ mod tests { fn token_pair(c: &Connection, id: &str) -> (Option, Option) { c.query_row( - "SELECT input_tokens, output_tokens FROM llm_calls WHERE id = ?", + "SELECT input_tokens, output_tokens FROM spans WHERE id = ?", duckdb::params![id], |r| Ok((r.get(0)?, r.get(1)?)), ) diff --git a/server/app/heron/src/main.rs b/server/app/heron/src/main.rs index ad909825..9e0c1607 100644 --- a/server/app/heron/src/main.rs +++ b/server/app/heron/src/main.rs @@ -531,7 +531,7 @@ async fn run_pipeline(cli: Cli) { // turn-tracker shard (writers) and the API's /api/agent-turns // handler (reader) so the console sees in-progress turns alongside // finalized ones without DB write amplification. - let active_turns = h_turn::new_active_turn_registry(); + let active_turns = h_turn::new_active_trace_registry(); // Process-wide gauge for the dashboard "Active Agent Turns" chart: // size of the in-progress turn registry at sample time. Registered on @@ -898,7 +898,7 @@ async fn run_pipeline(cli: Cli) { // the registry stays empty for the lifetime of this // process. Construct a fresh empty one so the API still // serves /api/agent-turns (returning DB rows only). - let api_active_turns = h_turn::new_active_turn_registry(); + let api_active_turns = h_turn::new_active_trace_registry(); Some(tokio::spawn(async move { let router = h_api::router( api_storage, diff --git a/server/app/heron/src/pipeline.rs b/server/app/heron/src/pipeline.rs index 5c3776c3..102660aa 100644 --- a/server/app/heron/src/pipeline.rs +++ b/server/app/heron/src/pipeline.rs @@ -4,7 +4,7 @@ //! llm → turn → **metrics** — with N sources fan-in through a //! [`RoutingSender`] that routes by `hash(source_id) % D` to D dispatcher //! channels (default D=1). Flow keys, HTTP reassembly state, -//! `LlmCall`/`AgentTurn` state, and the metrics aggregator's event-time +//! `LlmCall`/`Trace` state, and the metrics aggregator's event-time //! watermark never leak between pipelines. Only the storage sink is shared //! across pipelines so every record lands in the same DB tables. //! @@ -56,7 +56,7 @@ use h_protocol::{ }; use h_storage::StorageBackend; use h_turn::tracker::TrackerConfig; -use h_turn::AgentTurn; +use h_turn::Trace; /// Every task spawned by the pipeline is labeled so panic logs name the /// specific worker that died. Cheap strings — formatting happens only at @@ -127,7 +127,7 @@ impl Pipeline { storage: Arc, per_pipeline_metrics: &mut [MetricsSystem], shared_metrics: &mut MetricsSystem, - active_turns: h_turn::ActiveTurnRegistry, + active_turns: h_turn::ActiveTraceRegistry, classifier_cfg: ClassifierConfig, body_cap: h_common::config::BodyCapConfig, ) -> Self { @@ -148,7 +148,7 @@ impl Pipeline { let exchanges_cap = max_q(|q| q.storage_exchanges); let (calls_tx, calls_rx) = mpsc::channel::>(calls_cap); - let (turns_tx, turns_rx) = mpsc::channel::(turns_cap); + let (turns_tx, turns_rx) = mpsc::channel::(turns_cap); let (metrics_out_tx, metrics_out_rx) = mpsc::channel::(metrics_cap); let (http_exchanges_tx, http_exchanges_rx) = mpsc::channel::(exchanges_cap); diff --git a/server/app/heron/tests/pipeline_e2e.rs b/server/app/heron/tests/pipeline_e2e.rs index 778c44a4..c4268fbb 100644 --- a/server/app/heron/tests/pipeline_e2e.rs +++ b/server/app/heron/tests/pipeline_e2e.rs @@ -2,7 +2,7 @@ //! //! Drives `Pipeline::build` with a pcap fixture, waits for the EOF cascade to //! drain every stage, then opens a fresh DuckDB connection to verify that all -//! three tables (`llm_calls`, `agent_turns`, `llm_metrics`) contain the +//! three tables (`spans`, `traces`, `llm_metrics`) contain the //! expected rows. Unlike the per-stage integration tests, this one exercises //! the composition root and the storage sink together — regressions in //! channel wiring, shard-fan-out, or EOF propagation surface here. @@ -125,7 +125,7 @@ async fn run_pipeline_multi(fixture_names: &[&str]) -> Option<(TempDir, PathBuf) storage.clone(), &mut per_pipeline_metrics, &mut shared_metrics, - h_turn::new_active_turn_registry(), + h_turn::new_active_trace_registry(), h_llm::agent_classifier::ClassifierConfig::default(), h_common::config::BodyCapConfig::default(), ); @@ -189,18 +189,18 @@ async fn claude_cli_pcap_populates_all_three_tables() { let conn = Connection::open(&db_path).expect("reopen duckdb for verify"); - let calls = count(&conn, "llm_calls"); - let turns = count(&conn, "agent_turns"); + let calls = count(&conn, "spans"); + let turns = count(&conn, "traces"); let metrics = count(&conn, "llm_metrics"); eprintln!("e2e rows: calls={calls} turns={turns} metrics={metrics}"); - assert!(calls >= 1, "expected >=1 llm_calls, got {calls}"); - assert!(turns >= 1, "expected >=1 agent_turns, got {turns}"); + assert!(calls >= 1, "expected >=1 spans, got {calls}"); + assert!(turns >= 1, "expected >=1 traces, got {turns}"); assert!(metrics >= 1, "expected >=1 llm_metrics, got {metrics}"); // Wire-API ground truth: fixture is an anthropic Messages API capture. let call_wire_apis: Vec = conn - .prepare("SELECT DISTINCT wire_api FROM llm_calls") + .prepare("SELECT DISTINCT wire_api FROM spans") .unwrap() .query_map([], |r| r.get::<_, String>(0)) .unwrap() @@ -208,7 +208,7 @@ async fn claude_cli_pcap_populates_all_three_tables() { .collect(); assert!( call_wire_apis.iter().any(|p| p == wa::ANTHROPIC), - "expected anthropic in llm_calls wire_apis, got {call_wire_apis:?}" + "expected anthropic in spans wire_apis, got {call_wire_apis:?}" ); // A single complete claude-cli turn is the documented ground truth @@ -216,7 +216,7 @@ async fn claude_cli_pcap_populates_all_three_tables() { let (anthropic_turns, status, agent_kind): (i64, String, String) = conn .query_row( "SELECT COUNT(*), MIN(status), MIN(agent_kind) \ - FROM agent_turns WHERE wire_api = 'anthropic'", + FROM traces WHERE wire_api = 'anthropic'", [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), ) @@ -241,29 +241,29 @@ async fn claude_cli_pcap_populates_all_three_tables() { ); // A completed turn's final_call_id must reference a real row in - // llm_calls — catches future divergence between turn shard fan-out and + // spans — catches future divergence between turn shard fan-out and // call sink writes. let dangling_final_call: i64 = conn .query_row( - "SELECT COUNT(*) FROM agent_turns t \ + "SELECT COUNT(*) FROM traces t \ WHERE t.final_call_id IS NOT NULL \ - AND NOT EXISTS (SELECT 1 FROM llm_calls c WHERE c.id = t.final_call_id)", + AND NOT EXISTS (SELECT 1 FROM spans c WHERE c.id = t.final_call_id)", [], |r| r.get(0), ) .expect("dangling final_call_id query"); assert_eq!( dangling_final_call, 0, - "turn.final_call_id must reference an existing llm_calls row" + "turn.final_call_id must reference an existing spans row" ); // A complete anthropic turn must have call_count consistent with how - // many anthropic calls landed in llm_calls (≤, since some calls may + // many anthropic calls landed in spans (≤, since some calls may // belong to unfinalised turns in other fixtures). let (turn_call_count, anthropic_calls): (i64, i64) = conn .query_row( - "SELECT (SELECT MIN(call_count) FROM agent_turns WHERE wire_api = 'anthropic'), \ - (SELECT COUNT(*) FROM llm_calls WHERE wire_api = 'anthropic')", + "SELECT (SELECT MIN(call_count) FROM traces WHERE wire_api = 'anthropic'), \ + (SELECT COUNT(*) FROM spans WHERE wire_api = 'anthropic')", [], |r| Ok((r.get(0)?, r.get(1)?)), ) @@ -277,7 +277,7 @@ async fn claude_cli_pcap_populates_all_three_tables() { /// Drives two different pcap fixtures through two capture sources /// simultaneously and asserts that: /// -/// * Both wire APIs land in `llm_calls` — proves each sub-pipeline reached +/// * Both wire APIs land in `spans` — proves each sub-pipeline reached /// the shared sink independently. /// * The anthropic-only fixture produces exactly 1 complete anthropic turn /// (matches the single-source E2E's ground truth), which rules out @@ -297,7 +297,7 @@ async fn two_pcaps_isolated_but_metrics_merged() { let conn = Connection::open(&db_path).expect("reopen duckdb for verify"); let wire_apis: Vec = conn - .prepare("SELECT DISTINCT wire_api FROM llm_calls ORDER BY 1") + .prepare("SELECT DISTINCT wire_api FROM spans ORDER BY 1") .unwrap() .query_map([], |r| r.get::<_, String>(0)) .unwrap() @@ -305,11 +305,11 @@ async fn two_pcaps_isolated_but_metrics_merged() { .collect(); assert!( wire_apis.iter().any(|p| p == wa::ANTHROPIC), - "expected anthropic in llm_calls wire_apis, got {wire_apis:?}" + "expected anthropic in spans wire_apis, got {wire_apis:?}" ); assert!( wire_apis.iter().any(|p| p == wa::OPENAI_RESPONSES), - "expected openai-responses in llm_calls wire_apis, got {wire_apis:?}" + "expected openai-responses in spans wire_apis, got {wire_apis:?}" ); // The anthropic fixture alone produces exactly 1 complete turn. If the @@ -317,7 +317,7 @@ async fn two_pcaps_isolated_but_metrics_merged() { // tracker, flow-key collisions, …), this count would drift. let anthropic_turns: i64 = conn .query_row( - "SELECT COUNT(*) FROM agent_turns \ + "SELECT COUNT(*) FROM traces \ WHERE wire_api = 'anthropic' AND status = 'complete'", [], |r| r.get(0), @@ -333,7 +333,7 @@ async fn two_pcaps_isolated_but_metrics_merged() { // anthropic one. let openai_turns: i64 = conn .query_row( - "SELECT COUNT(*) FROM agent_turns WHERE wire_api = 'openai-responses'", + "SELECT COUNT(*) FROM traces WHERE wire_api = 'openai-responses'", [], |r| r.get(0), ) diff --git a/server/config/default.toml b/server/config/default.toml index d020bbf5..f5766743 100644 --- a/server/config/default.toml +++ b/server/config/default.toml @@ -141,15 +141,16 @@ flush_interval_ms = 200 # active defaults, shown for reference). Set `enabled = false` to opt out, or # set any field to `0` to make that table never expire. # -# Constraint: `turns` must not outlive `calls` (validate enforces -# `turns <= calls`, with `calls = 0` treated as infinite). Surviving turns -# whose llm_calls have been pruned would render as empty/partial in the -# detail view, since `agent_turns.call_ids` references rows in `llm_calls`. +# Constraint: `traces` must not outlive `spans` (validate enforces +# `traces <= spans`, with `spans = 0` treated as infinite). Surviving traces +# whose spans have been pruned would render as empty/partial in the +# detail view, since `traces.span_ids` references rows in `spans`. +# The pre-rename keys `calls`/`turns` are still accepted as deprecated aliases. # [storage.retention] # enabled = true # check_interval_secs = 3600 -# calls = 30 # LLM call detail (request/response bodies); caps `turns` -# turns = 30 # Agent turn summaries; must be <= calls (or set calls = 0) +# spans = 30 # Per-call detail (request/response bodies); caps `traces` +# traces = 30 # Agent-trace summaries; must be <= spans (or set spans = 0) # http_exchanges = 7 # Raw HTTP transport records (headers + bodies) — bulkiest table # # Per-granularity retention for llm_metrics. Missing keys fall back to the diff --git a/server/h-api/src/lib.rs b/server/h-api/src/lib.rs index d8f491cd..c79eab78 100644 --- a/server/h-api/src/lib.rs +++ b/server/h-api/src/lib.rs @@ -26,7 +26,7 @@ use h_common::error::{AppError, Result}; use h_common::internal_metrics::{AggregateHistory, MetricsSvc}; use h_pcap_extract::PipelineRoot; use h_storage::StorageBackend; -use h_turn::ActiveTurnRegistry; +use h_turn::ActiveTraceRegistry; /// Carriers for `/api/internal-metrics` — every per-pipeline `MetricsSvc` /// plus the cross-pipeline (storage) one. Build this in `main.rs` after @@ -74,14 +74,14 @@ pub struct ApiHealthContext { /// Composite state for the `/api/agent-turns*` routes. Carries both the /// storage backend (for finalized rows) and the in-memory -/// `ActiveTurnRegistry` (for in-progress rows). The list handler unions +/// `ActiveTraceRegistry` (for in-progress rows). The list handler unions /// the two; detail / calls only need storage. We pass the composite /// around for all three handlers so the registry is available where it /// matters without a separate router for the list endpoint. #[derive(Clone)] pub struct ApiAgentTurnsContext { pub storage: Arc, - pub active_turns: ActiveTurnRegistry, + pub active_turns: ActiveTraceRegistry, } /// Bind the API server listener. Call this before spawning so bind errors @@ -102,7 +102,7 @@ pub fn router( runtime_config: ApiRuntimeConfigContext, health: ApiHealthContext, pcap_roots: Arc>, - active_turns: ActiveTurnRegistry, + active_turns: ActiveTraceRegistry, ) -> Router { let internal_metrics_routes = Router::new() .route( @@ -134,35 +134,50 @@ pub fn router( .route("/api/pcap/extract", get(routes::pcap_extract::handler)) .with_state(pcap_roots); - // Agent-turns routes use a composite state (storage + in-memory - // active-turn registry) so the list handler can union finalized - // (DuckDB) and in-progress (RAM) rows in one response. detail/calls - // only need storage but ride on the same composite for plumbing - // simplicity. - let agent_turns_state = ApiAgentTurnsContext { + // Trace routes use a composite state (storage + in-memory active-trace + // registry) so the list handler can union finalized (DuckDB) and + // in-progress (RAM) rows in one response. detail/spans only need storage + // but ride on the same composite for plumbing simplicity. + let traces_state = ApiAgentTurnsContext { storage: storage.clone(), active_turns, }; - let agent_turns_routes = Router::new() - .route("/api/agent-turns", get(routes::agent_turns::list)) + // Canonical OpenTelemetry-aligned routes. + let traces_routes = Router::new() + .route("/api/traces", get(routes::traces::list)) + .route("/api/traces/summary", get(routes::traces::summary)) + .route("/api/traces/activity", get(routes::traces::activity)) + .route("/api/traces/{id}", get(routes::traces::detail)) + .route("/api/traces/{id}/spans", get(routes::traces::calls)) .route( - "/api/agent-turns/summary", - get(routes::agent_turns::summary), - ) - .route( - "/api/agent-turns/activity", - get(routes::agent_turns::activity), - ) - .route("/api/agent-turns/{id}", get(routes::agent_turns::detail)) - .route( - "/api/agent-turns/{id}/calls", - get(routes::agent_turns::calls), + "/api/traces/{id}/proxy-view", + get(routes::traces::proxy_view), ) + .with_state(traces_state.clone()); + // Deprecated pre-rename aliases → identical handlers, stamped with an + // RFC 8594 `Deprecation` header pointing at the canonical path. Kept one + // release cycle for the external consumer still calling `/api/agent-turns`. + let traces_alias_routes = Router::new() + .route("/api/agent-turns", get(routes::traces::list)) + .route("/api/agent-turns/summary", get(routes::traces::summary)) + .route("/api/agent-turns/activity", get(routes::traces::activity)) + .route("/api/agent-turns/{id}", get(routes::traces::detail)) + .route("/api/agent-turns/{id}/calls", get(routes::traces::calls)) .route( "/api/agent-turns/{id}/proxy-view", - get(routes::agent_turns::proxy_view), + get(routes::traces::proxy_view), ) - .with_state(agent_turns_state); + .with_state(traces_state) + .layer(axum::middleware::map_response(deprecate_traces)); + + // Spans: canonical `/api/spans` lives on the main storage router below; + // the deprecated `/api/llm-calls` alias rides its own sub-router so it can + // carry the Deprecation header without affecting the canonical path. + let spans_alias_routes = Router::new() + .route("/api/llm-calls", get(routes::spans::list)) + .route("/api/llm-calls/{id}", get(routes::spans::detail)) + .with_state(storage.clone()) + .layer(axum::middleware::map_response(deprecate_spans)); let capture_routes = Router::new().route( "/api/capture/interfaces", @@ -193,8 +208,8 @@ pub fn router( "/api/metrics/finish-reasons", get(routes::metrics::finish_reasons), ) - .route("/api/llm-calls", get(routes::llm_calls::list)) - .route("/api/llm-calls/{id}", get(routes::llm_calls::detail)) + .route("/api/spans", get(routes::spans::list)) + .route("/api/spans/{id}", get(routes::spans::detail)) .route("/api/http-exchanges", get(routes::http_exchanges::list)) .route( "/api/http-exchanges/{id}", @@ -218,7 +233,31 @@ pub fn router( .merge(runtime_config_routes) .merge(health_routes) .merge(pcap_extract_routes) - .merge(agent_turns_routes) + .merge(traces_routes) + .merge(traces_alias_routes) + .merge(spans_alias_routes) .merge(capture_routes) .layer(CorsLayer::permissive()) } + +/// Stamp the RFC 8594 `Deprecation` header (+ a `Link` to the canonical +/// successor) on responses served from a deprecated pre-rename alias route. +async fn deprecate_traces(mut res: axum::response::Response) -> axum::response::Response { + let h = res.headers_mut(); + h.insert("deprecation", axum::http::HeaderValue::from_static("true")); + h.insert( + "link", + axum::http::HeaderValue::from_static("; rel=\"successor-version\""), + ); + res +} + +async fn deprecate_spans(mut res: axum::response::Response) -> axum::response::Response { + let h = res.headers_mut(); + h.insert("deprecation", axum::http::HeaderValue::from_static("true")); + h.insert( + "link", + axum::http::HeaderValue::from_static("; rel=\"successor-version\""), + ); + res +} diff --git a/server/h-api/src/routes/agent_sessions.rs b/server/h-api/src/routes/agent_sessions.rs index 3edcc0f8..3b8cd93c 100644 --- a/server/h-api/src/routes/agent_sessions.rs +++ b/server/h-api/src/routes/agent_sessions.rs @@ -4,7 +4,7 @@ use axum::extract::State; use axum::response::IntoResponse; use serde::Deserialize; use h_storage::query::{ - decode_session_cursor, decode_session_turns_cursor, SessionListQuery, SessionTurnsQuery, + decode_session_cursor, decode_session_turns_cursor, SessionListQuery, SessionTracesQuery, }; use h_storage::StorageBackend; @@ -93,12 +93,12 @@ pub async fn turns( _ => None, }; - let query = SessionTurnsQuery { + let query = SessionTracesQuery { source_id, session_id, cursor, page_size: params.page_size.clamp(1, 200), }; - let page = storage.query_session_turns(&query).await?; + let page = storage.query_session_traces(&query).await?; Ok(ApiResponse::ok(page)) } diff --git a/server/h-api/src/routes/export.rs b/server/h-api/src/routes/export.rs index ed6e61e2..789ddac1 100644 --- a/server/h-api/src/routes/export.rs +++ b/server/h-api/src/routes/export.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use axum::body::Body; use axum::extract::State; use axum::http::{Response, StatusCode}; -use h_storage::query::{TurnDetail, TurnsQuery}; +use h_storage::query::{TraceDetail, TracesQuery}; use h_storage::StorageBackend; use serde::Deserialize; use serde_json::{json, Value}; @@ -46,7 +46,7 @@ pub struct ExportTrajectoryParams { pub session_id: Option, } -/// Mirrors `agent_turns::TurnsParams` (the list filters) so a batch export +/// Mirrors `traces::TracesParams` (the list filters) so a batch export /// covers exactly the turns the user is looking at. #[derive(Debug, Deserialize)] pub struct ExportBatchParams { @@ -86,7 +86,7 @@ pub async fn single( .filter(|s| !s.is_empty()) .ok_or_else(|| ApiError::InvalidParam("turn scope requires turn_id".into()))?; let detail = storage - .query_turn_by_id(turn_id) + .query_trace_by_id(turn_id) .await? .ok_or_else(|| ApiError::NotFound(format!("turn not found: {turn_id}")))?; (detail, "turn", format!("trajectory-{turn_id}.jsonl")) @@ -130,7 +130,7 @@ pub async fn batch( .collect::, _>>()?; let page_size = p.limit.unwrap_or(1000).clamp(1, MAX_BATCH_TURNS); - let query = TurnsQuery { + let query = TracesQuery { time_range: to_time_range(p.start, p.end)?, filter: to_dimension_filter(&p.wire_api, &p.model, &p.server_ip, &None), client_ips: parse_csv(&p.client_ip), @@ -144,12 +144,12 @@ pub async fn batch( include_proxy_hops: p.include_proxy_hops, }; - let page = storage.query_turns(&query).await?; + let page = storage.query_traces(&query).await?; let total = page.items.len(); let mut lines: Vec = Vec::new(); for item in &page.items { - // TurnListItem lacks final_call_id; resolve the full detail per turn. - let Some(detail) = storage.query_turn_by_id(&item.turn_id).await? else { + // TraceListItem lacks final_call_id; resolve the full detail per turn. + let Some(detail) = storage.query_trace_by_id(&item.turn_id).await? else { continue; }; match line_from_turn_detail(&storage, &detail, "turn").await { @@ -174,9 +174,9 @@ async fn resolve_session_last_turn( storage: &Arc, source_id: &str, session_id: &str, -) -> Result, ApiError> { +) -> Result, ApiError> { let page = storage - .query_session_turns(&h_storage::query::SessionTurnsQuery { + .query_session_traces(&h_storage::query::SessionTracesQuery { source_id: source_id.to_string(), session_id: session_id.to_string(), cursor: None, @@ -186,10 +186,10 @@ async fn resolve_session_last_turn( let Some(last) = page.items.first() else { return Ok(None); }; - Ok(storage.query_turn_by_id(&last.turn_id).await?) + Ok(storage.query_trace_by_id(&last.turn_id).await?) } -/// Reconstruct one trajectory line from a resolved `TurnDetail`. +/// Reconstruct one trajectory line from a resolved `TraceDetail`. /// /// Outer `Result` = a genuine storage/internal failure (500 on the single path; /// counted as skipped on the batch path). Inner `Result` = `Ok(line)` or @@ -197,14 +197,14 @@ async fn resolve_session_last_turn( /// truncated/missing body) the caller turns into a 400 (single) or a skip (batch). async fn line_from_turn_detail( storage: &Arc, - detail: &TurnDetail, + detail: &TraceDetail, scope_label: &str, ) -> Result, ApiError> { let Some(final_call_id) = detail.final_call_id.clone() else { return Ok(Err("turn has no terminal call".to_string())); }; let calls = storage - .query_calls_by_ids(&[final_call_id.clone()], true) + .query_spans_by_ids(&[final_call_id.clone()], true) .await?; let Some(call) = calls.into_iter().next() else { return Ok(Err("terminal call body unavailable".to_string())); diff --git a/server/h-api/src/routes/mod.rs b/server/h-api/src/routes/mod.rs index 6661d34c..a4436df3 100644 --- a/server/h-api/src/routes/mod.rs +++ b/server/h-api/src/routes/mod.rs @@ -1,5 +1,4 @@ pub mod agent_sessions; -pub mod agent_turns; pub mod capture_interfaces; pub mod capture_sources; pub mod export; @@ -7,8 +6,9 @@ pub mod filters; pub mod health; pub mod http_exchanges; pub mod internal_metrics; -pub mod llm_calls; pub mod metrics; pub mod pcap_extract; pub mod runtime_config; pub mod services; +pub mod spans; +pub mod traces; diff --git a/server/h-api/src/routes/llm_calls.rs b/server/h-api/src/routes/spans.rs similarity index 94% rename from server/h-api/src/routes/llm_calls.rs rename to server/h-api/src/routes/spans.rs index 2071c65c..d8cb9e48 100644 --- a/server/h-api/src/routes/llm_calls.rs +++ b/server/h-api/src/routes/spans.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use axum::extract::State; use axum::response::IntoResponse; use serde::Deserialize; -use h_storage::query::CallsQuery; +use h_storage::query::SpansQuery; use h_storage::StorageBackend; use crate::extractors::{Path, Query}; @@ -11,7 +11,7 @@ use crate::params::*; use crate::response::{ApiError, ApiResponse}; #[derive(Debug, Deserialize)] -pub struct CallsParams { +pub struct SpansParams { pub start: i64, pub end: i64, #[serde(default)] @@ -61,7 +61,7 @@ fn default_page_size() -> u32 { pub async fn list( State(storage): State>, - Query(params): Query, + Query(params): Query, ) -> Result { let page_size = params.page_size.min(200); let status_codes: Vec = parse_csv(¶ms.status_code) @@ -91,7 +91,7 @@ pub async fn list( } }; - let query = CallsQuery { + let query = SpansQuery { time_range: to_time_range(params.start, params.end)?, filter: to_dimension_filter(¶ms.wire_api, ¶ms.model, ¶ms.server_ip, &None), status_codes, @@ -106,7 +106,7 @@ pub async fn list( page_size, }; - let page = storage.query_calls(&query).await?; + let page = storage.query_spans(&query).await?; Ok(ApiResponse::ok(page)) } @@ -114,7 +114,7 @@ pub async fn detail( State(storage): State>, Path(id): Path, ) -> Result { - match storage.query_call_by_id(&id).await? { + match storage.query_span_by_id(&id).await? { Some(detail) => Ok(ApiResponse::ok(detail)), None => Err(ApiError::NotFound(format!("call not found: {id}"))), } diff --git a/server/h-api/src/routes/agent_turns.rs b/server/h-api/src/routes/traces.rs similarity index 93% rename from server/h-api/src/routes/agent_turns.rs rename to server/h-api/src/routes/traces.rs index e3d43b95..9b0596d7 100644 --- a/server/h-api/src/routes/agent_turns.rs +++ b/server/h-api/src/routes/traces.rs @@ -4,11 +4,11 @@ use axum::extract::State; use axum::response::IntoResponse; use serde::Deserialize; use h_storage::query::{ - AgentActivityPoint, AgentActivityQuery, AgentKindSummary, AgentSummaryQuery, CallDetail, + AgentActivityPoint, AgentActivityQuery, AgentKindSummary, AgentSummaryQuery, SpanDetail, HeaderDiffEntry, HeaderDiffKind, HeaderValueByLeg, LatencyBreakdown, ModelRewrite, - ProxyViewMember, ProxyViewResponse, TurnDetail, TurnListItem, TurnsQuery, + ProxyViewMember, ProxyViewResponse, TraceDetail, TraceListItem, TracesQuery, }; -use h_turn::AgentTurn; +use h_turn::Trace; use crate::extractors::{Path, Query}; use crate::params::*; @@ -16,7 +16,7 @@ use crate::response::{ApiError, ApiResponse}; use crate::ApiAgentTurnsContext; #[derive(Debug, Deserialize)] -pub struct TurnsParams { +pub struct TracesParams { pub start: i64, pub end: i64, #[serde(default)] @@ -65,7 +65,7 @@ fn default_page_size() -> u32 { pub async fn list( State(ctx): State, - Query(params): Query, + Query(params): Query, ) -> Result { let page_size = params.page_size.min(200); @@ -77,7 +77,7 @@ pub async fn list( }) .collect::, _>>()?; - let query = TurnsQuery { + let query = TracesQuery { time_range: to_time_range(params.start, params.end)?, filter: to_dimension_filter(¶ms.wire_api, ¶ms.model, ¶ms.server_ip, &None), client_ips: parse_csv(¶ms.client_ip), @@ -91,7 +91,7 @@ pub async fn list( include_proxy_hops: params.include_proxy_hops, }; - let mut page = ctx.storage.query_turns(&query).await?; + let mut page = ctx.storage.query_traces(&query).await?; // Snapshot the in-memory active-turn registry, filter by the same // params the SQL query used, and prepend matching rows to page 1 so @@ -120,8 +120,8 @@ pub async fn list( /// Read the active-turn registry under its read lock, filter by the /// query's time range, dimension filter, status, agent_kind, and -/// client_ip lists, and convert the survivors to `TurnListItem`s. -fn collect_in_progress(ctx: &ApiAgentTurnsContext, query: &TurnsQuery) -> Vec { +/// client_ip lists, and convert the survivors to `TraceListItem`s. +fn collect_in_progress(ctx: &ApiAgentTurnsContext, query: &TracesQuery) -> Vec { let map = match ctx.active_turns.read() { Ok(g) => g, Err(_) => return Vec::new(), // poisoned lock — degrade to empty @@ -130,7 +130,7 @@ fn collect_in_progress(ctx: &ApiAgentTurnsContext, query: &TurnsQuery) -> Vec = map + let mut out: Vec = map .values() .filter(|t| { // Time-range overlap with the requested window. An in-progress @@ -163,17 +163,17 @@ fn collect_in_progress(ctx: &ApiAgentTurnsContext, query: &TurnsQuery) -> Vec TurnListItem { - TurnListItem { +fn agent_turn_to_list_item(t: &Trace) -> TraceListItem { + TraceListItem { turn_id: t.turn_id.clone(), source_id: t.source_id.clone(), session_id: t.session_id.clone(), - // `TurnListItem.start_time`/`end_time` are MILLISECONDS — the DB query + // `TraceListItem.start_time`/`end_time` are MILLISECONDS — the DB query // path returns `epoch_ms(start_time)`, and the console renders the field // with `new Date(ms)`. The in-memory turn carries MICROSECONDS, so divide // by 1000 here; emitting the raw µs made in-progress turns render as the @@ -209,12 +209,12 @@ fn agent_turn_to_list_item(t: &AgentTurn) -> TurnListItem { } /// Wire-api / model / server-ip dimension filter — mirrors -/// `query_turns`'s SQL WHERE clause. Each list is an OR group; an empty +/// `query_traces`'s SQL WHERE clause. Each list is an OR group; an empty /// list means "any". The turn matches if for every non-empty list at /// least one entry matches the corresponding turn field (with `model` /// matching against the turn's `models_used` set since a turn can span /// several models). -fn matches_filter(t: &AgentTurn, f: &h_storage::query::DimensionFilter) -> bool { +fn matches_filter(t: &Trace, f: &h_storage::query::DimensionFilter) -> bool { if !f.wire_apis.is_empty() && !f.wire_apis.iter().any(|w| w == &t.wire_api) { return false; } @@ -245,20 +245,20 @@ pub async fn detail( return Ok(ApiResponse::ok(agent_turn_to_detail(t.clone()))); } } - match ctx.storage.query_turn_by_id(&turn_id).await? { + match ctx.storage.query_trace_by_id(&turn_id).await? { Some(detail) => Ok(ApiResponse::ok(detail)), None => Err(ApiError::NotFound(format!("turn not found: {turn_id}"))), } } -/// Convert a snapshot `AgentTurn` to a `TurnDetail`-shaped payload. The -/// DB-side `query_turn_by_id` returns a richer `TurnDetail` that pulls +/// Convert a snapshot `Trace` to a `TraceDetail`-shaped payload. The +/// DB-side `query_trace_by_id` returns a richer `TraceDetail` that pulls /// full bodies from referenced `llm_calls` rows; for in-progress turns /// we don't have those joins, so the previews and counts are returned /// as-is. The frontend tolerates the lighter payload (preview-only). -fn agent_turn_to_detail(t: AgentTurn) -> h_storage::query::TurnDetail { - use h_storage::query::TurnDetail; - TurnDetail { +fn agent_turn_to_detail(t: Trace) -> h_storage::query::TraceDetail { + use h_storage::query::TraceDetail; + TraceDetail { turn_id: t.turn_id, source_id: t.source_id, session_id: t.session_id, @@ -285,7 +285,7 @@ fn agent_turn_to_detail(t: AgentTurn) -> h_storage::query::TurnDetail { user_input: t.user_input_preview, final_call_id: t.final_call_id, final_answer: t.final_answer_preview, - call_ids: t.call_ids, + span_ids: t.span_ids, metadata: Some(t.metadata), tool_surfaces: t.tool_surfaces.iter().map(|s| s.to_string()).collect(), tool_call_total: t.tool_call_total, @@ -299,7 +299,7 @@ fn agent_turn_to_detail(t: AgentTurn) -> h_storage::query::TurnDetail { } #[derive(Debug, Default, Deserialize)] -pub struct CallsParams { +pub struct SpansParams { /// `lite=1` strips request_body, response_body, request_headers, /// response_headers from the response so a mega-turn (hundreds of /// agentic iterations × hundreds of KB body each) doesn't OOM the @@ -312,33 +312,33 @@ pub struct CallsParams { pub async fn calls( State(ctx): State, Path(turn_id): Path, - Query(params): Query, + Query(params): Query, ) -> Result { let include_bodies = params.lite == 0; - // In-progress turns: pull call_ids from the in-memory registry + // In-progress turns: pull span_ids from the in-memory registry // snapshot, then ask storage to fetch the matching `llm_calls` // rows. A call may be ingested into the tracker microseconds before // its row gets flushed from `WriteBuffer` to DuckDB; in that // narrow window the call is missing from the result and shows up on // the next refresh. Total lag is bounded by storage.flush_interval_ms // (200 ms after PR #5). - let in_progress_call_ids: Option> = ctx + let in_progress_span_ids: Option> = ctx .active_turns .read() .ok() - .and_then(|map| map.get(&turn_id).map(|t| t.call_ids.clone())); + .and_then(|map| map.get(&turn_id).map(|t| t.span_ids.clone())); - if let Some(call_ids) = in_progress_call_ids { + if let Some(span_ids) = in_progress_span_ids { let items = ctx .storage - .query_calls_by_ids(&call_ids, include_bodies) + .query_spans_by_ids(&span_ids, include_bodies) .await?; return Ok(ApiResponse::ok(items)); } let items = ctx .storage - .query_turn_calls(&turn_id, include_bodies) + .query_trace_spans(&turn_id, include_bodies) .await?; Ok(ApiResponse::ok(items)) } @@ -356,7 +356,7 @@ pub async fn proxy_view( State(ctx): State, Path(turn_id): Path, ) -> Result { - let turn = match ctx.storage.query_turn_by_id(&turn_id).await? { + let turn = match ctx.storage.query_trace_by_id(&turn_id).await? { Some(t) => t, None => return Err(ApiError::NotFound(format!("turn not found: {turn_id}"))), }; @@ -404,7 +404,7 @@ pub async fn proxy_view( for pid in &peer_ids { // Resolve each peer's role by fetching its metadata.proxy.role. // Cheap relative to the request bodies we're about to pull. - let peer_turn = ctx.storage.query_turn_by_id(pid).await?; + let peer_turn = ctx.storage.query_trace_by_id(pid).await?; let role = peer_turn .as_ref() .and_then(|t| t.metadata.as_ref()) @@ -423,16 +423,16 @@ pub async fn proxy_view( .then_with(|| a_id.cmp(b_id)) }); - // Fetch each member's TurnDetail + first call body. + // Fetch each member's TraceDetail + first call body. let mut members: Vec = Vec::with_capacity(order.len()); for (tid, role) in &order { - let detail = match ctx.storage.query_turn_by_id(tid).await? { + let detail = match ctx.storage.query_trace_by_id(tid).await? { Some(t) => t, None => continue, // peer evicted by retention since the sweep — skip }; - let first_call_id = detail.call_ids.first().cloned(); - let first_call: Option = match first_call_id { - Some(ref id) => ctx.storage.query_call_by_id(id).await?, + let first_call_id = detail.span_ids.first().cloned(); + let first_call: Option = match first_call_id { + Some(ref id) => ctx.storage.query_span_by_id(id).await?, None => None, }; members.push(member_from(detail, role.clone(), first_call)); @@ -466,9 +466,9 @@ fn role_sort_key(role: &str) -> u8 { } fn member_from( - detail: TurnDetail, + detail: TraceDetail, role: String, - first_call: Option, + first_call: Option, ) -> ProxyViewMember { let ( request_headers, @@ -829,8 +829,8 @@ mod proxy_view_tests { assert!(parse_headers_json(Some("garbage")).is_empty()); } - fn turn_with_start_us(start_us: i64) -> h_turn::AgentTurn { - h_turn::AgentTurn { + fn turn_with_start_us(start_us: i64) -> h_turn::Trace { + h_turn::Trace { source_id: String::new(), turn_id: "t".into(), session_id: "s".into(), @@ -849,13 +849,13 @@ mod proxy_view_tests { total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, total_cost_usd: None, - status: h_turn::TurnStatus::InProgress, + status: h_turn::TraceStatus::InProgress, final_finish_reason: None, user_input_preview: None, user_call_id: None, final_answer_preview: None, final_call_id: None, - call_ids: vec![], + span_ids: vec![], metadata: serde_json::json!({}), tool_surfaces: vec![], tool_call_total: 0, @@ -866,8 +866,8 @@ mod proxy_view_tests { // ───────────────────── timestamp-unit suite ───────────────────── // - // The active-turn registry holds `AgentTurn`s in MICROSECONDS, but the - // console-facing `TurnListItem`/`TurnDetail.start_time`/`end_time` are + // The active-turn registry holds `Trace`s in MICROSECONDS, but the + // console-facing `TraceListItem`/`TraceDetail.start_time`/`end_time` are // MILLISECONDS — the DB query path returns `epoch_ms(start_time)` and the // UI renders with `new Date(ms)`. These tests pin that boundary so an // in-progress (active-registry) turn never again diverges from a finalized diff --git a/server/h-api/tests/duckdb_routes.rs b/server/h-api/tests/duckdb_routes.rs index 58a96612..e19ab532 100644 --- a/server/h-api/tests/duckdb_routes.rs +++ b/server/h-api/tests/duckdb_routes.rs @@ -91,7 +91,7 @@ async fn finish_reasons_endpoint_returns_one_series_per_raw_value() { test_runtime_config_context(), test_health_context(), std::sync::Arc::new(vec![]), - h_turn::new_active_turn_registry(), + h_turn::new_active_trace_registry(), ); // start/end are seconds (matches existing /api/metrics/* convention). @@ -168,7 +168,7 @@ async fn finish_reasons_endpoint_accepts_csv_wire_api_filter() { test_runtime_config_context(), test_health_context(), std::sync::Arc::new(vec![]), - h_turn::new_active_turn_registry(), + h_turn::new_active_trace_registry(), ); let start_s = (ts / 1_000_000) - 1; @@ -252,7 +252,7 @@ async fn finish_reasons_endpoint_filters_by_server_ip() { test_runtime_config_context(), test_health_context(), std::sync::Arc::new(vec![]), - h_turn::new_active_turn_registry(), + h_turn::new_active_trace_registry(), ); let start_s = (ts / 1_000_000) - 1; @@ -366,7 +366,7 @@ async fn timeseries_endpoint_backfills_full_grid_for_sparse_data() { test_runtime_config_context(), test_health_context(), std::sync::Arc::new(vec![]), - h_turn::new_active_turn_registry(), + h_turn::new_active_trace_registry(), ); let start_s = ts / 1_000_000; @@ -422,7 +422,7 @@ async fn timeseries_endpoint_emits_grid_when_no_rows_exist() { test_runtime_config_context(), test_health_context(), std::sync::Arc::new(vec![]), - h_turn::new_active_turn_registry(), + h_turn::new_active_trace_registry(), ); let start_s = 1_700_000_040i64; @@ -461,7 +461,7 @@ async fn finish_reasons_endpoint_rejects_invalid_granularity() { test_runtime_config_context(), test_health_context(), std::sync::Arc::new(vec![]), - h_turn::new_active_turn_registry(), + h_turn::new_active_trace_registry(), ); let resp = app .oneshot( @@ -475,7 +475,7 @@ async fn finish_reasons_endpoint_rejects_invalid_granularity() { assert_eq!(resp.status(), StatusCode::BAD_REQUEST); } -// ---------- /api/llm-calls* ---------- +// ---------- /api/spans* (canonical) + /api/llm-calls* (deprecated alias) ---------- #[tokio::test] async fn invalid_status_code_returns_json_envelope() { @@ -490,13 +490,13 @@ async fn invalid_status_code_returns_json_envelope() { test_runtime_config_context(), test_health_context(), std::sync::Arc::new(vec![]), - h_turn::new_active_turn_registry(), + h_turn::new_active_trace_registry(), ); let resp = app .oneshot( Request::builder() - .uri("/api/llm-calls?start=0&end=1&status_code=200,abc") + .uri("/api/spans?start=0&end=1&status_code=200,abc") .body(Body::empty()) .unwrap(), ) @@ -534,13 +534,13 @@ async fn contains_params_parse() { test_runtime_config_context(), test_health_context(), std::sync::Arc::new(vec![]), - h_turn::new_active_turn_registry(), + h_turn::new_active_trace_registry(), ); let resp = app .oneshot( Request::builder() - .uri("/api/llm-calls?start=0&end=1&client_ip=10.0.0.1&request_path=/v1/chat") + .uri("/api/spans?start=0&end=1&client_ip=10.0.0.1&request_path=/v1/chat") .body(Body::empty()) .unwrap(), ) @@ -550,6 +550,51 @@ async fn contains_params_parse() { assert_eq!(resp.status(), StatusCode::OK); } +/// The deprecated pre-rename aliases (`/api/llm-calls`, `/api/agent-turns`) +/// still resolve to the same handlers AND carry the RFC 8594 `Deprecation` +/// header + a `Link` to the canonical successor, so clients can detect they +/// should migrate to `/api/spans` / `/api/traces`. +#[tokio::test] +async fn deprecated_aliases_work_and_carry_deprecation_header() { + let backend = DuckDbBackend::open(":memory:").unwrap(); + ::init(&backend) + .await + .unwrap(); + let storage: std::sync::Arc = std::sync::Arc::new(backend); + let app = router( + storage, + test_metrics_context(), + test_runtime_config_context(), + test_health_context(), + std::sync::Arc::new(vec![]), + h_turn::new_active_trace_registry(), + ); + + for (alias, successor) in [ + ("/api/llm-calls?start=0&end=1", ""), + ("/api/agent-turns?start=0&end=1", ""), + ] { + let resp = app + .clone() + .oneshot(Request::builder().uri(alias).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK, "alias {alias} must still serve"); + assert_eq!( + resp.headers().get("deprecation").map(|v| v.to_str().unwrap()), + Some("true"), + "alias {alias} must carry Deprecation: true", + ); + assert!( + resp.headers() + .get("link") + .map(|v| v.to_str().unwrap().contains(successor)) + .unwrap_or(false), + "alias {alias} Link must point at {successor}", + ); + } +} + /// `/api/metrics/timeseries?tool_surface=...` must SUM only the rows whose /// `tool_surface` column matches one of the CSV values, excluding other /// surfaces. An invalid value returns 400 instead of silently degrading to @@ -637,7 +682,7 @@ async fn metrics_filters_by_tool_surface() { test_runtime_config_context(), test_health_context(), std::sync::Arc::new(vec![]), - h_turn::new_active_turn_registry(), + h_turn::new_active_trace_registry(), ); let start_s = ts / 1_000_000; diff --git a/server/h-common/src/config.rs b/server/h-common/src/config.rs index a9f0f12d..7161170f 100644 --- a/server/h-common/src/config.rs +++ b/server/h-common/src/config.rs @@ -488,7 +488,7 @@ pub struct QueueConfig { /// llm stage → shared storage sink (LlmCall records) #[serde(default = "default_queue_capacity")] pub storage_calls: usize, - /// turn stage → shared storage sink (AgentTurn records) + /// turn stage → shared storage sink (Trace records) #[serde(default = "default_queue_capacity")] pub storage_turns: usize, /// metrics stage → shared storage sink (LlmMetric records) @@ -599,12 +599,14 @@ pub struct RetentionConfig { pub enabled: bool, #[serde(default = "default_retention_check_interval_secs")] pub check_interval_secs: u64, - /// Max age in days for `llm_calls`. `0` = never expire. - #[serde(default = "default_calls_retention_days")] - pub calls: u32, - /// Max age in days for `agent_turns`. `0` = never expire. - #[serde(default = "default_turns_retention_days")] - pub turns: u32, + /// Max age in days for `spans` (formerly `calls`). `0` = never expire. + /// The legacy `calls` key is still accepted via serde alias. + #[serde(default = "default_spans_retention_days", alias = "calls")] + pub spans: u32, + /// Max age in days for `traces` (formerly `turns`). `0` = never expire. + /// The legacy `turns` key is still accepted via serde alias. + #[serde(default = "default_traces_retention_days", alias = "turns")] + pub traces: u32, /// Max age in days for `http_exchanges`. `0` = never expire. Raw headers + /// bodies make this the bulkiest table, so a short forensics window keeps /// storage bounded. @@ -631,8 +633,8 @@ impl Default for RetentionConfig { Self { enabled: default_retention_enabled(), check_interval_secs: default_retention_check_interval_secs(), - calls: default_calls_retention_days(), - turns: default_turns_retention_days(), + spans: default_spans_retention_days(), + traces: default_traces_retention_days(), http_exchanges: default_http_exchanges_retention_days(), metrics: HashMap::new(), unknown_granularities: Vec::new(), @@ -644,13 +646,13 @@ fn default_retention_enabled() -> bool { true } -fn default_calls_retention_days() -> u32 { +fn default_spans_retention_days() -> u32 { 30 } -fn default_turns_retention_days() -> u32 { - // Must satisfy turns <= calls (see ConfigIssue::TurnsRetentionExceedsCalls). - // Kept equal to calls so the default deploy is consistent without forcing +fn default_traces_retention_days() -> u32 { + // Must satisfy traces <= spans (see ConfigIssue::TracesRetentionExceedsSpans). + // Kept equal to spans so the default deploy is consistent without forcing // operators to think about the dependency. 30 } @@ -1033,14 +1035,14 @@ pub enum ConfigIssue { /// Fail validation hard so the operator sees the problem before deploy. UnsafePcapDumpPipelineName { pipeline: String }, /// `agent_turns` retention outlives `llm_calls` retention, so the - /// no-JOIN turn-detail read (`agent_turns.call_ids` → `llm_calls` + /// no-JOIN turn-detail read (`agent_turns.span_ids` → `llm_calls` /// IN-lookup) returns empty/partial calls for surviving turns once the - /// calls sweep crosses their `request_time`. `turns_days = 0` is the + /// calls sweep crosses their `request_time`. `traces_days = 0` is the /// sentinel for "never expire" (which always violates a finite - /// `calls_days`); finite-vs-finite triggers when `turns_days > calls_days`. - /// Only emitted when `calls_days > 0` — infinite calls retention can + /// `spans_days`); finite-vs-finite triggers when `traces_days > spans_days`. + /// Only emitted when `spans_days > 0` — infinite calls retention can /// satisfy any turns retention. - TurnsRetentionExceedsCalls { turns_days: u32, calls_days: u32 }, + TracesRetentionExceedsSpans { traces_days: u32, spans_days: u32 }, } impl ConfigIssue { @@ -1058,7 +1060,7 @@ impl ConfigIssue { | Self::StoragePathParentUnwritable { .. } | Self::UnknownRetentionGranularity(_) | Self::UnsafePcapDumpPipelineName { .. } - | Self::TurnsRetentionExceedsCalls { .. } => IssueSeverity::Error, + | Self::TracesRetentionExceedsSpans { .. } => IssueSeverity::Error, } } } @@ -1125,21 +1127,21 @@ impl std::fmt::Display for ConfigIssue { sanitization, or '.' / '..'); the runtime cannot build a \ dump directory path" ), - Self::TurnsRetentionExceedsCalls { - turns_days, - calls_days, + Self::TracesRetentionExceedsSpans { + traces_days, + spans_days, } => { - let turns_str = if *turns_days == 0 { + let turns_str = if *traces_days == 0 { "never expire".to_string() } else { - format!("{turns_days}d") + format!("{traces_days}d") }; write!( f, - "storage.retention.turns ({turns_str}) outlives \ - storage.retention.calls ({calls_days}d): turns whose \ - llm_calls have been pruned will show empty/partial call \ - lists. Set turns <= calls (or set calls = 0 for infinite)." + "storage.retention.traces ({turns_str}) outlives \ + storage.retention.spans ({spans_days}d): traces whose \ + spans have been pruned will show empty/partial span \ + lists. Set traces <= spans (or set spans = 0 for infinite)." ) } } @@ -1266,17 +1268,17 @@ impl AppConfig { issues.push(ConfigIssue::UnknownRetentionGranularity(unknown.clone())); } - // agent_turns references llm_calls via JSON call_ids; the no-JOIN + // agent_turns references llm_calls via JSON span_ids; the no-JOIN // turn-detail read trusts that referenced calls still exist. If turns // outlive calls, surviving turns end up pointing at deleted call ids // and the detail view shows empty/partial results. 0 = never expire, - // so finite calls_days with any larger (or 0) turns_days is broken. - let calls_days = self.storage.retention.calls; - let turns_days = self.storage.retention.turns; - if calls_days > 0 && (turns_days == 0 || turns_days > calls_days) { - issues.push(ConfigIssue::TurnsRetentionExceedsCalls { - turns_days, - calls_days, + // so finite spans_days with any larger (or 0) traces_days is broken. + let spans_days = self.storage.retention.spans; + let traces_days = self.storage.retention.traces; + if spans_days > 0 && (traces_days == 0 || traces_days > spans_days) { + issues.push(ConfigIssue::TracesRetentionExceedsSpans { + traces_days, + spans_days, }); } @@ -1339,9 +1341,9 @@ mod phase2_tests { let cfg = RetentionConfig::default(); assert!(cfg.enabled); assert_eq!(cfg.check_interval_secs, 3600); - assert_eq!(cfg.calls, 30); - // turns must not exceed calls — see ConfigIssue::TurnsRetentionExceedsCalls. - assert_eq!(cfg.turns, 30); + assert_eq!(cfg.spans, 30); + // turns must not exceed calls — see ConfigIssue::TracesRetentionExceedsSpans. + assert_eq!(cfg.traces, 30); assert_eq!(cfg.http_exchanges, 7); // metrics map stays empty — per-granularity defaults are merged at // policy-build time in h-storage so users can override one label @@ -1353,9 +1355,9 @@ mod phase2_tests { fn storage_config_embeds_retention_defaults() { let cfg = StorageConfig::default(); assert!(cfg.retention.enabled); - assert_eq!(cfg.retention.calls, 30); - // turns must not exceed calls — see ConfigIssue::TurnsRetentionExceedsCalls. - assert_eq!(cfg.retention.turns, 30); + assert_eq!(cfg.retention.spans, 30); + // turns must not exceed calls — see ConfigIssue::TracesRetentionExceedsSpans. + assert_eq!(cfg.retention.traces, 30); } #[test] @@ -1378,8 +1380,8 @@ mod phase2_tests { .expect("deserialize retention"); assert!(cfg.enabled); assert_eq!(cfg.check_interval_secs, 60); - assert_eq!(cfg.calls, 7); - assert_eq!(cfg.turns, 30); + assert_eq!(cfg.spans, 7); + assert_eq!(cfg.traces, 30); assert_eq!(cfg.metrics.get("10s"), Some(&1)); assert_eq!(cfg.metrics.get("1m"), Some(&7)); assert_eq!(cfg.metrics.get("1h"), Some(&365)); @@ -1720,19 +1722,19 @@ mod phase2_tests { assert!( issues.iter().any(|i| matches!( i, - ConfigIssue::TurnsRetentionExceedsCalls { - turns_days: 30, - calls_days: 7 + ConfigIssue::TracesRetentionExceedsSpans { + traces_days: 30, + spans_days: 7 } )), - "expected TurnsRetentionExceedsCalls(30, 7), got {issues:?}" + "expected TracesRetentionExceedsSpans(30, 7), got {issues:?}" ); } #[test] fn validate_turns_infinite_with_calls_finite_is_error() { // turns = 0 (never expire) and calls > 0 → turns outlive every call. - // Detected as the same issue (sentinel turns_days = 0). + // Detected as the same issue (sentinel traces_days = 0). let toml = r#" [[pipeline]] name = "p" @@ -1749,12 +1751,12 @@ mod phase2_tests { assert!( issues.iter().any(|i| matches!( i, - ConfigIssue::TurnsRetentionExceedsCalls { - turns_days: 0, - calls_days: 7 + ConfigIssue::TracesRetentionExceedsSpans { + traces_days: 0, + spans_days: 7 } )), - "expected TurnsRetentionExceedsCalls(0, 7), got {issues:?}" + "expected TracesRetentionExceedsSpans(0, 7), got {issues:?}" ); } @@ -1777,8 +1779,8 @@ mod phase2_tests { assert!( !issues .iter() - .any(|i| matches!(i, ConfigIssue::TurnsRetentionExceedsCalls { .. })), - "expected no TurnsRetentionExceedsCalls, got {issues:?}" + .any(|i| matches!(i, ConfigIssue::TracesRetentionExceedsSpans { .. })), + "expected no TracesRetentionExceedsSpans, got {issues:?}" ); } @@ -1800,8 +1802,8 @@ mod phase2_tests { assert!( !issues .iter() - .any(|i| matches!(i, ConfigIssue::TurnsRetentionExceedsCalls { .. })), - "expected no TurnsRetentionExceedsCalls, got {issues:?}" + .any(|i| matches!(i, ConfigIssue::TracesRetentionExceedsSpans { .. })), + "expected no TracesRetentionExceedsSpans, got {issues:?}" ); } @@ -1821,7 +1823,7 @@ mod phase2_tests { assert!( !issues .iter() - .any(|i| matches!(i, ConfigIssue::TurnsRetentionExceedsCalls { .. })), + .any(|i| matches!(i, ConfigIssue::TracesRetentionExceedsSpans { .. })), "default config raised retention constraint: {issues:?}" ); } diff --git a/server/h-common/src/internal_metrics.rs b/server/h-common/src/internal_metrics.rs index 931a24bf..856385d5 100644 --- a/server/h-common/src/internal_metrics.rs +++ b/server/h-common/src/internal_metrics.rs @@ -220,7 +220,7 @@ define_metrics! { TurnHeartbeatsDropped => { kind: Counter, group: Turn, short: "turn_heartbeats_dropped" }, TurnActive => { kind: Gauge, group: Turn, short: "turn_calls_buffered" }, // Process-wide registry of in-progress agent turns (size of - // ActiveTurnRegistry). Distinct from `turn_calls_buffered`, which sums + // ActiveTraceRegistry). Distinct from `turn_calls_buffered`, which sums // pending LLM calls inside per-session turn buffers — that gauge counts // calls, not conversations. `agent_turns_open` is the truthful // "concurrent in-flight agent turns" signal and is what the dashboard diff --git a/server/h-llm/src/agents/generic.rs b/server/h-llm/src/agents/generic.rs index a082f02b..ab23395d 100644 --- a/server/h-llm/src/agents/generic.rs +++ b/server/h-llm/src/agents/generic.rs @@ -1,5 +1,5 @@ //! Generic agent profile — synthesizes a session id from request / response -//! payload tool-call anchors, so Heron can still produce `AgentTurn`s for +//! payload tool-call anchors, so Heron can still produce `Trace`s for //! header-less tool-using LLM traffic across supported wire APIs. //! //! `agent_kind == "generic"` is wire-api-agnostic; downstream consumers read diff --git a/server/h-llm/src/agents/opencode.rs b/server/h-llm/src/agents/opencode.rs index 6281bc29..a44886be 100644 --- a/server/h-llm/src/agents/opencode.rs +++ b/server/h-llm/src/agents/opencode.rs @@ -14,7 +14,7 @@ //! stable across the same chat session because the conversation prefix //! grows between user turns — so a long opencode chat fragments into //! several `gen-*` session ids and the UI shows each user turn as a -//! detached 1-call AgentTurn. Pulling `x-session-affinity` straight off +//! detached 1-call Trace. Pulling `x-session-affinity` straight off //! the wire keeps the whole conversation under one stable session. use crate::agent_primitives::{AgentPrimitives, SystemPromptMarkers}; diff --git a/server/h-llm/src/model.rs b/server/h-llm/src/model.rs index 8268fad0..642210ae 100644 --- a/server/h-llm/src/model.rs +++ b/server/h-llm/src/model.rs @@ -96,7 +96,7 @@ pub struct LlmCall { pub struct AgentCallInfo { /// Short stable agent name (e.g. `"claude-cli"`). Doubles as the profile /// selector (look up via `AgentProfileRegistry::find_by_name`) and the - /// persisted `AgentTurn.agent_kind` storage value. + /// persisted `Trace.agent_kind` storage value. pub agent_kind: &'static str, pub session_id: String, /// `Some(name)` if this call belongs to a sub-agent (e.g. `"task"` for diff --git a/server/h-llm/src/profile.rs b/server/h-llm/src/profile.rs index 8db10eac..d47e1dc4 100644 --- a/server/h-llm/src/profile.rs +++ b/server/h-llm/src/profile.rs @@ -69,7 +69,7 @@ impl TestCall { /// agent client (e.g. `claude-cli`, `codex-cli`). pub trait AgentProfile: Send + Sync { /// Short stable name (e.g. `"claude-cli"`). Persisted to storage as - /// `AgentTurn.agent_kind`. + /// `Trace.agent_kind`. fn name(&self) -> &'static str; /// Return true iff this profile handles the given call. diff --git a/server/h-storage-clickhouse/src/calls.rs b/server/h-storage-clickhouse/src/calls.rs index d8f12939..3a9c2bbf 100644 --- a/server/h-storage-clickhouse/src/calls.rs +++ b/server/h-storage-clickhouse/src/calls.rs @@ -1,4 +1,4 @@ -//! `llm_calls` table I/O — write + paginated / by-id / by-id-list reads. +//! `spans` table I/O — write + paginated / by-id / by-id-list reads. use clickhouse::Row; use serde::Deserialize; @@ -28,7 +28,7 @@ struct CountRow { n: u64, } -/// Row shape for the paginated `query_calls` list (mirrors the DuckDB SELECT). +/// Row shape for the paginated `query_spans` list (mirrors the DuckDB SELECT). #[derive(Row, Deserialize)] struct CallListRow { id: String, @@ -59,7 +59,7 @@ struct CallListRow { } #[derive(Row, Deserialize)] -struct CallDetailRow { +struct SpanDetailRow { id: String, source_id: String, request_time_ms: i64, @@ -123,13 +123,13 @@ struct TurnCallRow { } impl ClickHouseBackend { - pub(crate) async fn write_calls(&self, calls: Vec) -> Result<()> { + pub(crate) async fn write_spans(&self, calls: Vec) -> Result<()> { let rows: Vec = calls.into_iter().map(CallRow::from).collect(); - insert_all!(self.client, "llm_calls", CallRow, rows); + insert_all!(self.client, "spans", CallRow, rows); Ok(()) } - pub(crate) async fn query_calls(&self, query: &CallsQuery) -> Result { + pub(crate) async fn query_spans(&self, query: &SpansQuery) -> Result { if !VALID_SORT_FIELDS.contains(&query.sort_by.as_str()) { return Err(h_common::error::AppError::Storage(format!( "invalid sort_by field: {}", @@ -186,10 +186,10 @@ impl ClickHouseBackend { let total = self .client - .query(&format!("SELECT count() AS n FROM llm_calls WHERE {where_sql}")) + .query(&format!("SELECT count() AS n FROM spans WHERE {where_sql}")) .fetch_one::() .await - .map_err(|e| ch_err("query_calls count", e))? + .map_err(|e| ch_err("query_spans count", e))? .n; let offset = (query.page.saturating_sub(1)) as u64 * query.page_size as u64; @@ -199,7 +199,7 @@ impl ClickHouseBackend { input_tokens, output_tokens, client_ip, server_ip, server_port, request_path, \ response_body, is_agent_request, tool_surface, agent_topology, tool_call_count, \ tool_names_json, process_pid, process_comm, process_exe \ - FROM llm_calls WHERE {where_sql} \ + FROM spans WHERE {where_sql} \ ORDER BY {} {sort_order} LIMIT {} OFFSET {offset}", query.sort_by, query.page_size, ); @@ -208,13 +208,13 @@ impl ClickHouseBackend { .query(&items_sql) .fetch_all::() .await - .map_err(|e| ch_err("query_calls items", e))?; + .map_err(|e| ch_err("query_spans items", e))?; let items = rows.into_iter().map(call_list_item).collect(); - Ok(CallsPage { total, items }) + Ok(SpansPage { total, items }) } - pub(crate) async fn query_call_by_id(&self, id: &str) -> Result> { + pub(crate) async fn query_span_by_id(&self, id: &str) -> Result> { let sql = format!( "SELECT id, source_id, \ toUnixTimestamp64Milli(request_time) AS request_time_ms, \ @@ -225,47 +225,47 @@ impl ClickHouseBackend { client_ip, client_port, server_ip, server_port, request_body, response_body, \ request_headers, response_headers, is_agent_request, tool_surface, agent_topology, \ tool_call_count, tool_names_json, process_pid, process_comm, process_exe \ - FROM llm_calls WHERE id = '{}' LIMIT 1", + FROM spans WHERE id = '{}' LIMIT 1", escape_str(id) ); let row = self .client .query(&sql) - .fetch_all::() + .fetch_all::() .await - .map_err(|e| ch_err("query_call_by_id", e))? + .map_err(|e| ch_err("query_span_by_id", e))? .into_iter() .next(); Ok(row.map(call_detail)) } - pub(crate) async fn query_turn_calls( + pub(crate) async fn query_trace_spans( &self, turn_id: &str, include_bodies: bool, - ) -> Result> { - // No-JOIN two-step: resolve the turn's ordered call_ids, then fetch. - let call_ids = self.turn_call_ids(turn_id).await?; - self.read_calls_by_ids(&call_ids, include_bodies).await + ) -> Result> { + // No-JOIN two-step: resolve the turn's ordered span_ids, then fetch. + let span_ids = self.turn_span_ids(turn_id).await?; + self.read_calls_by_ids(&span_ids, include_bodies).await } - pub(crate) async fn query_calls_by_ids( + pub(crate) async fn query_spans_by_ids( &self, - call_ids: &[String], + span_ids: &[String], include_bodies: bool, - ) -> Result> { - self.read_calls_by_ids(call_ids, include_bodies).await + ) -> Result> { + self.read_calls_by_ids(span_ids, include_bodies).await } - /// Read `agent_turns.call_ids` (JSON array) for one turn. `FINAL` so the + /// Read `traces.span_ids` (JSON array) for one turn. `FINAL` so the /// latest ReplacingMergeTree version wins. - async fn turn_call_ids(&self, turn_id: &str) -> Result> { + async fn turn_span_ids(&self, turn_id: &str) -> Result> { #[derive(Row, Deserialize)] struct CallIdsRow { - call_ids: String, + span_ids: String, } let sql = format!( - "SELECT call_ids FROM agent_turns FINAL WHERE turn_id = '{}' LIMIT 1", + "SELECT span_ids FROM traces FINAL WHERE turn_id = '{}' LIMIT 1", escape_str(turn_id) ); let row = self @@ -273,25 +273,25 @@ impl ClickHouseBackend { .query(&sql) .fetch_all::() .await - .map_err(|e| ch_err("turn_call_ids", e))? + .map_err(|e| ch_err("turn_span_ids", e))? .into_iter() .next(); Ok(row - .map(|r| parse_json_string_list(Some(&r.call_ids))) + .map(|r| parse_json_string_list(Some(&r.span_ids))) .unwrap_or_default()) } - /// Shared "fetch calls by id list" — used by `query_turn_calls` (ids from - /// the persisted `agent_turns.call_ids`) and `query_calls_by_ids` (ids from + /// Shared "fetch calls by id list" — used by `query_trace_spans` (ids from + /// the persisted `traces.span_ids`) and `query_spans_by_ids` (ids from /// the in-memory active-turn registry). Calls not yet flushed simply don't /// return. Lite mode (`include_bodies = false`) selects NULL for the four /// heavy body/header fields. async fn read_calls_by_ids( &self, - call_ids: &[String], + span_ids: &[String], include_bodies: bool, - ) -> Result> { - if call_ids.is_empty() { + ) -> Result> { + if span_ids.is_empty() { return Ok(Vec::new()); } let body_columns = if include_bodies { @@ -315,9 +315,9 @@ impl ClickHouseBackend { wire_api, model, status_code, is_stream, finish_reason, ttft_ms, e2e_latency_ms, \ input_tokens, output_tokens, request_path, client_ip, client_port, server_ip, \ server_port, {body_columns} \ - FROM llm_calls WHERE id IN ({}) \ + FROM spans WHERE id IN ({}) \ ORDER BY request_time ASC, complete_time ASC", - sql_in_list(call_ids), + sql_in_list(span_ids), ); let rows = self .client @@ -335,7 +335,7 @@ impl ClickHouseBackend { r.output_tokens, r.response_body.as_deref(), ); - TurnCallItem { + TraceSpanItem { id: r.id, sequence: (i as u32) + 1, request_time: r.request_time_ms, @@ -386,12 +386,12 @@ fn row_process(pid: Option, comm: Option, exe: Option) -> O }) } -fn call_list_item(r: CallListRow) -> CallListItem { +fn call_list_item(r: CallListRow) -> SpanListItem { let tokens_estimated = derive_tokens_estimated(r.input_tokens, r.output_tokens, r.response_body.as_deref()); let tool_names = parse_json_string_list(r.tool_names_json.as_deref()); let process = row_process(r.process_pid, r.process_comm, r.process_exe); - CallListItem { + SpanListItem { id: r.id, source_id: r.source_id, request_time: r.request_time_ms, @@ -418,12 +418,12 @@ fn call_list_item(r: CallListRow) -> CallListItem { } } -fn call_detail(r: CallDetailRow) -> CallDetail { +fn call_detail(r: SpanDetailRow) -> SpanDetail { let tokens_estimated = derive_tokens_estimated(r.input_tokens, r.output_tokens, r.response_body.as_deref()); let tool_names = parse_json_string_list(r.tool_names_json.as_deref()); let process = row_process(r.process_pid, r.process_comm, r.process_exe); - CallDetail { + SpanDetail { id: r.id, source_id: r.source_id, request_time: r.request_time_ms, diff --git a/server/h-storage-clickhouse/src/distincts.rs b/server/h-storage-clickhouse/src/distincts.rs index 09215ce2..65e9880f 100644 --- a/server/h-storage-clickhouse/src/distincts.rs +++ b/server/h-storage-clickhouse/src/distincts.rs @@ -5,7 +5,7 @@ //! The first three read the pre-aggregated `llm_metrics` table and exclude the //! `'*'` rollup tier; `query_distinct_finish_reasons` does the same against //! `llm_finish_metrics`. `query_distinct_agent_kinds` reads the mutable -//! `agent_turns` table (so it uses `FINAL`) and optionally drops proxy-hop rows +//! `traces` table (so it uses `FINAL`) and optionally drops proxy-hop rows //! by inspecting the `metadata` JSON. use clickhouse::Row; @@ -77,7 +77,7 @@ impl ClickHouseBackend { &self, query: &DistinctAgentKindsQuery, ) -> Result> { - // `agent_turns` is a ReplacingMergeTree, so reads must use FINAL to see + // `traces` is a ReplacingMergeTree, so reads must use FINAL to see // the latest version per turn_id. let mut where_parts = vec![time_where( "start_time", @@ -119,7 +119,7 @@ impl ClickHouseBackend { } let sql = format!( - "SELECT DISTINCT agent_kind AS v FROM agent_turns FINAL \ + "SELECT DISTINCT agent_kind AS v FROM traces FINAL \ WHERE {} ORDER BY agent_kind", where_parts.join(" AND ") ); diff --git a/server/h-storage-clickhouse/src/it.rs b/server/h-storage-clickhouse/src/it.rs index a4f8f19b..c991b20b 100644 --- a/server/h-storage-clickhouse/src/it.rs +++ b/server/h-storage-clickhouse/src/it.rs @@ -75,10 +75,10 @@ macro_rules! require_backend { } const ALL_TABLES: &[&str] = &[ - "llm_calls", + "spans", "llm_metrics", "llm_finish_metrics", - "agent_turns", + "traces", "http_exchanges", ]; @@ -108,7 +108,7 @@ pub(crate) mod fixtures { use h_protocol::model::{HttpRequestData, HttpResponseData}; use h_protocol::net::FlowKey; use h_protocol::HttpExchange; - use h_turn::{AgentTurn, TurnStatus}; + use h_turn::{Trace, TraceStatus}; pub(crate) fn sample_call(id: &str, request_time_us: i64) -> LlmCall { LlmCall { @@ -250,9 +250,9 @@ pub(crate) mod fixtures { turn_id: &str, session_id: &str, start_us: i64, - call_ids: Vec<&str>, - ) -> AgentTurn { - AgentTurn { + span_ids: Vec<&str>, + ) -> Trace { + Trace { source_id: "src-0".into(), turn_id: turn_id.into(), session_id: session_id.into(), @@ -263,7 +263,7 @@ pub(crate) mod fixtures { start_time_us: start_us, end_time_us: start_us + 5_000_000, duration_ms: 5_000, - call_count: call_ids.len() as u32, + call_count: span_ids.len() as u32, models_used: vec!["gpt-4".into()], subagents_used: vec![], total_input_tokens: 100, @@ -271,13 +271,13 @@ pub(crate) mod fixtures { total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, total_cost_usd: None, - status: TurnStatus::Complete, + status: TraceStatus::Complete, final_finish_reason: Some("stop".into()), user_input_preview: Some("hello".into()), - user_call_id: call_ids.first().map(|s| s.to_string()), + user_call_id: span_ids.first().map(|s| s.to_string()), final_answer_preview: Some("world".into()), - final_call_id: call_ids.last().map(|s| s.to_string()), - call_ids: call_ids.into_iter().map(String::from).collect(), + final_call_id: span_ids.last().map(|s| s.to_string()), + span_ids: span_ids.into_iter().map(String::from).collect(), metadata: serde_json::json!({}), tool_surfaces: vec![], tool_call_total: 0, @@ -337,9 +337,9 @@ async fn write_paths_round_trip_all_tables() { let ts = 1_700_000_000_000_000_i64; backend - .write_calls(vec![fixtures::sample_call("call-1", ts)]) + .write_spans(vec![fixtures::sample_call("call-1", ts)]) .await - .expect("write_calls"); + .expect("write_spans"); backend .write_metrics(vec![fixtures::sample_metric("1m", ts)]) .await @@ -349,9 +349,9 @@ async fn write_paths_round_trip_all_tables() { .await .expect("write_finish_metrics"); backend - .write_turns(vec![fixtures::sample_turn("turn-1", "sess-1", ts, vec!["call-1"])]) + .write_traces(vec![fixtures::sample_turn("turn-1", "sess-1", ts, vec!["call-1"])]) .await - .expect("write_turns"); + .expect("write_traces"); backend .write_exchanges(vec![fixtures::sample_exchange("xchg-1", ts)]) .await @@ -367,7 +367,7 @@ async fn write_paths_round_trip_all_tables() { .client .query( "SELECT id, model, input_tokens, toUnixTimestamp64Micro(request_time) AS request_time_us \ - FROM llm_calls WHERE id = 'call-1'", + FROM spans WHERE id = 'call-1'", ) .fetch_one::() .await @@ -379,18 +379,18 @@ async fn write_paths_round_trip_all_tables() { } #[tokio::test] -async fn query_calls_paginates_filters_and_by_id() { +async fn query_spans_paginates_filters_and_by_id() { let backend = require_backend!("heron_it_calls"); let ts = 1_700_000_000_000_000_i64; backend - .write_calls(vec![ + .write_spans(vec![ fixtures::sample_call("c1", ts), fixtures::sample_call("c2", ts + 60_000_000), ]) .await .unwrap(); - let base = CallsQuery { + let base = SpansQuery { time_range: TimeRange { start_us: ts - 1, end_us: ts + 120_000_000 }, filter: DimensionFilter::default(), status_codes: vec![], @@ -404,42 +404,42 @@ async fn query_calls_paginates_filters_and_by_id() { page: 1, page_size: 10, }; - let page = backend.query_calls(&base).await.unwrap(); + let page = backend.query_spans(&base).await.unwrap(); assert_eq!(page.total, 2); assert_eq!(page.items.len(), 2); assert_eq!(page.items[0].id, "c2", "DESC by request_time"); assert_eq!(page.items[0].request_time, (ts + 60_000_000) / 1000); // Filter narrows to one row. - let filtered = CallsQuery { + let filtered = SpansQuery { server_ports: vec![8080], request_path_contains: Some("chat/completions".into()), ..base.clone() }; - assert_eq!(backend.query_calls(&filtered).await.unwrap().total, 2); - let none = CallsQuery { server_ports: vec![9999], ..base.clone() }; - assert_eq!(backend.query_calls(&none).await.unwrap().total, 0); + assert_eq!(backend.query_spans(&filtered).await.unwrap().total, 2); + let none = SpansQuery { server_ports: vec![9999], ..base.clone() }; + assert_eq!(backend.query_spans(&none).await.unwrap().total, 0); - let detail = backend.query_call_by_id("c1").await.unwrap().expect("c1"); + let detail = backend.query_span_by_id("c1").await.unwrap().expect("c1"); assert_eq!(detail.model, "gpt-4"); assert_eq!(detail.total_tokens, Some(150)); assert_eq!(detail.request_time, ts / 1000); - assert!(backend.query_call_by_id("missing").await.unwrap().is_none()); + assert!(backend.query_span_by_id("missing").await.unwrap().is_none()); } #[tokio::test] -async fn query_turn_calls_no_join_two_step() { +async fn query_trace_spans_no_join_two_step() { let backend = require_backend!("heron_it_turncalls"); let ts = 1_700_000_000_000_000_i64; backend - .write_calls(vec![ + .write_spans(vec![ fixtures::sample_call("tc1", ts), fixtures::sample_call("tc2", ts + 1_000_000), ]) .await .unwrap(); backend - .write_turns(vec![fixtures::sample_turn( + .write_traces(vec![fixtures::sample_turn( "turn-x", "sess-x", ts, @@ -448,18 +448,18 @@ async fn query_turn_calls_no_join_two_step() { .await .unwrap(); - let full = backend.query_turn_calls("turn-x", true).await.unwrap(); + let full = backend.query_trace_spans("turn-x", true).await.unwrap(); assert_eq!(full.len(), 2); assert_eq!(full[0].id, "tc1", "ordered by request_time ASC"); assert_eq!(full[0].sequence, 1); assert!(full[0].request_body.is_some(), "bodies included"); - let lite = backend.query_turn_calls("turn-x", false).await.unwrap(); + let lite = backend.query_trace_spans("turn-x", false).await.unwrap(); assert_eq!(lite.len(), 2); assert!(lite[0].request_body.is_none(), "lite drops bodies"); let by_ids = backend - .query_calls_by_ids(&["tc2".to_string()], true) + .query_spans_by_ids(&["tc2".to_string()], true) .await .unwrap(); assert_eq!(by_ids.len(), 1); @@ -550,15 +550,15 @@ async fn metrics_reads_smoke() { async fn agent_and_turns_reads_smoke() { let backend = require_backend!("heron_it_turns"); backend - .write_calls(vec![fixtures::sample_call("ac1", TS)]) + .write_spans(vec![fixtures::sample_call("ac1", TS)]) .await .unwrap(); backend - .write_turns(vec![fixtures::sample_turn("t1", "s1", TS, vec!["ac1"])]) + .write_traces(vec![fixtures::sample_turn("t1", "s1", TS, vec!["ac1"])]) .await .unwrap(); - // agent summary/activity (over agent_turns FINAL). + // agent summary/activity (over traces FINAL). let summ = backend .query_agent_summary(&AgentSummaryQuery { time_range: full_range() }) .await @@ -575,9 +575,9 @@ async fn agent_and_turns_reads_smoke() { .unwrap(); assert_eq!(act.len(), 1); - // query_turns + by_id. + // query_traces + by_id. let page = backend - .query_turns(&TurnsQuery { + .query_traces(&TracesQuery { time_range: full_range(), filter: DimensionFilter::default(), client_ips: vec![], @@ -596,32 +596,32 @@ async fn agent_and_turns_reads_smoke() { assert_eq!(page.items[0].turn_id, "t1"); assert_eq!(page.items[0].primary_model.as_deref(), Some("gpt-4")); - let detail = backend.query_turn_by_id("t1").await.unwrap().expect("t1"); - assert_eq!(detail.call_ids, vec!["ac1".to_string()]); + let detail = backend.query_trace_by_id("t1").await.unwrap().expect("t1"); + assert_eq!(detail.span_ids, vec!["ac1".to_string()]); // pair candidates: the un-proxied turn shows up. let cands = backend.query_pair_candidates(TS - 1, TS + 3_600_000_000).await.unwrap(); assert_eq!(cands.len(), 1); assert_eq!(cands[0].turn_id, "t1"); - // update_turn_metadata: merge a proxy role, then it's excluded from + // update_trace_metadata: merge a proxy role, then it's excluded from // candidates and visible in the detail metadata — and the row count stays 1 // (ReplacingMergeTree FINAL dedup). backend - .update_turn_metadata("t1", serde_json::json!({"proxy": {"role": "proxy_in"}})) + .update_trace_metadata("t1", serde_json::json!({"proxy": {"role": "proxy_in"}})) .await .unwrap(); - assert_eq!(count(&backend, "agent_turns FINAL").await, 1); - let after = backend.query_turn_by_id("t1").await.unwrap().expect("t1"); + assert_eq!(count(&backend, "traces FINAL").await, 1); + let after = backend.query_trace_by_id("t1").await.unwrap().expect("t1"); assert_eq!( after.metadata.as_ref().and_then(|m| m.pointer("/proxy/role")).and_then(|v| v.as_str()), Some("proxy_in") ); let cands2 = backend.query_pair_candidates(TS - 1, TS + 3_600_000_000).await.unwrap(); assert_eq!(cands2.len(), 0, "proxy-tagged turn excluded from candidates"); - // update_turn_metadata on a missing turn is a silent no-op. + // update_trace_metadata on a missing turn is a silent no-op. backend - .update_turn_metadata("nope", serde_json::json!({"x": 1})) + .update_trace_metadata("nope", serde_json::json!({"x": 1})) .await .unwrap(); } @@ -630,7 +630,7 @@ async fn agent_and_turns_reads_smoke() { async fn sessions_reads_smoke() { let backend = require_backend!("heron_it_sessions"); backend - .write_turns(vec![ + .write_traces(vec![ fixtures::sample_turn("st1", "sess-A", TS, vec!["c1"]), fixtures::sample_turn("st2", "sess-A", TS + 10_000_000, vec!["c2"]), ]) @@ -659,7 +659,7 @@ async fn sessions_reads_smoke() { assert_eq!(detail.turn_count, 2); let turns = backend - .query_session_turns(&SessionTurnsQuery { + .query_session_traces(&SessionTracesQuery { source_id: "src-0".into(), session_id: "sess-A".into(), cursor: None, @@ -719,7 +719,7 @@ async fn distincts_reads_smoke() { .await .unwrap(); backend - .write_turns(vec![fixtures::sample_turn("dt1", "ds1", TS, vec!["dc1"])]) + .write_traces(vec![fixtures::sample_turn("dt1", "ds1", TS, vec!["dc1"])]) .await .unwrap(); @@ -743,7 +743,7 @@ async fn distincts_reads_smoke() { async fn services_reads_smoke() { let backend = require_backend!("heron_it_services"); backend - .write_calls(vec![ + .write_spans(vec![ fixtures::sample_call("sc1", TS), fixtures::sample_call("sc2", TS + 1_000_000), ]) @@ -776,29 +776,29 @@ async fn retention_deletes_old_rows() { use std::time::SystemTime; let backend = require_backend!("heron_it_retention"); backend - .write_calls(vec![fixtures::sample_call("rc1", TS)]) + .write_spans(vec![fixtures::sample_call("rc1", TS)]) .await .unwrap(); backend - .write_turns(vec![fixtures::sample_turn("rt1", "rs1", TS, vec!["rc1"])]) + .write_traces(vec![fixtures::sample_turn("rt1", "rs1", TS, vec!["rc1"])]) .await .unwrap(); backend .write_metrics(vec![fixtures::sample_metric("1m", TS)]) .await .unwrap(); - assert_eq!(count(&backend, "llm_calls").await, 1); + assert_eq!(count(&backend, "spans").await, 1); // Cutoff = now → everything older than now (the 2023 fixtures) is deleted. let policy = RetentionPolicy { - calls_before: Some(SystemTime::now()), - turns_before: Some(SystemTime::now()), + spans_before: Some(SystemTime::now()), + traces_before: Some(SystemTime::now()), http_exchanges_before: None, metrics_before: vec![("1m".to_string(), SystemTime::now())], }; let report = backend.apply_retention(policy).await.unwrap(); - assert_eq!(report.calls_deleted, 1); - assert_eq!(report.turns_deleted, 1); + assert_eq!(report.spans_deleted, 1); + assert_eq!(report.traces_deleted, 1); assert_eq!(report.metrics_deleted.get("1m").copied(), Some(1)); - assert_eq!(count(&backend, "llm_calls").await, 0, "old calls swept"); + assert_eq!(count(&backend, "spans").await, 0, "old calls swept"); } diff --git a/server/h-storage-clickhouse/src/lib.rs b/server/h-storage-clickhouse/src/lib.rs index 8d6dca7d..47012965 100644 --- a/server/h-storage-clickhouse/src/lib.rs +++ b/server/h-storage-clickhouse/src/lib.rs @@ -36,7 +36,7 @@ use h_common::error::Result; use h_llm::model::LlmCall; use h_metrics::model::{LlmFinishMetric, LlmMetric}; use h_protocol::HttpExchange; -use h_turn::{AgentTurn, PairCandidate}; +use h_turn::{Trace, PairCandidate}; use h_storage::query::*; use h_storage::retention::{RetentionPolicy, RetentionReport}; @@ -87,8 +87,8 @@ impl StorageBackend for ClickHouseBackend { schema::init(self).await } - async fn write_calls(&self, calls: Vec) -> Result<()> { - ClickHouseBackend::write_calls(self, calls).await + async fn write_spans(&self, calls: Vec) -> Result<()> { + ClickHouseBackend::write_spans(self, calls).await } async fn write_metrics(&self, metrics: Vec) -> Result<()> { @@ -99,8 +99,8 @@ impl StorageBackend for ClickHouseBackend { ClickHouseBackend::write_finish_metrics(self, metrics).await } - async fn write_turns(&self, turns: Vec) -> Result<()> { - ClickHouseBackend::write_turns(self, turns).await + async fn write_traces(&self, turns: Vec) -> Result<()> { + ClickHouseBackend::write_traces(self, turns).await } async fn write_exchanges(&self, exchanges: Vec) -> Result<()> { @@ -171,36 +171,36 @@ impl StorageBackend for ClickHouseBackend { ClickHouseBackend::query_finish_reasons(self, query).await } - async fn query_calls(&self, query: &CallsQuery) -> Result { - ClickHouseBackend::query_calls(self, query).await + async fn query_spans(&self, query: &SpansQuery) -> Result { + ClickHouseBackend::query_spans(self, query).await } - async fn query_call_by_id(&self, id: &str) -> Result> { - ClickHouseBackend::query_call_by_id(self, id).await + async fn query_span_by_id(&self, id: &str) -> Result> { + ClickHouseBackend::query_span_by_id(self, id).await } - async fn query_turns(&self, query: &TurnsQuery) -> Result { - ClickHouseBackend::query_turns(self, query).await + async fn query_traces(&self, query: &TracesQuery) -> Result { + ClickHouseBackend::query_traces(self, query).await } - async fn query_turn_by_id(&self, turn_id: &str) -> Result> { - ClickHouseBackend::query_turn_by_id(self, turn_id).await + async fn query_trace_by_id(&self, turn_id: &str) -> Result> { + ClickHouseBackend::query_trace_by_id(self, turn_id).await } - async fn query_turn_calls( + async fn query_trace_spans( &self, turn_id: &str, include_bodies: bool, - ) -> Result> { - ClickHouseBackend::query_turn_calls(self, turn_id, include_bodies).await + ) -> Result> { + ClickHouseBackend::query_trace_spans(self, turn_id, include_bodies).await } - async fn query_calls_by_ids( + async fn query_spans_by_ids( &self, - call_ids: &[String], + span_ids: &[String], include_bodies: bool, - ) -> Result> { - ClickHouseBackend::query_calls_by_ids(self, call_ids, include_bodies).await + ) -> Result> { + ClickHouseBackend::query_spans_by_ids(self, span_ids, include_bodies).await } async fn query_sessions(&self, query: &SessionListQuery) -> Result { @@ -215,11 +215,11 @@ impl StorageBackend for ClickHouseBackend { ClickHouseBackend::query_session_by_id(self, source_id, session_id).await } - async fn query_session_turns( + async fn query_session_traces( &self, - query: &SessionTurnsQuery, - ) -> Result { - ClickHouseBackend::query_session_turns(self, query).await + query: &SessionTracesQuery, + ) -> Result { + ClickHouseBackend::query_session_traces(self, query).await } async fn query_distinct_wire_apis(&self) -> Result> { @@ -257,11 +257,11 @@ impl StorageBackend for ClickHouseBackend { ClickHouseBackend::query_pair_candidates(self, start_us, end_us).await } - async fn update_turn_metadata(&self, turn_id: &str, patch: serde_json::Value) -> Result<()> { - ClickHouseBackend::update_turn_metadata(self, turn_id, patch).await + async fn update_trace_metadata(&self, turn_id: &str, patch: serde_json::Value) -> Result<()> { + ClickHouseBackend::update_trace_metadata(self, turn_id, patch).await } - // checkpoint_turns_writer / reopen_all_connections use the trait's default + // checkpoint_traces_writer / reopen_all_connections use the trait's default // no-op impls: the clickhouse `Client` is a cheap HTTP pool with no // in-process MVCC/index state to compact or reopen. } diff --git a/server/h-storage-clickhouse/src/metrics.rs b/server/h-storage-clickhouse/src/metrics.rs index d255e339..306f4a0b 100644 --- a/server/h-storage-clickhouse/src/metrics.rs +++ b/server/h-storage-clickhouse/src/metrics.rs @@ -1,6 +1,6 @@ //! `llm_metrics` + `llm_finish_metrics` table I/O — aggregation writes plus the //! read-side time-series / summary / model-axis / finish-reason rollups and the -//! agent-distribution / activity aggregates over `agent_turns`. +//! agent-distribution / activity aggregates over `traces`. //! //! The dynamic-column time-series SELECT (the field count varies per request) //! is expressed as a single `[expr, ...] AS vals` array projection of @@ -511,7 +511,7 @@ impl ClickHouseBackend { sum(total_output_tokens) AS total_output_tokens, \ toNullable(avg(duration_ms)) AS avg_duration_ms, \ toUnixTimestamp64Milli(max(start_time)) AS last_seen_ms \ - FROM agent_turns FINAL \ + FROM traces FINAL \ WHERE {ts_pred} \ GROUP BY agent_kind \ ORDER BY turn_count DESC" @@ -563,7 +563,7 @@ impl ClickHouseBackend { toInt64(toUnixTimestamp(toStartOfInterval(start_time, INTERVAL {bucket} SECOND))) * 1000 AS ts, \ agent_kind, \ count() AS turn_count \ - FROM agent_turns FINAL \ + FROM traces FINAL \ WHERE {ts_pred} \ GROUP BY ts, agent_kind \ ORDER BY ts ASC, agent_kind ASC" diff --git a/server/h-storage-clickhouse/src/retention.rs b/server/h-storage-clickhouse/src/retention.rs index a19601d4..16eef9c0 100644 --- a/server/h-storage-clickhouse/src/retention.rs +++ b/server/h-storage-clickhouse/src/retention.rs @@ -72,14 +72,14 @@ impl ClickHouseBackend { // Tables touched this sweep — used to drive the optional OPTIMIZE FINAL. let mut swept: Vec<&'static str> = Vec::new(); - // llm_calls — keyed on request_time. - if let Some(cutoff) = policy.calls_before { + // spans — keyed on request_time. + if let Some(cutoff) = policy.spans_before { let us = cutoff_micros(cutoff)?; let predicate = format!("request_time < fromUnixTimestamp64Micro({us})"); - report.calls_deleted = self.count_where("llm_calls", &predicate).await?; - self.exec(&format!("DELETE FROM llm_calls WHERE {predicate}")) + report.spans_deleted = self.count_where("spans", &predicate).await?; + self.exec(&format!("DELETE FROM spans WHERE {predicate}")) .await?; - swept.push("llm_calls"); + swept.push("spans"); } // http_exchanges — keyed on request_time. @@ -92,14 +92,14 @@ impl ClickHouseBackend { swept.push("http_exchanges"); } - // agent_turns — keyed on end_time. - if let Some(cutoff) = policy.turns_before { + // traces — keyed on end_time. + if let Some(cutoff) = policy.traces_before { let us = cutoff_micros(cutoff)?; let predicate = format!("end_time < fromUnixTimestamp64Micro({us})"); - report.turns_deleted = self.count_where("agent_turns", &predicate).await?; - self.exec(&format!("DELETE FROM agent_turns WHERE {predicate}")) + report.traces_deleted = self.count_where("traces", &predicate).await?; + self.exec(&format!("DELETE FROM traces WHERE {predicate}")) .await?; - swept.push("agent_turns"); + swept.push("traces"); } // Per-granularity metrics sweep. For each (label, cutoff) pair, delete diff --git a/server/h-storage-clickhouse/src/rows.rs b/server/h-storage-clickhouse/src/rows.rs index ad68655d..f104684b 100644 --- a/server/h-storage-clickhouse/src/rows.rs +++ b/server/h-storage-clickhouse/src/rows.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; use h_llm::model::LlmCall; use h_metrics::model::{LlmFinishMetric, LlmMetric}; use h_protocol::HttpExchange; -use h_turn::AgentTurn; +use h_turn::Trace; use h_storage::convert::headers_to_json; @@ -57,6 +57,10 @@ pub(crate) struct CallRow { pub process_pid: Option, pub process_comm: Option, pub process_exe: Option, + /// OTel span kind. Every wire-captured span is an LLM call today; the + /// column is forward-looking for wire-visible tool spans. Tail field to + /// match the `spans` table column order (and the DuckDB layout). + pub kind: String, } impl From for CallRow { @@ -101,6 +105,7 @@ impl From for CallRow { process_pid: c.process.as_ref().map(|p| p.pid), process_comm: c.process.as_ref().map(|p| p.comm.clone()), process_exe: c.process.as_ref().and_then(|p| p.exe.clone()), + kind: "llm".into(), } } } @@ -265,7 +270,8 @@ pub(crate) struct TurnRow { pub user_call_id: Option, pub final_answer_preview: Option, pub final_call_id: Option, - pub call_ids: String, + // Maps to the `span_ids` column (RowBinary insert names columns by field). + pub span_ids: String, pub metadata: Option, pub tool_surfaces_json: Option, pub tool_call_total: u32, @@ -274,15 +280,15 @@ pub(crate) struct TurnRow { pub _version: u64, } -impl From for TurnRow { - fn from(t: AgentTurn) -> Self { +impl From for TurnRow { + fn from(t: Trace) -> Self { let tool_surfaces_json = { let strings: Vec = t.tool_surfaces.iter().map(|s| s.to_string()).collect(); serde_json::to_string(&strings).unwrap_or_else(|_| "[]".to_string()) }; let suspicious_skills_json = serde_json::to_string(&t.suspicious_skills).unwrap_or_else(|_| "[]".to_string()); - // Initial finalize version = end_time (micros). `update_turn_metadata` + // Initial finalize version = end_time (micros). `update_trace_metadata` // re-inserts with a strictly-greater wall-clock-micros version so the // ReplacingMergeTree keeps the latest metadata. let version = t.end_time_us.max(0) as u64; @@ -311,7 +317,7 @@ impl From for TurnRow { user_call_id: t.user_call_id, final_answer_preview: t.final_answer_preview, final_call_id: t.final_call_id, - call_ids: serde_json::to_string(&t.call_ids).unwrap_or_default(), + span_ids: serde_json::to_string(&t.span_ids).unwrap_or_default(), metadata: Some(t.metadata.to_string()), tool_surfaces_json: Some(tool_surfaces_json), tool_call_total: t.tool_call_total, diff --git a/server/h-storage-clickhouse/src/schema.rs b/server/h-storage-clickhouse/src/schema.rs index ef61bf6f..171ad0f4 100644 --- a/server/h-storage-clickhouse/src/schema.rs +++ b/server/h-storage-clickhouse/src/schema.rs @@ -10,7 +10,7 @@ //! rows are SUM/MAX-aggregated at read, one row per bucket from the //! aggregator, so no dedup is needed. //! * `agent_turns` — `ReplacingMergeTree(_version)`: it is the only mutated -//! table (`update_turn_metadata`), so reads use `FINAL` and updates +//! table (`update_trace_metadata`), so reads use `FINAL` and updates //! re-insert the whole row with a higher `_version`. //! //! Timestamps are `DateTime64(6, 'UTC')`; the `clickhouse` crate maps a Rust @@ -29,7 +29,7 @@ use crate::client::ch_err; use crate::ClickHouseBackend; const CREATE_LLM_CALLS: &str = " -CREATE TABLE IF NOT EXISTS llm_calls ( +CREATE TABLE IF NOT EXISTS spans ( id String, source_id String DEFAULT '', client_ip String, @@ -66,7 +66,8 @@ CREATE TABLE IF NOT EXISTS llm_calls ( body_bytes_dropped UInt64 DEFAULT 0, process_pid Nullable(UInt32), process_comm Nullable(String), - process_exe Nullable(String) + process_exe Nullable(String), + kind String DEFAULT 'llm' ) ENGINE = MergeTree ORDER BY (request_time, id) "; @@ -138,7 +139,7 @@ ORDER BY (granularity, timestamp, finish_reason, wire_api, model, server_ip) "; const CREATE_AGENT_TURNS: &str = " -CREATE TABLE IF NOT EXISTS agent_turns ( +CREATE TABLE IF NOT EXISTS traces ( turn_id String, source_id String DEFAULT '', session_id String, @@ -163,7 +164,7 @@ CREATE TABLE IF NOT EXISTS agent_turns ( user_call_id Nullable(String), final_answer_preview Nullable(String), final_call_id Nullable(String), - call_ids String, + span_ids String, metadata Nullable(String), tool_surfaces_json Nullable(String), tool_call_total UInt32 DEFAULT 0, @@ -219,6 +220,38 @@ pub(crate) async fn init(backend: &ClickHouseBackend) -> Result<()> { .await .map_err(|e| ch_err("create database", e))?; + // Step 1.5: Phase 8 migration — OpenTelemetry rename (agent_turns -> traces, + // llm_calls -> spans, traces.call_ids -> span_ids, + spans.kind). Runs + // before the CREATE loop so a migrated DB no-ops the `CREATE TABLE IF NOT + // EXISTS spans/traces` below instead of creating empty tables that collide + // with the rename. ClickHouse `RENAME TABLE` has no `IF EXISTS` form, so + // each rename is guarded by a system.tables existence check to stay + // idempotent across repeated init() calls. + if table_exists(backend, "agent_turns").await? && !table_exists(backend, "traces").await? { + backend.exec("RENAME TABLE agent_turns TO traces").await?; + info!("phase8 migration: renamed agent_turns -> traces"); + } + if table_exists(backend, "llm_calls").await? && !table_exists(backend, "spans").await? { + backend.exec("RENAME TABLE llm_calls TO spans").await?; + info!("phase8 migration: renamed llm_calls -> spans"); + } + if column_exists(backend, "traces", "call_ids").await? + && !column_exists(backend, "traces", "span_ids").await? + { + backend + .exec("ALTER TABLE traces RENAME COLUMN call_ids TO span_ids") + .await?; + info!("phase8 migration: renamed traces.call_ids -> span_ids"); + } + // `kind` on a migrated `spans` (ClickHouse supports `ADD COLUMN IF NOT + // EXISTS` with a NOT-NULL String DEFAULT). On a fresh DB `spans` does not + // exist yet — the CREATE below supplies `kind` — so guard on existence. + if table_exists(backend, "spans").await? { + backend + .exec("ALTER TABLE spans ADD COLUMN IF NOT EXISTS kind String DEFAULT 'llm'") + .await?; + } + // Step 2: create every table on the db-scoped client. Idempotent. for ddl in CREATE_TABLES { backend.exec(ddl).await?; @@ -228,6 +261,40 @@ pub(crate) async fn init(backend: &ClickHouseBackend) -> Result<()> { Ok(()) } +/// Whether `name` exists as a table in the backend's current database. Guards +/// the Phase 8 (no-`IF EXISTS`) `RENAME TABLE` statements for idempotency. +async fn table_exists(backend: &ClickHouseBackend, name: &str) -> Result { + let sql = format!( + "SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = '{}'", + name.replace('\'', "\\'") + ); + let n: u64 = backend + .client + .query(&sql) + .fetch_one::() + .await + .map_err(|e| ch_err("table_exists", e))?; + Ok(n > 0) +} + +/// Whether `table`.`column` exists in the current database. Companion to +/// `table_exists` for the Phase 8 column-rename guard. +async fn column_exists(backend: &ClickHouseBackend, table: &str, column: &str) -> Result { + let sql = format!( + "SELECT count() FROM system.columns \ + WHERE database = currentDatabase() AND table = '{}' AND name = '{}'", + table.replace('\'', "\\'"), + column.replace('\'', "\\'") + ); + let n: u64 = backend + .client + .query(&sql) + .fetch_one::() + .await + .map_err(|e| ch_err("column_exists", e))?; + Ok(n > 0) +} + /// Backtick-quote a ClickHouse identifier, doubling embedded backticks. /// Database names come from config (operator-controlled), but quoting keeps /// names with hyphens / reserved words valid. diff --git a/server/h-storage-clickhouse/src/services.rs b/server/h-storage-clickhouse/src/services.rs index 584f1dda..d232797b 100644 --- a/server/h-storage-clickhouse/src/services.rs +++ b/server/h-storage-clickhouse/src/services.rs @@ -1,4 +1,4 @@ -//! "Services" view reads — aggregate `llm_calls` by `(server_ip, server_port)` +//! "Services" view reads — aggregate `spans` by `(server_ip, server_port)` //! plus the service-topology graph. Ports of the DuckDB `query_services` / //! `query_services_topology` (see `h-storage-duckdb/src/metrics.rs`); same //! aggregation, percentiles, app classification, and `__clients__` super-node @@ -19,8 +19,8 @@ //! the whole window: a cheap inner subquery picks the recent ids per //! endpoint (`LIMIT N BY`, no body columns) and the outer fetches bodies for //! just those ids (`id IN (...)`). See `fetch_app_samples`. -//! * No JOINs (project rule). The DuckDB topology JOINs `agent_turns` to -//! `llm_calls` on the turn's first `call_ids` entry; we reimplement that as +//! * No JOINs (project rule). The DuckDB topology JOINs `traces` to +//! `spans` on the turn's first `span_ids` entry; we reimplement that as //! a no-JOIN two-step: read the turns + first call_id, fetch those calls, //! then map back in Rust. See `query_services_topology`. @@ -124,15 +124,15 @@ struct NodeAggRow { call_count: u64, } -/// One turn's proxy metadata + its first call_id, read from `agent_turns`. -/// `call_ids` is the raw JSON-array String column; `first_call_id` is the first +/// One turn's proxy metadata + its first call_id, read from `traces`. +/// `span_ids` is the raw JSON-array String column; `first_call_id` is the first /// element, extracted server-side (`JSONExtractArrayRaw` + `arrayElement`) but /// re-cleaned in Rust to strip the JSON quoting. #[derive(Row, Deserialize)] struct TurnEndpointRow { proxy_role: String, pair_id: String, - call_ids: String, + span_ids: String, } /// `(server_ip, server_port, client_ip)` for one llm_call, used to resolve a @@ -146,7 +146,7 @@ struct CallEndpointRow { } impl ClickHouseBackend { - /// "Services" view — aggregate `llm_calls` by `(server_ip, server_port)`. + /// "Services" view — aggregate `spans` by `(server_ip, server_port)`. /// Port of the DuckDB `query_services`; see that fn + `StorageBackend:: /// query_services` for motivation (port is not on `llm_metrics`). pub(crate) async fn query_services(&self, query: &ServicesQuery) -> Result> { @@ -197,7 +197,7 @@ impl ClickHouseBackend { toNullable(toFloat64(quantileTDigest(0.95)(e2e_latency_ms))) AS e2e_p95_ms, toUnixTimestamp64Milli(min(request_time)) AS first_seen_ms, toUnixTimestamp64Milli(max(request_time)) AS last_seen_ms - FROM llm_calls + FROM spans WHERE {where_sql} GROUP BY server_ip, server_port ORDER BY {} {sort_order} @@ -298,7 +298,7 @@ impl ClickHouseBackend { server_port, groupUniqArray(16)(request_path) AS request_paths, groupUniqArray(32)(finish_reason) AS finish_reasons - FROM llm_calls + FROM spans WHERE {sample_where} GROUP BY server_ip, server_port" ); @@ -333,9 +333,9 @@ impl ClickHouseBackend { toNullable(response_headers) AS response_headers, toNullable(request_headers) AS request_headers, request_body, response_body - FROM llm_calls + FROM spans WHERE id IN ( - SELECT id FROM llm_calls + SELECT id FROM spans WHERE {sample_where} ORDER BY request_time DESC LIMIT 5 BY server_ip, server_port @@ -387,8 +387,8 @@ impl ClickHouseBackend { /// Build the service-topology graph for the Path view. Port of the DuckDB /// `query_services_topology` — same node set, proxy / inferred / client /// edges, and `__clients__` super-node. The DuckDB original resolves a - /// turn's `(server_ip, server_port)` via a JOIN from `agent_turns` to - /// `llm_calls` on the turn's first `call_ids` entry; the project's no-JOIN + /// turn's `(server_ip, server_port)` via a JOIN from `traces` to + /// `spans` on the turn's first `span_ids` entry; the project's no-JOIN /// rule means we instead read the turns + their first call_id, fetch those /// calls in one `IN (...)` lookup, and map back in Rust (see below). pub(crate) async fn query_services_topology( @@ -412,7 +412,7 @@ impl ClickHouseBackend { groupUniqArray(32)(model) AS models, groupUniqArray(16)(request_path) AS request_paths, count() AS call_count - FROM llm_calls + FROM spans WHERE {nodes_where} GROUP BY server_ip, server_port" ); @@ -459,29 +459,29 @@ impl ClickHouseBackend { } // --- Resolve each turn's endpoint via its first call_id. The DuckDB - // query JOINs agent_turns → llm_calls on the first call_ids entry; the + // query JOINs traces → spans on the first span_ids entry; the // no-JOIN rule means we do this as a two-step: // 1. read every turn in the window with its proxy role / pair_id and - // its first call_id (parsed from the call_ids JSON in Rust), + // its first call_id (parsed from the span_ids JSON in Rust), // 2. fetch those calls in one IN (...) lookup, build an // id → (server_ip, server_port, client_ip) map, // 3. join the two in Rust. // DIVERGENCE FROM DUCKDB: the original used an in-SQL JOIN / - // correlated extract; we materialize the first call_ids set and do a + // correlated extract; we materialize the first span_ids set and do a // point-lookup batch instead. The resulting (turn, endpoint) pairs are // identical. // - // agent_turns is ReplacingMergeTree → FINAL so the latest row wins. + // traces is ReplacingMergeTree → FINAL so the latest row wins. // Time filter on start_time matches the DuckDB predicate. We pull the - // first call id server-side via arrayElement(JSONExtract(call_ids, + // first call id server-side via arrayElement(JSONExtract(span_ids, // 'Array(String)'), 1); parsing in Rust as a fallback for robustness. let turns_where = time_where("start_time", start_us, end_us); let turns_sql = format!( "SELECT JSONExtractString(coalesce(metadata, ''), 'proxy', 'role') AS proxy_role, JSONExtractString(coalesce(metadata, ''), 'proxy', 'pair_id') AS pair_id, - call_ids AS call_ids - FROM agent_turns FINAL + span_ids AS span_ids + FROM traces FINAL WHERE {turns_where}" ); let turn_rows = self @@ -500,7 +500,7 @@ impl ClickHouseBackend { let mut turn_infos: Vec = Vec::with_capacity(turn_rows.len()); let mut wanted_ids: HashSet = HashSet::new(); for t in turn_rows { - let ids = parse_json_string_list(Some(&t.call_ids)); + let ids = parse_json_string_list(Some(&t.span_ids)); if let Some(first) = ids.into_iter().next() { if first.is_empty() { continue; @@ -514,14 +514,14 @@ impl ClickHouseBackend { } } - // Fetch the endpoints for those first call_ids in one batch. + // Fetch the endpoints for those first span_ids in one batch. let mut endpoint_by_id: HashMap = HashMap::new(); if !wanted_ids.is_empty() { let id_list: Vec = wanted_ids.into_iter().collect(); let in_list = crate::sql::sql_in_list(&id_list); let calls_sql = format!( "SELECT id, server_ip, server_port, client_ip \ - FROM llm_calls WHERE id IN ({in_list})" + FROM spans WHERE id IN ({in_list})" ); let call_rows = self .client diff --git a/server/h-storage-clickhouse/src/sessions.rs b/server/h-storage-clickhouse/src/sessions.rs index 61f98222..d811704f 100644 --- a/server/h-storage-clickhouse/src/sessions.rs +++ b/server/h-storage-clickhouse/src/sessions.rs @@ -1,19 +1,19 @@ -//! Session-scoped queries. Sessions are a view over `agent_turns` grouped by +//! Session-scoped queries. Sessions are a view over `traces` grouped by //! `(source_id, session_id)` — no schema of their own. Ported from the DuckDB //! backend (`h-storage-duckdb/src/sessions.rs`); aggregates, windowing, and //! cursor semantics match it column-for-column. //! -//! `agent_turns` is a `ReplacingMergeTree(_version)`, so every read here uses -//! `FROM agent_turns FINAL` to ensure only the latest version per `turn_id` -//! participates in the aggregates (otherwise stale pre-`update_turn_metadata` +//! `traces` is a `ReplacingMergeTree(_version)`, so every read here uses +//! `FROM traces FINAL` to ensure only the latest version per `turn_id` +//! participates in the aggregates (otherwise stale pre-`update_trace_metadata` //! rows would double-count tokens / costs and skew MIN/MAX). //! -//! Divergence from DuckDB — `query_session_turns` user_input / final_answer: +//! Divergence from DuckDB — `query_session_traces` user_input / final_answer: //! the DuckDB backend reconstructs the FULL user_input / final_answer by //! re-running each agent profile's body extractor over the referenced call //! bodies (`extract_full_text_batch`, which needs a duckdb `Connection` and is //! not available here). For shape parity this backend populates -//! `SessionTurnItem.user_input` / `final_answer` best-effort from the turn's +//! `SessionTraceItem.user_input` / `final_answer` best-effort from the turn's //! stored `user_input_preview` / `final_answer_preview` columns instead. These //! may be truncated (preview strings end with `…`) where the DuckDB backend //! would return the full text. See the per-row comment below. @@ -58,7 +58,7 @@ struct SessionAggRow { first_call_id: Option, } -/// One page row for `query_session_turns`. Mirrors the DuckDB SELECT column +/// One page row for `query_session_traces`. Mirrors the DuckDB SELECT column /// list; preview/call-id columns are carried so we can populate /// `user_input` / `final_answer` best-effort (see module divergence note). #[derive(Row, Deserialize)] @@ -131,7 +131,7 @@ impl ClickHouseBackend { let step1_sql = format!( "SELECT source_id, session_id, \ toUnixTimestamp64Milli(max(end_time)) AS last_ms \ - FROM agent_turns FINAL \ + FROM traces FINAL \ WHERE {where_sql} \ GROUP BY source_id, session_id{having_sql} \ ORDER BY max(end_time) DESC, source_id DESC, session_id DESC \ @@ -186,7 +186,7 @@ impl ClickHouseBackend { total_cache_read_input_tokens, total_cache_creation_input_tokens, \ total_cost_usd, agent_kind, user_input_preview, user_call_id, \ row_number() OVER (PARTITION BY source_id, session_id ORDER BY start_time) AS rn \ - FROM agent_turns FINAL \ + FROM traces FINAL \ WHERE (source_id, session_id) IN ({pairs_sql}) \ ) t \ GROUP BY source_id, session_id" @@ -276,7 +276,7 @@ impl ClickHouseBackend { total_cache_read_input_tokens, total_cache_creation_input_tokens, \ total_cost_usd, agent_kind, user_input_preview, user_call_id, \ row_number() OVER (PARTITION BY source_id, session_id ORDER BY start_time) AS rn \ - FROM agent_turns FINAL \ + FROM traces FINAL \ WHERE source_id = '{}' AND session_id = '{}' \ ) t \ GROUP BY source_id, session_id \ @@ -314,10 +314,10 @@ impl ClickHouseBackend { })) } - pub(crate) async fn query_session_turns( + pub(crate) async fn query_session_traces( &self, - query: &SessionTurnsQuery, - ) -> Result { + query: &SessionTracesQuery, + ) -> Result { let page_size = query.page_size.max(1); let limit = (page_size as u64) + 1; @@ -336,7 +336,7 @@ impl ClickHouseBackend { }; // FINAL so only the latest version per turn is returned (matters after - // the pair sweeper's update_turn_metadata re-inserts). + // the pair sweeper's update_trace_metadata re-inserts). let sql = format!( "SELECT turn_id, source_id, session_id, \ toUnixTimestamp64Milli(start_time) AS start_ms, \ @@ -347,7 +347,7 @@ impl ClickHouseBackend { status, final_finish_reason, \ user_input_preview, final_answer_preview, \ tool_surfaces_json, tool_call_total, agent_topology, suspicious_skills_json \ - FROM agent_turns FINAL \ + FROM traces FINAL \ WHERE source_id = '{}' AND session_id = '{}'{cursor_sql} \ ORDER BY start_time DESC, turn_id DESC \ LIMIT {limit}", @@ -360,7 +360,7 @@ impl ClickHouseBackend { .query(&sql) .fetch_all::() .await - .map_err(|e| ch_err("query_session_turns", e))?; + .map_err(|e| ch_err("query_session_traces", e))?; // Fetch+1 pattern: if we got page_size + 1 rows, there's a next page. let has_more = rows.len() as u64 > page_size as u64; @@ -368,7 +368,7 @@ impl ClickHouseBackend { rows.truncate(page_size as usize); } - let items: Vec = rows + let items: Vec = rows .into_iter() .map(|r| { let models_used = parse_json_string_list(r.models_used.as_deref()); @@ -386,7 +386,7 @@ impl ClickHouseBackend { // available here). We populate these best-effort from the stored // preview columns; values may be truncated where DuckDB would // return full text. - SessionTurnItem { + SessionTraceItem { turn_id: r.turn_id, source_id: r.source_id, session_id: r.session_id, @@ -414,7 +414,7 @@ impl ClickHouseBackend { let next_cursor = if has_more { items.last().map(|last| { - encode_session_turns_cursor(&SessionTurnsCursor { + encode_session_turns_cursor(&SessionTracesCursor { start_time_us: last.start_time.saturating_mul(1000), turn_id: last.turn_id.clone(), }) @@ -423,6 +423,6 @@ impl ClickHouseBackend { None }; - Ok(SessionTurnsPage { items, next_cursor }) + Ok(SessionTracesPage { items, next_cursor }) } } diff --git a/server/h-storage-clickhouse/src/turns.rs b/server/h-storage-clickhouse/src/turns.rs index 9bc6479e..c5d69f3d 100644 --- a/server/h-storage-clickhouse/src/turns.rs +++ b/server/h-storage-clickhouse/src/turns.rs @@ -1,9 +1,9 @@ -//! `agent_turns` table I/O — write, paginated query, by-id detail, pair-sweeper -//! support (`query_pair_candidates` / `update_turn_metadata`). +//! `traces` table I/O — write, paginated query, by-id detail, pair-sweeper +//! support (`query_pair_candidates` / `update_trace_metadata`). //! -//! `agent_turns` is `ReplacingMergeTree(_version)`: it is the only mutated +//! `traces` is `ReplacingMergeTree(_version)`: it is the only mutated //! table. Writes insert with `_version = end_time` (micros); reads use `FINAL`; -//! `update_turn_metadata` reads the full row (FINAL), merges the JSON patch, and +//! `update_trace_metadata` reads the full row (FINAL), merges the JSON patch, and //! re-inserts the whole row with a wall-clock-micros `_version` so the latest //! metadata wins on the next FINAL read. @@ -15,17 +15,17 @@ use serde::Deserialize; use h_common::error::{AppError, Result}; use h_storage::convert::parse_json_string_list; use h_storage::query::*; -use h_turn::{AgentTurn, PairCandidate}; +use h_turn::{Trace, PairCandidate}; use crate::client::{ch_err, insert_all}; use crate::rows::TurnRow; use crate::sql::{escape_str, sql_in_list, time_where}; use crate::ClickHouseBackend; -/// Full `agent_turns` column list in `TurnRow` field order, with the two +/// Full `traces` column list in `TurnRow` field order, with the two /// `DateTime64(6)` columns surfaced as `i64` micros so they deserialize into /// `TurnRow`'s `i64` fields and round-trip on re-insert. Used by -/// `update_turn_metadata`'s read-modify-write. +/// `update_trace_metadata`'s read-modify-write. const TURN_ROW_SELECT: &str = "turn_id, source_id, session_id, wire_api, agent_kind, \ client_ip, server_ip, \ toUnixTimestamp64Micro(start_time) AS start_time, \ @@ -35,7 +35,7 @@ const TURN_ROW_SELECT: &str = "turn_id, source_id, session_id, wire_api, agent_k total_cache_read_input_tokens, total_cache_creation_input_tokens, \ total_cost_usd, status, final_finish_reason, \ user_input_preview, user_call_id, final_answer_preview, final_call_id, \ - call_ids, metadata, tool_surfaces_json, tool_call_total, agent_topology, \ + span_ids, metadata, tool_surfaces_json, tool_call_total, agent_topology, \ suspicious_skills_json, _version"; /// Read `metadata.proxy.{role, peer_turn_id, peer_turn_ids}` out of a row's @@ -104,7 +104,7 @@ struct TurnListRow { } #[derive(Row, Deserialize)] -struct TurnDetailRow { +struct TraceDetailRow { turn_id: String, source_id: String, session_id: String, @@ -127,7 +127,7 @@ struct TurnDetailRow { user_call_id: Option, final_answer_preview: Option, final_call_id: Option, - call_ids: String, + span_ids: String, metadata: Option, client_ip: String, server_ip: String, @@ -160,13 +160,13 @@ struct CountRow { } impl ClickHouseBackend { - pub(crate) async fn write_turns(&self, turns: Vec) -> Result<()> { + pub(crate) async fn write_traces(&self, turns: Vec) -> Result<()> { let rows: Vec = turns.into_iter().map(TurnRow::from).collect(); - insert_all!(self.client, "agent_turns", TurnRow, rows); + insert_all!(self.client, "traces", TurnRow, rows); Ok(()) } - pub(crate) async fn query_turns(&self, query: &TurnsQuery) -> Result { + pub(crate) async fn query_traces(&self, query: &TracesQuery) -> Result { const VALID_SORT_FIELDS: &[&str] = &[ "start_time", "end_time", @@ -213,14 +213,14 @@ impl ClickHouseBackend { where_parts.push(format!("client_ip IN ({})", sql_in_list(&query.client_ips))); } if !query.server_ports.is_empty() { - // agent_turns has no server_port; resolve via the turn's first - // call_id against llm_calls. ClickHouse can't do the DuckDB + // traces has no server_port; resolve via the turn's first + // call_id against spans. ClickHouse can't do the DuckDB // correlated EXISTS, so use an uncorrelated IN-subquery (still // not a JOIN): the turn's first call_id ∈ { calls on those ports }. let ports: Vec = query.server_ports.iter().map(|p| p.to_string()).collect(); where_parts.push(format!( - "arrayElement(JSONExtract(call_ids, 'Array(String)'), 1) IN \ - (SELECT id FROM llm_calls WHERE server_port IN ({}))", + "arrayElement(JSONExtract(span_ids, 'Array(String)'), 1) IN \ + (SELECT id FROM spans WHERE server_port IN ({}))", ports.join(", ") )); } @@ -242,11 +242,11 @@ impl ClickHouseBackend { let total = self .client .query(&format!( - "SELECT count() AS n FROM agent_turns FINAL WHERE {where_sql}" + "SELECT count() AS n FROM traces FINAL WHERE {where_sql}" )) .fetch_one::() .await - .map_err(|e| ch_err("query_turns count", e))? + .map_err(|e| ch_err("query_traces count", e))? .n; let offset = (query.page.saturating_sub(1)) as u64 * query.page_size as u64; @@ -258,7 +258,7 @@ impl ClickHouseBackend { total_input_tokens, total_output_tokens, status, final_finish_reason, \ user_input_preview, final_answer_preview, client_ip, server_ip, metadata, \ tool_surfaces_json, tool_call_total, agent_topology, suspicious_skills_json \ - FROM agent_turns FINAL WHERE {where_sql} \ + FROM traces FINAL WHERE {where_sql} \ ORDER BY {} {sort_order} LIMIT {} OFFSET {offset}", query.sort_by, query.page_size, ); @@ -267,7 +267,7 @@ impl ClickHouseBackend { .query(&items_sql) .fetch_all::() .await - .map_err(|e| ch_err("query_turns items", e))?; + .map_err(|e| ch_err("query_traces items", e))?; let items = rows .into_iter() @@ -282,7 +282,7 @@ impl ClickHouseBackend { .as_deref() .and_then(|s| serde_json::from_str(s).ok()) .unwrap_or_default(); - TurnListItem { + TraceListItem { turn_id: r.turn_id, source_id: r.source_id, session_id: r.session_id, @@ -312,10 +312,10 @@ impl ClickHouseBackend { } }) .collect(); - Ok(TurnsPage { total, items }) + Ok(TracesPage { total, items }) } - pub(crate) async fn query_turn_by_id(&self, turn_id: &str) -> Result> { + pub(crate) async fn query_trace_by_id(&self, turn_id: &str) -> Result> { let sql = format!( "SELECT turn_id, source_id, session_id, wire_api, agent_kind, \ toUnixTimestamp64Milli(start_time) AS start_time_ms, \ @@ -325,24 +325,24 @@ impl ClickHouseBackend { total_cache_read_input_tokens, total_cache_creation_input_tokens, \ total_cost_usd, status, final_finish_reason, \ user_input_preview, user_call_id, final_answer_preview, final_call_id, \ - call_ids, metadata, client_ip, server_ip, \ + span_ids, metadata, client_ip, server_ip, \ tool_surfaces_json, tool_call_total, agent_topology, suspicious_skills_json \ - FROM agent_turns FINAL WHERE turn_id = '{}' LIMIT 1", + FROM traces FINAL WHERE turn_id = '{}' LIMIT 1", escape_str(turn_id) ); let row = self .client .query(&sql) - .fetch_all::() + .fetch_all::() .await - .map_err(|e| ch_err("query_turn_by_id", e))? + .map_err(|e| ch_err("query_trace_by_id", e))? .into_iter() .next(); let Some(r) = row else { return Ok(None) }; let models_used = parse_json_string_list(r.models_used.as_deref()); let subagents_used = parse_json_string_list(r.subagents_used.as_deref()); - let call_ids = parse_json_string_list(Some(&r.call_ids)); + let span_ids = parse_json_string_list(Some(&r.span_ids)); let metadata = r .metadata .as_deref() @@ -357,7 +357,7 @@ impl ClickHouseBackend { // re-running the agent profile extractor over the referenced call // bodies (the DuckDB path's `extract_full_text`). We surface the stored // previews best-effort; truncated previews (ending `…`) stay truncated. - Ok(Some(TurnDetail { + Ok(Some(TraceDetail { turn_id: r.turn_id, source_id: r.source_id, session_id: r.session_id, @@ -382,7 +382,7 @@ impl ClickHouseBackend { user_input: r.user_input_preview, final_call_id: r.final_call_id, final_answer: r.final_answer_preview, - call_ids, + span_ids, metadata, tool_surfaces, tool_call_total: r.tool_call_total, @@ -403,7 +403,7 @@ impl ClickHouseBackend { toUnixTimestamp64Micro(end_time) AS end_time_us, \ call_count, total_input_tokens, total_output_tokens, \ final_finish_reason, models_used, client_ip, server_ip \ - FROM agent_turns FINAL \ + FROM traces FINAL \ WHERE {ts_pred} \ AND JSONExtractString(coalesce(metadata, ''), 'proxy', 'role') = '' \ ORDER BY start_time ASC" @@ -437,7 +437,7 @@ impl ClickHouseBackend { .collect()) } - pub(crate) async fn update_turn_metadata( + pub(crate) async fn update_trace_metadata( &self, turn_id: &str, patch: serde_json::Value, @@ -446,7 +446,7 @@ impl ClickHouseBackend { // (FINAL = latest version), shallow-merge the patch into metadata, and // re-insert with a strictly-greater `_version` (wall-clock micros). let sql = format!( - "SELECT {TURN_ROW_SELECT} FROM agent_turns FINAL WHERE turn_id = '{}' LIMIT 1", + "SELECT {TURN_ROW_SELECT} FROM traces FINAL WHERE turn_id = '{}' LIMIT 1", escape_str(turn_id) ); let existing = self @@ -454,7 +454,7 @@ impl ClickHouseBackend { .query(&sql) .fetch_all::() .await - .map_err(|e| ch_err("update_turn_metadata read", e))? + .map_err(|e| ch_err("update_trace_metadata read", e))? .into_iter() .next(); let Some(mut row) = existing else { @@ -478,7 +478,7 @@ impl ClickHouseBackend { row.metadata = Some(base.to_string()); row._version = now_micros(); - insert_all!(self.client, "agent_turns", TurnRow, vec![row]); + insert_all!(self.client, "traces", TurnRow, vec![row]); Ok(()) } } diff --git a/server/h-storage-duckdb/src/calls.rs b/server/h-storage-duckdb/src/calls.rs index c37d72f6..1c50d5ef 100644 --- a/server/h-storage-duckdb/src/calls.rs +++ b/server/h-storage-duckdb/src/calls.rs @@ -1,4 +1,4 @@ -//! `llm_calls` table I/O — write, query (paginated / by-id / by-id-list). +//! `spans` table I/O — write, query (paginated / by-id / by-id-list). use duckdb::types::{TimeUnit, Value}; use duckdb::Connection; @@ -122,22 +122,22 @@ fn read_process( })) } -/// Shared "fetch calls by id list" — used by both `query_turn_calls` -/// (which derives the ids from the persisted `agent_turns.call_ids`) -/// and `query_calls_by_ids` (which receives the ids directly from the -/// API for in-progress turns whose call_ids live in the in-memory +/// Shared "fetch calls by id list" — used by both `query_trace_spans` +/// (which derives the ids from the persisted `traces.span_ids`) +/// and `query_spans_by_ids` (which receives the ids directly from the +/// API for in-progress turns whose span_ids live in the in-memory /// active-turn registry). Calls not yet flushed from `WriteBuffer` to -/// `llm_calls` simply don't return — caller treats that as "show +/// `spans` simply don't return — caller treats that as "show /// fewer rows on this refresh, more on the next one." fn read_calls_by_ids_sync( conn: &Connection, - call_ids: &[String], + span_ids: &[String], include_bodies: bool, -) -> Result> { - if call_ids.is_empty() { +) -> Result> { + if span_ids.is_empty() { return Ok(Vec::new()); } - let placeholders = std::iter::repeat_n("?", call_ids.len()) + let placeholders = std::iter::repeat_n("?", span_ids.len()) .collect::>() .join(", "); // Lite mode (`include_bodies = false`) selects `NULL` for the four @@ -145,7 +145,7 @@ fn read_calls_by_ids_sync( // pages off disk and the rows never need to be transferred to // Rust as Strings. The downstream `tokens_estimated` derivation // falls back to `false` when response_body is missing — documented - // on `StorageBackend::query_turn_calls`. + // on `StorageBackend::query_trace_spans`. let body_columns: &str = if include_bodies { "request_body, response_body, request_headers, response_headers" } else { @@ -166,7 +166,7 @@ fn read_calls_by_ids_sync( request_path, client_ip, client_port, server_ip, server_port, {body_columns} - FROM llm_calls + FROM spans WHERE id IN ({placeholders}) ORDER BY request_time ASC, complete_time ASC" ); @@ -175,7 +175,7 @@ fn read_calls_by_ids_sync( .prepare(&sql) .map_err(|e| AppError::Storage(format!("failed to prepare turn_calls step2: {e}")))?; let mut rows = stmt - .query(duckdb::params_from_iter(call_ids.iter())) + .query(duckdb::params_from_iter(span_ids.iter())) .map_err(|e| AppError::Storage(format!("failed to execute turn_calls step2: {e}")))?; let mut items = Vec::new(); @@ -185,7 +185,7 @@ fn read_calls_by_ids_sync( .map_err(|e| AppError::Storage(format!("row error: {e}")))? { seq += 1; - items.push(TurnCallItem { + items.push(TraceSpanItem { id: row .get(0) .map_err(|e| AppError::Storage(format!("read error: {e}")))?, @@ -272,7 +272,7 @@ fn read_calls_by_ids_sync( } impl DuckDbBackend { - pub(crate) async fn write_calls(&self, calls: Vec) -> Result<()> { + pub(crate) async fn write_spans(&self, calls: Vec) -> Result<()> { if calls.is_empty() { return Ok(()); } @@ -286,7 +286,7 @@ impl DuckDbBackend { return Err(crate::fault_injection::disk_full_error()); } } - let conn = self.write_calls_conn.clone(); + let conn = self.write_spans_conn.clone(); tokio::task::spawn_blocking(move || { // Serialize/format outside the writer Mutex so the lock is held // only for the append + flush. @@ -296,7 +296,7 @@ impl DuckDbBackend { .lock() .map_err(|e| AppError::Storage(format!("failed to lock writer: {e}")))?; let mut appender = conn - .appender("llm_calls") + .appender("spans") .map_err(|e| AppError::Storage(format!("failed to create appender: {e}")))?; for p in &prepared { appender @@ -338,6 +338,10 @@ impl DuckDbBackend { p.process_pid, p.process_comm, p.process_exe, + // `kind` (tail column). Every wire-captured span is an + // LLM call today; written as a literal so the domain + // model stays free of a field that has one value. + "llm", ]) .map_err(|e| AppError::Storage(format!("failed to append call: {e}")))?; } @@ -350,7 +354,7 @@ impl DuckDbBackend { .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? } - pub(crate) async fn query_calls(&self, query: &CallsQuery) -> Result { + pub(crate) async fn query_spans(&self, query: &SpansQuery) -> Result { const VALID_SORT_FIELDS: &[&str] = &[ "request_time", "status_code", @@ -459,7 +463,7 @@ impl DuckDbBackend { let sort_by = &query.sort_by; // COUNT query - let count_sql = format!("SELECT COUNT(*) FROM llm_calls WHERE {where_sql}"); + let count_sql = format!("SELECT COUNT(*) FROM spans WHERE {where_sql}"); let mut count_stmt = conn .prepare(&count_sql) .map_err(|e| AppError::Storage(format!("failed to prepare count query: {e}")))?; @@ -476,7 +480,7 @@ impl DuckDbBackend { client_ip, server_ip, server_port, request_path, response_body, \ is_agent_request, tool_surface, agent_topology, tool_call_count, tool_names_json, \ process_pid, process_comm, process_exe \ - FROM llm_calls WHERE {where_sql} \ + FROM spans WHERE {where_sql} \ ORDER BY {sort_by} {sort_order} \ LIMIT {limit} OFFSET {offset}" ); @@ -514,7 +518,7 @@ impl DuckDbBackend { .unwrap_or_default(); let process = read_process(row, 22, 23, 24) .map_err(|e| AppError::Storage(format!("read error: {e}")))?; - items.push(CallListItem { + items.push(SpanListItem { id: row .get(0) .map_err(|e| AppError::Storage(format!("read error: {e}")))?, @@ -579,13 +583,13 @@ impl DuckDbBackend { }); } - Ok(CallsPage { total, items }) + Ok(SpansPage { total, items }) }) .await .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? } - pub(crate) async fn query_call_by_id(&self, id: &str) -> Result> { + pub(crate) async fn query_span_by_id(&self, id: &str) -> Result> { let conn = self.read_pool.acquire().await?; let id = id.to_string(); @@ -607,7 +611,7 @@ impl DuckDbBackend { is_agent_request, tool_surface, agent_topology, tool_call_count, tool_names_json, process_pid, process_comm, process_exe - FROM llm_calls + FROM spans WHERE id = ? LIMIT 1 "; @@ -627,7 +631,7 @@ impl DuckDbBackend { .as_deref() .and_then(|s| serde_json::from_str(s).ok()) .unwrap_or_default(); - Ok(CallDetail { + Ok(SpanDetail { id: row.get(0)?, source_id: row.get(1)?, request_time: row.get(2)?, @@ -676,20 +680,20 @@ impl DuckDbBackend { .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? } - pub(crate) async fn query_turn_calls( + pub(crate) async fn query_trace_spans( &self, turn_id: &str, include_bodies: bool, - ) -> Result> { + ) -> Result> { let conn = self.read_pool.acquire().await?; let turn_id = turn_id.to_string(); tokio::task::spawn_blocking(move || { - // Step 1: fetch the turn's call_ids by PK. JSON-array column is - // parsed with the same helper as query_turn_by_id. - let call_ids_raw: Option = { + // Step 1: fetch the turn's span_ids by PK. JSON-array column is + // parsed with the same helper as query_trace_by_id. + let span_ids_raw: Option = { let mut stmt = conn - .prepare("SELECT call_ids FROM agent_turns WHERE turn_id = ?") + .prepare("SELECT span_ids FROM traces WHERE turn_id = ?") .map_err(|e| { AppError::Storage(format!("failed to prepare turn_calls step1: {e}")) })?; @@ -706,28 +710,28 @@ impl DuckDbBackend { } }; - let call_ids = parse_json_string_list(call_ids_raw.as_deref()); - // Step 2 lives in a shared helper so `query_calls_by_ids` (used - // by the API for in-progress turns whose call_ids come from + let span_ids = parse_json_string_list(span_ids_raw.as_deref()); + // Step 2 lives in a shared helper so `query_spans_by_ids` (used + // by the API for in-progress turns whose span_ids come from // the in-memory registry) can reuse the exact same projection. - read_calls_by_ids_sync(&conn, &call_ids, include_bodies) + read_calls_by_ids_sync(&conn, &span_ids, include_bodies) }) .await .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? } - pub(crate) async fn query_calls_by_ids( + pub(crate) async fn query_spans_by_ids( &self, - call_ids: &[String], + span_ids: &[String], include_bodies: bool, - ) -> Result> { - if call_ids.is_empty() { + ) -> Result> { + if span_ids.is_empty() { return Ok(Vec::new()); } let conn = self.read_pool.acquire().await?; - let call_ids: Vec = call_ids.to_vec(); + let span_ids: Vec = span_ids.to_vec(); tokio::task::spawn_blocking(move || { - read_calls_by_ids_sync(&conn, &call_ids, include_bodies) + read_calls_by_ids_sync(&conn, &span_ids, include_bodies) }) .await .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? @@ -794,16 +798,16 @@ mod tests { } #[tokio::test] - async fn test_write_calls_single() { + async fn test_write_spans_single() { let backend = in_memory(); backend.init().await.unwrap(); let call = sample_call(); - backend.write_calls(vec![call]).await.unwrap(); + backend.write_spans(vec![call]).await.unwrap(); let conn = backend.test_conn().lock().unwrap(); let mut stmt = conn - .prepare("SELECT id, model, is_stream, input_tokens FROM llm_calls") + .prepare("SELECT id, model, is_stream, input_tokens FROM spans") .unwrap(); let row = stmt .query_row([], |row| { @@ -822,16 +826,16 @@ mod tests { } #[tokio::test] - async fn test_write_calls_new_fields() { + async fn test_write_spans_new_fields() { let backend = in_memory(); backend.init().await.unwrap(); let call = sample_call(); - backend.write_calls(vec![call]).await.unwrap(); + backend.write_spans(vec![call]).await.unwrap(); let conn = backend.test_conn().lock().unwrap(); let mut stmt = conn - .prepare("SELECT response_id, request_headers, response_headers FROM llm_calls") + .prepare("SELECT response_id, request_headers, response_headers FROM spans") .unwrap(); let (resp_id, req_hdr, resp_hdr) = stmt .query_row([], |row| { @@ -854,36 +858,36 @@ mod tests { } #[tokio::test] - async fn test_write_calls_id_present() { + async fn test_write_spans_id_present() { let backend = in_memory(); backend.init().await.unwrap(); let call = sample_call(); - backend.write_calls(vec![call]).await.unwrap(); + backend.write_spans(vec![call]).await.unwrap(); let conn = backend.test_conn().lock().unwrap(); - let mut stmt = conn.prepare("SELECT id FROM llm_calls").unwrap(); + let mut stmt = conn.prepare("SELECT id FROM spans").unwrap(); let id: String = stmt.query_row([], |row| row.get(0)).unwrap(); assert_eq!(id, "01912345-6789-7abc-def0-123456789abc"); } #[tokio::test] - async fn test_write_calls_empty_batch() { + async fn test_write_spans_empty_batch() { let backend = in_memory(); backend.init().await.unwrap(); - backend.write_calls(vec![]).await.unwrap(); + backend.write_spans(vec![]).await.unwrap(); } #[tokio::test] - async fn test_query_calls_basic() { + async fn test_query_spans_basic() { let backend = in_memory(); backend.init().await.unwrap(); let call = sample_call(); let call_time = call.request_time; - backend.write_calls(vec![call]).await.unwrap(); + backend.write_spans(vec![call]).await.unwrap(); - let query = CallsQuery { + let query = SpansQuery { time_range: TimeRange { start_us: call_time - 1, end_us: call_time + 1_000_000, @@ -901,7 +905,7 @@ mod tests { page_size: 10, }; - let page = backend.query_calls(&query).await.unwrap(); + let page = backend.query_spans(&query).await.unwrap(); assert_eq!(page.total, 1); assert_eq!(page.items.len(), 1); assert_eq!(page.items[0].id, "01912345-6789-7abc-def0-123456789abc"); @@ -912,7 +916,7 @@ mod tests { } #[tokio::test] - async fn test_query_calls_filter_status_code() { + async fn test_query_spans_filter_status_code() { let backend = in_memory(); backend.init().await.unwrap(); @@ -925,9 +929,9 @@ mod tests { call_429.status_code = Some(429); let call_time = call_200.request_time; - backend.write_calls(vec![call_200, call_429]).await.unwrap(); + backend.write_spans(vec![call_200, call_429]).await.unwrap(); - let query = CallsQuery { + let query = SpansQuery { time_range: TimeRange { start_us: call_time - 1, end_us: call_time + 1_000_000, @@ -945,7 +949,7 @@ mod tests { page_size: 10, }; - let page = backend.query_calls(&query).await.unwrap(); + let page = backend.query_spans(&query).await.unwrap(); assert_eq!(page.total, 1); assert_eq!(page.items.len(), 1); assert_eq!(page.items[0].id, "call-429"); @@ -953,16 +957,16 @@ mod tests { } #[tokio::test] - async fn test_query_call_by_id() { + async fn test_query_span_by_id() { let backend = in_memory(); backend.init().await.unwrap(); let call = sample_call(); - backend.write_calls(vec![call]).await.unwrap(); + backend.write_spans(vec![call]).await.unwrap(); // Query by existing id let detail = backend - .query_call_by_id("01912345-6789-7abc-def0-123456789abc") + .query_span_by_id("01912345-6789-7abc-def0-123456789abc") .await .unwrap(); assert!(detail.is_some()); @@ -980,7 +984,7 @@ mod tests { assert!(detail.response_headers.is_some()); // Query nonexistent id - let not_found = backend.query_call_by_id("does-not-exist").await.unwrap(); + let not_found = backend.query_span_by_id("does-not-exist").await.unwrap(); assert!(not_found.is_none()); } } diff --git a/server/h-storage-duckdb/src/concurrent_tests.rs b/server/h-storage-duckdb/src/concurrent_tests.rs index 63889d24..03b783cd 100644 --- a/server/h-storage-duckdb/src/concurrent_tests.rs +++ b/server/h-storage-duckdb/src/concurrent_tests.rs @@ -9,10 +9,10 @@ use h_llm::model::{ApiType, LlmCall}; use h_llm::wire_apis as wa; use h_metrics::model::LlmMetric; use h_storage::query::{ - DimensionFilter, DistinctAgentKindsQuery, TimeRange, TurnsQuery, + DimensionFilter, DistinctAgentKindsQuery, TimeRange, TracesQuery, }; use h_storage::StorageBackend; -use h_turn::{AgentTurn, TurnStatus}; +use h_turn::{Trace, TraceStatus}; fn mk_call(i: usize) -> LlmCall { LlmCall { @@ -54,8 +54,8 @@ fn mk_call(i: usize) -> LlmCall { } } -fn mk_turn(i: usize) -> AgentTurn { - AgentTurn { +fn mk_turn(i: usize) -> Trace { + Trace { source_id: String::new(), turn_id: format!("turn-{i:08}"), session_id: format!("session-{}", i % 10), @@ -74,13 +74,13 @@ fn mk_turn(i: usize) -> AgentTurn { total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, total_cost_usd: None, - status: TurnStatus::Complete, + status: TraceStatus::Complete, final_finish_reason: None, user_input_preview: None, user_call_id: None, final_answer_preview: None, final_call_id: None, - call_ids: vec![format!("call-{i:08}")], + span_ids: vec![format!("call-{i:08}")], metadata: serde_json::json!({}), tool_surfaces: vec![], tool_call_total: 0, @@ -160,7 +160,7 @@ async fn concurrent_writes_to_three_tables() { let calls_task = tokio::spawn(async move { for b in 0..BATCHES { let batch: Vec<_> = (0..PER_TABLE).map(|i| mk_call(b * PER_TABLE + i)).collect(); - calls_backend.write_calls(batch).await.unwrap(); + calls_backend.write_spans(batch).await.unwrap(); } }); @@ -168,7 +168,7 @@ async fn concurrent_writes_to_three_tables() { let turns_task = tokio::spawn(async move { for b in 0..BATCHES { let batch: Vec<_> = (0..PER_TABLE).map(|i| mk_turn(b * PER_TABLE + i)).collect(); - turns_backend.write_turns(batch).await.unwrap(); + turns_backend.write_traces(batch).await.unwrap(); } }); @@ -190,10 +190,10 @@ async fn concurrent_writes_to_three_tables() { let expected = (PER_TABLE * BATCHES) as i64; let conn = backend.test_conn().lock().unwrap(); let calls_count: i64 = conn - .query_row("SELECT COUNT(*) FROM llm_calls", [], |r| r.get(0)) + .query_row("SELECT COUNT(*) FROM spans", [], |r| r.get(0)) .unwrap(); let turns_count: i64 = conn - .query_row("SELECT COUNT(*) FROM agent_turns", [], |r| r.get(0)) + .query_row("SELECT COUNT(*) FROM traces", [], |r| r.get(0)) .unwrap(); let metrics_count: i64 = conn .query_row("SELECT COUNT(*) FROM llm_metrics", [], |r| r.get(0)) @@ -216,10 +216,10 @@ async fn llm_call_round_trip_with_agent_fields() { call.tool_names = vec!["Read".to_string(), "Edit".to_string()]; let call_id = call.id.clone(); - backend.write_calls(vec![call]).await.unwrap(); + backend.write_spans(vec![call]).await.unwrap(); let back = backend - .query_call_by_id(&call_id) + .query_span_by_id(&call_id) .await .unwrap() .expect("call should round-trip"); @@ -256,12 +256,12 @@ async fn reopen_all_connections_keeps_reads_and_writes_alive() { Arc::new(DuckDbBackend::open_with_pool(path.to_str().unwrap(), 2).unwrap()); backend.init().await.unwrap(); - backend.write_turns(vec![mk_turn(0)]).await.unwrap(); + backend.write_traces(vec![mk_turn(0)]).await.unwrap(); backend.reopen_all_connections().await.unwrap(); // Read path 1: turn list query — goes through the read pool. - let turns_q = TurnsQuery { + let turns_q = TracesQuery { time_range: TimeRange { start_us: 1_700_000_000_000_000 - 1_000_000, end_us: 1_700_000_000_000_000 + 60_000_000_000, @@ -277,10 +277,10 @@ async fn reopen_all_connections_keeps_reads_and_writes_alive() { page_size: 50, include_proxy_hops: false, }; - let page = backend.query_turns(&turns_q).await.unwrap(); + let page = backend.query_traces(&turns_q).await.unwrap(); assert_eq!( page.total, 1, - "query_turns after reopen must see the pre-reopen row" + "query_traces after reopen must see the pre-reopen row" ); // Read path 2: distinct agent kinds — different code path but @@ -304,12 +304,12 @@ async fn reopen_all_connections_keeps_reads_and_writes_alive() { // Write path on every writer mutex — proves all four writers // were rebuilt, not just turns. - backend.write_turns(vec![mk_turn(1)]).await.unwrap(); - backend.write_calls(vec![mk_call(0)]).await.unwrap(); + backend.write_traces(vec![mk_turn(1)]).await.unwrap(); + backend.write_spans(vec![mk_call(0)]).await.unwrap(); backend.write_metrics(vec![mk_metric(0)]).await.unwrap(); // And re-confirm the post-reopen reads pick up the new row. - let page2 = backend.query_turns(&turns_q).await.unwrap(); + let page2 = backend.query_traces(&turns_q).await.unwrap(); assert_eq!( page2.total, 2, "post-reopen writes must persist and be queryable" @@ -340,7 +340,7 @@ async fn reopen_recovers_from_injected_duckdb_invalidate() { backend.init().await.unwrap(); // Baseline write, before any fault — exercises the writer set. - backend.write_turns(vec![mk_turn(0)]).await.unwrap(); + backend.write_traces(vec![mk_turn(0)]).await.unwrap(); // ARM: every write path now synthesizes a FATAL invalidation error // identical in shape to what DuckDB returns when its in-process @@ -350,18 +350,18 @@ async fn reopen_recovers_from_injected_duckdb_invalidate() { assert!(backend.fault_set().should_fire(FaultPoint::DuckDbInvalidate)); let err = backend - .write_turns(vec![mk_turn(1)]) + .write_traces(vec![mk_turn(1)]) .await - .expect_err("write_turns must surface the injected FATAL"); + .expect_err("write_traces must surface the injected FATAL"); assert!( format!("{err}").contains("FATAL"), "injected error message must mention FATAL: got {err}" ); let err = backend - .write_calls(vec![mk_call(0)]) + .write_spans(vec![mk_call(0)]) .await - .expect_err("write_calls must surface the injected FATAL"); + .expect_err("write_spans must surface the injected FATAL"); assert!(format!("{err}").contains("FATAL")); let err = backend @@ -387,11 +387,11 @@ async fn reopen_recovers_from_injected_duckdb_invalidate() { // every reader path (turn list, distinct-agent-kinds — backed by // the pool that `reopen_all_connections` is supposed to have fully // replaced) must serve the new row plus the pre-fault baseline. - backend.write_turns(vec![mk_turn(2)]).await.unwrap(); - backend.write_calls(vec![mk_call(0)]).await.unwrap(); + backend.write_traces(vec![mk_turn(2)]).await.unwrap(); + backend.write_spans(vec![mk_call(0)]).await.unwrap(); backend.write_metrics(vec![mk_metric(0)]).await.unwrap(); - let turns_q = TurnsQuery { + let turns_q = TracesQuery { time_range: TimeRange { start_us: 1_700_000_000_000_000 - 1_000_000, end_us: 1_700_000_000_000_000 + 60_000_000_000, @@ -407,7 +407,7 @@ async fn reopen_recovers_from_injected_duckdb_invalidate() { page_size: 50, include_proxy_hops: false, }; - let page = backend.query_turns(&turns_q).await.unwrap(); + let page = backend.query_traces(&turns_q).await.unwrap(); assert_eq!( page.total, 2, "post-recovery list query must see baseline turn-0 + new turn-2" @@ -446,9 +446,9 @@ async fn read_pool_poisoned_fault_propagates_to_reads() { let backend = Arc::new(DuckDbBackend::open_with_pool(path.to_str().unwrap(), 2).unwrap()); backend.init().await.unwrap(); - backend.write_turns(vec![mk_turn(0)]).await.unwrap(); + backend.write_traces(vec![mk_turn(0)]).await.unwrap(); - let turns_q = TurnsQuery { + let turns_q = TracesQuery { time_range: TimeRange { start_us: 1_700_000_000_000_000 - 1_000_000, end_us: 1_700_000_000_000_000 + 60_000_000_000, @@ -468,7 +468,7 @@ async fn read_pool_poisoned_fault_propagates_to_reads() { { let _guard = FaultGuard::arm(backend.fault_set(), FaultPoint::ReadPoolPoisoned); let err = backend - .query_turns(&turns_q) + .query_traces(&turns_q) .await .expect_err("query must surface poisoned-pool error"); assert!( @@ -478,7 +478,7 @@ async fn read_pool_poisoned_fault_propagates_to_reads() { } // After guard drops, queries work again. - let page = backend.query_turns(&turns_q).await.unwrap(); + let page = backend.query_traces(&turns_q).await.unwrap(); assert_eq!(page.total, 1); } @@ -527,7 +527,7 @@ async fn chaos_burst( for r in 0..rounds { let base = id_base + (w * rounds + r) * CHAOS_BATCH; let batch: Vec<_> = (0..CHAOS_BATCH).map(|i| mk_call(base + i)).collect(); - match be.write_calls(batch).await { + match be.write_spans(batch).await { Ok(_) => { ok.fetch_add(1, Ordering::Relaxed); } @@ -549,7 +549,7 @@ async fn chaos_burst( fn count_calls(backend: &DuckDbBackend) -> usize { let conn = backend.test_conn().lock().unwrap(); let n: i64 = conn - .query_row("SELECT COUNT(*) FROM llm_calls", [], |r| r.get(0)) + .query_row("SELECT COUNT(*) FROM spans", [], |r| r.get(0)) .unwrap(); n as usize } diff --git a/server/h-storage-duckdb/src/distincts.rs b/server/h-storage-duckdb/src/distincts.rs index a5b8d978..0107c866 100644 --- a/server/h-storage-duckdb/src/distincts.rs +++ b/server/h-storage-duckdb/src/distincts.rs @@ -120,7 +120,7 @@ impl DuckDbBackend { } let sql = format!( - "SELECT DISTINCT agent_kind FROM agent_turns WHERE {} ORDER BY agent_kind", + "SELECT DISTINCT agent_kind FROM traces WHERE {} ORDER BY agent_kind", where_parts.join(" AND ") ); let mut stmt = conn.prepare(&sql).map_err(|e| { @@ -153,7 +153,7 @@ mod tests { use h_metrics::model::LlmMetric; use h_storage::query::{DimensionFilter, DistinctAgentKindsQuery, TimeRange}; use h_storage::StorageBackend; - use h_turn::{AgentTurn, TurnStatus}; + use h_turn::{Trace, TraceStatus}; fn in_memory() -> DuckDbBackend { DuckDbBackend::open(":memory:").unwrap() @@ -220,8 +220,8 @@ mod tests { server_ip: &str, start_us: i64, metadata: serde_json::Value, - ) -> AgentTurn { - AgentTurn { + ) -> Trace { + Trace { source_id: String::new(), turn_id: turn_id.into(), session_id: "s1".into(), @@ -240,13 +240,13 @@ mod tests { total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, total_cost_usd: None, - status: TurnStatus::Complete, + status: TraceStatus::Complete, final_finish_reason: Some("complete".into()), user_input_preview: Some("hello".into()), user_call_id: None, final_answer_preview: Some("world".into()), final_call_id: None, - call_ids: vec!["call-1".into()], + span_ids: vec!["call-1".into()], metadata, tool_surfaces: vec![], tool_call_total: 0, @@ -341,7 +341,7 @@ mod tests { let base = 1_700_000_000_000_000_i64; backend - .write_turns(vec![ + .write_traces(vec![ sample_turn( "t-openclaw", "openclaw", diff --git a/server/h-storage-duckdb/src/fault_injection.rs b/server/h-storage-duckdb/src/fault_injection.rs index de85f1e0..36da1e3f 100644 --- a/server/h-storage-duckdb/src/fault_injection.rs +++ b/server/h-storage-duckdb/src/fault_injection.rs @@ -21,7 +21,7 @@ //! //! ```ignore //! let _guard = FaultGuard::arm(&backend, FaultPoint::DuckDbInvalidate); -//! let err = backend.write_turns(vec![mk_turn(0)]).await.unwrap_err(); +//! let err = backend.write_traces(vec![mk_turn(0)]).await.unwrap_err(); //! // _guard's Drop disarms automatically — no leakage into sibling tests. //! ``` diff --git a/server/h-storage-duckdb/src/lib.rs b/server/h-storage-duckdb/src/lib.rs index 66d932a5..8bd24b3d 100644 --- a/server/h-storage-duckdb/src/lib.rs +++ b/server/h-storage-duckdb/src/lib.rs @@ -23,7 +23,7 @@ use h_common::error::{AppError, Result}; use h_llm::model::LlmCall; use h_metrics::model::{LlmFinishMetric, LlmMetric}; use h_protocol::HttpExchange; -use h_turn::{AgentTurn, PairCandidate}; +use h_turn::{Trace, PairCandidate}; use h_storage::query::*; use h_storage::retention::{RetentionPolicy, RetentionReport}; @@ -46,8 +46,8 @@ const DEFAULT_READ_POOL_SIZE: usize = 4; /// A small pool of reader connections is cloned from the calls writer. /// Queries never contend with writes on any of the writer Mutexes. pub struct DuckDbBackend { - pub(crate) write_calls_conn: Arc>, - pub(crate) write_turns_conn: Arc>, + pub(crate) write_spans_conn: Arc>, + pub(crate) write_traces_conn: Arc>, pub(crate) write_metrics_conn: Arc>, pub(crate) write_exchanges_conn: Arc>, pub(crate) read_pool: ReadPool, @@ -113,8 +113,8 @@ impl DuckDbBackend { let fault_set = fault_injection::FaultSet::new(); Ok(Self { - write_calls_conn: Arc::new(StdMutex::new(calls_writer)), - write_turns_conn: Arc::new(StdMutex::new(turns_writer)), + write_spans_conn: Arc::new(StdMutex::new(calls_writer)), + write_traces_conn: Arc::new(StdMutex::new(turns_writer)), write_metrics_conn: Arc::new(StdMutex::new(metrics_writer)), write_exchanges_conn: Arc::new(StdMutex::new(exchanges_writer)), read_pool: ReadPool::new( @@ -137,7 +137,7 @@ impl DuckDbBackend { #[cfg(test)] pub(crate) fn test_conn(&self) -> &StdMutex { - &self.write_calls_conn + &self.write_spans_conn } } @@ -147,8 +147,8 @@ impl StorageBackend for DuckDbBackend { schema::init(self).await } - async fn write_calls(&self, calls: Vec) -> Result<()> { - DuckDbBackend::write_calls(self, calls).await + async fn write_spans(&self, calls: Vec) -> Result<()> { + DuckDbBackend::write_spans(self, calls).await } async fn write_exchanges(&self, exchanges: Vec) -> Result<()> { @@ -171,8 +171,8 @@ impl StorageBackend for DuckDbBackend { DuckDbBackend::write_finish_metrics(self, metrics).await } - async fn write_turns(&self, turns: Vec) -> Result<()> { - DuckDbBackend::write_turns(self, turns).await + async fn write_traces(&self, turns: Vec) -> Result<()> { + DuckDbBackend::write_traces(self, turns).await } async fn query_metrics_timeseries( @@ -228,36 +228,36 @@ impl StorageBackend for DuckDbBackend { DuckDbBackend::query_finish_reasons(self, query).await } - async fn query_calls(&self, query: &CallsQuery) -> Result { - DuckDbBackend::query_calls(self, query).await + async fn query_spans(&self, query: &SpansQuery) -> Result { + DuckDbBackend::query_spans(self, query).await } - async fn query_call_by_id(&self, id: &str) -> Result> { - DuckDbBackend::query_call_by_id(self, id).await + async fn query_span_by_id(&self, id: &str) -> Result> { + DuckDbBackend::query_span_by_id(self, id).await } - async fn query_turns(&self, query: &TurnsQuery) -> Result { - DuckDbBackend::query_turns(self, query).await + async fn query_traces(&self, query: &TracesQuery) -> Result { + DuckDbBackend::query_traces(self, query).await } - async fn query_turn_by_id(&self, turn_id: &str) -> Result> { - DuckDbBackend::query_turn_by_id(self, turn_id).await + async fn query_trace_by_id(&self, turn_id: &str) -> Result> { + DuckDbBackend::query_trace_by_id(self, turn_id).await } - async fn query_turn_calls( + async fn query_trace_spans( &self, turn_id: &str, include_bodies: bool, - ) -> Result> { - DuckDbBackend::query_turn_calls(self, turn_id, include_bodies).await + ) -> Result> { + DuckDbBackend::query_trace_spans(self, turn_id, include_bodies).await } - async fn query_calls_by_ids( + async fn query_spans_by_ids( &self, - call_ids: &[String], + span_ids: &[String], include_bodies: bool, - ) -> Result> { - DuckDbBackend::query_calls_by_ids(self, call_ids, include_bodies).await + ) -> Result> { + DuckDbBackend::query_spans_by_ids(self, span_ids, include_bodies).await } async fn query_sessions(&self, query: &SessionListQuery) -> Result { @@ -272,8 +272,8 @@ impl StorageBackend for DuckDbBackend { DuckDbBackend::query_session_by_id(self, source_id, session_id).await } - async fn query_session_turns(&self, query: &SessionTurnsQuery) -> Result { - DuckDbBackend::query_session_turns(self, query).await + async fn query_session_traces(&self, query: &SessionTracesQuery) -> Result { + DuckDbBackend::query_session_traces(self, query).await } async fn query_distinct_wire_apis(&self) -> Result> { @@ -311,12 +311,12 @@ impl StorageBackend for DuckDbBackend { DuckDbBackend::query_pair_candidates(self, start_us, end_us).await } - async fn update_turn_metadata(&self, turn_id: &str, patch: serde_json::Value) -> Result<()> { - DuckDbBackend::update_turn_metadata(self, turn_id, patch).await + async fn update_trace_metadata(&self, turn_id: &str, patch: serde_json::Value) -> Result<()> { + DuckDbBackend::update_trace_metadata(self, turn_id, patch).await } - async fn checkpoint_turns_writer(&self) -> Result<()> { - let conn = self.write_turns_conn.clone(); + async fn checkpoint_traces_writer(&self) -> Result<()> { + let conn = self.write_traces_conn.clone(); tokio::task::spawn_blocking(move || { let conn = conn .lock() @@ -332,8 +332,8 @@ impl StorageBackend for DuckDbBackend { async fn reopen_all_connections(&self) -> Result<()> { let path = self.db_path.clone(); let pool_size = self.read_pool_size; - let calls_arc = self.write_calls_conn.clone(); - let turns_arc = self.write_turns_conn.clone(); + let calls_arc = self.write_spans_conn.clone(); + let turns_arc = self.write_traces_conn.clone(); let metrics_arc = self.write_metrics_conn.clone(); let exchanges_arc = self.write_exchanges_conn.clone(); let read_pool = self.read_pool.clone(); diff --git a/server/h-storage-duckdb/src/metrics.rs b/server/h-storage-duckdb/src/metrics.rs index 35741c24..a6efd07c 100644 --- a/server/h-storage-duckdb/src/metrics.rs +++ b/server/h-storage-duckdb/src/metrics.rs @@ -874,7 +874,7 @@ impl DuckDbBackend { server_port, CAST(list_distinct(array_agg(request_path))[1:16] AS JSON)::VARCHAR AS paths_json, CAST(list_distinct(array_agg(finish_reason))[1:32] AS JSON)::VARCHAR AS finish_reasons_json - FROM llm_calls + FROM spans WHERE request_time >= ? AND request_time < ? GROUP BY server_ip, server_port"; @@ -932,7 +932,7 @@ impl DuckDbBackend { PARTITION BY server_ip, server_port ORDER BY request_time DESC ) AS rn - FROM llm_calls + FROM spans WHERE request_time >= ? AND request_time < ? ) WHERE rn <= 5"; @@ -1008,7 +1008,7 @@ impl DuckDbBackend { Ok(out) } - /// "Services" view — aggregate `llm_calls` by `(server_ip, + /// "Services" view — aggregate `spans` by `(server_ip, /// server_port)`. See `StorageBackend::query_services` for the /// motivation (port is not on `llm_metrics`). pub(crate) async fn query_services(&self, query: &ServicesQuery) -> Result> { @@ -1063,7 +1063,7 @@ impl DuckDbBackend { // List-of-VARCHAR columns come back as JSON strings (DuckDB's // rust binding has no `FromSql` for `Vec`). We cast // to JSON in SQL and then `parse_json_string_list` them on - // the Rust side — same pattern used by `agent_turns.models_used`. + // the Rust side — same pattern used by `traces.models_used`. // // For app classification we sample one `request_headers` / // `response_headers` blob per group. `arg_min(col, @@ -1099,7 +1099,7 @@ impl DuckDbBackend { quantile_cont(e2e_latency_ms, 0.95) AS e2e_p95_ms, epoch_ms(MIN(request_time)) AS first_seen_ms, epoch_ms(MAX(request_time)) AS last_seen_ms - FROM llm_calls + FROM spans WHERE request_time >= ? AND request_time < ? GROUP BY server_ip, server_port ORDER BY {sort_by} {sort_order} @@ -1221,14 +1221,14 @@ impl DuckDbBackend { /// `query_services`, just the columns we need). Models / app /// come along so the UI can label and color nodes the same /// way the Table view does, no second fetch. - /// * Proxy edges — `agent_turns` carries + /// * Proxy edges — `traces` carries /// `metadata.proxy.pair_id` from the pair sweeper. For every /// `proxy_in` turn we look up the sibling `proxy_out` turn(s) /// by `pair_id`. The `proxy_in.server_endpoint → /// proxy_out.server_endpoint` pair becomes one edge, counted - /// by number of paired turns. `agent_turns` has no + /// by number of paired turns. `traces` has no /// `server_port`, so we look it up via the turn's first - /// `call_ids` entry against `llm_calls`. + /// `span_ids` entry against `spans`. /// * Client edges — any service that has a non-`proxy_out` turn /// in the window gets a virtual edge from `__clients__`. So /// entry-point services (no inbound proxy hop) still appear @@ -1261,7 +1261,7 @@ impl DuckDbBackend { CAST(list_distinct(array_agg(model))[1:32] AS JSON)::VARCHAR AS models_json, CAST(list_distinct(array_agg(request_path))[1:16] AS JSON)::VARCHAR AS paths_json, COUNT(*) AS call_count - FROM llm_calls + FROM spans WHERE request_time >= ? AND request_time < ? GROUP BY server_ip, server_port"; @@ -1333,7 +1333,7 @@ impl DuckDbBackend { // --- Proxy edges: pair the turn's endpoint with its // sibling's. We resolve (server_ip, server_port) for each - // turn via its first call_id; `agent_turns.server_ip` + // turn via its first call_id; `traces.server_ip` // alone isn't enough since the table doesn't carry // server_port. Filtering by `proxy.role` is done after // the join because some turns have NULL metadata. @@ -1350,9 +1350,9 @@ impl DuckDbBackend { c.server_port, json_extract_string(t.metadata, '$.proxy.role') AS proxy_role, json_extract_string(t.metadata, '$.proxy.pair_id') AS pair_id - FROM agent_turns t - JOIN llm_calls c - ON c.id = json_extract_string(t.call_ids, '$[0]') + FROM traces t + JOIN spans c + ON c.id = json_extract_string(t.span_ids, '$[0]') WHERE t.start_time >= ? AND t.start_time < ? ) SELECT @@ -1430,9 +1430,9 @@ impl DuckDbBackend { c.server_ip AS to_ip, c.server_port AS to_port, COALESCE(json_extract_string(t.metadata, '$.proxy.role'), '') AS proxy_role - FROM agent_turns t - JOIN llm_calls c - ON c.id = json_extract_string(t.call_ids, '$[0]') + FROM traces t + JOIN spans c + ON c.id = json_extract_string(t.span_ids, '$[0]') WHERE t.start_time >= ? AND t.start_time < ? ) SELECT caller_ip, to_ip, to_port, COUNT(*) AS turn_count @@ -1655,7 +1655,7 @@ impl DuckDbBackend { .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? } - /// Aggregate agent_turns by agent_kind over the window — drives + /// Aggregate traces by agent_kind over the window — drives /// the Overview distribution horizontal-bar chart. pub(crate) async fn query_agent_summary( &self, @@ -1673,7 +1673,7 @@ impl DuckDbBackend { COALESCE(SUM(total_output_tokens), 0)::UBIGINT AS total_output_tokens, AVG(duration_ms) AS avg_duration_ms, epoch_ms(MAX(start_time)) AS last_seen_ms - FROM agent_turns + FROM traces WHERE start_time >= ? AND start_time < ? GROUP BY agent_kind ORDER BY turn_count DESC"; @@ -1747,7 +1747,7 @@ impl DuckDbBackend { epoch_ms(time_bucket(INTERVAL {bucket} SECOND, start_time)) AS ts, agent_kind, COUNT(*) AS turn_count - FROM agent_turns + FROM traces WHERE start_time >= ? AND start_time < ? GROUP BY ts, agent_kind ORDER BY ts ASC, agent_kind ASC" diff --git a/server/h-storage-duckdb/src/retention.rs b/server/h-storage-duckdb/src/retention.rs index 2398721f..130ad1fa 100644 --- a/server/h-storage-duckdb/src/retention.rs +++ b/server/h-storage-duckdb/src/retention.rs @@ -8,26 +8,26 @@ use crate::DuckDbBackend; impl DuckDbBackend { pub(crate) async fn apply_retention(&self, policy: RetentionPolicy) -> Result { - let calls_conn = self.write_calls_conn.clone(); - let turns_conn = self.write_turns_conn.clone(); + let calls_conn = self.write_spans_conn.clone(); + let turns_conn = self.write_traces_conn.clone(); let metrics_conn = self.write_metrics_conn.clone(); let exchanges_conn = self.write_exchanges_conn.clone(); tokio::task::spawn_blocking(move || { let mut report = RetentionReport::default(); - if let Some(cutoff) = policy.calls_before { + if let Some(cutoff) = policy.spans_before { let ts = timestamp_value(cutoff)?; let conn = calls_conn .lock() .map_err(|e| AppError::Storage(format!("failed to lock calls writer: {e}")))?; let n = conn .execute( - "DELETE FROM llm_calls WHERE request_time < ?1", + "DELETE FROM spans WHERE request_time < ?1", duckdb::params![ts], ) - .map_err(|e| AppError::Storage(format!("failed to delete llm_calls: {e}")))?; - report.calls_deleted = n as u64; + .map_err(|e| AppError::Storage(format!("failed to delete spans: {e}")))?; + report.spans_deleted = n as u64; } if let Some(cutoff) = policy.http_exchanges_before { @@ -46,18 +46,18 @@ impl DuckDbBackend { report.http_exchanges_deleted = n as u64; } - if let Some(cutoff) = policy.turns_before { + if let Some(cutoff) = policy.traces_before { let ts = timestamp_value(cutoff)?; let conn = turns_conn .lock() .map_err(|e| AppError::Storage(format!("failed to lock turns writer: {e}")))?; let n = conn .execute( - "DELETE FROM agent_turns WHERE end_time < ?1", + "DELETE FROM traces WHERE end_time < ?1", duckdb::params![ts], ) - .map_err(|e| AppError::Storage(format!("failed to delete agent_turns: {e}")))?; - report.turns_deleted = n as u64; + .map_err(|e| AppError::Storage(format!("failed to delete traces: {e}")))?; + report.traces_deleted = n as u64; } for (label, cutoff) in &policy.metrics_before { @@ -113,7 +113,7 @@ mod tests { use h_metrics::model::{LlmFinishMetric, LlmMetric}; use h_storage::retention::RetentionPolicy; use h_storage::StorageBackend; - use h_turn::{AgentTurn, TurnStatus}; + use h_turn::{Trace, TraceStatus}; fn mk_call(id: &str, request_time_us: i64) -> LlmCall { LlmCall { @@ -155,8 +155,8 @@ mod tests { } } - fn mk_turn(id: &str, start_us: i64, duration_ms: u64) -> AgentTurn { - AgentTurn { + fn mk_turn(id: &str, start_us: i64, duration_ms: u64) -> Trace { + Trace { source_id: String::new(), turn_id: id.into(), session_id: "s".into(), @@ -175,13 +175,13 @@ mod tests { total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, total_cost_usd: None, - status: TurnStatus::Complete, + status: TraceStatus::Complete, final_finish_reason: None, user_input_preview: None, user_call_id: None, final_answer_preview: None, final_call_id: None, - call_ids: vec![id.into()], + span_ids: vec![id.into()], metadata: serde_json::json!({}), tool_surfaces: vec![], tool_call_total: 0, @@ -275,7 +275,7 @@ mod tests { // Calls: 1 old (30d), 1 new (1h). backend - .write_calls(vec![ + .write_spans(vec![ mk_call("c-old", now_us - 30 * day_us), mk_call("c-new", now_us - 3600 * 1_000_000), ]) @@ -284,7 +284,7 @@ mod tests { // Turns: 1 old (end_time 31d ago), 1 new (today). backend - .write_turns(vec![ + .write_traces(vec![ mk_turn("t-old", now_us - 31 * day_us, 1000), mk_turn("t-new", now_us - 3600 * 1_000_000, 1000), ]) @@ -325,8 +325,8 @@ mod tests { .unwrap(); let policy = RetentionPolicy { - calls_before: Some(now - Duration::from_secs(7 * 86_400)), - turns_before: Some(now - Duration::from_secs(14 * 86_400)), + spans_before: Some(now - Duration::from_secs(7 * 86_400)), + traces_before: Some(now - Duration::from_secs(14 * 86_400)), http_exchanges_before: None, metrics_before: vec![ ("10s".to_string(), now - Duration::from_secs(86_400)), @@ -337,8 +337,8 @@ mod tests { }; let report = backend.apply_retention(policy).await.unwrap(); - assert_eq!(report.calls_deleted, 1); - assert_eq!(report.turns_deleted, 1); + assert_eq!(report.spans_deleted, 1); + assert_eq!(report.traces_deleted, 1); assert_eq!(report.metrics_deleted.get("10s"), Some(&1)); assert_eq!(report.metrics_deleted.get("1m"), Some(&1)); assert_eq!(report.metrics_deleted.get("5m"), Some(&1)); @@ -346,10 +346,10 @@ mod tests { let conn = backend.test_conn().lock().unwrap(); let calls_count: i64 = conn - .query_row("SELECT COUNT(*) FROM llm_calls", [], |r| r.get(0)) + .query_row("SELECT COUNT(*) FROM spans", [], |r| r.get(0)) .unwrap(); let turns_count: i64 = conn - .query_row("SELECT COUNT(*) FROM agent_turns", [], |r| r.get(0)) + .query_row("SELECT COUNT(*) FROM traces", [], |r| r.get(0)) .unwrap(); let total_metrics: i64 = conn .query_row("SELECT COUNT(*) FROM llm_metrics", [], |r| r.get(0)) diff --git a/server/h-storage-duckdb/src/schema.rs b/server/h-storage-duckdb/src/schema.rs index 4422cd0b..af06fb94 100644 --- a/server/h-storage-duckdb/src/schema.rs +++ b/server/h-storage-duckdb/src/schema.rs @@ -7,7 +7,7 @@ use h_common::error::{AppError, Result}; use crate::DuckDbBackend; const CREATE_LLM_CALLS: &str = " -CREATE TABLE IF NOT EXISTS llm_calls ( +CREATE TABLE IF NOT EXISTS spans ( id VARCHAR NOT NULL PRIMARY KEY, source_id VARCHAR NOT NULL DEFAULT '', client_ip VARCHAR NOT NULL, @@ -44,7 +44,8 @@ CREATE TABLE IF NOT EXISTS llm_calls ( body_bytes_dropped UBIGINT NOT NULL DEFAULT 0, process_pid UINTEGER, process_comm VARCHAR, - process_exe VARCHAR + process_exe VARCHAR, + kind VARCHAR NOT NULL DEFAULT 'llm' ); "; @@ -117,7 +118,7 @@ CREATE INDEX IF NOT EXISTS idx_llm_finish_metrics_ts "; const CREATE_LLM_TURNS: &str = " -CREATE TABLE IF NOT EXISTS agent_turns ( +CREATE TABLE IF NOT EXISTS traces ( turn_id VARCHAR NOT NULL PRIMARY KEY, source_id VARCHAR NOT NULL DEFAULT '', session_id VARCHAR NOT NULL, @@ -142,7 +143,7 @@ CREATE TABLE IF NOT EXISTS agent_turns ( user_call_id VARCHAR, final_answer_preview VARCHAR, final_call_id VARCHAR, - call_ids JSON NOT NULL, + span_ids JSON NOT NULL, metadata VARCHAR, tool_surfaces_json VARCHAR, tool_call_total UINTEGER NOT NULL DEFAULT 0, @@ -178,20 +179,49 @@ CREATE TABLE IF NOT EXISTS http_exchanges ( pub(crate) async fn init(backend: &DuckDbBackend) -> Result<()> { // Any writer works — they share the same DuckDB instance. Using the // calls writer keeps init deterministic. - let conn = backend.write_calls_conn.clone(); + let conn = backend.write_spans_conn.clone(); tokio::task::spawn_blocking(move || { let conn = conn .lock() .map_err(|e| AppError::Storage(format!("failed to lock writer: {e}")))?; + + // Phase 8 migration: OpenTelemetry rename (agent_turns -> traces, + // llm_calls -> spans, traces.call_ids -> span_ids). Runs BEFORE any + // CREATE TABLE so that on a migrated DB the `CREATE TABLE IF NOT EXISTS + // spans/traces` statements below see the renamed tables and no-op, + // instead of creating empty tables that would collide with the rename. + // DuckDB's RENAME has no `IF EXISTS` form, so each rename is guarded by + // detect-then-rename (old present AND new absent) to stay idempotent + // across repeated init() calls. The `kind` column is added later + // (post-CREATE), where `spans` is guaranteed to exist. + if table_exists(&conn, "agent_turns") && !table_exists(&conn, "traces") { + match conn.execute_batch("ALTER TABLE agent_turns RENAME TO traces;") { + Ok(()) => info!("phase8 migration: renamed agent_turns -> traces"), + Err(e) => tracing::warn!(error = %e, "phase8 migration: rename agent_turns -> traces failed (non-fatal)"), + } + } + if table_exists(&conn, "llm_calls") && !table_exists(&conn, "spans") { + match conn.execute_batch("ALTER TABLE llm_calls RENAME TO spans;") { + Ok(()) => info!("phase8 migration: renamed llm_calls -> spans"), + Err(e) => tracing::warn!(error = %e, "phase8 migration: rename llm_calls -> spans failed (non-fatal)"), + } + } + if column_exists(&conn, "traces", "call_ids") && !column_exists(&conn, "traces", "span_ids") { + match conn.execute_batch("ALTER TABLE traces RENAME COLUMN call_ids TO span_ids;") { + Ok(()) => info!("phase8 migration: renamed traces.call_ids -> span_ids"), + Err(e) => tracing::warn!(error = %e, "phase8 migration: rename traces.call_ids -> span_ids failed (non-fatal)"), + } + } + conn.execute_batch(CREATE_LLM_CALLS) - .map_err(|e| AppError::Storage(format!("failed to create llm_calls: {e}")))?; + .map_err(|e| AppError::Storage(format!("failed to create spans: {e}")))?; conn.execute_batch(CREATE_LLM_METRICS) .map_err(|e| AppError::Storage(format!("failed to create llm_metrics: {e}")))?; conn.execute_batch(CREATE_LLM_FINISH_METRICS).map_err(|e| { AppError::Storage(format!("failed to create llm_finish_metrics: {e}")) })?; conn.execute_batch(CREATE_LLM_TURNS) - .map_err(|e| AppError::Storage(format!("failed to create agent_turns: {e}")))?; + .map_err(|e| AppError::Storage(format!("failed to create traces: {e}")))?; conn.execute_batch(CREATE_HTTP_EXCHANGES) .map_err(|e| AppError::Storage(format!("failed to create http_exchanges: {e}")))?; @@ -284,7 +314,7 @@ pub(crate) async fn init(backend: &DuckDbBackend) -> Result<()> { } } - // Phase 3 collapsed TurnStatus to Complete | Incomplete. Migrate + // Phase 3 collapsed TraceStatus to Complete | Incomplete. Migrate // legacy values: // 'length' -> 'complete' (max_tokens IS a wire terminal) // 'failed'/'cancelled' -> 'incomplete' (no wire terminal landed) @@ -292,11 +322,11 @@ pub(crate) async fn init(backend: &DuckDbBackend) -> Result<()> { // lives in agent_turns.final_finish_reason, so no information // loss beyond the status-axis collapse. match conn.execute_batch( - "UPDATE agent_turns SET status='complete' WHERE status = 'length';\n\ - UPDATE agent_turns SET status='incomplete' WHERE status IN ('failed', 'cancelled');", + "UPDATE traces SET status='complete' WHERE status = 'length';\n\ + UPDATE traces SET status='incomplete' WHERE status IN ('failed', 'cancelled');", ) { Ok(()) => tracing::debug!( - "phase3 migration: agent_turns.status legacy values rewritten (or absent)" + "phase3 migration: traces.status legacy values rewritten (or absent)" ), Err(e) => tracing::warn!( error = %e, @@ -323,11 +353,11 @@ pub(crate) async fn init(backend: &DuckDbBackend) -> Result<()> { // writes stay aligned. Fresh installs still get NOT NULL via // CREATE TABLE. let agent_columns = [ - "ALTER TABLE llm_calls ADD COLUMN IF NOT EXISTS is_agent_request BOOLEAN DEFAULT FALSE;", - "ALTER TABLE llm_calls ADD COLUMN IF NOT EXISTS tool_surface VARCHAR;", - "ALTER TABLE llm_calls ADD COLUMN IF NOT EXISTS agent_topology VARCHAR;", - "ALTER TABLE llm_calls ADD COLUMN IF NOT EXISTS tool_call_count UINTEGER DEFAULT 0;", - "ALTER TABLE llm_calls ADD COLUMN IF NOT EXISTS tool_names_json VARCHAR;", + "ALTER TABLE spans ADD COLUMN IF NOT EXISTS is_agent_request BOOLEAN DEFAULT FALSE;", + "ALTER TABLE spans ADD COLUMN IF NOT EXISTS tool_surface VARCHAR;", + "ALTER TABLE spans ADD COLUMN IF NOT EXISTS agent_topology VARCHAR;", + "ALTER TABLE spans ADD COLUMN IF NOT EXISTS tool_call_count UINTEGER DEFAULT 0;", + "ALTER TABLE spans ADD COLUMN IF NOT EXISTS tool_names_json VARCHAR;", ]; for stmt in agent_columns { match conn.execute_batch(stmt) { @@ -351,10 +381,10 @@ pub(crate) async fn init(backend: &DuckDbBackend) -> Result<()> { // of the table in both CREATE and ALTER so the appender's positional // writes stay aligned. Fresh installs get NOT NULL via CREATE TABLE. match conn.execute_batch( - "ALTER TABLE llm_calls ADD COLUMN IF NOT EXISTS body_bytes_dropped UBIGINT DEFAULT 0;", + "ALTER TABLE spans ADD COLUMN IF NOT EXISTS body_bytes_dropped UBIGINT DEFAULT 0;", ) { Ok(()) => tracing::debug!( - "phase6 migration: llm_calls.body_bytes_dropped added (or already present)" + "phase6 migration: spans.body_bytes_dropped added (or already present)" ), Err(e) => { tracing::info!("phase6 migration: llm_calls.body_bytes_dropped add skipped: {e}") @@ -370,9 +400,9 @@ pub(crate) async fn init(backend: &DuckDbBackend) -> Result<()> { // at the tail of the table in both CREATE and ALTER so the appender's // positional writes stay aligned. let process_columns = [ - "ALTER TABLE llm_calls ADD COLUMN IF NOT EXISTS process_pid UINTEGER;", - "ALTER TABLE llm_calls ADD COLUMN IF NOT EXISTS process_comm VARCHAR;", - "ALTER TABLE llm_calls ADD COLUMN IF NOT EXISTS process_exe VARCHAR;", + "ALTER TABLE spans ADD COLUMN IF NOT EXISTS process_pid UINTEGER;", + "ALTER TABLE spans ADD COLUMN IF NOT EXISTS process_comm VARCHAR;", + "ALTER TABLE spans ADD COLUMN IF NOT EXISTS process_exe VARCHAR;", ]; for stmt in process_columns { match conn.execute_batch(stmt) { @@ -392,10 +422,10 @@ pub(crate) async fn init(backend: &DuckDbBackend) -> Result<()> { // abort the rest. // Same NOT NULL-omission rationale as the llm_calls block above. let turn_agent_columns = [ - "ALTER TABLE agent_turns ADD COLUMN IF NOT EXISTS tool_surfaces_json VARCHAR;", - "ALTER TABLE agent_turns ADD COLUMN IF NOT EXISTS tool_call_total UINTEGER DEFAULT 0;", - "ALTER TABLE agent_turns ADD COLUMN IF NOT EXISTS agent_topology VARCHAR;", - "ALTER TABLE agent_turns ADD COLUMN IF NOT EXISTS suspicious_skills_json VARCHAR;", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS tool_surfaces_json VARCHAR;", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS tool_call_total UINTEGER DEFAULT 0;", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS agent_topology VARCHAR;", + "ALTER TABLE traces ADD COLUMN IF NOT EXISTS suspicious_skills_json VARCHAR;", ]; for stmt in turn_agent_columns { match conn.execute_batch(stmt) { @@ -421,6 +451,19 @@ pub(crate) async fn init(backend: &DuckDbBackend) -> Result<()> { ), } + // Phase 8 (kind column): added here, post-CREATE, where `spans` is + // guaranteed to exist (fresh CREATE or Phase-8 rename above). Added + // nullable-with-default for the same DuckDB reason as Phase 5/6/7 + // (DuckDB rejects `ADD COLUMN ... NOT NULL DEFAULT`). It sits at the + // tail of the table in both CREATE and ALTER so the appender's + // positional writes stay aligned. Fresh installs get NOT NULL via + // CREATE TABLE. Today every span is an LLM call (`kind='llm'`); the + // column is forward-looking for wire-visible tool spans. + match conn.execute_batch("ALTER TABLE spans ADD COLUMN IF NOT EXISTS kind VARCHAR DEFAULT 'llm';") { + Ok(()) => tracing::debug!("phase8 migration: spans.kind added (or already present)"), + Err(e) => tracing::info!("phase8 migration: spans.kind add skipped: {e}"), + } + info!("storage tables initialized"); Ok(()) }) @@ -440,11 +483,36 @@ fn rollup_empty_but_calls_present(conn: &duckdb::Connection) -> bool { return false; } let call_rows: i64 = conn - .query_row("SELECT COUNT(*) FROM llm_calls", [], |r| r.get(0)) + .query_row("SELECT COUNT(*) FROM spans", [], |r| r.get(0)) .unwrap_or(0); call_rows > 0 } +/// Whether a table with `name` exists in the current DuckDB database. +/// Used by the Phase 8 OTel rename to guard the (no-`IF EXISTS`) RENAME +/// statements so init() stays idempotent. +fn table_exists(conn: &duckdb::Connection, name: &str) -> bool { + conn.query_row( + "SELECT COUNT(*) FROM duckdb_tables() WHERE table_name = ?", + [name], + |r| r.get::<_, i64>(0), + ) + .map(|n| n > 0) + .unwrap_or(false) +} + +/// Whether `table`.`column` exists. Companion to `table_exists` for the +/// Phase 8 column rename guard. +fn column_exists(conn: &duckdb::Connection, table: &str, column: &str) -> bool { + conn.query_row( + "SELECT COUNT(*) FROM duckdb_columns() WHERE table_name = ? AND column_name = ?", + [table, column], + |r| r.get::<_, i64>(0), + ) + .map(|n| n > 0) + .unwrap_or(false) +} + /// Build the SQL that re-aggregates llm_calls into llm_metrics at one /// granularity. Mirrors `WindowBucket::on_call_complete`. One /// simplification: @@ -539,7 +607,7 @@ fn backfill_sql(granularity_label: &str, time_bucket_interval: &str) -> String { THEN (EPOCH_US(complete_time) - EPOCH_US(response_time)) / 1000.0 / output_tokens END, 0.99), NULL AS tool_surface - FROM llm_calls + FROM spans GROUP BY time_bucket({time_bucket_interval}, request_time), source_id, @@ -613,7 +681,7 @@ mod tests { backend.init().await.unwrap(); let conn = backend.test_conn().lock().unwrap(); - let mut stmt = conn.prepare("SELECT COUNT(*) FROM llm_calls").unwrap(); + let mut stmt = conn.prepare("SELECT COUNT(*) FROM spans").unwrap(); let count: i64 = stmt.query_row([], |row| row.get(0)).unwrap(); assert_eq!(count, 0); diff --git a/server/h-storage-duckdb/src/sessions.rs b/server/h-storage-duckdb/src/sessions.rs index 49d18672..406ac061 100644 --- a/server/h-storage-duckdb/src/sessions.rs +++ b/server/h-storage-duckdb/src/sessions.rs @@ -1,4 +1,4 @@ -//! Session-scoped queries. Sessions are a view over `agent_turns` +//! Session-scoped queries. Sessions are a view over `traces` //! grouped by `(source_id, session_id)` — no schema of their own. use h_common::error::{AppError, Result}; @@ -56,7 +56,7 @@ impl DuckDbBackend { let step1_sql = format!( "SELECT source_id, session_id, epoch_ms(MAX(end_time)) AS last_ms \ - FROM agent_turns \ + FROM traces \ WHERE {where_sql} \ GROUP BY source_id, session_id{having_sql} \ ORDER BY MAX(end_time) DESC, source_id DESC, session_id DESC \ @@ -141,7 +141,7 @@ impl DuckDbBackend { total_cache_read_input_tokens, total_cache_creation_input_tokens, \ total_cost_usd, agent_kind, user_input_preview, user_call_id, \ ROW_NUMBER() OVER (PARTITION BY source_id, session_id ORDER BY start_time) AS rn \ - FROM agent_turns \ + FROM traces \ WHERE (source_id, session_id) IN ({pairs_sql}) \ ) t \ GROUP BY source_id, session_id" @@ -268,7 +268,7 @@ impl DuckDbBackend { total_cache_read_input_tokens, total_cache_creation_input_tokens, \ total_cost_usd, agent_kind, user_input_preview, user_call_id, \ ROW_NUMBER() OVER (PARTITION BY source_id, session_id ORDER BY start_time) AS rn \ - FROM agent_turns \ + FROM traces \ WHERE source_id = ? AND session_id = ? \ ) t \ GROUP BY source_id, session_id"; @@ -340,10 +340,10 @@ impl DuckDbBackend { .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? } - pub(crate) async fn query_session_turns( + pub(crate) async fn query_session_traces( &self, - query: &SessionTurnsQuery, - ) -> Result { + query: &SessionTracesQuery, + ) -> Result { let conn = self.read_pool.acquire().await?; let query = query.clone(); @@ -362,7 +362,7 @@ impl DuckDbBackend { (String::new(), None) }; - // Paging query. SELECT returns SessionTurnItem columns + preview + + // Paging query. SELECT returns SessionTraceItem columns + preview + // call_id for each side so we know whether to run full-text // extraction below. let sql = format!( @@ -376,7 +376,7 @@ impl DuckDbBackend { user_input_preview, user_call_id, \ final_answer_preview, final_call_id, \ tool_surfaces_json, tool_call_total, agent_topology, suspicious_skills_json \ - FROM agent_turns \ + FROM traces \ WHERE source_id = ? AND session_id = ?{cursor_sql} \ ORDER BY start_time DESC, turn_id DESC \ LIMIT {limit}" @@ -504,7 +504,7 @@ impl DuckDbBackend { let user_map = extract_full_text_batch(&conn, ExtractKind::User, &need_user); let asst_map = extract_full_text_batch(&conn, ExtractKind::Assistant, &need_assistant); - let mut items: Vec = Vec::with_capacity(fetched.len()); + let mut items: Vec = Vec::with_capacity(fetched.len()); for t in fetched { let ( turn_id, @@ -551,7 +551,7 @@ impl DuckDbBackend { .and_then(|s| serde_json::from_str(s).ok()) .unwrap_or_default(); - items.push(SessionTurnItem { + items.push(SessionTraceItem { turn_id, source_id, session_id, @@ -578,7 +578,7 @@ impl DuckDbBackend { let next_cursor = if has_more { items.last().map(|last| { - encode_session_turns_cursor(&SessionTurnsCursor { + encode_session_turns_cursor(&SessionTracesCursor { start_time_us: last.start_time.saturating_mul(1000), turn_id: last.turn_id.clone(), }) @@ -587,7 +587,7 @@ impl DuckDbBackend { None }; - Ok(SessionTurnsPage { items, next_cursor }) + Ok(SessionTracesPage { items, next_cursor }) }) .await .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? @@ -600,7 +600,7 @@ mod tests { use h_llm::wire_apis as wa; use h_storage::query::{SessionListQuery, TimeRange}; use h_storage::StorageBackend; - use h_turn::{AgentTurn, TurnStatus}; + use h_turn::{Trace, TraceStatus}; fn in_memory() -> DuckDbBackend { DuckDbBackend::open(":memory:").unwrap() @@ -611,8 +611,8 @@ mod tests { session_id: &str, agent_kind: &str, start_us: i64, - ) -> AgentTurn { - AgentTurn { + ) -> Trace { + Trace { source_id: String::new(), turn_id: turn_id.into(), session_id: session_id.into(), @@ -631,13 +631,13 @@ mod tests { total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, total_cost_usd: None, - status: TurnStatus::Complete, + status: TraceStatus::Complete, final_finish_reason: Some("complete".into()), user_input_preview: Some("hello".into()), user_call_id: None, final_answer_preview: Some("world".into()), final_call_id: None, - call_ids: vec!["call-1".into()], + span_ids: vec!["call-1".into()], metadata: serde_json::json!({}), tool_surfaces: vec![], tool_call_total: 0, @@ -658,7 +658,7 @@ mod tests { // Create 4 sessions with different agent_kinds backend - .write_turns(vec![ + .write_traces(vec![ // Session 1: claude-cli sample_turn("t1", "s1", "claude-cli", base + 1_000_000), // Session 2: codex-cli diff --git a/server/h-storage-duckdb/src/turns.rs b/server/h-storage-duckdb/src/turns.rs index d14a306e..f999385d 100644 --- a/server/h-storage-duckdb/src/turns.rs +++ b/server/h-storage-duckdb/src/turns.rs @@ -1,9 +1,9 @@ -//! `agent_turns` table I/O — write, paginated query, by-id detail. +//! `traces` table I/O — write, paginated query, by-id detail. use duckdb::types::{TimeUnit, Value}; use h_common::error::{AppError, Result}; use h_storage::query::*; -use h_turn::AgentTurn; +use h_turn::Trace; use crate::util::{extract_full_text, parse_json_string_list, us_to_timestamp, ExtractKind}; use crate::DuckDbBackend; @@ -70,7 +70,7 @@ struct PreparedTurn { user_call_id: Option, final_answer_preview: Option, final_call_id: Option, - call_ids: String, + span_ids: String, metadata: String, tool_surfaces_json: String, tool_call_total: u32, @@ -78,7 +78,7 @@ struct PreparedTurn { suspicious_skills_json: String, } -fn prepare_turn(t: AgentTurn) -> PreparedTurn { +fn prepare_turn(t: Trace) -> PreparedTurn { // Serialize tool_surfaces as a JSON array of snake_case strings. let tool_surfaces_json = { let strings: Vec = t.tool_surfaces.iter().map(|s| s.to_string()).collect(); @@ -111,7 +111,7 @@ fn prepare_turn(t: AgentTurn) -> PreparedTurn { user_call_id: t.user_call_id, final_answer_preview: t.final_answer_preview, final_call_id: t.final_call_id, - call_ids: serde_json::to_string(&t.call_ids).unwrap_or_default(), + span_ids: serde_json::to_string(&t.span_ids).unwrap_or_default(), metadata: t.metadata.to_string(), tool_surfaces_json, tool_call_total: t.tool_call_total, @@ -121,7 +121,7 @@ fn prepare_turn(t: AgentTurn) -> PreparedTurn { } impl DuckDbBackend { - pub(crate) async fn write_turns(&self, turns: Vec) -> Result<()> { + pub(crate) async fn write_traces(&self, turns: Vec) -> Result<()> { if turns.is_empty() { return Ok(()); } @@ -135,7 +135,7 @@ impl DuckDbBackend { return Err(crate::fault_injection::disk_full_error()); } } - let conn = self.write_turns_conn.clone(); + let conn = self.write_traces_conn.clone(); tokio::task::spawn_blocking(move || { let prepared: Vec = turns.into_iter().map(prepare_turn).collect(); @@ -143,7 +143,7 @@ impl DuckDbBackend { .lock() .map_err(|e| AppError::Storage(format!("failed to lock writer: {e}")))?; let mut appender = conn - .appender("agent_turns") + .appender("traces") .map_err(|e| AppError::Storage(format!("failed to create turns appender: {e}")))?; for p in &prepared { appender @@ -172,7 +172,7 @@ impl DuckDbBackend { p.user_call_id, p.final_answer_preview, p.final_call_id, - p.call_ids, + p.span_ids, p.metadata, p.tool_surfaces_json, p.tool_call_total, @@ -190,7 +190,7 @@ impl DuckDbBackend { .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? } - pub(crate) async fn query_turns(&self, query: &TurnsQuery) -> Result { + pub(crate) async fn query_traces(&self, query: &TracesQuery) -> Result { const VALID_SORT_FIELDS: &[&str] = &[ "start_time", "end_time", @@ -270,15 +270,15 @@ impl DuckDbBackend { where_parts.push(format!("client_ip IN ({})", list.join(", "))); } if !query.server_ports.is_empty() { - // agent_turns has no server_port column, so we resolve it - // via the turn's first call_id against llm_calls — same + // traces has no server_port column, so we resolve it + // via the turn's first call_id against spans — same // shortcut the topology query uses. EXISTS is cheaper // than a JOIN here because we never select from the // joined row. let list: Vec = query.server_ports.iter().map(|p| p.to_string()).collect(); where_parts.push(format!( - "EXISTS (SELECT 1 FROM llm_calls c \ - WHERE c.id = json_extract_string(agent_turns.call_ids, '$[0]') \ + "EXISTS (SELECT 1 FROM spans c \ + WHERE c.id = json_extract_string(traces.span_ids, '$[0]') \ AND c.server_port IN ({}))", list.join(", ") )); @@ -310,7 +310,7 @@ impl DuckDbBackend { let where_sql = where_parts.join(" AND "); let sort_by = &query.sort_by; - let count_sql = format!("SELECT COUNT(*) FROM agent_turns WHERE {where_sql}"); + let count_sql = format!("SELECT COUNT(*) FROM traces WHERE {where_sql}"); let mut count_stmt = conn .prepare(&count_sql) .map_err(|e| AppError::Storage(format!("failed to prepare count query: {e}")))?; @@ -328,7 +328,7 @@ impl DuckDbBackend { final_finish_reason, user_input_preview, final_answer_preview, \ client_ip, server_ip, metadata, \ tool_surfaces_json, tool_call_total, agent_topology, suspicious_skills_json \ - FROM agent_turns WHERE {where_sql} \ + FROM traces WHERE {where_sql} \ ORDER BY {sort_by} {sort_order} \ LIMIT {limit} OFFSET {offset}" ); @@ -367,7 +367,7 @@ impl DuckDbBackend { .as_deref() .and_then(|s| serde_json::from_str(s).ok()) .unwrap_or_default(); - items.push(TurnListItem { + items.push(TraceListItem { turn_id: row .get(0) .map_err(|e| AppError::Storage(format!("read error: {e}")))?, @@ -436,13 +436,13 @@ impl DuckDbBackend { }); } - Ok(TurnsPage { total, items }) + Ok(TracesPage { total, items }) }) .await .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? } - pub(crate) async fn query_turn_by_id(&self, turn_id: &str) -> Result> { + pub(crate) async fn query_trace_by_id(&self, turn_id: &str) -> Result> { let conn = self.read_pool.acquire().await?; let turn_id = turn_id.to_string(); @@ -457,10 +457,10 @@ impl DuckDbBackend { total_cost_usd, status, final_finish_reason, user_input_preview, user_call_id, final_answer_preview, final_call_id, - call_ids, metadata, + span_ids, metadata, client_ip, server_ip, tool_surfaces_json, tool_call_total, agent_topology, suspicious_skills_json - FROM agent_turns + FROM traces WHERE turn_id = ? "; @@ -493,7 +493,7 @@ impl DuckDbBackend { row.get::<_, Option>(19)?, // user_call_id row.get::<_, Option>(20)?, // final_answer_preview row.get::<_, Option>(21)?, // final_call_id - row.get::<_, Option>(22)?, // call_ids (JSON) + row.get::<_, Option>(22)?, // span_ids (JSON) row.get::<_, Option>(23)?, // metadata row.get::<_, String>(24)?, // client_ip row.get::<_, String>(25)?, // server_ip @@ -537,7 +537,7 @@ impl DuckDbBackend { user_call_id, final_answer_preview, final_call_id, - call_ids_raw, + span_ids_raw, metadata_raw, client_ip, server_ip, @@ -549,7 +549,7 @@ impl DuckDbBackend { let models_used = parse_json_string_list(models_used_raw.as_deref()); let subagents_used = parse_json_string_list(subagents_used_raw.as_deref()); - let call_ids = parse_json_string_list(call_ids_raw.as_deref()); + let span_ids = parse_json_string_list(span_ids_raw.as_deref()); let metadata = metadata_raw .as_deref() .and_then(|s| serde_json::from_str::(s).ok()); @@ -562,7 +562,7 @@ impl DuckDbBackend { // `truncate_preview` in h-turn appends `…` only when it truncates, // so a preview that does not end in `…` is already the full text — - // skip the llm_calls lookup + profile re-extraction in that case. + // skip the spans lookup + profile re-extraction in that case. let user_input = match user_input_preview.as_deref() { Some(p) if !p.ends_with('…') => user_input_preview.clone(), _ => extract_full_text( @@ -584,7 +584,7 @@ impl DuckDbBackend { .or_else(|| final_answer_preview.clone()), }; - Ok(Some(TurnDetail { + Ok(Some(TraceDetail { turn_id, source_id, session_id, @@ -609,7 +609,7 @@ impl DuckDbBackend { user_input, final_call_id, final_answer, - call_ids, + span_ids, metadata, tool_surfaces, tool_call_total, @@ -621,7 +621,7 @@ impl DuckDbBackend { .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? } - /// Light projection of `agent_turns` rows for pair-detection. Skips + /// Light projection of `traces` rows for pair-detection. Skips /// rows whose `metadata` already encodes a `proxy.role`. pub(crate) async fn query_pair_candidates( &self, @@ -645,7 +645,7 @@ impl DuckDbBackend { final_finish_reason, models_used, client_ip, server_ip - FROM agent_turns + FROM traces WHERE start_time >= ? AND start_time < ? AND (metadata IS NULL @@ -715,11 +715,11 @@ impl DuckDbBackend { .map_err(|e| AppError::Storage(format!("spawn_blocking failed: {e}")))? } - /// Read-modify-write `agent_turns.metadata` on a single row: merge + /// Read-modify-write `traces.metadata` on a single row: merge /// `patch` (shallow, top-level key replacement) into the existing /// JSON object. No-op if `turn_id` doesn't exist — the sweeper may /// race finalization and a target turn can be momentarily absent. - pub(crate) async fn update_turn_metadata( + pub(crate) async fn update_trace_metadata( &self, turn_id: &str, patch: serde_json::Value, @@ -734,7 +734,7 @@ impl DuckDbBackend { return Err(crate::fault_injection::disk_full_error()); } } - let conn = self.write_turns_conn.clone(); + let conn = self.write_traces_conn.clone(); let turn_id = turn_id.to_string(); tokio::task::spawn_blocking(move || { let conn = conn @@ -742,7 +742,7 @@ impl DuckDbBackend { .map_err(|e| AppError::Storage(format!("failed to lock writer: {e}")))?; let existing: Option> = conn .query_row( - "SELECT metadata FROM agent_turns WHERE turn_id = ?", + "SELECT metadata FROM traces WHERE turn_id = ?", duckdb::params![turn_id], |r| r.get(0), ) @@ -768,7 +768,7 @@ impl DuckDbBackend { }; let merged_str = merged.to_string(); conn.execute( - "UPDATE agent_turns SET metadata = ? WHERE turn_id = ?", + "UPDATE traces SET metadata = ? WHERE turn_id = ?", duckdb::params![merged_str, turn_id], ) .map_err(|e| AppError::Storage(format!("update turn metadata: {e}")))?; @@ -787,7 +787,7 @@ mod tests { use h_llm::wire_apis as wa; use h_storage::query::*; use h_storage::StorageBackend; - use h_turn::{AgentTurn, TurnStatus}; + use h_turn::{Trace, TraceStatus}; fn sample_turn( turn_id: &str, @@ -797,10 +797,10 @@ mod tests { start_us: i64, duration_ms: u64, call_count: u32, - call_ids: Vec<&str>, - status: TurnStatus, - ) -> AgentTurn { - AgentTurn { + span_ids: Vec<&str>, + status: TraceStatus, + ) -> Trace { + Trace { source_id: String::new(), turn_id: turn_id.into(), session_id: session_id.into(), @@ -825,7 +825,7 @@ mod tests { user_call_id: None, final_answer_preview: Some("world".into()), final_call_id: None, - call_ids: call_ids.into_iter().map(String::from).collect(), + span_ids: span_ids.into_iter().map(String::from).collect(), metadata: serde_json::json!({}), tool_surfaces: vec![], tool_call_total: 0, @@ -887,13 +887,13 @@ mod tests { 1500, 3, vec!["call-42"], - TurnStatus::Complete, + TraceStatus::Complete, ); - backend.write_turns(vec![turn]).await.unwrap(); + backend.write_traces(vec![turn]).await.unwrap(); } - fn base_turns_query() -> TurnsQuery { - TurnsQuery { + fn base_turns_query() -> TracesQuery { + TracesQuery { time_range: TimeRange { start_us: 1_700_000_000_000_000 - 1, end_us: 1_800_000_000_000_000, @@ -912,7 +912,7 @@ mod tests { } #[tokio::test] - async fn query_turns_filters_and_paginates() { + async fn query_traces_filters_and_paginates() { let backend = DuckDbBackend::open(":memory:").unwrap(); backend.init().await.unwrap(); @@ -927,7 +927,7 @@ mod tests { 100, 1, vec!["c1"], - TurnStatus::Complete, + TraceStatus::Complete, ), sample_turn( "t2", @@ -938,7 +938,7 @@ mod tests { 200, 2, vec!["c2", "c3"], - TurnStatus::Complete, + TraceStatus::Complete, ), sample_turn( "t3", @@ -949,7 +949,7 @@ mod tests { 300, 3, vec!["c4"], - TurnStatus::Incomplete, + TraceStatus::Incomplete, ), sample_turn( "t4", @@ -960,13 +960,13 @@ mod tests { 400, 4, vec!["c5"], - TurnStatus::Complete, + TraceStatus::Complete, ), ]; - backend.write_turns(turns).await.unwrap(); + backend.write_traces(turns).await.unwrap(); // No filter: all 4 turns, default sort_by=start_time DESC - let page = backend.query_turns(&base_turns_query()).await.unwrap(); + let page = backend.query_traces(&base_turns_query()).await.unwrap(); assert_eq!(page.total, 4); assert_eq!(page.items.len(), 4); assert_eq!(page.items[0].turn_id, "t4"); @@ -977,23 +977,23 @@ mod tests { // wire_api filter let mut q = base_turns_query(); q.filter.wire_apis = vec![wa::ANTHROPIC.into()]; - let page = backend.query_turns(&q).await.unwrap(); + let page = backend.query_traces(&q).await.unwrap(); assert_eq!(page.total, 1); assert_eq!(page.items[0].turn_id, "t2"); // Model filter via list_has_any — should include t1 and t4 (both list gpt-4) let mut q = base_turns_query(); q.filter.models = vec!["gpt-4".into()]; - let page = backend.query_turns(&q).await.unwrap(); + let page = backend.query_traces(&q).await.unwrap(); assert_eq!(page.total, 2); let ids: Vec<_> = page.items.iter().map(|t| t.turn_id.clone()).collect(); assert!(ids.contains(&"t1".to_string())); assert!(ids.contains(&"t4".to_string())); - // Status filter (TurnStatus Display: incomplete) + // Status filter (TraceStatus Display: incomplete) let mut q = base_turns_query(); q.statuses = vec!["incomplete".into()]; - let page = backend.query_turns(&q).await.unwrap(); + let page = backend.query_traces(&q).await.unwrap(); assert_eq!(page.total, 1); assert_eq!(page.items[0].turn_id, "t3"); @@ -1001,7 +1001,7 @@ mod tests { let mut q = base_turns_query(); q.sort_by = "duration_ms".into(); q.sort_order = "asc".into(); - let page = backend.query_turns(&q).await.unwrap(); + let page = backend.query_traces(&q).await.unwrap(); let durations: Vec<_> = page.items.iter().map(|t| t.duration_ms).collect(); assert_eq!(durations, vec![100, 200, 300, 400]); @@ -1009,22 +1009,22 @@ mod tests { let mut q = base_turns_query(); q.page_size = 2; q.page = 1; - let page1 = backend.query_turns(&q).await.unwrap(); + let page1 = backend.query_traces(&q).await.unwrap(); assert_eq!(page1.total, 4); assert_eq!(page1.items.len(), 2); q.page = 2; - let page2 = backend.query_turns(&q).await.unwrap(); + let page2 = backend.query_traces(&q).await.unwrap(); assert_eq!(page2.items.len(), 2); assert_ne!(page1.items[0].turn_id, page2.items[0].turn_id); // Invalid sort field is rejected let mut q = base_turns_query(); q.sort_by = "bogus".into(); - assert!(backend.query_turns(&q).await.is_err()); + assert!(backend.query_traces(&q).await.is_err()); } #[tokio::test] - async fn query_turn_by_id_hit_and_miss() { + async fn query_trace_by_id_hit_and_miss() { let backend = DuckDbBackend::open(":memory:").unwrap(); backend.init().await.unwrap(); @@ -1037,29 +1037,29 @@ mod tests { 1500, 2, vec!["call-a", "call-b"], - TurnStatus::Complete, + TraceStatus::Complete, ); - backend.write_turns(vec![turn]).await.unwrap(); + backend.write_traces(vec![turn]).await.unwrap(); - let hit = backend.query_turn_by_id("t-detail").await.unwrap(); + let hit = backend.query_trace_by_id("t-detail").await.unwrap(); let d = hit.expect("turn exists"); assert_eq!(d.turn_id, "t-detail"); assert_eq!(d.models_used, vec!["claude-sonnet", "claude-haiku"]); - assert_eq!(d.call_ids, vec!["call-a", "call-b"]); + assert_eq!(d.span_ids, vec!["call-a", "call-b"]); // With no user_call_id/final_call_id, full text falls back to previews. assert_eq!(d.user_input.as_deref(), Some("hello")); assert_eq!(d.final_answer.as_deref(), Some("world")); - let miss = backend.query_turn_by_id("does-not-exist").await.unwrap(); + let miss = backend.query_trace_by_id("does-not-exist").await.unwrap(); assert!(miss.is_none()); } #[tokio::test] - async fn query_turn_by_id_skips_calls_lookup_when_preview_complete() { + async fn query_trace_by_id_skips_calls_lookup_when_preview_complete() { let backend = DuckDbBackend::open(":memory:").unwrap(); backend.init().await.unwrap(); - // A matching llm_calls row exists with full-body text that differs from + // A matching spans row exists with full-body text that differs from // the preview. If the optimization works, we return the preview // (short, no trailing `…`) and never touch the body. let base = 1_700_000_000_000_000_i64; @@ -1072,7 +1072,7 @@ mod tests { asst_call.response_body = Some(r#"{"content":[{"type":"text","text":"DB-ASSISTANT-FULL"}]}"#.into()); backend - .write_calls(vec![user_call, asst_call]) + .write_spans(vec![user_call, asst_call]) .await .unwrap(); @@ -1085,30 +1085,30 @@ mod tests { 1500, 2, vec!["c-user", "c-asst"], - TurnStatus::Complete, + TraceStatus::Complete, ); turn.user_input_preview = Some("hi".into()); turn.user_call_id = Some("c-user".into()); turn.final_answer_preview = Some("bye".into()); turn.final_call_id = Some("c-asst".into()); - backend.write_turns(vec![turn]).await.unwrap(); + backend.write_traces(vec![turn]).await.unwrap(); let d = backend - .query_turn_by_id("t-short") + .query_trace_by_id("t-short") .await .unwrap() .expect("turn exists"); - // Preview is returned as-is; no llm_calls lookup happened. + // Preview is returned as-is; no spans lookup happened. assert_eq!(d.user_input.as_deref(), Some("hi")); assert_eq!(d.final_answer.as_deref(), Some("bye")); } #[tokio::test] - async fn query_turn_by_id_reads_full_text_when_preview_truncated() { + async fn query_trace_by_id_reads_full_text_when_preview_truncated() { let backend = DuckDbBackend::open(":memory:").unwrap(); backend.init().await.unwrap(); - // Truncated previews (ending in `…`) must fall through to the llm_calls + // Truncated previews (ending in `…`) must fall through to the spans // lookup and return the full body text. let base = 1_700_000_000_000_000_i64; let full_user: String = "u".repeat(600); @@ -1130,7 +1130,7 @@ mod tests { .to_string(), ); backend - .write_calls(vec![user_call, asst_call]) + .write_spans(vec![user_call, asst_call]) .await .unwrap(); @@ -1145,16 +1145,16 @@ mod tests { 1500, 2, vec!["c-user", "c-asst"], - TurnStatus::Complete, + TraceStatus::Complete, ); turn.user_input_preview = Some(truncated_user); turn.user_call_id = Some("c-user".into()); turn.final_answer_preview = Some(truncated_asst); turn.final_call_id = Some("c-asst".into()); - backend.write_turns(vec![turn]).await.unwrap(); + backend.write_traces(vec![turn]).await.unwrap(); let d = backend - .query_turn_by_id("t-long") + .query_trace_by_id("t-long") .await .unwrap() .expect("turn exists"); @@ -1163,7 +1163,7 @@ mod tests { } #[tokio::test] - async fn query_turn_calls_orders_and_sequences() { + async fn query_trace_spans_orders_and_sequences() { let backend = DuckDbBackend::open(":memory:").unwrap(); backend.init().await.unwrap(); @@ -1173,10 +1173,10 @@ mod tests { mk_call_with_time("call-b", base + 2_000_000), mk_call_with_time("call-a", base + 1_000_000), mk_call_with_time("call-c", base + 3_000_000), - // Extra call not in the turn's call_ids — must be excluded. + // Extra call not in the turn's span_ids — must be excluded. mk_call_with_time("call-other", base + 500_000), ]; - backend.write_calls(calls).await.unwrap(); + backend.write_spans(calls).await.unwrap(); let turn = sample_turn( "t-calls", @@ -1187,11 +1187,11 @@ mod tests { 3000, 3, vec!["call-a", "call-b", "call-c"], - TurnStatus::Complete, + TraceStatus::Complete, ); - backend.write_turns(vec![turn]).await.unwrap(); + backend.write_traces(vec![turn]).await.unwrap(); - let items = backend.query_turn_calls("t-calls", true).await.unwrap(); + let items = backend.query_trace_spans("t-calls", true).await.unwrap(); assert_eq!(items.len(), 3); assert_eq!(items[0].id, "call-a"); assert_eq!(items[0].sequence, 1); @@ -1205,7 +1205,7 @@ mod tests { // identical. Use this same fixture to verify the contract: ids, // sequence, timing, etc. all match; only the 4 heavy fields // come back as None. - let lite = backend.query_turn_calls("t-calls", false).await.unwrap(); + let lite = backend.query_trace_spans("t-calls", false).await.unwrap(); assert_eq!(lite.len(), 3); for (full, lite) in items.iter().zip(lite.iter()) { assert_eq!(full.id, lite.id); @@ -1220,7 +1220,7 @@ mod tests { // Unknown turn → empty vec (not error). let empty = backend - .query_turn_calls("no-such-turn", true) + .query_trace_spans("no-such-turn", true) .await .unwrap(); assert!(empty.is_empty()); @@ -1231,7 +1231,7 @@ mod tests { session_id: &str, start_us: i64, user_input: Option<&str>, - ) -> AgentTurn { + ) -> Trace { let mut t = sample_turn( turn_id, session_id, @@ -1241,7 +1241,7 @@ mod tests { 500, 1, vec![turn_id], - TurnStatus::Complete, + TraceStatus::Complete, ); t.user_input_preview = user_input.map(String::from); t.user_call_id = user_input.map(|_| format!("call-{turn_id}")); @@ -1260,7 +1260,7 @@ mod tests { let base = 1_700_000_000_000_000_i64; let us = |secs: i64| base + secs * 1_000_000; backend - .write_turns(vec![ + .write_traces(vec![ sample_turn_for_session("t1a", "S1", us(10), Some("first S1")), sample_turn_for_session("t1b", "S1", us(50), None), sample_turn_for_session("t2a", "S2", us(30), Some("first S2")), @@ -1350,7 +1350,7 @@ mod tests { let base = 1_700_000_000_000_000_i64; let us = |secs: i64| base + secs * 1_000_000; backend - .write_turns(vec![ + .write_traces(vec![ sample_turn_for_session("ta", "SX", us(10), Some("opener")), sample_turn_for_session("tb", "SX", us(20), None), sample_turn_for_session("tc", "SX", us(30), None), @@ -1372,7 +1372,7 @@ mod tests { // Turns list: ordered by start_time DESC. let turns = backend - .query_session_turns(&SessionTurnsQuery { + .query_session_traces(&SessionTracesQuery { source_id: String::new(), session_id: "SX".into(), cursor: None, @@ -1388,7 +1388,7 @@ mod tests { } #[tokio::test] - async fn query_session_turns_cursor_pagination() { + async fn query_session_traces_cursor_pagination() { use h_storage::query::decode_session_turns_cursor; let backend = DuckDbBackend::open(":memory:").unwrap(); @@ -1399,7 +1399,7 @@ mod tests { // no full-text extraction round-trip is triggered. let base = 1_700_000_000_000_000_i64; let us = |secs: i64| base + secs * 1_000_000; - let turns: Vec = (0..5) + let turns: Vec = (0..5) .map(|i| { sample_turn_for_session( &format!("turn-{i}"), @@ -1409,11 +1409,11 @@ mod tests { ) }) .collect(); - backend.write_turns(turns).await.unwrap(); + backend.write_traces(turns).await.unwrap(); // Page 1: newest 2 (turn-4, turn-3). let p1 = backend - .query_session_turns(&SessionTurnsQuery { + .query_session_traces(&SessionTracesQuery { source_id: String::new(), session_id: "S-CURSOR".into(), cursor: None, @@ -1428,7 +1428,7 @@ mod tests { // Page 2: turn-2, turn-1. let p2 = backend - .query_session_turns(&SessionTurnsQuery { + .query_session_traces(&SessionTracesQuery { source_id: String::new(), session_id: "S-CURSOR".into(), cursor: decode_session_turns_cursor(&cursor1), @@ -1443,7 +1443,7 @@ mod tests { // Page 3: turn-0, no next cursor. let p3 = backend - .query_session_turns(&SessionTurnsQuery { + .query_session_traces(&SessionTracesQuery { source_id: String::new(), session_id: "S-CURSOR".into(), cursor: decode_session_turns_cursor(&cursor2), @@ -1457,11 +1457,11 @@ mod tests { } #[tokio::test] - async fn query_session_turns_extracts_full_text_when_preview_truncated() { + async fn query_session_traces_extracts_full_text_when_preview_truncated() { let backend = DuckDbBackend::open(":memory:").unwrap(); backend.init().await.unwrap(); - // Build llm_calls carrying real bodies that the Anthropic profile + // Build spans carrying real bodies that the Anthropic profile // extractor can parse. Bodies must be long enough that the preview is // `…`-terminated (i.e. > 500 chars so the stored preview is truncated). let base = 1_700_000_000_000_000_i64; @@ -1486,7 +1486,7 @@ mod tests { .to_string(), ); backend - .write_calls(vec![user_call, asst_call]) + .write_spans(vec![user_call, asst_call]) .await .unwrap(); @@ -1502,16 +1502,16 @@ mod tests { 1500, 2, vec!["sc-user", "sc-asst"], - TurnStatus::Complete, + TraceStatus::Complete, ); turn.user_input_preview = Some(truncated_user); turn.user_call_id = Some("sc-user".into()); turn.final_answer_preview = Some(truncated_asst); turn.final_call_id = Some("sc-asst".into()); - backend.write_turns(vec![turn]).await.unwrap(); + backend.write_traces(vec![turn]).await.unwrap(); let page = backend - .query_session_turns(&SessionTurnsQuery { + .query_session_traces(&SessionTracesQuery { source_id: String::new(), session_id: "S-EXTRACT".into(), cursor: None, @@ -1547,7 +1547,7 @@ mod tests { 1000, 1, vec!["c1"], - TurnStatus::Complete, + TraceStatus::Complete, ); let mut t2 = sample_turn( "t2", @@ -1558,7 +1558,7 @@ mod tests { 1000, 1, vec!["c2"], - TurnStatus::Complete, + TraceStatus::Complete, ); // t2 already paired — sweeper should skip it. t2.metadata = serde_json::json!({ @@ -1568,7 +1568,7 @@ mod tests { "peer_turn_id": "tX", } }); - backend.write_turns(vec![t1, t2]).await.unwrap(); + backend.write_traces(vec![t1, t2]).await.unwrap(); let cands = backend .query_pair_candidates(base - 1, base + 2_000_000_000) @@ -1582,7 +1582,7 @@ mod tests { } #[tokio::test] - async fn update_turn_metadata_merges_into_existing_object() { + async fn update_trace_metadata_merges_into_existing_object() { let backend = DuckDbBackend::open(":memory:").unwrap(); backend.init().await.unwrap(); let mut turn = sample_turn( @@ -1594,11 +1594,11 @@ mod tests { 1500, 1, vec!["c1"], - TurnStatus::Complete, + TraceStatus::Complete, ); // Pre-existing metadata key — must survive the patch. turn.metadata = serde_json::json!({"unrelated": "preserve_me"}); - backend.write_turns(vec![turn]).await.unwrap(); + backend.write_traces(vec![turn]).await.unwrap(); let patch = serde_json::json!({ "proxy": { @@ -1607,9 +1607,9 @@ mod tests { "peer_turn_id": "tB", } }); - backend.update_turn_metadata("tA", patch).await.unwrap(); + backend.update_trace_metadata("tA", patch).await.unwrap(); - let detail = backend.query_turn_by_id("tA").await.unwrap().unwrap(); + let detail = backend.query_trace_by_id("tA").await.unwrap().unwrap(); let meta = detail.metadata.expect("metadata json"); assert_eq!( meta.get("unrelated"), @@ -1620,7 +1620,7 @@ mod tests { } #[tokio::test] - async fn query_turns_hides_proxy_hops_by_default_and_surfaces_them_with_flag() { + async fn query_traces_hides_proxy_hops_by_default_and_surfaces_them_with_flag() { let backend = DuckDbBackend::open(":memory:").unwrap(); backend.init().await.unwrap(); let base = 1_700_000_000_000_000_i64; @@ -1635,7 +1635,7 @@ mod tests { 1500, 1, vec!["c_in"], - TurnStatus::Complete, + TraceStatus::Complete, ); t_in.metadata = serde_json::json!({ "proxy": {"role": "proxy_in", "pair_id": "p1", "peer_turn_id": "t_out"} @@ -1649,7 +1649,7 @@ mod tests { 1500, 1, vec!["c_out"], - TurnStatus::Complete, + TraceStatus::Complete, ); t_out.metadata = serde_json::json!({ "proxy": {"role": "proxy_out", "pair_id": "p1", "peer_turn_id": "t_in"} @@ -1663,10 +1663,10 @@ mod tests { 1500, 1, vec!["c_d"], - TurnStatus::Complete, + TraceStatus::Complete, ); backend - .write_turns(vec![t_in.clone(), t_out.clone(), t_direct.clone()]) + .write_traces(vec![t_in.clone(), t_out.clone(), t_direct.clone()]) .await .unwrap(); @@ -1675,7 +1675,7 @@ mod tests { q.time_range.start_us = base - 1; q.time_range.end_us = base + 1_000_000_000; q.include_proxy_hops = false; - let page = backend.query_turns(&q).await.unwrap(); + let page = backend.query_traces(&q).await.unwrap(); let ids: Vec = page.items.iter().map(|i| i.turn_id.clone()).collect(); assert!(ids.contains(&"t_in".to_string())); assert!(ids.contains(&"t_direct".to_string())); @@ -1695,20 +1695,20 @@ mod tests { // Flag flipped — every row is returned including proxy_out. q.include_proxy_hops = true; - let page = backend.query_turns(&q).await.unwrap(); + let page = backend.query_traces(&q).await.unwrap(); let ids: Vec = page.items.iter().map(|i| i.turn_id.clone()).collect(); assert!(ids.contains(&"t_out".to_string())); assert_eq!(page.total, 3); } #[tokio::test] - async fn update_turn_metadata_is_noop_when_turn_absent() { + async fn update_trace_metadata_is_noop_when_turn_absent() { let backend = DuckDbBackend::open(":memory:").unwrap(); backend.init().await.unwrap(); // No turn written; update must succeed silently. let patch = serde_json::json!({"proxy": {"role": "proxy_in"}}); backend - .update_turn_metadata("never-existed", patch) + .update_trace_metadata("never-existed", patch) .await .expect("noop on missing row"); } diff --git a/server/h-storage-duckdb/src/util.rs b/server/h-storage-duckdb/src/util.rs index 16767f39..edda61eb 100644 --- a/server/h-storage-duckdb/src/util.rs +++ b/server/h-storage-duckdb/src/util.rs @@ -87,7 +87,7 @@ pub(crate) fn render_body_for_detail(bytes: Option>) -> Option { } } -/// Load the request_body / response_body of `call_id` from llm_calls and run +/// Load the request_body / response_body of `call_id` from spans and run /// it through the `agent_kind`-matched profile to produce the full user_input /// or final_answer text. Returns `None` if the call row is missing, the /// profile is not registered, or the extractor declines. @@ -102,8 +102,8 @@ pub(crate) fn extract_full_text( let profile = registry.find_by_name(agent_kind)?; let sql = match kind { - ExtractKind::User => "SELECT request_body, wire_api FROM llm_calls WHERE id = ?", - ExtractKind::Assistant => "SELECT response_body, wire_api FROM llm_calls WHERE id = ?", + ExtractKind::User => "SELECT request_body, wire_api FROM spans WHERE id = ?", + ExtractKind::Assistant => "SELECT response_body, wire_api FROM spans WHERE id = ?", }; let (body, wire_api_stored): (Option, String) = conn .query_row(sql, duckdb::params![call_id], |row| { @@ -168,7 +168,7 @@ pub(crate) fn extract_full_text( /// Batch version of `extract_full_text`. Given `(agent_kind, call_id)` pairs /// and an `ExtractKind` selecting which body column to read, issues a single -/// `SELECT ... WHERE id IN (...)` against `llm_calls` and runs each profile's +/// `SELECT ... WHERE id IN (...)` against `spans` and runs each profile's /// extractor to produce the final text. Returns a map keyed by `call_id`. /// /// - Missing call rows, unknown `wire_api`s, or extractors that decline are @@ -186,19 +186,19 @@ pub(crate) fn extract_full_text_batch( } // Build agent_kind lookup keyed by call_id (last-writer-wins if a call id - // appears twice — extremely unlikely given AgentTurn invariants). + // appears twice — extremely unlikely given Trace invariants). let mut agent_by_call: HashMap<&str, &str> = HashMap::new(); for (ak, cid) in requests { agent_by_call.insert(cid.as_str(), ak.as_str()); } - let call_ids: Vec<&str> = agent_by_call.keys().copied().collect(); + let span_ids: Vec<&str> = agent_by_call.keys().copied().collect(); let col = match kind { ExtractKind::User => "request_body", ExtractKind::Assistant => "response_body", }; - let placeholders = vec!["?"; call_ids.len()].join(","); - let sql = format!("SELECT id, wire_api, {col} FROM llm_calls WHERE id IN ({placeholders})"); + let placeholders = vec!["?"; span_ids.len()].join(","); + let sql = format!("SELECT id, wire_api, {col} FROM spans WHERE id IN ({placeholders})"); let registry = build_default_registry(); @@ -206,7 +206,7 @@ pub(crate) fn extract_full_text_batch( return out; }; let params: Vec<&dyn duckdb::ToSql> = - call_ids.iter().map(|s| s as &dyn duckdb::ToSql).collect(); + span_ids.iter().map(|s| s as &dyn duckdb::ToSql).collect(); let Ok(mut rows) = stmt.query(duckdb::params_from_iter(params.iter().copied())) else { return out; }; diff --git a/server/h-storage-duckdb/tests/migrations.rs b/server/h-storage-duckdb/tests/migrations.rs index ac199d9d..6efda8f7 100644 --- a/server/h-storage-duckdb/tests/migrations.rs +++ b/server/h-storage-duckdb/tests/migrations.rs @@ -28,7 +28,7 @@ use tempfile::TempDir; use h_common::process::ProcessInfo; use h_llm::model::{ApiType, LlmCall}; -use h_storage::query::{DimensionFilter, TimeRange, TurnsQuery}; +use h_storage::query::{DimensionFilter, TimeRange, TracesQuery}; use h_storage::StorageBackend; use h_storage_duckdb::DuckDbBackend; @@ -292,14 +292,15 @@ async fn phase5_adds_nullable_agent_columns_to_llm_calls_on_legacy_db() { backend.init().await.expect("init must reconcile legacy schema"); let conn = Connection::open(&path).unwrap(); - let cols = column_names(&conn, "llm_calls"); + // Post-init the table has been renamed `llm_calls` -> `spans` (Phase 8). + let cols = column_names(&conn, "spans"); // The three nullable Phase-5 columns can always be added via // `ALTER ADD COLUMN IF NOT EXISTS`. See FIXME below for the NOT NULL // columns DuckDB refuses to add to a populated/aged table. for expected in ["tool_surface", "agent_topology", "tool_names_json"] { assert!( cols.iter().any(|c| c == expected), - "post-migration llm_calls must contain {expected}, got: {cols:?}" + "post-migration spans must contain {expected}, got: {cols:?}" ); } } @@ -323,11 +324,11 @@ async fn phase5_adds_agent_not_null_columns_to_llm_calls_on_legacy_db() { backend.init().await.expect("init must reconcile legacy schema"); let conn = Connection::open(&path).unwrap(); - let cols = column_names(&conn, "llm_calls"); + let cols = column_names(&conn, "spans"); for expected in ["is_agent_request", "tool_call_count"] { assert!( cols.iter().any(|c| c == expected), - "post-migration llm_calls must contain {expected}, got: {cols:?}" + "post-migration spans must contain {expected}, got: {cols:?}" ); } } @@ -364,10 +365,10 @@ async fn phase6_adds_body_bytes_dropped_and_appender_stays_aligned_on_legacy_db( // Column landed. { let conn = Connection::open(&path).unwrap(); - let cols = column_names(&conn, "llm_calls"); + let cols = column_names(&conn, "spans"); assert!( cols.iter().any(|c| c == "body_bytes_dropped"), - "post-migration llm_calls must contain body_bytes_dropped, got: {cols:?}" + "post-migration spans must contain body_bytes_dropped, got: {cols:?}" ); } @@ -375,9 +376,9 @@ async fn phase6_adds_body_bytes_dropped_and_appender_stays_aligned_on_legacy_db( // read both trailing columns back. Distinct sentinels (7 vs 4242) catch a // positional swap. backend - .write_calls(vec![sample_call("call-phase6", 7, 4242)]) + .write_spans(vec![sample_call("call-phase6", 7, 4242)]) .await - .expect("write_calls must succeed against a migrated table"); + .expect("write_spans must succeed against a migrated table"); // Close the backend (checkpoints DuckDB to the file) before reading via a // fresh connection — a second live connection sees the pre-write snapshot. @@ -386,11 +387,11 @@ async fn phase6_adds_body_bytes_dropped_and_appender_stays_aligned_on_legacy_db( let conn = Connection::open(&path).unwrap(); let (tool_call_count, body_bytes_dropped): (u32, u64) = conn .query_row( - "SELECT tool_call_count, body_bytes_dropped FROM llm_calls WHERE id = 'call-phase6'", + "SELECT tool_call_count, body_bytes_dropped FROM spans WHERE id = 'call-phase6'", [], |r| Ok((r.get::<_, u32>(0)?, r.get::<_, u64>(1)?)), ) - .expect("row must be present after write_calls"); + .expect("row must be present after write_spans"); assert_eq!( tool_call_count, 7, "tool_call_count must round-trip (appender column alignment)" @@ -431,11 +432,11 @@ async fn phase7_adds_process_columns_and_appender_stays_aligned_on_legacy_db() { // Columns landed. { let conn = Connection::open(&path).unwrap(); - let cols = column_names(&conn, "llm_calls"); + let cols = column_names(&conn, "spans"); for col in ["process_pid", "process_comm", "process_exe"] { assert!( cols.iter().any(|c| c == col), - "post-migration llm_calls must contain {col}, got: {cols:?}" + "post-migration spans must contain {col}, got: {cols:?}" ); } } @@ -450,9 +451,9 @@ async fn phase7_adds_process_columns_and_appender_stays_aligned_on_legacy_db() { exe: Some("/usr/bin/python3.12".into()), }); backend - .write_calls(vec![call]) + .write_spans(vec![call]) .await - .expect("write_calls must succeed against a migrated table"); + .expect("write_spans must succeed against a migrated table"); drop(backend); @@ -460,7 +461,7 @@ async fn phase7_adds_process_columns_and_appender_stays_aligned_on_legacy_db() { let (pid, comm, exe, body_bytes_dropped): (Option, Option, Option, u64) = conn.query_row( "SELECT process_pid, process_comm, process_exe, body_bytes_dropped \ - FROM llm_calls WHERE id = 'call-phase7'", + FROM spans WHERE id = 'call-phase7'", [], |r| { Ok(( @@ -471,7 +472,7 @@ async fn phase7_adds_process_columns_and_appender_stays_aligned_on_legacy_db() { )) }, ) - .expect("row must be present after write_calls"); + .expect("row must be present after write_spans"); assert_eq!(pid, Some(31337), "process_pid must round-trip"); assert_eq!(comm.as_deref(), Some("python3"), "process_comm must round-trip"); assert_eq!( @@ -557,7 +558,7 @@ async fn phase3_rewrites_legacy_agent_turn_status_values() { // Inspect the raw status column post-migration. let conn = Connection::open(&path).unwrap(); let mut stmt = conn - .prepare("SELECT turn_id, status FROM agent_turns ORDER BY turn_id") + .prepare("SELECT turn_id, status FROM traces ORDER BY turn_id") .unwrap(); let rows: Vec<(String, String)> = stmt .query_map([], |r| Ok((r.get(0)?, r.get(1)?))) @@ -602,23 +603,23 @@ async fn init_is_idempotent_against_canonical_schema() { let cols_before = { let conn = Connection::open(&path).unwrap(); - column_names(&conn, "llm_calls") + column_names(&conn, "spans") }; backend.init().await.expect("re-init must succeed"); let cols_after = { let conn = Connection::open(&path).unwrap(); - column_names(&conn, "llm_calls") + column_names(&conn, "spans") }; assert_eq!( cols_before, cols_after, - "second init() must not drift the llm_calls column set" + "second init() must not drift the spans column set" ); // And the read path must still work. let _ = backend - .query_turns(&TurnsQuery { + .query_traces(&TracesQuery { time_range: TimeRange { start_us: 0, end_us: i64::MAX, @@ -635,5 +636,100 @@ async fn init_is_idempotent_against_canonical_schema() { include_proxy_hops: false, }) .await - .expect("query_turns must work after double-init"); + .expect("query_traces must work after double-init"); +} + +/// Phase-8 OTel rename: a complete pre-rename database (legacy `llm_calls` +/// plus `agent_turns` carrying `call_ids`) must migrate in place to `spans` +/// and `traces` with `span_ids`, the new `kind` column backfilled to 'llm' +/// on existing rows, and zero row loss. This is the core guard for the +/// rename — a same-name-only check would miss content/row-loss regressions. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn phase8_otel_rename_migrates_tables_columns_and_backfills_kind() { + let tmp = TempDir::new().unwrap(); + let path = synth_db( + &tmp, + &[LEGACY_LLM_CALLS_PRE_PHASE7, LEGACY_AGENT_TURNS_WITH_OLD_STATUS], + ); + + // Seed one row in each legacy table. The turn carries a known span list so + // we can assert the JSON survives the column rename byte-for-byte. + { + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "INSERT INTO llm_calls \ + (id, client_ip, client_port, server_ip, server_port, request_time, \ + wire_api, model, api_type, is_stream, request_path) VALUES \ + ('call-1', '1.1.1.1', 1, '2.2.2.2', 443, NOW(), \ + 'openai-chat', 'gpt-test', 'chat', false, '/v1/chat/completions');", + ) + .expect("seed legacy llm_calls row"); + conn.execute_batch( + "INSERT INTO agent_turns \ + (turn_id, session_id, wire_api, agent_kind, client_ip, server_ip, \ + start_time, end_time, duration_ms, call_count, \ + total_input_tokens, total_output_tokens, \ + total_cache_read_input_tokens, total_cache_creation_input_tokens, \ + status, call_ids) VALUES \ + ('turn-1', 's', 'openai-chat', 'test', '1.1.1.1', '2.2.2.2', \ + NOW(), NOW(), 0, 1, 0, 0, 0, 0, 'complete', '[\"call-1\",\"call-2\"]');", + ) + .expect("seed legacy agent_turns row"); + } + + let backend = Arc::new( + DuckDbBackend::open_with_pool(path.to_str().unwrap(), 2).expect("open backend"), + ); + backend + .init() + .await + .expect("init must reconcile + rename legacy schema"); + drop(backend); + + let conn = Connection::open(&path).unwrap(); + + // (1) Tables renamed: new names present, old names gone. + let tables: Vec = { + let mut stmt = conn + .prepare("SELECT table_name FROM duckdb_tables() ORDER BY table_name") + .unwrap(); + stmt.query_map([], |r| r.get::<_, String>(0)) + .unwrap() + .filter_map(|r| r.ok()) + .collect() + }; + assert!(tables.iter().any(|t| t == "spans"), "spans table must exist, got: {tables:?}"); + assert!(tables.iter().any(|t| t == "traces"), "traces table must exist, got: {tables:?}"); + assert!(!tables.iter().any(|t| t == "llm_calls"), "old llm_calls must be gone, got: {tables:?}"); + assert!(!tables.iter().any(|t| t == "agent_turns"), "old agent_turns must be gone, got: {tables:?}"); + + // (2) Column renamed call_ids -> span_ids, content preserved byte-for-byte. + let trace_cols = column_names(&conn, "traces"); + assert!(trace_cols.iter().any(|c| c == "span_ids"), "traces must have span_ids, got: {trace_cols:?}"); + assert!(!trace_cols.iter().any(|c| c == "call_ids"), "traces must not have call_ids, got: {trace_cols:?}"); + let span_ids: String = conn + .query_row("SELECT span_ids FROM traces WHERE turn_id = 'turn-1'", [], |r| r.get(0)) + .expect("migrated turn row must be present"); + assert_eq!( + span_ids, "[\"call-1\",\"call-2\"]", + "span_ids JSON must survive the rename byte-for-byte" + ); + + // (3) `kind` column added and backfilled to 'llm' on the pre-existing row. + let span_cols = column_names(&conn, "spans"); + assert!(span_cols.iter().any(|c| c == "kind"), "spans must have kind, got: {span_cols:?}"); + let kind: String = conn + .query_row("SELECT kind FROM spans WHERE id = 'call-1'", [], |r| r.get(0)) + .expect("migrated span row must be present"); + assert_eq!(kind, "llm", "kind must backfill to 'llm' on existing rows"); + + // (4) No row loss across the rename. + let span_rows: i64 = conn + .query_row("SELECT COUNT(*) FROM spans", [], |r| r.get(0)) + .unwrap(); + let trace_rows: i64 = conn + .query_row("SELECT COUNT(*) FROM traces", [], |r| r.get(0)) + .unwrap(); + assert_eq!(span_rows, 1, "the seeded span row must survive the rename"); + assert_eq!(trace_rows, 1, "the seeded trace row must survive the rename"); } diff --git a/server/h-storage/src/backend.rs b/server/h-storage/src/backend.rs index ce543fcc..91244cb4 100644 --- a/server/h-storage/src/backend.rs +++ b/server/h-storage/src/backend.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use h_llm::model::LlmCall; use h_metrics::model::{LlmFinishMetric, LlmMetric}; use h_protocol::HttpExchange; -use h_turn::{AgentTurn, PairCandidate}; +use h_turn::{Trace, PairCandidate}; use crate::query::*; use crate::retention::{RetentionPolicy, RetentionReport}; @@ -16,7 +16,7 @@ pub trait StorageBackend: Send + Sync { /// Batch-write LlmCall records. Takes ownership so the backend can move /// the batch into a blocking task without an extra clone. - async fn write_calls(&self, calls: Vec) -> Result<()>; + async fn write_spans(&self, calls: Vec) -> Result<()>; /// Batch-write LlmMetric records. async fn write_metrics(&self, metrics: Vec) -> Result<()>; @@ -25,8 +25,8 @@ pub trait StorageBackend: Send + Sync { /// `llm_finish_metrics` table. async fn write_finish_metrics(&self, metrics: Vec) -> Result<()>; - /// Batch-write AgentTurn records. - async fn write_turns(&self, turns: Vec) -> Result<()>; + /// Batch-write Trace records. + async fn write_traces(&self, turns: Vec) -> Result<()>; /// Batch-write HttpExchange records. Authoritative transport-layer record /// for all HTTP traffic (LLM + non-LLM). Soft-FK'd from `llm_calls` via @@ -37,7 +37,7 @@ pub trait StorageBackend: Send + Sync { async fn query_http_exchange_by_id(&self, id: &str) -> Result>; /// Paginated, filterable list of HTTP exchanges. Powers the HTTP - /// Exchanges page and mirrors `query_calls`'s shape. + /// Exchanges page and mirrors `query_spans`'s shape. async fn query_http_exchanges(&self, query: &HttpExchangesQuery) -> Result; async fn query_metrics_timeseries( @@ -100,10 +100,10 @@ pub trait StorageBackend: Send + Sync { &self, query: &FinishReasonsQuery, ) -> Result>; - async fn query_calls(&self, query: &CallsQuery) -> Result; - async fn query_call_by_id(&self, id: &str) -> Result>; - async fn query_turns(&self, query: &TurnsQuery) -> Result; - async fn query_turn_by_id(&self, turn_id: &str) -> Result>; + async fn query_spans(&self, query: &SpansQuery) -> Result; + async fn query_span_by_id(&self, id: &str) -> Result>; + async fn query_traces(&self, query: &TracesQuery) -> Result; + async fn query_trace_by_id(&self, turn_id: &str) -> Result>; /// `include_bodies = false` makes the four heavy fields /// (`request_body`, `response_body`, `request_headers`, /// `response_headers`) come back as `None`. On mega-turns (878 @@ -111,23 +111,23 @@ pub trait StorageBackend: Send + Sync { /// the body-bearing response freezes browsers; lite mode keeps the /// summary < 1 MB. `tokens_estimated` cannot be derived without /// the response body and defaults to `false` in lite mode. - async fn query_turn_calls( + async fn query_trace_spans( &self, turn_id: &str, include_bodies: bool, - ) -> Result>; - /// Sister of `query_turn_calls` for in-progress turns: the API - /// already knows the call_ids (from the in-memory active-turn + ) -> Result>; + /// Sister of `query_trace_spans` for in-progress turns: the API + /// already knows the span_ids (from the in-memory active-turn /// registry) and only needs Step 2 of the join. Returns the same - /// `TurnCallItem` shape so the frontend's calls panel renders + /// `TraceSpanItem` shape so the frontend's calls panel renders /// identically whether the turn is still in progress or finalized. /// Calls not yet flushed from `WriteBuffer` to `llm_calls` are /// silently skipped — they appear on the next refresh. - async fn query_calls_by_ids( + async fn query_spans_by_ids( &self, - call_ids: &[String], + span_ids: &[String], include_bodies: bool, - ) -> Result>; + ) -> Result>; /// Paginated session list (view over `agent_turns`; no materialised /// session table). A session is included when at least one of its turns @@ -146,7 +146,7 @@ pub trait StorageBackend: Send + Sync { /// Paginated list of the session's turns, ordered by start_time DESC. Not /// time-windowed — the session detail page shows the full history. - async fn query_session_turns(&self, query: &SessionTurnsQuery) -> Result; + async fn query_session_traces(&self, query: &SessionTracesQuery) -> Result; async fn query_distinct_wire_apis(&self) -> Result>; async fn query_distinct_models(&self) -> Result>; async fn query_distinct_server_ips(&self) -> Result>; @@ -193,18 +193,18 @@ pub trait StorageBackend: Send + Sync { /// Returns `Ok(())` even if `turn_id` doesn't exist — the sweeper /// races finalization and a turn may briefly be unwritten when the /// patch arrives. - async fn update_turn_metadata(&self, _turn_id: &str, _patch: serde_json::Value) -> Result<()> { + async fn update_trace_metadata(&self, _turn_id: &str, _patch: serde_json::Value) -> Result<()> { Ok(()) } /// Compact pending MVCC tombstones on the agent_turns writer. /// Called by the pair sweeper after each batch of - /// `update_turn_metadata` so the version chain stays short — + /// `update_trace_metadata` so the version chain stays short — /// high-frequency UPDATEs on an indexed table (PRIMARY KEY on /// `turn_id`) without checkpoints can hit a "Failed to delete all /// rows from index" FATAL inside DuckDB that poisons the entire /// process's connection. Default no-op for mock backends. - async fn checkpoint_turns_writer(&self) -> Result<()> { + async fn checkpoint_traces_writer(&self) -> Result<()> { Ok(()) } diff --git a/server/h-storage/src/classify.rs b/server/h-storage/src/classify.rs index 6575c763..16eaa691 100644 --- a/server/h-storage/src/classify.rs +++ b/server/h-storage/src/classify.rs @@ -157,7 +157,7 @@ pub fn classify_app( // -- vLLM signature in the request body. Agentic flows include // -- prior `assistant.tool_calls[].id` values in their message - // -- history, and vLLM-generated tool_call_ids carry the + // -- history, and vLLM-generated tool_span_ids carry the // -- `chatcmpl-tool-` prefix. Works even when the response is // -- SSE-streamed (response_body is NULL). if let Some(body) = sample_request_body { @@ -414,7 +414,7 @@ mod tests { fn vllm_via_request_body_tool_history() { // Streaming-only endpoint — response_body is null. Agentic // round N+1 sends prior assistant.tool_calls history back to - // the server, and vLLM's tool_call_ids carry `chatcmpl-tool-`. + // the server, and vLLM's tool_span_ids carry `chatcmpl-tool-`. let c = C { server_header: Some("uvicorn".into()), raw_response_headers_json: Some(hdrs(&[("server", "uvicorn")])), diff --git a/server/h-storage/src/convert.rs b/server/h-storage/src/convert.rs index 48d5332e..c73f2df0 100644 --- a/server/h-storage/src/convert.rs +++ b/server/h-storage/src/convert.rs @@ -60,7 +60,7 @@ pub fn headers_to_json(headers: &[(String, String)]) -> String { } /// Parse a JSON-encoded array-of-strings (as stored in agent_turns.models_used / -/// subagents_used / call_ids) into a `Vec`. Missing or malformed values +/// subagents_used / span_ids) into a `Vec`. Missing or malformed values /// degrade to an empty vec — the turn payload is still returnable. pub fn parse_json_string_list(raw: Option<&str>) -> Vec { match raw { diff --git a/server/h-storage/src/pair_sweeper.rs b/server/h-storage/src/pair_sweeper.rs index d0030c7b..9ad22268 100644 --- a/server/h-storage/src/pair_sweeper.rs +++ b/server/h-storage/src/pair_sweeper.rs @@ -4,7 +4,7 @@ //! On each tick it asks the storage backend for a window of finalized //! turns whose `metadata.proxy.role` is unset, runs the in-memory //! `pair_all` classifier, and writes the resulting pair annotations -//! back via `update_turn_metadata`. The sweep is fully idempotent: a +//! back via `update_trace_metadata`. The sweep is fully idempotent: a //! turn that is already paired is excluded by the backend query, so //! repeat sweeps converge. //! @@ -23,7 +23,7 @@ //! //! ### Why a sweeper (not inline at write-time) //! -//! When `write_turns` flushes a batch, the peer of a pair may not yet +//! When `write_traces` flushes a batch, the peer of a pair may not yet //! have arrived — it might still be sitting in another shard's buffer //! waiting for grace. Inline pairing at write time would systematically //! miss the first of any pair. A sweeper that scans recently-finalized @@ -84,7 +84,7 @@ pub fn spawn_pair_sweeper( // MVCC tombstone chain doesn't grow into the // DuckDB "Failed to delete all rows from index" // FATAL trap. Cheap when nothing to compact. - if let Err(e) = storage.checkpoint_turns_writer().await { + if let Err(e) = storage.checkpoint_traces_writer().await { tracing::warn!(error = %e, "pair-sweeper: post-sweep CHECKPOINT failed"); } } @@ -137,7 +137,7 @@ pub async fn sweep_once( let patch = g .metadata_for(&member.turn_id) .expect("member belongs to group it came from"); - storage.update_turn_metadata(&member.turn_id, patch).await?; + storage.update_trace_metadata(&member.turn_id, patch).await?; turns_tagged += 1; } } @@ -174,7 +174,7 @@ mod tests { use h_protocol::HttpExchange; use h_turn::proxy_pair::PairCandidate; - use h_turn::AgentTurn; + use h_turn::Trace; /// In-memory stub storage that holds the candidates the sweeper will /// see and records the metadata patches it writes back. Lets us test @@ -189,7 +189,7 @@ mod tests { async fn init(&self) -> Result<()> { Ok(()) } - async fn write_calls(&self, _: Vec) -> Result<()> { + async fn write_spans(&self, _: Vec) -> Result<()> { Ok(()) } async fn write_metrics(&self, _: Vec) -> Result<()> { @@ -198,7 +198,7 @@ mod tests { async fn write_finish_metrics(&self, _: Vec) -> Result<()> { Ok(()) } - async fn write_turns(&self, _: Vec) -> Result<()> { + async fn write_traces(&self, _: Vec) -> Result<()> { Ok(()) } async fn write_exchanges(&self, _: Vec) -> Result<()> { @@ -272,36 +272,36 @@ mod tests { ) -> Result> { Ok(vec![]) } - async fn query_calls(&self, _: &CallsQuery) -> Result { - Ok(CallsPage { + async fn query_spans(&self, _: &SpansQuery) -> Result { + Ok(SpansPage { total: 0, items: vec![], }) } - async fn query_call_by_id(&self, _: &str) -> Result> { + async fn query_span_by_id(&self, _: &str) -> Result> { Ok(None) } - async fn query_turns(&self, _: &TurnsQuery) -> Result { - Ok(TurnsPage { + async fn query_traces(&self, _: &TracesQuery) -> Result { + Ok(TracesPage { total: 0, items: vec![], }) } - async fn query_turn_by_id(&self, _: &str) -> Result> { + async fn query_trace_by_id(&self, _: &str) -> Result> { Ok(None) } - async fn query_turn_calls( + async fn query_trace_spans( &self, _: &str, _include_bodies: bool, - ) -> Result> { + ) -> Result> { Ok(vec![]) } - async fn query_calls_by_ids( + async fn query_spans_by_ids( &self, _: &[String], _include_bodies: bool, - ) -> Result> { + ) -> Result> { Ok(vec![]) } async fn query_sessions(&self, _: &SessionListQuery) -> Result { @@ -313,8 +313,8 @@ mod tests { async fn query_session_by_id(&self, _: &str, _: &str) -> Result> { Ok(None) } - async fn query_session_turns(&self, _: &SessionTurnsQuery) -> Result { - Ok(SessionTurnsPage { + async fn query_session_traces(&self, _: &SessionTracesQuery) -> Result { + Ok(SessionTracesPage { items: vec![], next_cursor: None, }) @@ -343,7 +343,7 @@ mod tests { async fn query_pair_candidates(&self, _: i64, _: i64) -> Result> { Ok(self.candidates.clone()) } - async fn update_turn_metadata( + async fn update_trace_metadata( &self, turn_id: &str, patch: serde_json::Value, @@ -387,7 +387,7 @@ mod tests { // The second test verifies the actual patch contents written // through the stub. Here we just assert pair detection ran end // to end — the sweeper only counts a pair after BOTH - // update_turn_metadata calls succeeded. + // update_trace_metadata calls succeeded. } #[tokio::test] diff --git a/server/h-storage/src/query.rs b/server/h-storage/src/query.rs index 0d0a406c..3051462b 100644 --- a/server/h-storage/src/query.rs +++ b/server/h-storage/src/query.rs @@ -184,7 +184,7 @@ pub struct AgentActivityPoint { } #[derive(Debug, Clone)] -pub struct CallsQuery { +pub struct SpansQuery { pub time_range: TimeRange, pub filter: DimensionFilter, pub status_codes: Vec, @@ -283,7 +283,7 @@ pub struct MetricsModelRow { } #[derive(Debug, Clone, Serialize)] -pub struct CallListItem { +pub struct SpanListItem { pub id: String, pub source_id: String, pub request_time: i64, @@ -316,9 +316,9 @@ pub struct CallListItem { } #[derive(Debug, Clone, Serialize)] -pub struct CallsPage { +pub struct SpansPage { pub total: u64, - pub items: Vec, + pub items: Vec, } #[derive(Debug, Clone)] @@ -371,15 +371,15 @@ pub struct HttpExchangesPage { } #[derive(Debug, Clone)] -pub struct TurnsQuery { +pub struct TracesQuery { pub time_range: TimeRange, pub filter: DimensionFilter, /// Per-call client IP filter. `DimensionFilter` only carries `server_ips` /// (the metrics-pre-aggregated dimension); client IP lives outside the - /// filter, parallel to `CallsQuery.client_ips`. + /// filter, parallel to `SpansQuery.client_ips`. pub client_ips: Vec, /// Per-call server port filter. `agent_turns` doesn't carry `server_port` - /// — we resolve it through the turn's first `call_ids` entry against + /// — we resolve it through the turn's first `span_ids` entry against /// `llm_calls`, same shortcut the topology query uses. pub server_ports: Vec, pub statuses: Vec, @@ -397,7 +397,7 @@ pub struct TurnsQuery { } #[derive(Debug, Clone, Serialize)] -pub struct TurnListItem { +pub struct TraceListItem { pub turn_id: String, pub source_id: String, pub session_id: String, @@ -447,18 +447,18 @@ pub struct TurnListItem { } #[derive(Debug, Clone, Serialize)] -pub struct TurnsPage { +pub struct TracesPage { pub total: u64, - pub items: Vec, + pub items: Vec, } /// One turn row returned by the session-turns endpoint. Identical to -/// `TurnListItem` except `user_input_preview` / `final_answer_preview` are +/// `TraceListItem` except `user_input_preview` / `final_answer_preview` are /// replaced by full-text `user_input` / `final_answer` (server-side /// reconstructed from the referenced call bodies, see -/// `query_session_turns` in `duckdb.rs`). +/// `query_session_traces` in `duckdb.rs`). #[derive(Debug, Clone, Serialize)] -pub struct SessionTurnItem { +pub struct SessionTraceItem { pub turn_id: String, pub source_id: String, pub session_id: String, @@ -487,15 +487,15 @@ pub struct SessionTurnItem { } #[derive(Debug, Clone, Serialize)] -pub struct SessionTurnsPage { - pub items: Vec, +pub struct SessionTracesPage { + pub items: Vec, /// Opaque cursor for the next page. `None` when the current page is the /// last one (fewer than `page_size` rows were returned). pub next_cursor: Option, } #[derive(Debug, Clone, Serialize)] -pub struct TurnDetail { +pub struct TraceDetail { pub turn_id: String, pub source_id: String, pub session_id: String, @@ -520,7 +520,7 @@ pub struct TurnDetail { pub user_input: Option, pub final_call_id: Option, pub final_answer: Option, - pub call_ids: Vec, + pub span_ids: Vec, pub metadata: Option, /// Distinct tool surfaces seen across the turn's calls. pub tool_surfaces: Vec, @@ -533,7 +533,7 @@ pub struct TurnDetail { } #[derive(Debug, Clone, Serialize)] -pub struct TurnCallItem { +pub struct TraceSpanItem { pub id: String, pub sequence: u32, pub request_time: i64, @@ -549,7 +549,7 @@ pub struct TurnCallItem { pub input_tokens: Option, pub output_tokens: Option, /// True when the row's tokens came from the fallback estimator. Mirrors - /// `CallListItem.tokens_estimated`. + /// `SpanListItem.tokens_estimated`. #[serde(default)] pub tokens_estimated: bool, pub request_path: String, @@ -746,12 +746,12 @@ fn hex_digit(b: u8) -> Option { /// side, so comparison `(start_time, turn_id) < (?, ?)` steps through pages /// without duplicates even when two turns share a microsecond. #[derive(Debug, Clone)] -pub struct SessionTurnsCursor { +pub struct SessionTracesCursor { pub start_time_us: i64, pub turn_id: String, } -pub fn encode_session_turns_cursor(c: &SessionTurnsCursor) -> String { +pub fn encode_session_turns_cursor(c: &SessionTracesCursor) -> String { let json = serde_json::json!({ "t": c.start_time_us, "k": c.turn_id }).to_string(); let mut out = String::with_capacity(json.len() * 2); for b in json.as_bytes() { @@ -761,7 +761,7 @@ pub fn encode_session_turns_cursor(c: &SessionTurnsCursor) -> String { out } -pub fn decode_session_turns_cursor(s: &str) -> Option { +pub fn decode_session_turns_cursor(s: &str) -> Option { if s.len() % 2 != 0 { return None; } @@ -775,17 +775,17 @@ pub fn decode_session_turns_cursor(s: &str) -> Option { let v: serde_json::Value = serde_json::from_slice(&decoded).ok()?; let start_time_us = v.get("t")?.as_i64()?; let turn_id = v.get("k")?.as_str()?.to_string(); - Some(SessionTurnsCursor { + Some(SessionTracesCursor { start_time_us, turn_id, }) } #[derive(Debug, Clone)] -pub struct SessionTurnsQuery { +pub struct SessionTracesQuery { pub source_id: String, pub session_id: String, - pub cursor: Option, + pub cursor: Option, pub page_size: u32, } @@ -888,7 +888,7 @@ pub struct ProxyViewResponse { } #[derive(Debug, Clone, Serialize)] -pub struct CallDetail { +pub struct SpanDetail { pub id: String, pub source_id: String, pub request_time: i64, @@ -905,7 +905,7 @@ pub struct CallDetail { pub output_tokens: Option, pub total_tokens: Option, /// True when the row's tokens came from the fallback estimator. See - /// `CallListItem.tokens_estimated`. + /// `SpanListItem.tokens_estimated`. #[serde(default)] pub tokens_estimated: bool, pub ttft_ms: Option, @@ -935,7 +935,7 @@ mod session_turns_cursor_tests { #[test] fn session_turns_cursor_roundtrip() { - let c = SessionTurnsCursor { + let c = SessionTracesCursor { start_time_us: 1_729_000_000_000_000, turn_id: "abc-123".to_string(), }; diff --git a/server/h-storage/src/retention.rs b/server/h-storage/src/retention.rs index ba5d763f..3f779baa 100644 --- a/server/h-storage/src/retention.rs +++ b/server/h-storage/src/retention.rs @@ -19,8 +19,8 @@ use crate::backend::StorageBackend; /// A concrete cutoff decision for each table. `None` means "skip". #[derive(Debug, Clone, Default)] pub struct RetentionPolicy { - pub calls_before: Option, - pub turns_before: Option, + pub spans_before: Option, + pub traces_before: Option, pub http_exchanges_before: Option, /// `(granularity_label, cutoff)` pairs. Only listed granularities are swept. pub metrics_before: Vec<(String, SystemTime)>, @@ -29,8 +29,8 @@ pub struct RetentionPolicy { impl RetentionPolicy { /// True when no tables have a cutoff — used to short-circuit the sweep. pub fn is_empty(&self) -> bool { - self.calls_before.is_none() - && self.turns_before.is_none() + self.spans_before.is_none() + && self.traces_before.is_none() && self.http_exchanges_before.is_none() && self.metrics_before.is_empty() } @@ -39,8 +39,8 @@ impl RetentionPolicy { /// Per-table row counts from a single sweep. #[derive(Debug, Clone, Default)] pub struct RetentionReport { - pub calls_deleted: u64, - pub turns_deleted: u64, + pub spans_deleted: u64, + pub traces_deleted: u64, pub http_exchanges_deleted: u64, /// Per-granularity deletion counts — one entry per swept label. pub metrics_deleted: HashMap, @@ -48,8 +48,8 @@ pub struct RetentionReport { impl RetentionReport { pub fn total(&self) -> u64 { - self.calls_deleted - + self.turns_deleted + self.spans_deleted + + self.traces_deleted + self.http_exchanges_deleted + self.metrics_deleted.values().sum::() } @@ -80,8 +80,8 @@ pub fn policy_from_config(cfg: &RetentionConfig, now: SystemTime) -> RetentionPo .collect(); RetentionPolicy { - calls_before: days_to_cutoff(cfg.calls), - turns_before: days_to_cutoff(cfg.turns), + spans_before: days_to_cutoff(cfg.spans), + traces_before: days_to_cutoff(cfg.traces), http_exchanges_before: days_to_cutoff(cfg.http_exchanges), metrics_before, } @@ -132,8 +132,8 @@ pub fn spawn_retention_task( Ok(report) => { if report.total() > 0 { info!( - calls = report.calls_deleted, - turns = report.turns_deleted, + calls = report.spans_deleted, + turns = report.traces_deleted, http_exchanges = report.http_exchanges_deleted, metrics = ?report.metrics_deleted, "retention: sweep complete" @@ -163,8 +163,8 @@ mod tests { fn make_cfg(calls: u32, turns: u32, metrics: &[(&str, u32)]) -> RetentionConfig { let mut cfg = RetentionConfig::default(); cfg.enabled = true; - cfg.calls = calls; - cfg.turns = turns; + cfg.spans = calls; + cfg.traces = turns; cfg.http_exchanges = 0; cfg.metrics = metrics .iter() @@ -178,8 +178,8 @@ mod tests { let cfg = make_cfg(0, 0, &[]); let now = SystemTime::now(); let policy = policy_from_config(&cfg, now); - assert!(policy.calls_before.is_none()); - assert!(policy.turns_before.is_none()); + assert!(policy.spans_before.is_none()); + assert!(policy.traces_before.is_none()); assert!(policy.http_exchanges_before.is_none()); assert!(policy.metrics_before.is_empty()); assert!(policy.is_empty()); @@ -214,8 +214,8 @@ mod tests { let cfg = make_cfg(7, 30, &[]); let now = SystemTime::now(); let policy = policy_from_config(&cfg, now); - let calls_before = policy.calls_before.expect("calls cutoff"); - let elapsed = now.duration_since(calls_before).expect("cutoff before now"); + let spans_before = policy.spans_before.expect("calls cutoff"); + let elapsed = now.duration_since(spans_before).expect("cutoff before now"); // Within 1 second of exactly 7*86400 seconds. let expected = Duration::from_secs(7 * 86_400); let delta = if elapsed > expected { diff --git a/server/h-storage/src/sink.rs b/server/h-storage/src/sink.rs index 67a83f1c..d98d31f4 100644 --- a/server/h-storage/src/sink.rs +++ b/server/h-storage/src/sink.rs @@ -11,7 +11,7 @@ use tokio::task::JoinHandle; use h_llm::model::LlmCall; use h_metrics::model::{LlmFinishMetric, LlmMetric, LlmMetricsBatch}; use h_protocol::HttpExchange; -use h_turn::AgentTurn; +use h_turn::Trace; use h_common::internal_metrics::{Metric, MetricsWorker}; @@ -38,7 +38,7 @@ impl Default for StorageSinkConfig { pub fn spawn_storage_sink_stage( config: StorageSinkConfig, calls_rx: mpsc::Receiver>, - turns_rx: mpsc::Receiver, + turns_rx: mpsc::Receiver, metrics_rx: mpsc::Receiver, http_exchanges_rx: mpsc::Receiver, backend: Arc, @@ -119,7 +119,7 @@ pub fn spawn_storage_sink_stage( calls_buffer .run(move |batch| { let b = calls_storage.clone(); - async move { b.write_calls(batch).await } + async move { b.write_spans(batch).await } }) .await; }); @@ -136,7 +136,7 @@ pub fn spawn_storage_sink_stage( turns_buffer .run(move |batch| { let b = turns_storage.clone(); - async move { b.write_turns(batch).await } + async move { b.write_traces(batch).await } }) .await; }); @@ -211,13 +211,13 @@ pub fn spawn_storage_sink_stage( mod tests { use super::*; use crate::query::{ - AgentActivityPoint, AgentActivityQuery, AgentKindSummary, AgentSummaryQuery, CallDetail, - CallsPage, CallsQuery, DistinctAgentKindsQuery, DistinctFinishReason, + AgentActivityPoint, AgentActivityQuery, AgentKindSummary, AgentSummaryQuery, SpanDetail, + SpansPage, SpansQuery, DistinctAgentKindsQuery, DistinctFinishReason, FinishReasonTimeseries, FinishReasonsQuery, HttpExchangeDetail, HttpExchangesPage, HttpExchangesQuery, MetricsModelRow, MetricsModelsQuery, MetricsSummaryQuery, MetricsSummaryRow, MetricsTimeseriesQuery, MetricsTimeseriesRow, ServiceRow, ServicesQuery, - ServicesTopology, ServicesTopologyQuery, SessionDetail, SessionListQuery, SessionTurnsPage, - SessionTurnsQuery, SessionsPage, TurnCallItem, TurnDetail, TurnsPage, TurnsQuery, + ServicesTopology, ServicesTopologyQuery, SessionDetail, SessionListQuery, SessionTracesPage, + SessionTracesQuery, SessionsPage, TraceSpanItem, TraceDetail, TracesPage, TracesQuery, }; use async_trait::async_trait; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -237,11 +237,11 @@ mod tests { async fn init(&self) -> Result<()> { Ok(()) } - async fn write_calls(&self, batch: Vec) -> Result<()> { + async fn write_spans(&self, batch: Vec) -> Result<()> { self.calls.fetch_add(batch.len(), Ordering::SeqCst); Ok(()) } - async fn write_turns(&self, batch: Vec) -> Result<()> { + async fn write_traces(&self, batch: Vec) -> Result<()> { self.turns.fetch_add(batch.len(), Ordering::SeqCst); Ok(()) } @@ -330,36 +330,36 @@ mod tests { ) -> Result> { Ok(vec![]) } - async fn query_calls(&self, _query: &CallsQuery) -> Result { - Ok(CallsPage { + async fn query_spans(&self, _query: &SpansQuery) -> Result { + Ok(SpansPage { total: 0, items: vec![], }) } - async fn query_call_by_id(&self, _id: &str) -> Result> { + async fn query_span_by_id(&self, _id: &str) -> Result> { Ok(None) } - async fn query_turns(&self, _query: &TurnsQuery) -> Result { - Ok(TurnsPage { + async fn query_traces(&self, _query: &TracesQuery) -> Result { + Ok(TracesPage { total: 0, items: vec![], }) } - async fn query_turn_by_id(&self, _turn_id: &str) -> Result> { + async fn query_trace_by_id(&self, _turn_id: &str) -> Result> { Ok(None) } - async fn query_turn_calls( + async fn query_trace_spans( &self, _turn_id: &str, _include_bodies: bool, - ) -> Result> { + ) -> Result> { Ok(vec![]) } - async fn query_calls_by_ids( + async fn query_spans_by_ids( &self, - _call_ids: &[String], + _span_ids: &[String], _include_bodies: bool, - ) -> Result> { + ) -> Result> { Ok(vec![]) } async fn query_sessions(&self, _query: &SessionListQuery) -> Result { @@ -375,11 +375,11 @@ mod tests { ) -> Result> { Ok(None) } - async fn query_session_turns( + async fn query_session_traces( &self, - _query: &SessionTurnsQuery, - ) -> Result { - Ok(SessionTurnsPage { + _query: &SessionTracesQuery, + ) -> Result { + Ok(SessionTracesPage { items: vec![], next_cursor: None, }) @@ -427,7 +427,7 @@ mod tests { let backend: Arc = Arc::new(counts); let (calls_tx, calls_rx) = mpsc::channel::>(16); - let (turns_tx, turns_rx) = mpsc::channel::(16); + let (turns_tx, turns_rx) = mpsc::channel::(16); let (metrics_tx, metrics_rx) = mpsc::channel::(16); let (exch_tx, exch_rx) = mpsc::channel::(16); @@ -563,8 +563,8 @@ mod tests { } } - fn dummy_turn(i: usize) -> AgentTurn { - AgentTurn { + fn dummy_turn(i: usize) -> Trace { + Trace { source_id: String::new(), turn_id: format!("t-{i}"), session_id: "s".into(), @@ -583,13 +583,13 @@ mod tests { total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, total_cost_usd: None, - status: h_turn::TurnStatus::Complete, + status: h_turn::TraceStatus::Complete, final_finish_reason: None, user_input_preview: None, user_call_id: None, final_answer_preview: None, final_call_id: None, - call_ids: vec![format!("c-{i}")], + span_ids: vec![format!("c-{i}")], metadata: serde_json::json!({}), tool_surfaces: vec![], tool_call_total: 0, diff --git a/server/h-turn/src/lib.rs b/server/h-turn/src/lib.rs index d2cd68ac..ddb489e5 100644 --- a/server/h-turn/src/lib.rs +++ b/server/h-turn/src/lib.rs @@ -1,4 +1,4 @@ -//! Turn grouping: aggregates `LlmCall` into `AgentTurn` per agent session. +//! Turn grouping: aggregates `LlmCall` into `Trace` per agent session. //! //! Header-explicit only — calls without a matching `AgentProfile` do not //! participate in turn grouping. @@ -9,7 +9,7 @@ pub mod stage; pub mod test_support; pub mod tracker; -pub use model::{new_active_turn_registry, ActiveTurnRegistry, AgentTurn, TurnKey, TurnStatus}; +pub use model::{new_active_trace_registry, ActiveTraceRegistry, Trace, TraceKey, TraceStatus}; pub use proxy_pair::{ candidate_from_turn, group_all, GroupMember, PairCandidate, ProxyGroup, ProxyRole, MAX_REQ_TIME_GAP_US, MIRROR_TIME_TOLERANCE_US, diff --git a/server/h-turn/src/model.rs b/server/h-turn/src/model.rs index 72f3a664..dac6a04a 100644 --- a/server/h-turn/src/model.rs +++ b/server/h-turn/src/model.rs @@ -9,7 +9,7 @@ use std::sync::{Arc, RwLock}; /// For Anthropic (implicit): turn_id is generated by TurnTracker on turn start; /// session_id comes from the X-Claude-Code-Session-Id header. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct TurnKey { +pub struct TraceKey { pub source_id: String, pub session_id: String, pub turn_id: String, @@ -19,7 +19,7 @@ pub struct TurnKey { /// /// * `InProgress` — at least one call has been ingested for this turn, no /// wire-level terminal has landed yet. Lives only in the in-memory -/// [`ActiveTurnRegistry`]; never written to the `agent_turns` table. +/// [`ActiveTraceRegistry`]; never written to the `agent_turns` table. /// * `Complete` — a main-agent terminal call (e.g. `end_turn`, /// `max_tokens`, `refusal`) landed and the buffer's grace window expired. /// Persisted to the `agent_turns` table. @@ -29,18 +29,18 @@ pub struct TurnKey { /// The wire-level reason (e.g. `end_turn`) lives separately in /// `final_finish_reason: Option`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TurnStatus { +pub enum TraceStatus { InProgress, Complete, Incomplete, } -impl fmt::Display for TurnStatus { +impl fmt::Display for TraceStatus { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - TurnStatus::InProgress => write!(f, "in_progress"), - TurnStatus::Complete => write!(f, "complete"), - TurnStatus::Incomplete => write!(f, "incomplete"), + TraceStatus::InProgress => write!(f, "in_progress"), + TraceStatus::Complete => write!(f, "complete"), + TraceStatus::Incomplete => write!(f, "incomplete"), } } } @@ -67,17 +67,17 @@ impl fmt::Display for TurnStatus { /// `/api/agent-turns` until either the next ingest re-creates them or /// the turn finalizes and writes to DuckDB. This matches the existing /// in-memory `SessionBuffer` behavior — both were already RAM-only. -pub type ActiveTurnRegistry = Arc>>; +pub type ActiveTraceRegistry = Arc>>; -/// Construct an empty `ActiveTurnRegistry` ready to share between +/// Construct an empty `ActiveTraceRegistry` ready to share between /// trackers and the API. -pub fn new_active_turn_registry() -> ActiveTurnRegistry { +pub fn new_active_trace_registry() -> ActiveTraceRegistry { Arc::new(RwLock::new(HashMap::new())) } /// Aggregated record for one agent turn (user input → final assistant output). #[derive(Debug, Clone)] -pub struct AgentTurn { +pub struct Trace { pub source_id: String, pub turn_id: String, pub session_id: String, @@ -104,7 +104,7 @@ pub struct AgentTurn { pub total_cache_creation_input_tokens: u64, pub total_cost_usd: Option, // None when pricing unknown - pub status: TurnStatus, + pub status: TraceStatus, pub final_finish_reason: Option, // raw provider finish_reason of last call /// Up to ~500 chars of the user prompt that opened this turn. Full body @@ -122,7 +122,7 @@ pub struct AgentTurn { /// Ordered list of `LlmCall.id` values that belong to this turn. /// Populated by `TurnTracker::ingest` in ingest order. Replaces the /// Phase-1 `LlmCall.turn_id` back-reference. - pub call_ids: Vec, + pub span_ids: Vec, pub metadata: serde_json::Value, // future extension point; empty object by default @@ -137,11 +137,11 @@ pub struct AgentTurn { pub suspicious_skills: Vec, } -impl fmt::Display for AgentTurn { +impl fmt::Display for Trace { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "[AgentTurn] turn={} session={} agent={} calls={} dur={}ms status={}", + "[Trace] turn={} session={} agent={} calls={} dur={}ms status={}", self.turn_id, self.session_id, self.agent_kind, @@ -160,12 +160,12 @@ mod tests { #[test] fn turn_key_equality_and_hash() { use std::collections::HashSet; - let k1 = TurnKey { + let k1 = TraceKey { source_id: String::new(), session_id: "s".into(), turn_id: "t".into(), }; - let k2 = TurnKey { + let k2 = TraceKey { source_id: String::new(), session_id: "s".into(), turn_id: "t".into(), @@ -177,13 +177,13 @@ mod tests { #[test] fn status_display() { - assert_eq!(TurnStatus::Complete.to_string(), "complete"); - assert_eq!(TurnStatus::Incomplete.to_string(), "incomplete"); + assert_eq!(TraceStatus::Complete.to_string(), "complete"); + assert_eq!(TraceStatus::Incomplete.to_string(), "incomplete"); } #[test] fn agent_turn_display_includes_key_fields() { - let turn = AgentTurn { + let turn = Trace { source_id: String::new(), turn_id: "t1".into(), session_id: "s1".into(), @@ -202,13 +202,13 @@ mod tests { total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, total_cost_usd: None, - status: TurnStatus::Complete, + status: TraceStatus::Complete, final_finish_reason: Some("complete".into()), user_input_preview: None, user_call_id: None, final_answer_preview: None, final_call_id: None, - call_ids: vec![], + span_ids: vec![], metadata: serde_json::json!({}), tool_surfaces: vec![], tool_call_total: 0, @@ -223,8 +223,8 @@ mod tests { } #[test] - fn agent_turn_has_call_ids() { - let turn = AgentTurn { + fn agent_turn_has_span_ids() { + let turn = Trace { source_id: String::new(), turn_id: "t".into(), session_id: "s".into(), @@ -243,20 +243,20 @@ mod tests { total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, total_cost_usd: None, - status: TurnStatus::Complete, + status: TraceStatus::Complete, final_finish_reason: None, user_input_preview: None, user_call_id: None, final_answer_preview: None, final_call_id: None, - call_ids: vec!["call-1".into(), "call-2".into()], + span_ids: vec!["call-1".into(), "call-2".into()], metadata: serde_json::json!({}), tool_surfaces: vec![], tool_call_total: 0, agent_topology: None, suspicious_skills: vec![], }; - assert_eq!(turn.call_ids.len(), 2); - assert_eq!(turn.call_ids[0], "call-1"); + assert_eq!(turn.span_ids.len(), 2); + assert_eq!(turn.span_ids[0], "call-1"); } } diff --git a/server/h-turn/src/proxy_pair.rs b/server/h-turn/src/proxy_pair.rs index 2934d12b..4eda8aef 100644 --- a/server/h-turn/src/proxy_pair.rs +++ b/server/h-turn/src/proxy_pair.rs @@ -1,4 +1,4 @@ -//! Passive llmproxy pair detection — folds 2+ `AgentTurn` records that +//! Passive llmproxy pair detection — folds 2+ `Trace` records that //! represent the same logical LLM call observed at different network //! vantage points. //! @@ -6,7 +6,7 @@ //! //! 1. **Real proxy hops** — e.g. an external client → haproxy_glm5 container //! → sglang container. Both legs cross interfaces Heron captures -//! so each becomes its own `AgentTurn`. The proxy_in leg strictly +//! so each becomes its own `Trace`. The proxy_in leg strictly //! contains the proxy_out leg in event time. //! //! 2. **Multi-interface double-capture** — libpcap on `any` interface @@ -42,7 +42,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::model::AgentTurn; +use crate::model::Trace; /// Role of a turn inside its group. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -101,7 +101,7 @@ pub const MAX_REQ_TIME_GAP_US: i64 = 100_000; /// (start_gap 2ms, end_gap 1ms) still classifies as strict-nesting. pub const MIRROR_TIME_TOLERANCE_US: i64 = 500; -/// Light fingerprint of an `AgentTurn` carrying just the fields the +/// Light fingerprint of an `Trace` carrying just the fields the /// pairing rule needs. Pulled from DB via a narrow projection so the /// sweeper doesn't materialize every column. #[derive(Debug, Clone, PartialEq, Eq)] @@ -356,10 +356,10 @@ pub fn group_all(set: &[PairCandidate]) -> Vec { groups } -/// Build a `PairCandidate` from an `AgentTurn`, used by callers that have +/// Build a `PairCandidate` from an `Trace`, used by callers that have /// the full turn in memory (e.g. unit tests). Production callers build /// candidates directly from a DB projection. -pub fn candidate_from_turn(t: &AgentTurn) -> PairCandidate { +pub fn candidate_from_turn(t: &Trace) -> PairCandidate { PairCandidate { turn_id: t.turn_id.clone(), session_id: t.session_id.clone(), diff --git a/server/h-turn/src/stage.rs b/server/h-turn/src/stage.rs index b63ec65a..6922654b 100644 --- a/server/h-turn/src/stage.rs +++ b/server/h-turn/src/stage.rs @@ -13,7 +13,7 @@ use tokio::task::JoinHandle; use h_common::internal_metrics::{Metric, MetricsSystem}; use h_llm::model::TurnShardInput; -use crate::model::{ActiveTurnRegistry, AgentTurn}; +use crate::model::{ActiveTraceRegistry, Trace}; use crate::tracker::{TrackerConfig, TurnEvent, TurnTracker}; /// Spawn one turn-tracker task per shard (inferred from `shard_rxs.len()`). @@ -26,9 +26,9 @@ use crate::tracker::{TrackerConfig, TurnEvent, TurnTracker}; pub fn spawn_turn_stage( tracker_cfg: TrackerConfig, shard_rxs: Vec>, - turns_tx: mpsc::Sender, + turns_tx: mpsc::Sender, metrics_sys: &mut MetricsSystem, - active_registry: Option, + active_registry: Option, ) -> Vec> { assert!( !shard_rxs.is_empty(), @@ -41,7 +41,7 @@ pub fn spawn_turn_stage( // a finalized turn. NOTE: this counts CALLS, not open turns; a single // conversation with N concurrent calls contributes N. For the "open // agent turns" signal the dashboard uses, see `agent_turns_open` - // (registered on the global svc against ActiveTurnRegistry). + // (registered on the global svc against ActiveTraceRegistry). let active_gauges: Vec> = (0..shard_rxs.len()) .map(|_| Arc::new(AtomicU64::new(0))) .collect(); @@ -223,7 +223,7 @@ mod tests { #[tokio::test] async fn single_shard_produces_turn() { let (shard_tx, shard_rx) = mpsc::channel::(16); - let (turns_tx, mut turns_rx) = mpsc::channel::(16); + let (turns_tx, mut turns_rx) = mpsc::channel::(16); let mut metrics_sys = MetricsSystem::new(); spawn_turn_stage( @@ -263,7 +263,7 @@ mod tests { turns.push(t); } assert_eq!(turns.len(), 1, "one complete turn expected"); - assert_eq!(turns[0].call_ids, vec![id1, id2]); + assert_eq!(turns[0].span_ids, vec![id1, id2]); } #[tokio::test] @@ -275,7 +275,7 @@ mod tests { shard_txs.push(tx); shard_rxs.push(rx); } - let (turns_tx, mut turns_rx) = mpsc::channel::(64); + let (turns_tx, mut turns_rx) = mpsc::channel::(64); let mut metrics_sys = MetricsSystem::new(); spawn_turn_stage( @@ -317,13 +317,13 @@ mod tests { let sessions: std::collections::HashSet<_> = turns.iter().map(|t| t.session_id.clone()).collect(); assert_eq!(sessions.len(), 4); - assert!(turns.iter().all(|t| t.call_ids.len() == 2)); + assert!(turns.iter().all(|t| t.span_ids.len() == 2)); } #[tokio::test] #[should_panic(expected = "spawn_turn_stage: shard_rxs must be non-empty")] async fn panics_on_empty_shard_rxs() { - let (_turns_tx, _turns_rx) = mpsc::channel::(1); + let (_turns_tx, _turns_rx) = mpsc::channel::(1); let mut metrics_sys = MetricsSystem::new(); spawn_turn_stage( TrackerConfig::default(), diff --git a/server/h-turn/src/test_support.rs b/server/h-turn/src/test_support.rs index 920ff7b4..bfb68f04 100644 --- a/server/h-turn/src/test_support.rs +++ b/server/h-turn/src/test_support.rs @@ -11,7 +11,7 @@ use h_llm::model::{ApiType, LlmCall}; use h_llm::wire_apis as wa; use crate::tracker::{TrackerConfig, TurnEvent, TurnTracker}; -use crate::AgentTurn; +use crate::Trace; pub use h_llm::model::{AgentCall, AgentCallInfo}; @@ -106,10 +106,10 @@ pub fn make_call( } /// Feed `calls` through a `TurnTracker` (with zero grace) and return the -/// single finalized `AgentTurn`. The first call is forced into the user-start +/// single finalized `Trace`. The first call is forced into the user-start /// role and the last into the terminal role so the partition closes exactly /// once, no matter how the caller built each `AgentCall`. -pub fn feed_session_and_finalize(session_id: &str, calls: &[AgentCall]) -> AgentTurn { +pub fn feed_session_and_finalize(session_id: &str, calls: &[AgentCall]) -> Trace { assert!( !calls.is_empty(), "need at least one call to finalize a turn" @@ -139,7 +139,7 @@ pub fn feed_session_and_finalize(session_id: &str, calls: &[AgentCall]) -> Agent } events.extend(tracker.flush_all()); - let mut turns: Vec = events + let mut turns: Vec = events .into_iter() .map(|TurnEvent::Completed(t)| t) .collect(); diff --git a/server/h-turn/src/tracker.rs b/server/h-turn/src/tracker.rs index af97d07c..29dfcaca 100644 --- a/server/h-turn/src/tracker.rs +++ b/server/h-turn/src/tracker.rs @@ -3,7 +3,7 @@ //! Each `(source_id, session_id)` owns a `SessionBuffer` that holds calls //! sorted by `request_time` until a main-agent terminal call appears and its //! grace window elapses. On grace expiry the buffer is partitioned at each -//! terminal and every partition becomes one `AgentTurn`. Partitions that +//! terminal and every partition becomes one `Trace`. Partitions that //! contain no `is_user_turn_start = Some(true)` call are discarded. use std::collections::{BTreeMap, BTreeSet, HashMap}; @@ -16,7 +16,7 @@ use h_common::internal_metrics::{Metric, MetricsWorker}; use h_llm::agent_classifier::SuspiciousSignal; use h_llm::model::AgentCall; -use crate::model::{ActiveTurnRegistry, AgentTurn, TurnStatus}; +use crate::model::{ActiveTraceRegistry, Trace, TraceStatus}; use crate::SuspiciousSkillRollup; const FINAL_ANSWER_PREVIEW_CHARS: usize = 500; @@ -63,7 +63,7 @@ impl Default for TrackerConfig { /// events were removed when the buffer model replaced ActiveTurn (04b §6.4). #[derive(Debug, Clone)] pub enum TurnEvent { - Completed(AgentTurn), + Completed(Trace), } #[derive(Debug)] @@ -99,7 +99,7 @@ struct SessionBuffer { /// Stable `turn_id` for the in-progress turn currently assembling in /// this buffer. Minted lazily on the first ingest after the previous /// turn was finalized; reused by every snapshot inserted into the - /// `ActiveTurnRegistry` and by the eventual `Completed` turn so the + /// `ActiveTraceRegistry` and by the eventual `Completed` turn so the /// in-memory in-progress entry's id matches the row that gets /// persisted (lets the API drop the registry entry without a /// frontend "row jump"). Cleared on every finalize / sweep / flush. @@ -136,7 +136,7 @@ pub struct TurnTracker { /// upserts a snapshot into the registry, and every finalize / /// sweep / flush removes the entry. `None` is used by tests that /// don't care about the snapshot path. - active_registry: Option, + active_registry: Option, } impl TurnTracker { @@ -144,14 +144,14 @@ impl TurnTracker { Self::with_registry(config, metrics, None) } - /// Variant that wires an [`ActiveTurnRegistry`] into the tracker so + /// Variant that wires an [`ActiveTraceRegistry`] into the tracker so /// the API can read in-progress snapshots without going through the /// DB or a channel fan-out. Pipelines call this; tests that don't /// exercise the snapshot path use [`Self::new`]. pub fn with_registry( config: TrackerConfig, metrics: MetricsWorker, - active_registry: Option, + active_registry: Option, ) -> Self { Self { config, @@ -249,14 +249,14 @@ impl TurnTracker { } // Refresh the registry's snapshot for this turn. Cheap (one HashMap - // upsert + an AgentTurn alloc); skipped if no registry is wired (e.g. + // upsert + an Trace alloc); skipped if no registry is wired (e.g. // tests via `TurnTracker::new`). self.refresh_active_snapshot(&source_id, &session_id); self.flush_ready_buffers(now_wall) } - /// Build the in-progress AgentTurn snapshot for a given session and + /// Build the in-progress Trace snapshot for a given session and /// upsert it into the active registry. Truncates the partition at the /// first main-agent terminal so multi-turn buffers don't mix two /// turns' state into one snapshot. Applies the same discard rule as @@ -314,7 +314,7 @@ impl TurnTracker { let mut snap = build_turn( &partition, Some(turn_id.clone()), - Some(TurnStatus::InProgress), + Some(TraceStatus::InProgress), ); snap.source_id = source_id.to_string(); snap.session_id = session_id.to_string(); @@ -327,8 +327,8 @@ impl TurnTracker { /// Read-side helper used by tests to inspect the registry contents /// without going through the lock directly. Returns a clone of every /// in-progress snapshot currently registered. Production reads happen - /// in `h-api` directly against the `ActiveTurnRegistry` Arc. - pub fn snapshot_active(&self) -> Vec { + /// in `h-api` directly against the `ActiveTraceRegistry` Arc. + pub fn snapshot_active(&self) -> Vec { match &self.active_registry { Some(reg) => reg .read() @@ -773,7 +773,7 @@ fn push_unique(list: &mut Vec, value: String) { /// /// `turn_id_override`: caller-supplied id. Used by the in-progress /// snapshot path so every snapshot for the same in-flight turn shares -/// one id, and the eventual finalized `AgentTurn` carries the same id +/// one id, and the eventual finalized `Trace` carries the same id /// the registry has been advertising. `None` mints a fresh UUID. /// /// `status_override`: forces a status. Used by the in-progress snapshot @@ -797,8 +797,8 @@ fn push_unique(list: &mut Vec, value: String) { fn build_turn( calls: &[&AgentCall], turn_id_override: Option, - status_override: Option, -) -> AgentTurn { + status_override: Option, +) -> Trace { assert!(!calls.is_empty(), "build_turn requires at least one call"); let first = calls[0]; let source_id = first.call.source_id.clone(); @@ -823,7 +823,7 @@ fn build_turn( let duration_ms = ((end_time_us - start_time_us).max(0) / 1000) as u64; let call_count = calls.len() as u32; - let call_ids: Vec = calls.iter().map(|ic| ic.call.id.clone()).collect(); + let span_ids: Vec = calls.iter().map(|ic| ic.call.id.clone()).collect(); let mut models_used: Vec = Vec::new(); let mut subagents_used: Vec = Vec::new(); @@ -890,8 +890,8 @@ fn build_turn( // window — the row should flip to Complete only via the Completed // event, never via an in-progress snapshot). let status = status_override.unwrap_or_else(|| match terminal { - Some(_) => TurnStatus::Complete, - None => TurnStatus::Incomplete, + Some(_) => TraceStatus::Complete, + None => TraceStatus::Incomplete, }); let (final_answer_preview, final_call_id, final_finish_reason) = match terminal { @@ -951,7 +951,7 @@ fn build_turn( suspicious.retain(|s| seen.insert(s.tool_name.clone())); let tool_surfaces: Vec = surfaces.into_iter().collect(); - AgentTurn { + Trace { source_id, turn_id, session_id, @@ -976,7 +976,7 @@ fn build_turn( user_call_id, final_answer_preview, final_call_id, - call_ids, + span_ids, metadata: serde_json::json!({}), tool_surfaces, tool_call_total, @@ -1183,7 +1183,7 @@ mod tests { } } - fn drain_completed(events: Vec) -> Vec { + fn drain_completed(events: Vec) -> Vec { events .into_iter() .map(|TurnEvent::Completed(t)| t) @@ -1215,9 +1215,9 @@ mod tests { events.extend(t.ingest(ic(c2.clone(), id2))); let turns = drain_completed(events); assert_eq!(turns.len(), 1); - assert_eq!(turns[0].status, TurnStatus::Complete); + assert_eq!(turns[0].status, TraceStatus::Complete); assert_eq!(turns[0].call_count, 2); - assert_eq!(turns[0].call_ids, vec![c1.id, c2.id]); + assert_eq!(turns[0].span_ids, vec![c1.id, c2.id]); } #[test] @@ -1505,7 +1505,7 @@ mod tests { let events = t.ingest(ic(c, id)); let turns = drain_completed(events); assert_eq!(turns.len(), 1); - assert_eq!(turns[0].status, TurnStatus::Complete); + assert_eq!(turns[0].status, TraceStatus::Complete); assert_eq!(t.active_count(), 0); } @@ -1519,7 +1519,7 @@ mod tests { let id = call_info_for_anthropic(&c); let turns = drain_completed(t.ingest(ic(c, id))); assert_eq!(turns.len(), 1); - assert_eq!(turns[0].status, TurnStatus::Complete); + assert_eq!(turns[0].status, TraceStatus::Complete); assert_eq!( turns[0].final_finish_reason.as_deref(), Some("max_tokens"), @@ -1565,7 +1565,7 @@ mod tests { assert_eq!(t.active_count(), 1); let turns = drain_completed(t.flush_all()); assert_eq!(turns.len(), 1); - assert_eq!(turns[0].status, TurnStatus::Incomplete); + assert_eq!(turns[0].status, TraceStatus::Incomplete); // No terminal ⇒ no final_*. Guards against regressing to "last call by // request_time wins" which would mislabel the ToolUse call as final. assert!(turns[0].final_call_id.is_none()); @@ -1612,7 +1612,7 @@ mod tests { let id = call_info_for_codex(&c); let turns = drain_completed(t.ingest(ic(c, id))); assert_eq!(turns.len(), 1); - assert_eq!(turns[0].status, TurnStatus::Complete); + assert_eq!(turns[0].status, TraceStatus::Complete); } #[test] @@ -1653,7 +1653,7 @@ mod tests { assert_eq!(t.active_count(), 1); let turns = drain_completed(t.flush_all()); assert_eq!(turns.len(), 1); - assert_eq!(turns[0].status, TurnStatus::Incomplete); + assert_eq!(turns[0].status, TraceStatus::Incomplete); assert!(turns[0].final_call_id.is_none()); assert!(turns[0].final_finish_reason.is_none()); assert!(turns[0].final_answer_preview.is_none()); @@ -1679,12 +1679,12 @@ mod tests { let swept = drain_completed(t.advance_time_at(arrival + 1_000_000, &source_id, Instant::now())); assert_eq!(swept.len(), 1); - assert_eq!(swept[0].status, TurnStatus::Incomplete); + assert_eq!(swept[0].status, TraceStatus::Incomplete); assert_eq!(t.active_count(), 0); } #[test] - fn ingest_populates_call_ids_into_finalized_turn() { + fn ingest_populates_span_ids_into_finalized_turn() { let mut t = mk_tracker_no_grace(); // Two codex calls, second is terminal-output → grace fires. let mut c1 = codex_call("s1", "t1", "message", "completed"); @@ -1707,7 +1707,7 @@ mod tests { events.extend(t.ingest(ic(c2, id2))); let turns = drain_completed(events); assert_eq!(turns.len(), 1); - assert_eq!(turns[0].call_ids, vec!["c1".to_string(), "c2".to_string()]); + assert_eq!(turns[0].span_ids, vec!["c1".to_string(), "c2".to_string()]); } #[test] @@ -1733,17 +1733,17 @@ mod tests { } // ----------------------------------------------------------------- - // ActiveTurnRegistry — in-memory in-progress visibility (PR B / Plan C) + // ActiveTraceRegistry — in-memory in-progress visibility (PR B / Plan C) // ----------------------------------------------------------------- - fn mk_tracker_with_registry() -> (TurnTracker, crate::model::ActiveTurnRegistry) { - let reg = crate::model::new_active_turn_registry(); + fn mk_tracker_with_registry() -> (TurnTracker, crate::model::ActiveTraceRegistry) { + let reg = crate::model::new_active_trace_registry(); let t = TurnTracker::with_registry(TrackerConfig::default(), test_metrics(), Some(reg.clone())); (t, reg) } - fn registry_snapshot(reg: &crate::model::ActiveTurnRegistry) -> Vec { + fn registry_snapshot(reg: &crate::model::ActiveTraceRegistry) -> Vec { reg.read().unwrap().values().cloned().collect() } @@ -1764,7 +1764,7 @@ mod tests { let snaps = registry_snapshot(®); assert_eq!(snaps.len(), 1, "exactly one in-progress snapshot"); let s = &snaps[0]; - assert_eq!(s.status, TurnStatus::InProgress); + assert_eq!(s.status, TraceStatus::InProgress); assert_eq!(s.call_count, 1); assert_eq!(s.session_id, "S-reg"); assert!(s.final_finish_reason.is_none()); @@ -1794,7 +1794,7 @@ mod tests { let s = &snaps[0]; assert_eq!(s.turn_id, first_id, "turn_id stable across ingests"); assert_eq!(s.call_count, 3, "snapshot reflects cumulative state"); - assert_eq!(s.status, TurnStatus::InProgress); + assert_eq!(s.status, TraceStatus::InProgress); } #[test] @@ -1825,7 +1825,7 @@ mod tests { completed[0].turn_id, in_progress_id, "Completed reuses in-progress turn_id (UI sees in-place transition)" ); - assert_eq!(completed[0].status, TurnStatus::Complete); + assert_eq!(completed[0].status, TraceStatus::Complete); assert!( registry_snapshot(®).is_empty(), "registry entry must be removed once Completed lands in DB" @@ -1845,7 +1845,7 @@ mod tests { sweep_interval_us: 100_000, grace: Duration::from_millis(1_000), }; - let reg = crate::model::new_active_turn_registry(); + let reg = crate::model::new_active_trace_registry(); let mut t = TurnTracker::with_registry(cfg, test_metrics(), Some(reg.clone())); let now = std::time::Instant::now(); @@ -1858,7 +1858,7 @@ mod tests { let completed = drain_completed(events); assert_eq!(completed.len(), 1, "idle sweep emits exactly one Completed"); assert_eq!(completed[0].turn_id, in_progress_id); - assert_eq!(completed[0].status, TurnStatus::Incomplete); + assert_eq!(completed[0].status, TraceStatus::Incomplete); assert!( registry_snapshot(®).is_empty(), "idle sweep must drop the in-progress registry entry" @@ -1955,7 +1955,7 @@ mod tests { "every snapshot reflects the full per-session ingest count" ); assert!( - snaps.iter().all(|s| s.status == TurnStatus::InProgress), + snaps.iter().all(|s| s.status == TraceStatus::InProgress), "every snapshot is in_progress" ); diff --git a/server/h-turn/tests/corpus_golden.rs b/server/h-turn/tests/corpus_golden.rs index 25fa2b2a..125b490d 100644 --- a/server/h-turn/tests/corpus_golden.rs +++ b/server/h-turn/tests/corpus_golden.rs @@ -2,7 +2,7 @@ //! //! Data-driven: reads `testdata/pcaps/corpus.toml`, replays each committed //! fixture through the FULL pipeline (capture → protocol → llm → turn), projects -//! the extracted `LlmCall`/`AgentTurn` into a DETERMINISTIC JSON shape (no uuids, +//! the extracted `LlmCall`/`Trace` into a DETERMINISTIC JSON shape (no uuids, //! no timing fields), and compares it to `testdata/pcaps/golden/.json`. //! //! - Missing fixtures (or unsmudged git-LFS pointers) are SKIPPED, so the @@ -69,7 +69,7 @@ fn fixture(file: &str) -> Option { async fn run_pcap_collecting_calls( file: &str, -) -> Option<(Vec, Vec>)> { +) -> Option<(Vec, Vec>)> { let path = fixture(file)?; let mut metrics_sys = MetricsSystem::new(); @@ -122,7 +122,7 @@ async fn run_pcap_collecting_calls( } let (calls_tx, mut calls_rx) = mpsc::channel::>(queue_size); - let (turns_tx, mut turns_rx) = mpsc::channel::(queue_size); + let (turns_tx, mut turns_rx) = mpsc::channel::(queue_size); let (m_out_tx, mut m_out_rx) = mpsc::channel::(queue_size); spawn_flow_dispatcher(raw_rx, parsed_txs, "dispatcher", &mut metrics_sys); @@ -177,7 +177,7 @@ async fn run_pcap_collecting_calls( }); let metrics_drain = tokio::spawn(async move { while m_out_rx.recv().await.is_some() {} }); - let mut finalized: Vec = Vec::new(); + let mut finalized: Vec = Vec::new(); while let Some(turn) = turns_rx.recv().await { finalized.push(turn); } @@ -205,7 +205,7 @@ fn sorted_strings(v: &[String]) -> Vec { s } -fn project_turn(t: &h_turn::AgentTurn) -> serde_json::Value { +fn project_turn(t: &h_turn::Trace) -> serde_json::Value { let tool_surfaces: Vec = { let mut s: Vec = t.tool_surfaces.iter().map(|x| x.to_string()).collect(); s.sort(); @@ -315,7 +315,7 @@ fn sort_values(mut v: Vec) -> Vec { } fn build_golden( - turns: &[h_turn::AgentTurn], + turns: &[h_turn::Trace], calls: &[Arc], ) -> serde_json::Value { let turns_p = sort_values(turns.iter().map(project_turn).collect()); diff --git a/server/h-turn/tests/integration.rs b/server/h-turn/tests/integration.rs index 65c10a07..abba191f 100644 --- a/server/h-turn/tests/integration.rs +++ b/server/h-turn/tests/integration.rs @@ -13,7 +13,7 @@ use h_common::internal_metrics::{Metric, MetricsSystem}; use h_llm::wire_apis as wa; use h_protocol::{spawn_flow_dispatcher, spawn_http_joiner_stage, spawn_protocol_stage}; use h_turn::tracker::TrackerConfig; -use h_turn::TurnStatus; +use h_turn::TraceStatus; fn fixture(name: &str) -> Option { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -31,7 +31,7 @@ async fn run_pcap_full_sharded( flow_shards: usize, turn_shards: usize, metrics_shards: usize, -) -> Option> { +) -> Option> { let path = fixture(name)?; let mut metrics_sys = MetricsSystem::new(); @@ -86,7 +86,7 @@ async fn run_pcap_full_sharded( } let (calls_tx, mut calls_rx) = mpsc::channel::>(queue_size); - let (turns_tx, mut turns_rx) = mpsc::channel::(queue_size); + let (turns_tx, mut turns_rx) = mpsc::channel::(queue_size); let (m_out_tx, mut m_out_rx) = mpsc::channel::(queue_size); spawn_flow_dispatcher(raw_rx, parsed_txs, "dispatcher", &mut metrics_sys); @@ -137,7 +137,7 @@ async fn run_pcap_full_sharded( let calls_drain = tokio::spawn(async move { while calls_rx.recv().await.is_some() {} }); let metrics_drain = tokio::spawn(async move { while m_out_rx.recv().await.is_some() {} }); - let mut finalized: Vec = Vec::new(); + let mut finalized: Vec = Vec::new(); while let Some(turn) = turns_rx.recv().await { finalized.push(turn); } @@ -152,11 +152,11 @@ async fn run_pcap_sharded( name: &str, turn_shards: usize, metrics_shards: usize, -) -> Option> { +) -> Option> { run_pcap_full_sharded(name, 1, turn_shards, metrics_shards).await } -async fn run_pcap(name: &str) -> Option> { +async fn run_pcap(name: &str) -> Option> { run_pcap_sharded(name, 1, 1).await } @@ -166,7 +166,7 @@ async fn run_pcap(name: &str) -> Option> { /// accumulation). async fn run_pcap_collecting_calls( name: &str, -) -> Option<(Vec, Vec>)> { +) -> Option<(Vec, Vec>)> { let path = fixture(name)?; let mut metrics_sys = MetricsSystem::new(); @@ -219,7 +219,7 @@ async fn run_pcap_collecting_calls( } let (calls_tx, mut calls_rx) = mpsc::channel::>(queue_size); - let (turns_tx, mut turns_rx) = mpsc::channel::(queue_size); + let (turns_tx, mut turns_rx) = mpsc::channel::(queue_size); let (m_out_tx, mut m_out_rx) = mpsc::channel::(queue_size); spawn_flow_dispatcher(raw_rx, parsed_txs, "dispatcher", &mut metrics_sys); @@ -274,7 +274,7 @@ async fn run_pcap_collecting_calls( }); let metrics_drain = tokio::spawn(async move { while m_out_rx.recv().await.is_some() {} }); - let mut finalized: Vec = Vec::new(); + let mut finalized: Vec = Vec::new(); while let Some(turn) = turns_rx.recv().await { finalized.push(turn); } @@ -308,7 +308,7 @@ async fn claude_cli_messages_expects_one_complete_turn() { "expected 1 turn; got {}", anthropic.len() ); - assert_eq!(anthropic[0].status, TurnStatus::Complete); + assert_eq!(anthropic[0].status, TraceStatus::Complete); assert_eq!(anthropic[0].agent_kind, "claude-cli"); } @@ -384,7 +384,7 @@ async fn codex_cli_messages_multi_expects_two_turns() { // flush_all. Without the fix, both would be Incomplete (closed only at EOF). let complete_count = openai .iter() - .filter(|t| t.status == TurnStatus::Complete) + .filter(|t| t.status == TraceStatus::Complete) .count(); assert!( complete_count >= 1, @@ -551,7 +551,7 @@ async fn openclaw_multi_sessions_expects_two_sessions_four_turns() { assert_eq!(chat.len(), 4, "expected 4 turns; got {}", chat.len()); assert!(chat.iter().all(|t| t.agent_kind == "openclaw")); assert!( - chat.iter().all(|t| t.status == TurnStatus::Complete), + chat.iter().all(|t| t.status == TraceStatus::Complete), "all turns expected Complete" ); let sessions: std::collections::BTreeSet<_> = @@ -639,7 +639,7 @@ async fn openclaw_anthropic_parallel_tool_use_inputs_intact() { ); assert!(anthropic.iter().all(|t| t.agent_kind == "openclaw")); assert!( - anthropic.iter().all(|t| t.status == TurnStatus::Complete), + anthropic.iter().all(|t| t.status == TraceStatus::Complete), "all turns expected Complete" ); let sessions: std::collections::BTreeSet<_> = @@ -744,7 +744,7 @@ async fn hermes_openai_expects_hermes_main_only() { } assert_eq!(chat.len(), 1, "expected 1 turn; got {}", chat.len()); assert!( - chat.iter().all(|t| t.status == TurnStatus::Complete), + chat.iter().all(|t| t.status == TraceStatus::Complete), "main turn expected Complete", ); @@ -798,7 +798,7 @@ async fn hermes_openai_pcap_shard_parity() { } /// End-to-end unit test: two `LlmCall` records (no claude-cli UA) run through -/// `build_agent_call_info` + `TurnTracker` produce a single `AgentTurn` with +/// `build_agent_call_info` + `TurnTracker` produce a single `Trace` with /// `agent_kind == "generic"` and `session_id == "toolu_pcap"`. The Anthropic /// wire-api shape exercises the generic profile's `wa::ANTHROPIC` branch. /// No pcap fixture — calls are constructed directly for speed and determinism. @@ -813,7 +813,7 @@ async fn generic_profile_anthropic_two_call_session() { use h_llm::build_agent_call_info; use h_llm::model::{AgentCall, ApiType, LlmCall}; use h_turn::tracker::{TrackerConfig, TurnTracker}; - use h_turn::{TurnEvent, TurnStatus}; + use h_turn::{TurnEvent, TraceStatus}; fn make_call(req: &str, resp: &str, ts_us: i64, finish: Option<&str>) -> LlmCall { LlmCall { @@ -982,7 +982,7 @@ async fn generic_profile_anthropic_two_call_session() { "final_answer_preview" ); assert!( - matches!(t.status, TurnStatus::Complete), + matches!(t.status, TraceStatus::Complete), "status must be Complete, got {:?}", t.status ); diff --git a/server/h-turn/tests/reorder.rs b/server/h-turn/tests/reorder.rs index 25aa3fb5..d96c490d 100644 --- a/server/h-turn/tests/reorder.rs +++ b/server/h-turn/tests/reorder.rs @@ -18,7 +18,7 @@ use h_llm::agents; use h_llm::model::{AgentCall, ApiType, LlmCall}; use h_llm::wire_apis as wa; use h_turn::tracker::{TrackerConfig, TurnEvent, TurnTracker}; -use h_turn::{AgentTurn, TurnStatus}; +use h_turn::{Trace, TraceStatus}; // -------- helpers -------- @@ -154,7 +154,7 @@ fn agent_call(call: LlmCall) -> AgentCall { } /// Drain finalized turns from a slice of TurnEvents. -fn collect_turns(events: Vec) -> Vec { +fn collect_turns(events: Vec) -> Vec { events .into_iter() .map(|TurnEvent::Completed(t)| t) @@ -164,7 +164,7 @@ fn collect_turns(events: Vec) -> Vec { /// Drive a sequence of calls through the tracker in the given arrival order, /// then flush. Identity is built per-call via the production pipeline (see /// [`agent_call`]). Returns every Completed turn the tracker produced. -fn run_in_order(t: &mut TurnTracker, sequence: Vec) -> Vec { +fn run_in_order(t: &mut TurnTracker, sequence: Vec) -> Vec { let mut events = Vec::new(); for c in sequence { events.extend(t.ingest(agent_call(c))); @@ -200,9 +200,9 @@ fn bug_a_late_user_start_splits_turn() { turns.len() ); let turn = &turns[0]; - assert_eq!(turn.status, TurnStatus::Complete); + assert_eq!(turn.status, TraceStatus::Complete); assert_eq!(turn.call_count, 2); - assert_eq!(turn.call_ids, vec![c1.id.clone(), c2.id.clone()]); + assert_eq!(turn.span_ids, vec![c1.id.clone(), c2.id.clone()]); } // -------- B: 3-call reorder with the terminal in the middle -------- @@ -239,10 +239,10 @@ fn bug_b_state_corruption_when_terminal_arrives_before_predecessors() { "deeply reordered same-session calls must collapse into one turn" ); let turn = &turns[0]; - assert_eq!(turn.status, TurnStatus::Complete); + assert_eq!(turn.status, TraceStatus::Complete); assert_eq!(turn.call_count, 3); assert_eq!( - turn.call_ids, + turn.span_ids, vec![c1.id.clone(), c2.id.clone(), c3.id.clone()] ); } @@ -352,8 +352,8 @@ fn bug_d_late_call_after_finalize_is_orphan_not_phantom() { "late call older than the finalized high-water must be dropped, not start a phantom turn" ); let turn = &turns[0]; - assert_eq!(turn.status, TurnStatus::Complete); - assert_eq!(turn.call_ids, vec![c1.id.clone(), c2.id.clone()]); + assert_eq!(turn.status, TraceStatus::Complete); + assert_eq!(turn.span_ids, vec![c1.id.clone(), c2.id.clone()]); } // -------- E: a partition with no user_turn_start is discarded ----------- @@ -429,8 +429,8 @@ fn bug_f_heartbeat_advance_does_not_open_phantom_for_late_call() { "heartbeat-advanced clock plus a late call must not create a phantom turn" ); let turn = &turns[0]; - assert_eq!(turn.status, TurnStatus::Complete); - assert_eq!(turn.call_ids, vec![c1.id.clone(), c2.id.clone()]); + assert_eq!(turn.status, TraceStatus::Complete); + assert_eq!(turn.span_ids, vec![c1.id.clone(), c2.id.clone()]); } // -------- watermark isolation regressions (F1/F2/F3) -------------------- @@ -542,7 +542,7 @@ fn f1_intra_source_hb_does_not_orphan_same_session_laggard() { turns[0].call_count, 2, "terminal + laggard expected — orphaned implies grace was event-time fast-forwarded" ); - let mut ids = turns[0].call_ids.clone(); + let mut ids = turns[0].span_ids.clone(); ids.sort(); let mut want = vec![terminal.id.clone(), laggard.id.clone()]; want.sort(); diff --git a/server/h-turn/tests/rollup.rs b/server/h-turn/tests/rollup.rs index 83386cf8..70c327a5 100644 --- a/server/h-turn/tests/rollup.rs +++ b/server/h-turn/tests/rollup.rs @@ -1,5 +1,5 @@ //! Verify the turn-finalize rollup combines per-call agent fields into the -//! correct aggregate fields on `AgentTurn`. +//! correct aggregate fields on `Trace`. use h_common::agent::{AgentTopology, ToolSurface}; use h_llm::agent_classifier::SuspiciousSignal;