diff --git a/backend/app/routers/analysis.py b/backend/app/routers/analysis.py index d22e661..e317f86 100644 --- a/backend/app/routers/analysis.py +++ b/backend/app/routers/analysis.py @@ -367,10 +367,11 @@ def aggregate_tool(tool_id: str, db: DBSession = Depends(get_db)): all_sessions = db.query(Session).all() per_session = [] - for session in all_sessions: - tool_name_lower = tool.name.lower() - tool_display_names = {tv.display_name.lower() for tv in tool.versions} + all_call_records: list[dict] = [] + tool_name_lower = tool.name.lower() + tool_display_names = {tv.display_name.lower() for tv in tool.versions} + for session in all_sessions: matched = False for ev in session.events: if ev.type in ("tool_call", "tool_error", "hallucinated_tool_call"): @@ -381,9 +382,10 @@ def aggregate_tool(tool_id: str, db: DBSession = Depends(get_db)): break if matched: - metrics = _session_metrics_for_tool(session, tool) + metrics, call_records = _session_metrics_for_tool(session, tool) if metrics: per_session.append(metrics) + all_call_records.extend(call_records) if not per_session: return { @@ -395,6 +397,7 @@ def aggregate_tool(tool_id: str, db: DBSession = Depends(get_db)): "hallucinated_tool_calls": 0, "per_model": {}, "per_session": [], + "calls_history": [], } total_calls = sum(m["tool_calls"] for m in per_session) @@ -415,6 +418,13 @@ def aggregate_tool(tool_id: str, db: DBSession = Depends(get_db)): per_model[model]["sessions"] += 1 per_model[model]["total_tokens"] += m["total_tokens"] + # Sort call history in reverse chronological order (most recent first) + calls_history = sorted( + all_call_records, + key=lambda r: r["timestamp"] or "", + reverse=True, + ) + return { "tool_id": tool_id, "tool_name": tool.name, @@ -431,6 +441,7 @@ def aggregate_tool(tool_id: str, db: DBSession = Depends(get_db)): "total_tokens_stats": safe_stats(all_tokens), "per_model": per_model, "per_session": per_session, + "calls_history": calls_history, } @@ -444,8 +455,12 @@ def _get_model_name(session: Session) -> str: return "unknown" -def _session_metrics_for_tool(session: Session, tool: Tool) -> dict | None: - """Return per-session metrics for a specific tool, or None if tool was never called in this session.""" +def _session_metrics_for_tool(session: Session, tool: Tool) -> tuple[dict | None, list[dict]]: + """Return (per-session metrics, list of individual call records) for a specific tool. + + Returns (None, []) if tool was never called in this session. + Each call record captures input args, output result/error, status and timing. + """ events = session.events tool_calls = 0 tool_errors = 0 @@ -456,6 +471,13 @@ def _session_metrics_for_tool(session: Session, tool: Tool) -> dict | None: tool_name_lower = tool.name.lower() tool_display_names = {tv.display_name.lower() for tv in tool.versions} + model_name = _get_model_name(session) + + # Build a map from tool_call_id -> call record so we can attach the result + # after we see the corresponding tool_result / tool_error event. + pending_calls: dict[str, dict] = {} # tool_call_id -> partial record + call_records: list[dict] = [] + for ev in events: payload = json.loads(ev.payload) if isinstance(ev.payload, str) else ev.payload ev_name = (payload.get("name", "") or "").lower() @@ -470,15 +492,61 @@ def _session_metrics_for_tool(session: Session, tool: Tool) -> dict | None: if ev.token_usage: tu = json.loads(ev.token_usage) if isinstance(ev.token_usage, str) else ev.token_usage token_usage.append(tu) + + record: dict = { + "session_id": session.id, + "model_name": model_name, + "timestamp": ev.timestamp.isoformat() if ev.timestamp else None, + "tool_call_id": ev.tool_call_id, + "args": payload.get("parsed_args"), + "output": None, + "status": "success", # optimistic; overwritten on error + "latency_ms": ev.latency_ms, + } + call_records.append(record) + if ev.tool_call_id: + pending_calls[ev.tool_call_id] = record + elif ev.type == "tool_error": tool_errors += 1 + # Try to attach the error to the matching call record + matched = pending_calls.get(ev.tool_call_id or "") if ev.tool_call_id else None + if matched: + matched["status"] = "error" + matched["output"] = payload.get("error") or payload.get("errors") + else: + # Standalone error record (e.g. schema validation before tool_call emitted) + call_records.append({ + "session_id": session.id, + "model_name": model_name, + "timestamp": ev.timestamp.isoformat() if ev.timestamp else None, + "tool_call_id": ev.tool_call_id, + "args": payload.get("args"), + "output": payload.get("error") or payload.get("errors"), + "status": "error", + "latency_ms": ev.latency_ms, + }) + + elif ev.type == "tool_result": + matched = pending_calls.get(ev.tool_call_id or "") if ev.tool_call_id else None + if matched: + matched["output"] = payload.get("result") + elif ev.type == "hallucinated_tool_call": tool_hallucinated += 1 + call_records.append({ + "session_id": session.id, + "model_name": model_name, + "timestamp": ev.timestamp.isoformat() if ev.timestamp else None, + "tool_call_id": ev.tool_call_id, + "args": payload.get("args"), + "output": None, + "status": "hallucinated", + "latency_ms": ev.latency_ms, + }) if tool_calls == 0 and tool_errors == 0 and tool_hallucinated == 0: - return None - - model_name = _get_model_name(session) + return None, [] return { "session_id": session.id, @@ -496,4 +564,4 @@ def _session_metrics_for_tool(session: Session, tool: Tool) -> dict | None: "total_tokens": sum( tu.get("input_tokens", 0) + tu.get("output_tokens", 0) for tu in token_usage ), - } + }, call_records diff --git a/backend/seed_tool_history.py b/backend/seed_tool_history.py new file mode 100644 index 0000000..30e1303 --- /dev/null +++ b/backend/seed_tool_history.py @@ -0,0 +1,148 @@ +""" +Seed script: injects fake tool call history into harness.db so you can +test the Call History feature on the Tool Stats page without an API key. + +Usage: uv run python seed_tool_history.py +""" +import json +import sqlite3 +import uuid +from datetime import datetime, timedelta, timezone + +DB_PATH = "harness.db" + +# We'll use two existing built-in tools from the live DB. +TOOLS = [ + { + "tool_id": "78b5b858-414f-4766-9e7b-aa757ecf58d2", + "display_name": "summon_cat", + }, + { + "tool_id": "a6b04fe7-8da2-4045-a546-616c6b0703f1", + "display_name": "snake_oil", + }, +] + +# Grab an existing plan_version_id from the DB to attach sessions to. +con = sqlite3.connect(DB_PATH) +con.row_factory = sqlite3.Row +cur = con.cursor() + +cur.execute("SELECT id FROM plan_versions LIMIT 1") +pv_row = cur.fetchone() +if pv_row is None: + print("❌ No plan versions found in harness.db.") + print(" Please create at least one Plan first via the UI, then re-run this script.") + con.close() + exit(1) + +plan_version_id = pv_row["id"] +print(f"✅ Using plan_version_id: {plan_version_id}") + +now = datetime.now(timezone.utc) + +def _uuid(): + return str(uuid.uuid4()) + +def _iso(dt): + return dt.isoformat() + +def insert_session(cur, plan_version_id, started_at, status="completed"): + sid = _uuid() + cur.execute( + """INSERT INTO sessions + (id, plan_version_id, batch_id, batch_index, started_at, ended_at, + status, termination_reason, tool_order_used, totals) + VALUES (?,?,NULL,0,?,?,?,'completed_with_tool_call','[]','{}')""", + (sid, plan_version_id, _iso(started_at), _iso(started_at + timedelta(seconds=5)), status), + ) + return sid + +def insert_event(cur, session_id, seq, ev_type, payload, latency_ms=None, tool_call_id=None, ts=None): + ts = ts or now + cur.execute( + """INSERT INTO events + (id, session_id, sequence_no, timestamp, type, payload, latency_ms, token_usage, tool_call_id) + VALUES (?,?,?,?,?,?,?,NULL,?)""", + (_uuid(), session_id, seq, _iso(ts), ev_type, json.dumps(payload), latency_ms, tool_call_id), + ) + + +# ── Session 1: summon_cat – 2 successful calls, different incantations ──────── +s1_start = now - timedelta(hours=2) +s1 = insert_session(cur, plan_version_id, s1_start) + +tc1a = _uuid() +insert_event(cur, s1, 1, "tool_call", + {"name": "summon_cat", "parsed_args": {"incantation": "here kitty kitty", "tribute": "tuna"}, + "raw_args": '{"incantation":"here kitty kitty","tribute":"tuna"}'}, + latency_ms=312, tool_call_id=tc1a, ts=s1_start + timedelta(seconds=1)) +insert_event(cur, s1, 2, "tool_result", + {"name": "summon_cat", "result": {"cat_arrived": True, "cat_name": "Lord Whiskers the Unimpressed", + "cat_mood": "grudgingly curious", "did_accept_tribute": True}}, + tool_call_id=tc1a, ts=s1_start + timedelta(seconds=2)) + +tc1b = _uuid() +insert_event(cur, s1, 3, "tool_call", + {"name": "summon_cat", "parsed_args": {"incantation": "I offer you my debugging sorrows", "tribute": "catnip"}, + "raw_args": '{"incantation":"I offer you my debugging sorrows","tribute":"catnip"}'}, + latency_ms=487, tool_call_id=tc1b, ts=s1_start + timedelta(seconds=3)) +insert_event(cur, s1, 4, "tool_result", + {"name": "summon_cat", "result": {"cat_arrived": False, "cat_mood": "entirely unimpressed", + "parting_remark": "Catnip? Really? You disappoint me."}}, + tool_call_id=tc1b, ts=s1_start + timedelta(seconds=4)) + +print(f"✅ Session 1 (summon_cat × 2 success): {s1}") + +# ── Session 2: summon_cat – 1 error call ────────────────────────────────────── +s2_start = now - timedelta(hours=1) +s2 = insert_session(cur, plan_version_id, s2_start) + +tc2a = _uuid() +insert_event(cur, s2, 1, "tool_call", + {"name": "summon_cat", "parsed_args": {"incantation": "go cat go", "tribute": "belly_rub"}, + "raw_args": '{"incantation":"go cat go","tribute":"belly_rub"}'}, + latency_ms=152, tool_call_id=tc2a, ts=s2_start + timedelta(seconds=1)) +insert_event(cur, s2, 2, "tool_error", + {"name": "summon_cat", "error": "Cat portal timeout: the feline realm is temporarily unavailable"}, + tool_call_id=tc2a, ts=s2_start + timedelta(seconds=2)) + +print(f"✅ Session 2 (summon_cat error): {s2}") + +# ── Session 3: snake_oil – 1 success call ───────────────────────────────────── +s3_start = now - timedelta(minutes=15) +s3 = insert_session(cur, plan_version_id, s3_start) + +tc3a = _uuid() +insert_event(cur, s3, 1, "tool_call", + {"name": "snake_oil", "parsed_args": {"ailment": "chronic overthinking and deadline dread"}, + "raw_args": '{"ailment":"chronic overthinking and deadline dread"}'}, + latency_ms=98, tool_call_id=tc3a, ts=s3_start + timedelta(seconds=1)) +insert_event(cur, s3, 2, "tool_result", + {"name": "snake_oil", "result": {"prescription": "Two tablespoons of Dr. Ambrose's Miracle Elixir", + "cure_guaranteed": True, "scientifically_verified": False, + "price": "Your immortal soul (or $19.99, whichever is more convenient)"}}, + tool_call_id=tc3a, ts=s3_start + timedelta(seconds=2)) + +print(f"✅ Session 3 (snake_oil × 1 success): {s3}") + +# ── Session 4: most recent, summon_cat with hallucinated call ───────────────── +s4_start = now - timedelta(minutes=3) +s4 = insert_session(cur, plan_version_id, s4_start) + +tc4a = _uuid() +insert_event(cur, s4, 1, "hallucinated_tool_call", + {"name": "summon_cat", "args": {"incantation": "please just work", "tribute": "pizza"}}, + tool_call_id=tc4a, ts=s4_start + timedelta(seconds=1)) + +print(f"✅ Session 4 (summon_cat hallucinated): {s4}") + +con.commit() +con.close() + +print() +print("🎉 Done! Seed data inserted into harness.db.") +print() +print("👉 Now go to: http://localhost:5173/tools") +print(" Click the 📊 Stats icon next to 'summon_cat' to see the Call History.") +print(" The 'snake_oil' tool also has one call — check its Stats too!") diff --git a/backend/tests/test_analysis.py b/backend/tests/test_analysis.py new file mode 100644 index 0000000..56fc70a --- /dev/null +++ b/backend/tests/test_analysis.py @@ -0,0 +1,219 @@ +"""Tests for /analysis/tool/{tool_id} endpoint, specifically the calls_history field.""" +import json +import uuid +from datetime import datetime, timezone + +from app.models import Event, Plan, PlanVersion, Session + + +TOOL_PAYLOAD = { + "name": "search", + "description": "A search tool", + "tags": ["web"], + "version": { + "display_name": "search", + "model_facing_description": "Search the web", + "parameter_schema": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + "response_mode": "static", + "static_response": {"results": ["foo", "bar"]}, + }, +} + +_MCS = json.dumps( + { + "base_url": "https://api.example.com/v1", + "api_key_env": "FAKE_KEY", + "model_snapshot": "test-model", + "params": "{}", + "input_cost_per_1k": 0.0, + "output_cost_per_1k": 0.0, + } +) + +_RUN_SETTINGS = json.dumps( + {"max_turns": 20, "max_tool_calls": 50, "timeout_seconds": 300} +) + + +def _make_plan_version(db): + plan = Plan(name="test-plan") + db.add(plan) + db.flush() + pv = PlanVersion( + plan_id=plan.id, + version_number=1, + model_config_snapshot=_MCS, + system_prompt="", + user_prompt="Hello", + run_settings=_RUN_SETTINGS, + ) + db.add(pv) + db.flush() + return pv + + +def _make_session(db, pv, status="completed"): + session = Session(plan_version_id=pv.id, status=status) + session.started_at = datetime(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + session.ended_at = datetime(2026, 1, 1, 12, 0, 5, tzinfo=timezone.utc) + db.add(session) + db.flush() + return session + + +def _add_events(db, session_id, tool_display_name, args, result=None, error=None): + """Insert a tool_call + tool_result/tool_error event pair into the DB.""" + tc_id = str(uuid.uuid4()) + ts = datetime(2026, 1, 1, 12, 0, 1, tzinfo=timezone.utc) + + call_ev = Event( + id=str(uuid.uuid4()), + session_id=session_id, + sequence_no=1, + timestamp=ts, + type="tool_call", + payload=json.dumps({"name": tool_display_name, "parsed_args": args, "raw_args": json.dumps(args)}), + latency_ms=123, + tool_call_id=tc_id, + ) + db.add(call_ev) + + if error: + err_ev = Event( + id=str(uuid.uuid4()), + session_id=session_id, + sequence_no=2, + timestamp=ts, + type="tool_error", + payload=json.dumps({"name": tool_display_name, "error": error}), + latency_ms=None, + tool_call_id=tc_id, + ) + db.add(err_ev) + else: + res_ev = Event( + id=str(uuid.uuid4()), + session_id=session_id, + sequence_no=2, + timestamp=ts, + type="tool_result", + payload=json.dumps({"name": tool_display_name, "result": result}), + latency_ms=None, + tool_call_id=tc_id, + ) + db.add(res_ev) + + db.commit() + + +def test_tool_stats_calls_history_success(client, db): + """calls_history includes entries with args and output for successful tool calls.""" + # Create the tool + tool_resp = client.post("/api/tools", json=TOOL_PAYLOAD).json() + tool_id = tool_resp["id"] + tool_display_name = tool_resp["versions"][0]["display_name"] + + pv = _make_plan_version(db) + session = _make_session(db, pv) + _add_events(db, session.id, tool_display_name, args={"query": "hello"}, result={"results": ["a"]}) + + r = client.get(f"/api/analysis/tool/{tool_id}") + assert r.status_code == 200 + data = r.json() + + assert "calls_history" in data + assert len(data["calls_history"]) == 1 + entry = data["calls_history"][0] + assert entry["status"] == "success" + assert entry["args"] == {"query": "hello"} + assert entry["output"] == {"results": ["a"]} + assert entry["latency_ms"] == 123 + assert entry["session_id"] == session.id + + +def test_tool_stats_calls_history_error(client, db): + """calls_history correctly records error status and error output.""" + tool_resp = client.post("/api/tools", json=TOOL_PAYLOAD).json() + tool_id = tool_resp["id"] + tool_display_name = tool_resp["versions"][0]["display_name"] + + pv = _make_plan_version(db) + session = _make_session(db, pv) + _add_events(db, session.id, tool_display_name, args={"query": "bad"}, error="Tool failed: timeout") + + r = client.get(f"/api/analysis/tool/{tool_id}") + assert r.status_code == 200 + data = r.json() + + assert len(data["calls_history"]) == 1 + entry = data["calls_history"][0] + assert entry["status"] == "error" + assert "timeout" in entry["output"] + assert entry["args"] == {"query": "bad"} + + +def test_tool_stats_calls_history_reverse_chronological(client, db): + """calls_history is sorted most-recent first when a tool is called multiple times.""" + tool_resp = client.post("/api/tools", json=TOOL_PAYLOAD).json() + tool_id = tool_resp["id"] + tool_display_name = tool_resp["versions"][0]["display_name"] + + pv = _make_plan_version(db) + + # Session 1 (earlier) + s1 = _make_session(db, pv) + tc1_id = str(uuid.uuid4()) + early_ts = datetime(2026, 1, 1, 10, 0, 0, tzinfo=timezone.utc) + db.add(Event( + id=str(uuid.uuid4()), session_id=s1.id, sequence_no=1, + timestamp=early_ts, type="tool_call", + payload=json.dumps({"name": tool_display_name, "parsed_args": {"query": "first"}, "raw_args": '{"query":"first"}'}), + latency_ms=100, tool_call_id=tc1_id, + )) + db.add(Event( + id=str(uuid.uuid4()), session_id=s1.id, sequence_no=2, + timestamp=early_ts, type="tool_result", + payload=json.dumps({"name": tool_display_name, "result": {"r": 1}}), + tool_call_id=tc1_id, + )) + + # Session 2 (later) + s2 = _make_session(db, pv) + tc2_id = str(uuid.uuid4()) + late_ts = datetime(2026, 1, 2, 10, 0, 0, tzinfo=timezone.utc) + db.add(Event( + id=str(uuid.uuid4()), session_id=s2.id, sequence_no=1, + timestamp=late_ts, type="tool_call", + payload=json.dumps({"name": tool_display_name, "parsed_args": {"query": "second"}, "raw_args": '{"query":"second"}'}), + latency_ms=200, tool_call_id=tc2_id, + )) + db.add(Event( + id=str(uuid.uuid4()), session_id=s2.id, sequence_no=2, + timestamp=late_ts, type="tool_result", + payload=json.dumps({"name": tool_display_name, "result": {"r": 2}}), + tool_call_id=tc2_id, + )) + db.commit() + + r = client.get(f"/api/analysis/tool/{tool_id}") + assert r.status_code == 200 + history = r.json()["calls_history"] + + assert len(history) == 2 + # Most recent first + assert history[0]["args"]["query"] == "second" + assert history[1]["args"]["query"] == "first" + + +def test_tool_stats_calls_history_empty_when_no_sessions(client): + """calls_history is an empty list when the tool has never been called.""" + tool_resp = client.post("/api/tools", json=TOOL_PAYLOAD).json() + tool_id = tool_resp["id"] + + r = client.get(f"/api/analysis/tool/{tool_id}") + assert r.status_code == 200 + assert r.json()["calls_history"] == [] diff --git a/frontend/src/pages/ToolStats.tsx b/frontend/src/pages/ToolStats.tsx index cb4885e..26cb055 100644 --- a/frontend/src/pages/ToolStats.tsx +++ b/frontend/src/pages/ToolStats.tsx @@ -1,6 +1,6 @@ import { useQuery } from '@tanstack/react-query' import { useParams, Link } from 'react-router-dom' -import { ArrowLeft, BarChart2, Sparkles } from 'lucide-react' +import { ArrowLeft, BarChart2, Sparkles, Clock, CheckCircle2, XCircle, AlertTriangle } from 'lucide-react' import { api } from '../api/client' import { ModelCallsChart } from '../components/charts' @@ -12,6 +12,17 @@ interface PerModelStats { total_tokens: number } +interface ToolCallRecord { + session_id: string + model_name: string + timestamp: string | null + tool_call_id: string | null + args: Record | null + output: unknown + status: 'success' | 'error' | 'hallucinated' + latency_ms: number | null +} + interface ToolAnalysis { tool_id: string tool_name: string @@ -24,6 +35,7 @@ interface ToolAnalysis { total_tokens_stats: { mean: number; stdev: number; min: number; max: number } per_model: Record per_session: Record[] + calls_history: ToolCallRecord[] } function ChartCard({ title, subtitle, children }: { title: string; subtitle?: string; children?: React.ReactNode }) { @@ -262,6 +274,111 @@ export default function ToolStats() { )} + {/* Call History */} + {stats.calls_history && stats.calls_history.length > 0 && ( +
+

Call History

+

All past invocations, most recent first

+
+ {stats.calls_history.map((record, idx) => { + const statusColor = + record.status === 'success' + ? 'bg-emerald-100 text-emerald-700' + : record.status === 'error' + ? 'bg-rose-100 text-rose-700' + : 'bg-amber-100 text-amber-700' + + const StatusIcon = + record.status === 'success' + ? CheckCircle2 + : record.status === 'error' + ? XCircle + : AlertTriangle + + return ( +
+ {/* Header row */} +
+ + + {record.status} + + {record.model_name && ( + {record.model_name} + )} + + {record.session_id.slice(0, 8)}… + + + {record.latency_ms != null && ( + <> + + {record.latency_ms}ms + + )} + {record.timestamp && ( + + {new Intl.DateTimeFormat('en-US', { + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }).format(new Date(record.timestamp))} + + )} + +
+ + {/* Args */} + {record.args != null && ( +
+ + + + Inputs + +
+                            {JSON.stringify(record.args, null, 2)}
+                          
+
+ )} + + {/* Output */} + {record.output != null && ( +
+ + + + {record.status === 'error' ? 'Error' : 'Output'} + +
+                            {typeof record.output === 'string'
+                              ? record.output
+                              : JSON.stringify(record.output, null, 2)}
+                          
+
+ )} +
+ ) + })} +
+
+ )} )}