diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..11120da --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,19 @@ +{ + "permissions": { + "allow": [ + "Bash(python -c ' *)", + "Bash(python -m mypy --no-site-packages src/agenticlens/models/step.py)", + "Bash(pip uninstall *)", + "Bash(git check-ignore *)", + "Bash(python -m pytest tests/test_adapters_langchain.py -q)", + "Bash(python -m pytest -q tests/)", + "Bash(python -m ruff check benchmarks/shared/benchmark_runner.py)", + "Bash(git commit -m ' *)", + "Bash(mkdir -p \"C:\\\\Users\\\\manem\\\\AppData\\\\Local\\\\Temp\\\\claude\\\\e--agenticlens\\\\9e618684-551e-4264-8655-deac3e2e58bd\\\\scratchpad\")", + "Skill(artifact-design)", + "Skill(artifact-design:*)", + "Bash(python -c \"import os; print\\('OPENAI_API_KEY set:', bool\\(os.getenv\\('OPENAI_API_KEY'\\)\\)\\)\")", + "Bash(python examples/live_multiagent_travel_briefing.py)" + ] + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 048a832..6c56239 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: run: uv python install ${{ matrix.python-version }} - name: Install dependencies - run: uv sync --extra dev --python ${{ matrix.python-version }} + run: uv sync --extra dev --extra langchain --python ${{ matrix.python-version }} - name: Lint (ruff check) run: uv run ruff check . diff --git a/.gitignore b/.gitignore index a0dd274..c16cbf4 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,11 @@ site/ .env test/ +# Generated workflow reports from running examples/benchmarks locally +/*_report.json +/workflow.json +/report.json + # Local patent/provisional drafting artifacts docs/patents/ tools/build_agenticlens_provisional.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e980b5..71ccb53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ All notable changes to this project will be documented here. This project follows [Semantic Versioning](https://semver.org/). +## Unreleased + +### Added + +- `agenticlens.adapters.langchain.AgenticLensCallbackHandler`, an optional + (`pip install "agenticlens[langchain]"`) LangChain/LangGraph callback + handler that auto-instruments LLM, tool, and retriever calls as AgenticLens + steps without manual `step()` blocks. Documented in + `docs/langchain-integration.md`. +- `benchmarks/`, a cross-framework benchmark harness that profiles the same + refund-ticket workload through AutoGen, CrewAI, LangGraph, LlamaIndex, + Semantic Kernel, and native Python for an apples-to-apples + token/cost/latency comparison. Linked from the README. +- `examples/support_copilot.py`, `examples/multiagent_edge_cases_demo.py`, and + `examples/live_multiagent_travel_briefing.py`, additional practical, + edge-case, and live multi-agent profiling examples. + ## 0.4.0 - 2026-08-08 ### Added diff --git a/README.md b/README.md index f6a6c0e..1ee0e14 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,9 @@ The product idea is simple: - [Features](#features) - [Cost Calculation](#cost-calculation) - [Configuration Reference](#configuration-reference) +- [Framework Benchmarks](#framework-benchmarks) +- [Notebooks](#notebooks) +- [LangChain Integration](#langchain-integration) - [CLI Reference](#cli-reference) - [Current Limitations](#current-limitations) - [Development](#development) @@ -961,6 +964,9 @@ Other examples: - `examples/rag_customer_support_demo.py` - `examples/multiagent_support_demo.py` - `examples/multiagent_token_optimization_demo.py` +- `examples/support_copilot.py` — practical support workflow profiling example +- `examples/multiagent_edge_cases_demo.py` — edge-case instrumentation example +- `examples/live_multiagent_travel_briefing.py` — live provider multi-agent travel briefing demo - `examples/reference_workflows/langgraph_supervisor.py` — offline LangGraph supervisor - `examples/export_demo.py` — export to Markdown and Jira - `examples/live_evaluation_demo.py` — trusted live Python target for `evaluate-live` @@ -979,6 +985,28 @@ official framework repositories. See [docs/multi-agent-reference-workflows.md](docs/multi-agent-reference-workflows.md) for setup, source links, dependency isolation, and instrumentation boundaries. +## Framework Benchmarks + +`benchmarks/` runs the same practical refund-ticket workload through AutoGen, +CrewAI, LangGraph, LlamaIndex, Semantic Kernel, and native Python, profiling +each with AgenticLens to normalize tokens, cost, latency, tool calls, and +retrieved chunks across implementations. See +[benchmarks/results/benchmark_summary.md](benchmarks/results/benchmark_summary.md) +for the current comparison table. + +## Notebooks + +Beginner-friendly notebooks are available in: + +```text +notebooks/agenticlens_workflow_demo_beginner.ipynb +notebooks/agenticlens_multiagent_demo_beginner.ipynb +``` + +The notebooks walk through step-by-step workflow and multi-agent profiling, +token usage tables, latency and cost charts, saved AgenticLens artifacts, and +optimization analysis. + ## Exporting Reports ### Markdown @@ -1026,6 +1054,29 @@ Set credentials via environment variables for safety — see For sample output previews of all formats, see [docs/export-formats.md](docs/export-formats.md). +## LangChain Integration + +Auto-instrument a LangChain (or LangGraph) run via its callback system instead +of wrapping every call in `step()`: + +```bash +pip install "agenticlens[langchain]" +``` + +```python +from agenticlens import profile +from agenticlens.adapters.langchain import AgenticLensCallbackHandler + +with profile("My LangChain App") as workflow: + chain.invoke(inputs, config={"callbacks": [AgenticLensCallbackHandler()]}) +``` + +LLM calls, tool calls, and retriever calls are tracked automatically as +`llm_call`, `tool_call`, and `retriever` steps, with token usage extracted the +same way `s.record(...)` does. See +[docs/langchain-integration.md](docs/langchain-integration.md) for details on +what is and is not tracked. + ## CLI Reference Profile a Python script: diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/compare_results.py b/benchmarks/compare_results.py new file mode 100644 index 0000000..bd2ce78 --- /dev/null +++ b/benchmarks/compare_results.py @@ -0,0 +1,254 @@ +import json +from pathlib import Path + +import matplotlib.pyplot as plt +import pandas as pd + +REPORTS = { + "Native Python": "benchmarks/reports/native_python/support_refund_report.json", + "LangGraph": "benchmarks/reports/langgraph/support_refund_report.json", + "CrewAI": "benchmarks/reports/crewai/support_refund_report.json", + "AutoGen": "benchmarks/reports/autogen/support_refund_report.json", + "LlamaIndex": "benchmarks/reports/llamaindex/support_refund_report.json", + "Semantic Kernel": "benchmarks/reports/semantic_kernel/support_refund_report.json", +} + +RESULTS_DIR = Path("benchmarks/results") +RESULTS_DIR.mkdir(parents=True, exist_ok=True) + + +def load_report(path: str | Path) -> dict: + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"Report not found: {path}") + return json.loads(path.read_text(encoding="utf-8")) + + +def summarize_report(framework: str, report: dict) -> dict: + steps = report.get("steps", []) + + total_tokens = 0 + prompt_tokens = 0 + completion_tokens = 0 + total_cost = 0.0 + total_latency = 0.0 + tool_calls = 0 + retrieved_chunks = 0 + + highest_token_step = None + highest_step_tokens = -1 + + highest_cost_step = None + highest_step_cost = -1.0 + + for step in steps: + metrics = step.get("metrics") or {} + metadata = step.get("metadata") or {} + + step_tokens = metrics.get("total_tokens") or 0 + step_prompt_tokens = metrics.get("prompt_tokens") or 0 + step_completion_tokens = metrics.get("completion_tokens") or 0 + step_cost = metrics.get("cost") or 0.0 + step_latency = metrics.get("latency") or 0.0 + + total_tokens += step_tokens + prompt_tokens += step_prompt_tokens + completion_tokens += step_completion_tokens + total_cost += step_cost + total_latency += step_latency + + if step.get("type") == "tool_call": + tool_calls += 1 + + if step.get("type") == "retriever": + retrieved_chunks += metadata.get("chunk_count") or 0 + + if step_tokens > highest_step_tokens: + highest_step_tokens = step_tokens + highest_token_step = step.get("name") + + if step_cost > highest_step_cost: + highest_step_cost = step_cost + highest_cost_step = step.get("name") + + return { + "framework": framework, + "workflow_name": report.get("name"), + "step_count": len(steps), + "total_tokens": total_tokens, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_cost_usd": round(total_cost, 8), + "total_latency_sec": round(total_latency, 8), + "tool_calls": tool_calls, + "retrieved_chunks": retrieved_chunks, + "highest_token_step": highest_token_step, + "highest_step_tokens": highest_step_tokens, + "highest_cost_step": highest_cost_step, + "highest_step_cost_usd": round(highest_step_cost, 8), + } + + +def extract_step_rows(framework: str, report: dict) -> list[dict]: + rows = [] + + for step in report.get("steps", []): + metrics = step.get("metrics") or {} + metadata = step.get("metadata") or {} + + rows.append( + { + "framework": framework, + "workflow_name": report.get("name"), + "step_name": step.get("name"), + "step_type": step.get("type"), + "provider": step.get("provider"), + "model": step.get("model"), + "prompt_tokens": metrics.get("prompt_tokens") or 0, + "completion_tokens": metrics.get("completion_tokens") or 0, + "total_tokens": metrics.get("total_tokens") or 0, + "cost_usd": metrics.get("cost") or 0.0, + "latency_sec": metrics.get("latency") or 0.0, + "chunk_count": metadata.get("chunk_count"), + "tool_name": metadata.get("tool_name"), + } + ) + + return rows + + +def create_markdown_summary(summary_df: pd.DataFrame, output_path: Path) -> None: + lines = [ + "# AgenticLens Framework Benchmark Comparison", + "", + "Use case: Practical customer support refund workflow.", + "", + "The workflow includes:", + "", + "- ticket intent classification", + "- query rewriting", + "- refund policy retrieval", + "- order lookup", + "- refund eligibility check", + "- customer reply generation", + "", + "## Summary Results", + "", + "| Framework | Total Tokens | Prompt Tokens | Completion Tokens | Cost USD | " + "Latency Sec | Steps | Tool Calls | Retrieved Chunks | Highest Token Step |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", + ] + + for _, row in summary_df.iterrows(): + lines.append( + f"| {row['framework']} | " + f"{row['total_tokens']} | " + f"{row['prompt_tokens']} | " + f"{row['completion_tokens']} | " + f"${row['total_cost_usd']:.8f} | " + f"{row['total_latency_sec']:.8f} | " + f"{row['step_count']} | " + f"{row['tool_calls']} | " + f"{row['retrieved_chunks']} | " + f"{row['highest_token_step']} |" + ) + + lines.extend( + [ + "", + "## Key Finding", + "", + "The final customer reply step is the highest token-consuming step across " + "the benchmark runs.", + "", + "## Important Note", + "", + "These results are workload-specific. They should not be treated as a " + "universal ranking of frameworks.", + "The purpose is to show how AgenticLens can normalize and compare token, " + "cost, latency, retrieval, and tool-call metrics across framework " + "implementations.", + ] + ) + + output_path.write_text("\n".join(lines), encoding="utf-8") + + +def plot_total_tokens(summary_df: pd.DataFrame) -> None: + plt.figure(figsize=(10, 5)) + plt.bar(summary_df["framework"], summary_df["total_tokens"]) + plt.title("AgenticLens Benchmark: Total Tokens by Framework") + plt.xlabel("Framework") + plt.ylabel("Total Tokens") + plt.xticks(rotation=30, ha="right") + plt.tight_layout() + + output = RESULTS_DIR / "benchmark_tokens_chart.png" + plt.savefig(output) + plt.close() + + print(f"Saved token chart: {output}") + + +def plot_total_cost(summary_df: pd.DataFrame) -> None: + plt.figure(figsize=(10, 5)) + plt.bar(summary_df["framework"], summary_df["total_cost_usd"]) + plt.title("AgenticLens Benchmark: Estimated Cost by Framework") + plt.xlabel("Framework") + plt.ylabel("Estimated Cost USD") + plt.xticks(rotation=30, ha="right") + plt.tight_layout() + + output = RESULTS_DIR / "benchmark_cost_chart.png" + plt.savefig(output) + plt.close() + + print(f"Saved cost chart: {output}") + + +def main() -> None: + summary_rows = [] + step_rows = [] + + for framework, report_path in REPORTS.items(): + path = Path(report_path) + + if not path.exists(): + print(f"Skipping {framework}: report not found at {report_path}") + continue + + report = load_report(path) + + summary_rows.append(summarize_report(framework, report)) + step_rows.extend(extract_step_rows(framework, report)) + + if not summary_rows: + raise RuntimeError("No reports found. Run AgenticLens profile commands first.") + + summary_df = pd.DataFrame(summary_rows) + step_df = pd.DataFrame(step_rows) + + summary_df = summary_df.sort_values(by=["total_tokens", "framework"]) + + summary_csv = RESULTS_DIR / "benchmark_results.csv" + step_csv = RESULTS_DIR / "benchmark_step_breakdown.csv" + summary_md = RESULTS_DIR / "benchmark_summary.md" + + summary_df.to_csv(summary_csv, index=False) + step_df.to_csv(step_csv, index=False) + create_markdown_summary(summary_df, summary_md) + + plot_total_tokens(summary_df) + plot_total_cost(summary_df) + + print("\nBenchmark comparison complete.") + print(f"Summary CSV: {summary_csv}") + print(f"Step breakdown CSV: {step_csv}") + print(f"Markdown summary: {summary_md}") + + print("\nSummary:") + print(summary_df.to_string(index=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/datasets/orders.json b/benchmarks/datasets/orders.json new file mode 100644 index 0000000..9305a43 --- /dev/null +++ b/benchmarks/datasets/orders.json @@ -0,0 +1,29 @@ +[ + { + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + }, + { + "order_id": "B456", + "status": "delivered", + "delivered_days_ago": 45, + "package_opened": false, + "item_used": false, + "payment_method": "Mastercard ending 1111", + "tracking_updated_hours_ago": 72 + }, + { + "order_id": "C789", + "status": "in_transit", + "delivered_days_ago": null, + "package_opened": false, + "item_used": false, + "payment_method": "Visa ending 9999", + "tracking_updated_hours_ago": 52 + } +] \ No newline at end of file diff --git a/benchmarks/datasets/refund_policy_docs.json b/benchmarks/datasets/refund_policy_docs.json new file mode 100644 index 0000000..d0c1698 --- /dev/null +++ b/benchmarks/datasets/refund_policy_docs.json @@ -0,0 +1,42 @@ +[ + { + "doc_id": "refund_001", + "category": "refund", + "text": "Customers can request a refund within 30 days of delivery." + }, + { + "doc_id": "refund_002", + "category": "refund", + "text": "Items must be unused and in original packaging to qualify for a standard refund." + }, + { + "doc_id": "refund_003", + "category": "refund", + "text": "Opened items may require manual review unless the item is defective." + }, + { + "doc_id": "refund_004", + "category": "payment", + "text": "Refunds are processed to the original payment method." + }, + { + "doc_id": "refund_005", + "category": "payment", + "text": "Refunds may take 5 to 10 business days after approval." + }, + { + "doc_id": "shipping_001", + "category": "shipping", + "text": "Delivered orders are eligible for return review if the delivery date is within the return window." + }, + { + "doc_id": "tracking_001", + "category": "tracking", + "text": "If tracking is not updated for 48 hours, customers should contact support." + }, + { + "doc_id": "cancel_001", + "category": "cancellation", + "text": "Orders can be cancelled before shipment. Shipped orders cannot be cancelled." + } +] \ No newline at end of file diff --git a/benchmarks/datasets/support_cases.json b/benchmarks/datasets/support_cases.json new file mode 100644 index 0000000..30f773d --- /dev/null +++ b/benchmarks/datasets/support_cases.json @@ -0,0 +1,31 @@ +[ + { + "case_id": "refund_001", + "ticket": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "order_id": "A123", + "expected_facts": [ + "within 30 day refund window", + "opened package may require manual review", + "item was not used", + "refund may take 5 to 10 business days" + ] + }, + { + "case_id": "refund_002", + "ticket": "My order B456 arrived 45 days ago. Can I return it?", + "order_id": "B456", + "expected_facts": [ + "outside 30 day refund window", + "refund may not be eligible" + ] + }, + { + "case_id": "tracking_001", + "ticket": "My tracking has not updated for 2 days. What should I do?", + "order_id": "C789", + "expected_facts": [ + "tracking not updated for 48 hours", + "contact support" + ] + } +] \ No newline at end of file diff --git a/benchmarks/frameworks/__init__.py b/benchmarks/frameworks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/autogen/__init__.py b/benchmarks/frameworks/autogen/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/autogen/run_autogen.py b/benchmarks/frameworks/autogen/run_autogen.py new file mode 100644 index 0000000..02cb6f8 --- /dev/null +++ b/benchmarks/frameworks/autogen/run_autogen.py @@ -0,0 +1,133 @@ +import time + +from agenticlens import profile, step +from benchmarks.shared.support_tasks import ( + check_refund_eligibility, + classify_ticket, + generate_customer_reply, + lookup_order_tool, + retrieve_policy, + rewrite_query, +) + + +def main() -> None: + framework = "AutoGen" + + try: + from autogen_agentchat.agents import AssistantAgent + except ImportError as exc: + raise RuntimeError( + "AutoGen AgentChat is not installed. Run: pip install autogen-agentchat autogen-core" + ) from exc + + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + order_id = "A123" + + # Framework-specific agents. + # We instantiate agents for benchmark identity, but do not call a live model client here. + classifier_agent = AssistantAgent( + name="support_intent_classifier", + model_client=None, + ) + refund_agent = AssistantAgent( + name="refund_decision_agent", + model_client=None, + ) + response_agent = AssistantAgent( + name="customer_response_agent", + model_client=None, + ) + + with profile("Benchmark - AutoGen - Support Refund"): + with step( + "AutoGen - Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="autogen", + agent_name=classifier_agent.name, + ) as s: + start = time.time() + response = classify_ticket(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + intent = response.choices[0].message.content + + with step( + "AutoGen - Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="autogen", + ) as s: + start = time.time() + response = rewrite_query(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + query = response.choices[0].message.content + + with step( + "AutoGen - Retrieve Refund Policy", + type="retriever", + query=query, + framework="autogen", + ) as s: + chunks, policy_context, avg_tokens, latency = retrieve_policy(query, top_k=6) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = avg_tokens + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + + with step( + "AutoGen - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + framework="autogen", + ) as s: + order, latency = lookup_order_tool(order_id) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = order + + with step( + "AutoGen - Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="autogen", + agent_name=refund_agent.name, + ) as s: + start = time.time() + response = check_refund_eligibility(ticket, order, policy_context, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + decision = response.choices[0].message.content + s.step.metadata["intent"] = intent + + with step( + "AutoGen - Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="autogen", + agent_name=response_agent.name, + ) as s: + start = time.time() + response = generate_customer_reply(ticket, order, policy_context, decision, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/autogen/run_autogen_live.py b/benchmarks/frameworks/autogen/run_autogen_live.py new file mode 100644 index 0000000..5163537 --- /dev/null +++ b/benchmarks/frameworks/autogen/run_autogen_live.py @@ -0,0 +1,161 @@ +import asyncio +import time + +from agenticlens import profile, step +from benchmarks.shared.live_travel_tasks import ( + OPENAI_MODEL, + QUESTION, + TRIP, + USE_REAL_OPENAI, + LLMResponse, + classify_trip, + estimate_avg_tokens_per_chunk, + fetch_destination_summary, + fetch_exchange_rate, + fetch_weather, + geocode_city, + synthesize_briefing, +) + + +async def _run_agent(agent, task: str) -> tuple[str, int, int]: + result = await agent.run(task=task) + last = result.messages[-1] + usage = last.models_usage + prompt_tokens = usage.prompt_tokens if usage else 0 + completion_tokens = usage.completion_tokens if usage else 0 + return last.content, prompt_tokens, completion_tokens + + +def classify_trip_native(planner_agent) -> tuple[LLMResponse, float]: + """Real call runs through AssistantAgent.run() -- AutoGen's own + system-message + task framing, not a raw OpenAI SDK call.""" + if not USE_REAL_OPENAI: + return classify_trip("AutoGen") + + start = time.time() + content, prompt_tokens, completion_tokens = asyncio.run( + _run_agent(planner_agent, f"Classify this trip request in one short line:\n{QUESTION}") + ) + return LLMResponse(content, prompt_tokens, completion_tokens), (time.time() - start) + + +def synthesize_briefing_native( + briefing_agent, place: dict, weather: dict, fx: dict, summary: dict +) -> tuple[LLMResponse, float]: + if not USE_REAL_OPENAI: + return synthesize_briefing("AutoGen", place) + + start = time.time() + prompt = ( + f"Traveler question: {QUESTION}\n\n" + f"Live weather at {place['name']}: {weather}\n" + f"Live USD->JPY rate: {fx['rate']} (as of {fx['date']})\n" + f"Destination facts: {summary['extract']}\n\n" + "Write a concise, friendly travel briefing (3-4 sentences) using only this data." + ) + content, prompt_tokens, completion_tokens = asyncio.run(_run_agent(briefing_agent, prompt)) + return LLMResponse(content, prompt_tokens, completion_tokens), (time.time() - start) + + +def main() -> None: + framework = "AutoGen" + + try: + from autogen_agentchat.agents import AssistantAgent + except ImportError as exc: + raise RuntimeError( + "AutoGen AgentChat is not installed. Run: pip install autogen-agentchat autogen-core" + ) from exc + + model_client = None + if USE_REAL_OPENAI: + from autogen_ext.models.openai import OpenAIChatCompletionClient + + model_client = OpenAIChatCompletionClient(model=OPENAI_MODEL) + + planner_agent = AssistantAgent(name="trip_planner", model_client=model_client) + briefing_agent = AssistantAgent(name="briefing_writer", model_client=model_client) + + with profile(f"Benchmark - {framework} - Live Travel Briefing"): + with step( + f"{framework} - Classify Trip Request", + type="planner", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="autogen", + agent_name=planner_agent.name, + ) as s: + response, latency = classify_trip_native(planner_agent) + s.record(response) + s.step.metrics.latency = latency + + with step( + f"{framework} - Geocode Destination", + type="tool_call", + tool_name="open_meteo_geocoding", + tool_args={"city": TRIP["destination"]}, + framework="autogen", + ) as s: + place, latency = geocode_city(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = place + + with step( + f"{framework} - Fetch Live Weather", + type="tool_call", + tool_name="open_meteo_forecast", + tool_args={"lat": place["lat"], "lon": place["lon"]}, + framework="autogen", + ) as s: + weather, latency = fetch_weather(place["lat"], place["lon"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = weather + + with step( + f"{framework} - Fetch Live Exchange Rate", + type="tool_call", + tool_name="frankfurter_exchange_rate", + tool_args={"base": "USD", "target": "JPY"}, + framework="autogen", + ) as s: + fx, latency = fetch_exchange_rate("USD", "JPY") + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = fx + + with step( + f"{framework} - Retrieve Destination Facts", + type="retriever", + query=TRIP["destination"], + framework="autogen", + ) as s: + summary, paragraphs, latency = fetch_destination_summary(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(paragraphs) + s.step.metadata["avg_tokens_per_chunk"] = estimate_avg_tokens_per_chunk(paragraphs) + s.step.metadata["retrieved_doc_ids"] = [summary["title"]] + + with step( + f"{framework} - Synthesize Travel Briefing", + type="final_response", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="autogen", + agent_name=briefing_agent.name, + ) as s: + response, latency = synthesize_briefing_native( + briefing_agent, place, weather, fx, summary + ) + s.record(response) + s.step.metrics.latency = latency + + if USE_REAL_OPENAI: + asyncio.run(model_client.close()) + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/crewai/__init__.py b/benchmarks/frameworks/crewai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/crewai/run_crewai.py b/benchmarks/frameworks/crewai/run_crewai.py new file mode 100644 index 0000000..a4bc48d --- /dev/null +++ b/benchmarks/frameworks/crewai/run_crewai.py @@ -0,0 +1,181 @@ +import time + +from agenticlens import profile, step +from benchmarks.shared.support_tasks import ( + check_refund_eligibility, + classify_ticket, + generate_customer_reply, + lookup_order_tool, + retrieve_policy, + rewrite_query, +) + + +def main() -> None: + framework = "CrewAI" + + try: + from crewai import Agent, Crew, Process, Task + except ImportError as exc: + raise RuntimeError("CrewAI is not installed. Run: pip install crewai") from exc + + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + order_id = "A123" + + # Framework-specific objects. + # These make this a CrewAI benchmark adapter, while AgenticLens measures each business step. + classifier_agent = Agent( + role="Support Intent Classifier", + goal="Classify customer support tickets", + backstory="You classify customer tickets into support intents.", + verbose=False, + allow_delegation=False, + ) + + policy_agent = Agent( + role="Policy Retrieval Agent", + goal="Find relevant refund and tracking policies", + backstory="You retrieve policy evidence for support decisions.", + verbose=False, + allow_delegation=False, + ) + + refund_agent = Agent( + role="Refund Decision Agent", + goal="Decide refund eligibility using order and policy facts", + backstory="You apply company refund rules to customer orders.", + verbose=False, + allow_delegation=False, + ) + + response_agent = Agent( + role="Customer Response Agent", + goal="Write clear customer-facing replies", + backstory="You write helpful support responses.", + verbose=False, + allow_delegation=False, + ) + + tasks = [ + Task( + description="Classify the refund ticket intent.", + expected_output="Ticket intent and priority.", + agent=classifier_agent, + ), + Task( + description="Retrieve relevant refund policy chunks.", + expected_output="Relevant policy evidence.", + agent=policy_agent, + ), + Task( + description="Check refund eligibility.", + expected_output="Eligibility decision.", + agent=refund_agent, + ), + Task( + description="Generate customer reply.", + expected_output="Customer-facing answer.", + agent=response_agent, + ), + ] + + # We create the CrewAI crew so the benchmark records that this implementation uses CrewAI. + # We do not call crew.kickoff() in this deterministic benchmark because that would + # call a live LLM. + crew = Crew( + agents=[classifier_agent, policy_agent, refund_agent, response_agent], + tasks=tasks, + process=Process.sequential, + verbose=False, + ) + + with profile("Benchmark - CrewAI - Support Refund"): + with step( + "CrewAI - Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="crewai", + crew_agents=len(crew.agents), + ) as s: + start = time.time() + response = classify_ticket(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + intent = response.choices[0].message.content + + with step( + "CrewAI - Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="crewai", + ) as s: + start = time.time() + response = rewrite_query(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + query = response.choices[0].message.content + + with step( + "CrewAI - Retrieve Refund Policy", + type="retriever", + query=query, + framework="crewai", + ) as s: + chunks, policy_context, avg_tokens, latency = retrieve_policy(query, top_k=6) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = avg_tokens + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + + with step( + "CrewAI - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + framework="crewai", + ) as s: + order, latency = lookup_order_tool(order_id) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = order + + with step( + "CrewAI - Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="crewai", + ) as s: + start = time.time() + response = check_refund_eligibility(ticket, order, policy_context, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + decision = response.choices[0].message.content + s.step.metadata["intent"] = intent + + with step( + "CrewAI - Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="crewai", + ) as s: + start = time.time() + response = generate_customer_reply(ticket, order, policy_context, decision, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/crewai/run_crewai_live.py b/benchmarks/frameworks/crewai/run_crewai_live.py new file mode 100644 index 0000000..85a9afc --- /dev/null +++ b/benchmarks/frameworks/crewai/run_crewai_live.py @@ -0,0 +1,194 @@ +import time + +from agenticlens import profile, step +from benchmarks.shared.live_travel_tasks import ( + OPENAI_MODEL, + QUESTION, + TRIP, + USE_REAL_OPENAI, + LLMResponse, + classify_trip, + estimate_avg_tokens_per_chunk, + fetch_destination_summary, + fetch_exchange_rate, + fetch_weather, + geocode_city, + synthesize_briefing, +) + + +def classify_trip_native( + planner_agent, task_cls, crew_cls, process_cls +) -> tuple[LLMResponse, float]: + """Real call runs through Crew.kickoff() -- CrewAI wraps the task in its + own role/goal/backstory prompt template before it ever reaches the model, + so token usage genuinely differs from a raw SDK call.""" + if not USE_REAL_OPENAI: + return classify_trip("CrewAI") + + start = time.time() + task = task_cls( + description=f"Classify this trip request in one short line:\n{QUESTION}", + expected_output="A short trip intent classification.", + agent=planner_agent, + ) + crew = crew_cls( + agents=[planner_agent], + tasks=[task], + process=process_cls.sequential, + verbose=False, + ) + result = crew.kickoff() + usage = result.token_usage + return LLMResponse(str(result), usage.prompt_tokens, usage.completion_tokens), ( + time.time() - start + ) + + +def synthesize_briefing_native( + briefing_agent, + task_cls, + crew_cls, + process_cls, + place: dict, + weather: dict, + fx: dict, + summary: dict, +) -> tuple[LLMResponse, float]: + if not USE_REAL_OPENAI: + return synthesize_briefing("CrewAI", place) + + start = time.time() + prompt = ( + f"Traveler question: {QUESTION}\n\n" + f"Live weather at {place['name']}: {weather}\n" + f"Live USD->JPY rate: {fx['rate']} (as of {fx['date']})\n" + f"Destination facts: {summary['extract']}\n\n" + "Write a concise, friendly travel briefing (3-4 sentences) using only this data." + ) + task = task_cls( + description=prompt, + expected_output="A concise, friendly travel briefing.", + agent=briefing_agent, + ) + crew = crew_cls( + agents=[briefing_agent], + tasks=[task], + process=process_cls.sequential, + verbose=False, + ) + result = crew.kickoff() + usage = result.token_usage + return LLMResponse(str(result), usage.prompt_tokens, usage.completion_tokens), ( + time.time() - start + ) + + +def main() -> None: + framework = "CrewAI" + + try: + from crewai import LLM, Agent, Crew, Process, Task + except ImportError as exc: + raise RuntimeError("CrewAI is not installed. Run: pip install crewai") from exc + + llm = LLM(model=OPENAI_MODEL) if USE_REAL_OPENAI else None + + # Framework-specific objects. When a real key is available these agents + # are actually executed via crew.kickoff() below; otherwise they exist + # only for benchmark identity, matching the deterministic support-refund + # benchmark's design. + planner_agent = Agent( + role="Trip Planner", + goal="Classify a trip briefing request", + backstory="You classify traveler requests into structured trip intents.", + llm=llm, + verbose=False, + allow_delegation=False, + ) + briefing_agent = Agent( + role="Briefing Writer", + goal="Write a concise travel briefing from live data", + backstory="You synthesize weather, currency, and destination facts for travelers.", + llm=llm, + verbose=False, + allow_delegation=False, + ) + + with profile(f"Benchmark - {framework} - Live Travel Briefing"): + with step( + f"{framework} - Classify Trip Request", + type="planner", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="crewai", + ) as s: + response, latency = classify_trip_native(planner_agent, Task, Crew, Process) + s.record(response) + s.step.metrics.latency = latency + + with step( + f"{framework} - Geocode Destination", + type="tool_call", + tool_name="open_meteo_geocoding", + tool_args={"city": TRIP["destination"]}, + framework="crewai", + ) as s: + place, latency = geocode_city(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = place + + with step( + f"{framework} - Fetch Live Weather", + type="tool_call", + tool_name="open_meteo_forecast", + tool_args={"lat": place["lat"], "lon": place["lon"]}, + framework="crewai", + ) as s: + weather, latency = fetch_weather(place["lat"], place["lon"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = weather + + with step( + f"{framework} - Fetch Live Exchange Rate", + type="tool_call", + tool_name="frankfurter_exchange_rate", + tool_args={"base": "USD", "target": "JPY"}, + framework="crewai", + ) as s: + fx, latency = fetch_exchange_rate("USD", "JPY") + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = fx + + with step( + f"{framework} - Retrieve Destination Facts", + type="retriever", + query=TRIP["destination"], + framework="crewai", + ) as s: + summary, paragraphs, latency = fetch_destination_summary(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(paragraphs) + s.step.metadata["avg_tokens_per_chunk"] = estimate_avg_tokens_per_chunk(paragraphs) + s.step.metadata["retrieved_doc_ids"] = [summary["title"]] + + with step( + f"{framework} - Synthesize Travel Briefing", + type="final_response", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="crewai", + ) as s: + response, latency = synthesize_briefing_native( + briefing_agent, Task, Crew, Process, place, weather, fx, summary + ) + s.record(response) + s.step.metrics.latency = latency + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/langgraph/__init__.py b/benchmarks/frameworks/langgraph/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/langgraph/run_langgraph.py b/benchmarks/frameworks/langgraph/run_langgraph.py new file mode 100644 index 0000000..1b00f07 --- /dev/null +++ b/benchmarks/frameworks/langgraph/run_langgraph.py @@ -0,0 +1,245 @@ +import time +from typing import Any, TypedDict + +from agenticlens import profile, step +from benchmarks.shared.support_data import ( + build_policy_context, + estimate_avg_tokens_per_chunk, + lookup_order, + simple_retrieve, +) + + +class SupportState(TypedDict, total=False): + ticket: str + order_id: str + intent: str + query: str + chunks: list[dict[str, Any]] + policy_context: str + order: dict[str, Any] + decision: str + final_answer: str + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeMessage: + def __init__(self, content: str): + self.content = content + + +class FakeChoice: + def __init__(self, content: str): + self.message = FakeMessage(content) + + +class FakeResponse: + def __init__(self, content: str, prompt_tokens: int, completion_tokens: int): + self.usage = FakeUsage(prompt_tokens, completion_tokens) + self.choices = [FakeChoice(content)] + + +def classify_ticket_llm(ticket: str) -> FakeResponse: + return FakeResponse( + content="intent=refund_request; priority=normal", + prompt_tokens=190, + completion_tokens=30, + ) + + +def rewrite_query_llm(ticket: str) -> FakeResponse: + return FakeResponse( + content=( + "refund eligibility delivered order opened package unused item refund processing time" + ), + prompt_tokens=240, + completion_tokens=40, + ) + + +def refund_decision_llm(ticket: str, order: dict, policy_context: str) -> FakeResponse: + return FakeResponse( + content=( + "The order is within the 30-day refund window. " + "The item was not used, but the package was opened, so manual review may be required." + ), + prompt_tokens=780, + completion_tokens=110, + ) + + +def final_response_llm( + ticket: str, order: dict, policy_context: str, decision: str +) -> FakeResponse: + return FakeResponse( + content=( + "Your order is within the 30-day refund window. Since the package was opened, " + "the refund may need manual review. Because the item was not used, you may " + "still be eligible. If approved, the refund will return to your original " + "payment method and may take 5 to 10 business days." + ), + prompt_tokens=920, + completion_tokens=150, + ) + + +def classify_ticket_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=state["ticket"], + ) as s: + start = time.time() + response = classify_ticket_llm(state["ticket"]) + s.record(response) + s.step.metrics.latency = time.time() - start + + state["intent"] = response.choices[0].message.content + return state + + +def rewrite_query_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=state["ticket"], + ) as s: + start = time.time() + response = rewrite_query_llm(state["ticket"]) + s.record(response) + s.step.metrics.latency = time.time() - start + + state["query"] = response.choices[0].message.content + return state + + +def retrieve_policy_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Retrieve Refund Policy", + type="retriever", + query=state["query"], + ) as s: + start = time.time() + chunks = simple_retrieve(state["query"], top_k=6) + s.step.metrics.latency = time.time() - start + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = estimate_avg_tokens_per_chunk(chunks) + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + + state["chunks"] = chunks + state["policy_context"] = build_policy_context(chunks) + return state + + +def lookup_order_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": state["order_id"]}, + ) as s: + start = time.time() + order = lookup_order(state["order_id"]) + s.step.metrics.latency = time.time() - start + s.step.metadata["tool_result"] = order + + state["order"] = order + return state + + +def refund_decision_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=state["policy_context"], + ) as s: + start = time.time() + response = refund_decision_llm( + state["ticket"], + state["order"], + state["policy_context"], + ) + s.record(response) + s.step.metrics.latency = time.time() - start + + state["decision"] = response.choices[0].message.content + return state + + +def final_response_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=state["policy_context"], + ) as s: + start = time.time() + response = final_response_llm( + state["ticket"], + state["order"], + state["policy_context"], + state["decision"], + ) + s.record(response) + s.step.metrics.latency = time.time() - start + + state["final_answer"] = response.choices[0].message.content + return state + + +def main() -> None: + try: + from langgraph.graph import END, StateGraph + except ImportError as exc: + raise RuntimeError("LangGraph is not installed. Run: pip install langgraph") from exc + + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + + initial_state: SupportState = { + "ticket": ticket, + "order_id": "A123", + } + + graph = StateGraph(SupportState) + + graph.add_node("classify_ticket", classify_ticket_node) + graph.add_node("rewrite_query", rewrite_query_node) + graph.add_node("retrieve_policy", retrieve_policy_node) + graph.add_node("lookup_order", lookup_order_node) + graph.add_node("refund_decision", refund_decision_node) + graph.add_node("final_response", final_response_node) + + graph.set_entry_point("classify_ticket") + graph.add_edge("classify_ticket", "rewrite_query") + graph.add_edge("rewrite_query", "retrieve_policy") + graph.add_edge("retrieve_policy", "lookup_order") + graph.add_edge("lookup_order", "refund_decision") + graph.add_edge("refund_decision", "final_response") + graph.add_edge("final_response", END) + + app = graph.compile() + + with profile("Benchmark - LangGraph - Support Refund"): + final_state = app.invoke(initial_state) + + print(final_state["final_answer"]) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/langgraph/run_langgraph_live.py b/benchmarks/frameworks/langgraph/run_langgraph_live.py new file mode 100644 index 0000000..c008071 --- /dev/null +++ b/benchmarks/frameworks/langgraph/run_langgraph_live.py @@ -0,0 +1,203 @@ +import time +from typing import Any, TypedDict + +from agenticlens import profile, step +from benchmarks.shared.live_travel_tasks import ( + OPENAI_MODEL, + QUESTION, + TRIP, + USE_REAL_OPENAI, + LLMResponse, + classify_trip, + estimate_avg_tokens_per_chunk, + fetch_destination_summary, + fetch_exchange_rate, + fetch_weather, + geocode_city, + synthesize_briefing, +) + +if USE_REAL_OPENAI: + from langchain_openai import ChatOpenAI + + _llm = ChatOpenAI(model=OPENAI_MODEL) + + +def classify_trip_native() -> tuple[LLMResponse, float]: + """Real call goes through LangChain's ChatOpenAI -- its own message + formatting and response object, not a raw OpenAI SDK call.""" + if not USE_REAL_OPENAI: + return classify_trip("LangGraph") + + start = time.time() + ai_message = _llm.invoke(f"Classify this trip request in one short line:\n{QUESTION}") + usage = ai_message.usage_metadata or {} + return LLMResponse( + ai_message.content, usage.get("input_tokens", 0), usage.get("output_tokens", 0) + ), (time.time() - start) + + +def synthesize_briefing_native( + place: dict, weather: dict, fx: dict, summary: dict +) -> tuple[LLMResponse, float]: + if not USE_REAL_OPENAI: + return synthesize_briefing("LangGraph", place) + + start = time.time() + prompt = ( + f"Traveler question: {QUESTION}\n\n" + f"Live weather at {place['name']}: {weather}\n" + f"Live USD->JPY rate: {fx['rate']} (as of {fx['date']})\n" + f"Destination facts: {summary['extract']}\n\n" + "Write a concise, friendly travel briefing (3-4 sentences) using only this data." + ) + ai_message = _llm.invoke(prompt) + usage = ai_message.usage_metadata or {} + return LLMResponse( + ai_message.content, usage.get("input_tokens", 0), usage.get("output_tokens", 0) + ), (time.time() - start) + + +class TravelState(TypedDict, total=False): + intent: str + place: dict[str, Any] + weather: dict[str, Any] + fx: dict[str, Any] + summary: dict[str, Any] + final_answer: str + + +def classify_node(state: TravelState) -> TravelState: + with step( + "LangGraph - Classify Trip Request", + type="planner", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="langgraph", + ) as s: + response, latency = classify_trip_native() + s.record(response) + s.step.metrics.latency = latency + + state["intent"] = response.choices[0].message.content + return state + + +def geocode_node(state: TravelState) -> TravelState: + with step( + "LangGraph - Geocode Destination", + type="tool_call", + tool_name="open_meteo_geocoding", + tool_args={"city": TRIP["destination"]}, + framework="langgraph", + ) as s: + place, latency = geocode_city(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = place + + state["place"] = place + return state + + +def weather_node(state: TravelState) -> TravelState: + place = state["place"] + with step( + "LangGraph - Fetch Live Weather", + type="tool_call", + tool_name="open_meteo_forecast", + tool_args={"lat": place["lat"], "lon": place["lon"]}, + framework="langgraph", + ) as s: + weather, latency = fetch_weather(place["lat"], place["lon"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = weather + + state["weather"] = weather + return state + + +def fx_node(state: TravelState) -> TravelState: + with step( + "LangGraph - Fetch Live Exchange Rate", + type="tool_call", + tool_name="frankfurter_exchange_rate", + tool_args={"base": "USD", "target": "JPY"}, + framework="langgraph", + ) as s: + fx, latency = fetch_exchange_rate("USD", "JPY") + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = fx + + state["fx"] = fx + return state + + +def retrieve_node(state: TravelState) -> TravelState: + with step( + "LangGraph - Retrieve Destination Facts", + type="retriever", + query=TRIP["destination"], + framework="langgraph", + ) as s: + summary, paragraphs, latency = fetch_destination_summary(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(paragraphs) + s.step.metadata["avg_tokens_per_chunk"] = estimate_avg_tokens_per_chunk(paragraphs) + s.step.metadata["retrieved_doc_ids"] = [summary["title"]] + + state["summary"] = summary + return state + + +def synthesize_node(state: TravelState) -> TravelState: + with step( + "LangGraph - Synthesize Travel Briefing", + type="final_response", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="langgraph", + ) as s: + response, latency = synthesize_briefing_native( + state["place"], state["weather"], state["fx"], state["summary"] + ) + s.record(response) + s.step.metrics.latency = latency + + state["final_answer"] = response.choices[0].message.content + return state + + +def main() -> None: + try: + from langgraph.graph import END, StateGraph + except ImportError as exc: + raise RuntimeError("LangGraph is not installed. Run: pip install langgraph") from exc + + graph = StateGraph(TravelState) + graph.add_node("classify", classify_node) + graph.add_node("geocode", geocode_node) + graph.add_node("weather", weather_node) + graph.add_node("fx", fx_node) + graph.add_node("retrieve", retrieve_node) + graph.add_node("synthesize", synthesize_node) + + graph.set_entry_point("classify") + graph.add_edge("classify", "geocode") + graph.add_edge("geocode", "weather") + graph.add_edge("weather", "fx") + graph.add_edge("fx", "retrieve") + graph.add_edge("retrieve", "synthesize") + graph.add_edge("synthesize", END) + + app = graph.compile() + + with profile("Benchmark - LangGraph - Live Travel Briefing"): + final_state = app.invoke({}) + + print(final_state["final_answer"]) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/llamaindex/__init__.py b/benchmarks/frameworks/llamaindex/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/llamaindex/run_llamaindex.py b/benchmarks/frameworks/llamaindex/run_llamaindex.py new file mode 100644 index 0000000..8a8f265 --- /dev/null +++ b/benchmarks/frameworks/llamaindex/run_llamaindex.py @@ -0,0 +1,133 @@ +import time + +from agenticlens import profile, step +from benchmarks.shared.support_data import load_policy_docs +from benchmarks.shared.support_tasks import ( + check_refund_eligibility, + classify_ticket, + generate_customer_reply, + lookup_order_tool, + retrieve_policy, + rewrite_query, +) + + +def main() -> None: + framework = "LlamaIndex" + + try: + from llama_index.core import Document + except ImportError as exc: + raise RuntimeError("LlamaIndex is not installed. Run: pip install llama-index") from exc + + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + order_id = "A123" + + # Framework-specific indexing object. + # This builds a LlamaIndex document collection, but the deterministic benchmark + # uses shared retrieval so results stay comparable with other framework runs. + policy_docs = load_policy_docs() + documents = [ + Document(text=doc["text"], metadata={"doc_id": doc["doc_id"], "category": doc["category"]}) + for doc in policy_docs + ] + + # Do not build a real embedding index in the deterministic run because it may + # require model configuration. Keep this object as the framework-specific + # document representation. + index_metadata = { + "framework_documents": len(documents), + "index_type": "llamaindex_documents", + } + + with profile("Benchmark - LlamaIndex - Support Refund"): + with step( + "LlamaIndex - Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="llamaindex", + **index_metadata, + ) as s: + start = time.time() + response = classify_ticket(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + intent = response.choices[0].message.content + + with step( + "LlamaIndex - Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="llamaindex", + ) as s: + start = time.time() + response = rewrite_query(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + query = response.choices[0].message.content + + with step( + "LlamaIndex - Retrieve Refund Policy", + type="retriever", + query=query, + framework="llamaindex", + index_type="Document collection", + ) as s: + chunks, policy_context, avg_tokens, latency = retrieve_policy(query, top_k=6) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = avg_tokens + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + + with step( + "LlamaIndex - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + framework="llamaindex", + ) as s: + order, latency = lookup_order_tool(order_id) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = order + + with step( + "LlamaIndex - Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="llamaindex", + ) as s: + start = time.time() + response = check_refund_eligibility(ticket, order, policy_context, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + decision = response.choices[0].message.content + s.step.metadata["intent"] = intent + + with step( + "LlamaIndex - Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="llamaindex", + ) as s: + start = time.time() + response = generate_customer_reply(ticket, order, policy_context, decision, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/llamaindex/run_llamaindex_live.py b/benchmarks/frameworks/llamaindex/run_llamaindex_live.py new file mode 100644 index 0000000..3cd50c0 --- /dev/null +++ b/benchmarks/frameworks/llamaindex/run_llamaindex_live.py @@ -0,0 +1,147 @@ +import time + +from agenticlens import profile, step +from benchmarks.shared.live_travel_tasks import ( + OPENAI_MODEL, + QUESTION, + TRIP, + USE_REAL_OPENAI, + LLMResponse, + classify_trip, + estimate_avg_tokens_per_chunk, + fetch_destination_summary, + fetch_exchange_rate, + fetch_weather, + geocode_city, + synthesize_briefing, +) + + +def classify_trip_native(llm) -> tuple[LLMResponse, float]: + """Real call runs through LlamaIndex's LLM.complete() -- its own + completion wrapper, not a raw OpenAI SDK call.""" + if not USE_REAL_OPENAI: + return classify_trip("LlamaIndex") + + start = time.time() + resp = llm.complete(f"Classify this trip request in one short line:\n{QUESTION}") + kwargs = resp.additional_kwargs + return LLMResponse( + str(resp), kwargs.get("prompt_tokens", 0), kwargs.get("completion_tokens", 0) + ), (time.time() - start) + + +def synthesize_briefing_native( + llm, place: dict, weather: dict, fx: dict, summary: dict +) -> tuple[LLMResponse, float]: + if not USE_REAL_OPENAI: + return synthesize_briefing("LlamaIndex", place) + + start = time.time() + prompt = ( + f"Traveler question: {QUESTION}\n\n" + f"Live weather at {place['name']}: {weather}\n" + f"Live USD->JPY rate: {fx['rate']} (as of {fx['date']})\n" + f"Destination facts: {summary['extract']}\n\n" + "Write a concise, friendly travel briefing (3-4 sentences) using only this data." + ) + resp = llm.complete(prompt) + kwargs = resp.additional_kwargs + return LLMResponse( + str(resp), kwargs.get("prompt_tokens", 0), kwargs.get("completion_tokens", 0) + ), (time.time() - start) + + +def main() -> None: + framework = "LlamaIndex" + + try: + from llama_index.core import Document + except ImportError as exc: + raise RuntimeError("LlamaIndex is not installed. Run: pip install llama-index") from exc + + llm = None + if USE_REAL_OPENAI: + from llama_index.llms.openai import OpenAI as LlamaOpenAI + + llm = LlamaOpenAI(model=OPENAI_MODEL) + + with profile(f"Benchmark - {framework} - Live Travel Briefing"): + with step( + f"{framework} - Classify Trip Request", + type="planner", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="llamaindex", + ) as s: + response, latency = classify_trip_native(llm) + s.record(response) + s.step.metrics.latency = latency + + with step( + f"{framework} - Geocode Destination", + type="tool_call", + tool_name="open_meteo_geocoding", + tool_args={"city": TRIP["destination"]}, + framework="llamaindex", + ) as s: + place, latency = geocode_city(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = place + + with step( + f"{framework} - Fetch Live Weather", + type="tool_call", + tool_name="open_meteo_forecast", + tool_args={"lat": place["lat"], "lon": place["lon"]}, + framework="llamaindex", + ) as s: + weather, latency = fetch_weather(place["lat"], place["lon"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = weather + + with step( + f"{framework} - Fetch Live Exchange Rate", + type="tool_call", + tool_name="frankfurter_exchange_rate", + tool_args={"base": "USD", "target": "JPY"}, + framework="llamaindex", + ) as s: + fx, latency = fetch_exchange_rate("USD", "JPY") + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = fx + + with step( + f"{framework} - Retrieve Destination Facts", + type="retriever", + query=TRIP["destination"], + framework="llamaindex", + ) as s: + summary, paragraphs, latency = fetch_destination_summary(TRIP["destination"]) + # Framework-specific object: wrap the live Wikipedia extract as a + # LlamaIndex Document, matching how this framework represents + # retrieved context, without building a real embedding index. + document = Document(text=summary["extract"], metadata={"title": summary["title"]}) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(paragraphs) + s.step.metadata["avg_tokens_per_chunk"] = estimate_avg_tokens_per_chunk(paragraphs) + s.step.metadata["retrieved_doc_ids"] = [document.doc_id] + + with step( + f"{framework} - Synthesize Travel Briefing", + type="final_response", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="llamaindex", + ) as s: + response, latency = synthesize_briefing_native(llm, place, weather, fx, summary) + s.record(response) + s.step.metrics.latency = latency + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/native_python/__init__.py b/benchmarks/frameworks/native_python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/native_python/run_native.py b/benchmarks/frameworks/native_python/run_native.py new file mode 100644 index 0000000..b106738 --- /dev/null +++ b/benchmarks/frameworks/native_python/run_native.py @@ -0,0 +1,172 @@ +import sys +import time +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(PROJECT_ROOT)) + +from agenticlens import profile, step # noqa: E402 +from benchmarks.shared.support_data import ( # noqa: E402 + build_policy_context, + estimate_avg_tokens_per_chunk, + lookup_order, + simple_retrieve, +) + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeMessage: + def __init__(self, content: str): + self.content = content + + +class FakeChoice: + def __init__(self, content: str): + self.message = FakeMessage(content) + + +class FakeResponse: + def __init__(self, content: str, prompt_tokens: int, completion_tokens: int): + self.usage = FakeUsage(prompt_tokens, completion_tokens) + self.choices = [FakeChoice(content)] + + +def classify_ticket(ticket: str) -> FakeResponse: + return FakeResponse( + content="intent=refund_request; priority=normal", + prompt_tokens=180, + completion_tokens=25, + ) + + +def rewrite_query(ticket: str) -> FakeResponse: + return FakeResponse( + content=( + "refund eligibility delivered order opened package unused item refund processing time" + ), + prompt_tokens=220, + completion_tokens=35, + ) + + +def check_refund_eligibility(ticket: str, order: dict, policy_context: str) -> FakeResponse: + return FakeResponse( + content=( + "The order is within the 30-day refund window. " + "The item was not used, but the package was opened, so manual review may be required." + ), + prompt_tokens=720, + completion_tokens=95, + ) + + +def generate_customer_reply( + ticket: str, order: dict, policy_context: str, decision: str +) -> FakeResponse: + return FakeResponse( + content=( + "Your order is within the 30-day refund window. Since the package was opened, " + "the refund may need manual review. Because the item was not used, you may " + "still be eligible. If approved, the refund will return to your original " + "payment method and may take 5 to 10 business days." + ), + prompt_tokens=850, + completion_tokens=130, + ) + + +def main() -> None: + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + order_id = "A123" + + with profile("Benchmark - Native Python - Support Refund"): + with step( + "Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + ) as s: + start = time.time() + response = classify_ticket(ticket) + s.record(response) + s.step.metrics.latency = time.time() - start + intent = response.choices[0].message.content + + with step( + "Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + ) as s: + start = time.time() + response = rewrite_query(ticket) + s.record(response) + s.step.metrics.latency = time.time() - start + query = response.choices[0].message.content + + with step( + "Retrieve Refund Policy", + type="retriever", + query=query, + ) as s: + start = time.time() + chunks = simple_retrieve(query, top_k=6) + s.step.metrics.latency = time.time() - start + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = estimate_avg_tokens_per_chunk(chunks) + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + policy_context = build_policy_context(chunks) + + with step( + "Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + ) as s: + start = time.time() + order = lookup_order(order_id) + s.step.metrics.latency = time.time() - start + s.step.metadata["tool_result"] = order + + with step( + "Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + ) as s: + start = time.time() + response = check_refund_eligibility(ticket, order, policy_context) + s.record(response) + s.step.metrics.latency = time.time() - start + decision = response.choices[0].message.content + s.step.metadata["intent"] = intent + + with step( + "Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + ) as s: + start = time.time() + response = generate_customer_reply(ticket, order, policy_context, decision) + s.record(response) + s.step.metrics.latency = time.time() - start + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/native_python/run_native_live.py b/benchmarks/frameworks/native_python/run_native_live.py new file mode 100644 index 0000000..f337483 --- /dev/null +++ b/benchmarks/frameworks/native_python/run_native_live.py @@ -0,0 +1,154 @@ +import sys +import time +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(PROJECT_ROOT)) + +from agenticlens import profile, step # noqa: E402 +from benchmarks.shared.live_travel_tasks import ( # noqa: E402 + OPENAI_MODEL, + QUESTION, + TRIP, + USE_REAL_OPENAI, + LLMResponse, + classify_trip, + estimate_avg_tokens_per_chunk, + fetch_destination_summary, + fetch_exchange_rate, + fetch_weather, + geocode_city, + synthesize_briefing, +) + +if USE_REAL_OPENAI: + from openai import OpenAI + + client = OpenAI() + + +def classify_trip_native(framework: str) -> tuple[LLMResponse, float]: + """Raw OpenAI SDK call -- the "control" with no framework prompt wrapping.""" + if not USE_REAL_OPENAI: + return classify_trip(framework) + + start = time.time() + resp = client.chat.completions.create( + model=OPENAI_MODEL, + messages=[ + { + "role": "user", + "content": f"Classify this trip request in one short line:\n{QUESTION}", + } + ], + max_tokens=30, + ) + content = resp.choices[0].message.content + return LLMResponse(content, resp.usage.prompt_tokens, resp.usage.completion_tokens), ( + time.time() - start + ) + + +def synthesize_briefing_native( + framework: str, place: dict, weather: dict, fx: dict, summary: dict +) -> tuple[LLMResponse, float]: + if not USE_REAL_OPENAI: + return synthesize_briefing(framework, place) + + start = time.time() + prompt = ( + f"Traveler question: {QUESTION}\n\n" + f"Live weather at {place['name']}: {weather}\n" + f"Live USD->JPY rate: {fx['rate']} (as of {fx['date']})\n" + f"Destination facts: {summary['extract']}\n\n" + "Write a concise, friendly travel briefing (3-4 sentences) using only this data." + ) + resp = client.chat.completions.create( + model=OPENAI_MODEL, + messages=[{"role": "user", "content": prompt}], + max_tokens=180, + ) + content = resp.choices[0].message.content + return LLMResponse(content, resp.usage.prompt_tokens, resp.usage.completion_tokens), ( + time.time() - start + ) + + +def main() -> None: + framework = "Native Python" + + with profile(f"Benchmark - {framework} - Live Travel Briefing"): + with step( + f"{framework} - Classify Trip Request", + type="planner", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="native_python", + ) as s: + response, latency = classify_trip_native(framework) + s.record(response) + s.step.metrics.latency = latency + + with step( + f"{framework} - Geocode Destination", + type="tool_call", + tool_name="open_meteo_geocoding", + tool_args={"city": TRIP["destination"]}, + framework="native_python", + ) as s: + place, latency = geocode_city(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = place + + with step( + f"{framework} - Fetch Live Weather", + type="tool_call", + tool_name="open_meteo_forecast", + tool_args={"lat": place["lat"], "lon": place["lon"]}, + framework="native_python", + ) as s: + weather, latency = fetch_weather(place["lat"], place["lon"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = weather + + with step( + f"{framework} - Fetch Live Exchange Rate", + type="tool_call", + tool_name="frankfurter_exchange_rate", + tool_args={"base": "USD", "target": "JPY"}, + framework="native_python", + ) as s: + fx, latency = fetch_exchange_rate("USD", "JPY") + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = fx + + with step( + f"{framework} - Retrieve Destination Facts", + type="retriever", + query=TRIP["destination"], + framework="native_python", + ) as s: + summary, paragraphs, latency = fetch_destination_summary(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(paragraphs) + s.step.metadata["avg_tokens_per_chunk"] = estimate_avg_tokens_per_chunk(paragraphs) + s.step.metadata["retrieved_doc_ids"] = [summary["title"]] + + with step( + f"{framework} - Synthesize Travel Briefing", + type="final_response", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="native_python", + ) as s: + response, latency = synthesize_briefing_native(framework, place, weather, fx, summary) + s.record(response) + s.step.metrics.latency = latency + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/semantic_kernel/__init__.py b/benchmarks/frameworks/semantic_kernel/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py b/benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py new file mode 100644 index 0000000..39ecfac --- /dev/null +++ b/benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py @@ -0,0 +1,120 @@ +import time + +from agenticlens import profile, step +from benchmarks.shared.support_tasks import ( + check_refund_eligibility, + classify_ticket, + generate_customer_reply, + lookup_order_tool, + retrieve_policy, + rewrite_query, +) + + +def main() -> None: + framework = "Semantic Kernel" + + try: + import semantic_kernel as sk + except ImportError as exc: + raise RuntimeError( + "Semantic Kernel is not installed. Run: pip install semantic-kernel" + ) from exc + + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + order_id = "A123" + + # Framework-specific kernel object. + # This confirms the implementation is using the Semantic Kernel runtime surface. + kernel = sk.Kernel() + + with profile("Benchmark - Semantic Kernel - Support Refund"): + with step( + "Semantic Kernel - Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="semantic_kernel", + kernel_type=type(kernel).__name__, + ) as s: + start = time.time() + response = classify_ticket(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + intent = response.choices[0].message.content + + with step( + "Semantic Kernel - Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="semantic_kernel", + ) as s: + start = time.time() + response = rewrite_query(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + query = response.choices[0].message.content + + with step( + "Semantic Kernel - Retrieve Refund Policy", + type="retriever", + query=query, + framework="semantic_kernel", + ) as s: + chunks, policy_context, avg_tokens, latency = retrieve_policy(query, top_k=6) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = avg_tokens + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + + with step( + "Semantic Kernel - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + framework="semantic_kernel", + ) as s: + order, latency = lookup_order_tool(order_id) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = order + + with step( + "Semantic Kernel - Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="semantic_kernel", + ) as s: + start = time.time() + response = check_refund_eligibility(ticket, order, policy_context, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + decision = response.choices[0].message.content + s.step.metadata["intent"] = intent + + with step( + "Semantic Kernel - Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="semantic_kernel", + ) as s: + start = time.time() + response = generate_customer_reply(ticket, order, policy_context, decision, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/frameworks/semantic_kernel/run_semantic_kernel_live.py b/benchmarks/frameworks/semantic_kernel/run_semantic_kernel_live.py new file mode 100644 index 0000000..1ecf7e1 --- /dev/null +++ b/benchmarks/frameworks/semantic_kernel/run_semantic_kernel_live.py @@ -0,0 +1,160 @@ +import asyncio +import time + +from agenticlens import profile, step +from benchmarks.shared.live_travel_tasks import ( + OPENAI_MODEL, + QUESTION, + TRIP, + USE_REAL_OPENAI, + LLMResponse, + classify_trip, + estimate_avg_tokens_per_chunk, + fetch_destination_summary, + fetch_exchange_rate, + fetch_weather, + geocode_city, + synthesize_briefing, +) + + +async def _get_completion(service, prompt: str) -> tuple[str, int, int]: + from semantic_kernel.contents import ChatHistory + + history = ChatHistory() + history.add_user_message(prompt) + settings = service.get_prompt_execution_settings_class()() + result = await service.get_chat_message_content(chat_history=history, settings=settings) + usage = result.metadata.get("usage") + prompt_tokens = usage.prompt_tokens if usage else 0 + completion_tokens = usage.completion_tokens if usage else 0 + return str(result), prompt_tokens, completion_tokens + + +def classify_trip_native(service) -> tuple[LLMResponse, float]: + """Real call runs through Semantic Kernel's chat completion service -- + its own ChatHistory/settings wrapper, not a raw OpenAI SDK call.""" + if not USE_REAL_OPENAI: + return classify_trip("Semantic Kernel") + + start = time.time() + content, prompt_tokens, completion_tokens = asyncio.run( + _get_completion(service, f"Classify this trip request in one short line:\n{QUESTION}") + ) + return LLMResponse(content, prompt_tokens, completion_tokens), (time.time() - start) + + +def synthesize_briefing_native( + service, place: dict, weather: dict, fx: dict, summary: dict +) -> tuple[LLMResponse, float]: + if not USE_REAL_OPENAI: + return synthesize_briefing("Semantic Kernel", place) + + start = time.time() + prompt = ( + f"Traveler question: {QUESTION}\n\n" + f"Live weather at {place['name']}: {weather}\n" + f"Live USD->JPY rate: {fx['rate']} (as of {fx['date']})\n" + f"Destination facts: {summary['extract']}\n\n" + "Write a concise, friendly travel briefing (3-4 sentences) using only this data." + ) + content, prompt_tokens, completion_tokens = asyncio.run(_get_completion(service, prompt)) + return LLMResponse(content, prompt_tokens, completion_tokens), (time.time() - start) + + +def main() -> None: + framework = "Semantic Kernel" + + try: + import semantic_kernel as sk + except ImportError as exc: + raise RuntimeError( + "Semantic Kernel is not installed. Run: pip install semantic-kernel" + ) from exc + + # Framework-specific kernel object. + # This confirms the implementation is using the Semantic Kernel runtime surface. + kernel = sk.Kernel() + service = None + if USE_REAL_OPENAI: + from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion + + service = OpenAIChatCompletion(ai_model_id=OPENAI_MODEL, service_id="chat") + kernel.add_service(service) + + with profile(f"Benchmark - {framework} - Live Travel Briefing"): + with step( + f"{framework} - Classify Trip Request", + type="planner", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="semantic_kernel", + kernel_type=type(kernel).__name__, + ) as s: + response, latency = classify_trip_native(service) + s.record(response) + s.step.metrics.latency = latency + + with step( + f"{framework} - Geocode Destination", + type="tool_call", + tool_name="open_meteo_geocoding", + tool_args={"city": TRIP["destination"]}, + framework="semantic_kernel", + ) as s: + place, latency = geocode_city(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = place + + with step( + f"{framework} - Fetch Live Weather", + type="tool_call", + tool_name="open_meteo_forecast", + tool_args={"lat": place["lat"], "lon": place["lon"]}, + framework="semantic_kernel", + ) as s: + weather, latency = fetch_weather(place["lat"], place["lon"]) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = weather + + with step( + f"{framework} - Fetch Live Exchange Rate", + type="tool_call", + tool_name="frankfurter_exchange_rate", + tool_args={"base": "USD", "target": "JPY"}, + framework="semantic_kernel", + ) as s: + fx, latency = fetch_exchange_rate("USD", "JPY") + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = fx + + with step( + f"{framework} - Retrieve Destination Facts", + type="retriever", + query=TRIP["destination"], + framework="semantic_kernel", + ) as s: + summary, paragraphs, latency = fetch_destination_summary(TRIP["destination"]) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(paragraphs) + s.step.metadata["avg_tokens_per_chunk"] = estimate_avg_tokens_per_chunk(paragraphs) + s.step.metadata["retrieved_doc_ids"] = [summary["title"]] + + with step( + f"{framework} - Synthesize Travel Briefing", + type="final_response", + provider="openai", + model=OPENAI_MODEL, + prompt=QUESTION, + framework="semantic_kernel", + ) as s: + response, latency = synthesize_briefing_native(service, place, weather, fx, summary) + s.record(response) + s.step.metrics.latency = latency + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/reports/autogen/live_travel_report.json b/benchmarks/reports/autogen/live_travel_report.json new file mode 100644 index 0000000..4953146 --- /dev/null +++ b/benchmarks/reports/autogen/live_travel_report.json @@ -0,0 +1,159 @@ +{ + "id": "b04b682a-246a-4fd9-af57-686d8dffd425", + "name": "Benchmark - AutoGen - Live Travel Briefing", + "start_time": "2026-07-15T01:19:35.638526Z", + "end_time": "2026-07-15T01:19:37.104054Z", + "steps": [ + { + "id": "b40495a2-1d2d-408e-a3e1-c8493545553a", + "name": "AutoGen - Classify Trip Request", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 140, + "completion_tokens": 28, + "total_tokens": 168, + "latency": 4.9299997044727206e-05, + "ttft": null, + "cost": 3.78e-05 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "autogen", + "agent_name": "trip_planner" + } + }, + { + "id": "f4e2f55f-e8a9-40e3-91d2-b906ccdf0a77", + "name": "AutoGen - Geocode Destination", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.6200604000041494, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_geocoding", + "tool_args": { + "city": "Tokyo" + }, + "framework": "autogen", + "tool_result": { + "name": "Tokyo", + "country": "Japan", + "lat": 35.6895, + "lon": 139.69171 + } + } + }, + { + "id": "042ac083-62af-416b-9a99-1898a610d08f", + "name": "AutoGen - Fetch Live Weather", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.563434600000619, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_forecast", + "tool_args": { + "lat": 35.6895, + "lon": 139.69171 + }, + "framework": "autogen", + "tool_result": { + "time": "2026-07-15T01:15", + "interval": 900, + "temperature_2m": 29.9, + "weather_code": 2, + "wind_speed_10m": 4.2 + } + } + }, + { + "id": "03e29004-9279-422f-b8cf-75e49ae84e4f", + "name": "AutoGen - Fetch Live Exchange Rate", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.13430779999907827, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "frankfurter_exchange_rate", + "tool_args": { + "base": "USD", + "target": "JPY" + }, + "framework": "autogen", + "tool_result": { + "base": "USD", + "date": "2026-07-14", + "rate": 162.22 + } + } + }, + { + "id": "6d66e28e-fede-462d-a46c-6cb0797d7cab", + "name": "AutoGen - Retrieve Destination Facts", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.1468003999980283, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "Tokyo", + "framework": "autogen", + "chunk_count": 3, + "avg_tokens_per_chunk": 28, + "retrieved_doc_ids": [ + "Tokyo" + ] + } + }, + { + "id": "c88e0c8a-ca75-47ab-83d3-2d44e816e9c8", + "name": "AutoGen - Synthesize Travel Briefing", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 460, + "completion_tokens": 140, + "total_tokens": 600, + "latency": 7.349999941652641e-05, + "ttft": null, + "cost": 0.00015299999999999998 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "autogen", + "agent_name": "briefing_writer" + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/autogen/support_refund_report.json b/benchmarks/reports/autogen/support_refund_report.json new file mode 100644 index 0000000..9cb4f82 --- /dev/null +++ b/benchmarks/reports/autogen/support_refund_report.json @@ -0,0 +1,148 @@ +{ + "id": "df496e14-4707-4a7f-af36-2bff85cd766b", + "name": "Benchmark - AutoGen - Support Refund", + "start_time": "2026-07-15T00:01:48.858563Z", + "end_time": "2026-07-15T00:01:48.859799Z", + "steps": [ + { + "id": "be51f5cb-cea3-42b1-af99-f1224b7ce963", + "name": "AutoGen - Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 180, + "completion_tokens": 25, + "total_tokens": 205, + "latency": 7.470000127796084e-05, + "ttft": null, + "cost": 4.199999999999999e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "autogen", + "agent_name": "support_intent_classifier" + } + }, + { + "id": "4edb1b45-facc-4250-8002-b400135b143a", + "name": "AutoGen - Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 220, + "completion_tokens": 35, + "total_tokens": 255, + "latency": 1.8200000340584666e-05, + "ttft": null, + "cost": 5.4e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "autogen" + } + }, + { + "id": "f60327ec-dcca-4d4c-b774-9343a5b40c1c", + "name": "AutoGen - Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.0004650000009860378, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "framework": "autogen", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "2ac9061a-2673-4edf-b8ec-a8db5bc9e367", + "name": "AutoGen - Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00032500000088475645, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "framework": "autogen", + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "b75bdf65-197f-4e79-a0b1-c8c43037980c", + "name": "AutoGen - Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 720, + "completion_tokens": 95, + "total_tokens": 815, + "latency": 3.119999746559188e-05, + "ttft": null, + "cost": 0.00016499999999999997 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "autogen", + "agent_name": "refund_decision_agent", + "intent": "framework=AutoGen; intent=refund_request; priority=normal" + } + }, + { + "id": "e971d586-169f-4103-8a6c-a482bcf79d5e", + "name": "AutoGen - Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 850, + "completion_tokens": 130, + "total_tokens": 980, + "latency": 1.3900000340072438e-05, + "ttft": null, + "cost": 0.00020549999999999998 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "autogen", + "agent_name": "customer_response_agent" + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/crewai/live_travel_report.json b/benchmarks/reports/crewai/live_travel_report.json new file mode 100644 index 0000000..9567fcd --- /dev/null +++ b/benchmarks/reports/crewai/live_travel_report.json @@ -0,0 +1,158 @@ +{ + "id": "6c794877-9bdc-4878-9592-3edd823e21cc", + "name": "Benchmark - CrewAI - Live Travel Briefing", + "start_time": "2026-07-15T01:19:32.652529Z", + "end_time": "2026-07-15T01:19:34.231540Z", + "steps": [ + { + "id": "d900ba7d-9780-418a-9b72-10a0855731fd", + "name": "CrewAI - Classify Trip Request", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 140, + "completion_tokens": 28, + "total_tokens": 168, + "latency": 4.7299996367655694e-05, + "ttft": null, + "cost": 3.78e-05 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "crewai", + "crew_agents": 2 + } + }, + { + "id": "c0e67687-1a98-4266-a5b2-a1705aa48bc0", + "name": "CrewAI - Geocode Destination", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.5961224999991828, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_geocoding", + "tool_args": { + "city": "Tokyo" + }, + "framework": "crewai", + "tool_result": { + "name": "Tokyo", + "country": "Japan", + "lat": 35.6895, + "lon": 139.69171 + } + } + }, + { + "id": "e21fe3ed-60b9-4dcb-8658-c20240b42156", + "name": "CrewAI - Fetch Live Weather", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.5814734000014141, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_forecast", + "tool_args": { + "lat": 35.6895, + "lon": 139.69171 + }, + "framework": "crewai", + "tool_result": { + "time": "2026-07-15T01:15", + "interval": 900, + "temperature_2m": 29.9, + "weather_code": 2, + "wind_speed_10m": 4.2 + } + } + }, + { + "id": "8b3546f3-6bb9-45b0-875c-d40b4fd94574", + "name": "CrewAI - Fetch Live Exchange Rate", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.2329119999994873, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "frankfurter_exchange_rate", + "tool_args": { + "base": "USD", + "target": "JPY" + }, + "framework": "crewai", + "tool_result": { + "base": "USD", + "date": "2026-07-14", + "rate": 162.22 + } + } + }, + { + "id": "71ac4785-7b2c-4e70-9bf7-247b7772bacb", + "name": "CrewAI - Retrieve Destination Facts", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.1676378999982262, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "Tokyo", + "framework": "crewai", + "chunk_count": 3, + "avg_tokens_per_chunk": 28, + "retrieved_doc_ids": [ + "Tokyo" + ] + } + }, + { + "id": "7d2db832-002d-48f7-97cb-8792a32f4d9f", + "name": "CrewAI - Synthesize Travel Briefing", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 460, + "completion_tokens": 140, + "total_tokens": 600, + "latency": 6.949999806238338e-05, + "ttft": null, + "cost": 0.00015299999999999998 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "crewai" + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/crewai/support_refund_report.json b/benchmarks/reports/crewai/support_refund_report.json new file mode 100644 index 0000000..c3c2e47 --- /dev/null +++ b/benchmarks/reports/crewai/support_refund_report.json @@ -0,0 +1,146 @@ +{ + "id": "af870a32-4467-43a7-9f71-bdd67c3ccf9c", + "name": "Benchmark - CrewAI - Support Refund", + "start_time": "2026-07-15T00:01:46.504093Z", + "end_time": "2026-07-15T00:01:46.504997Z", + "steps": [ + { + "id": "eebd56da-d5c5-432e-88f2-cc971baa2ad4", + "name": "CrewAI - Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 180, + "completion_tokens": 25, + "total_tokens": 205, + "latency": 6.55999974696897e-05, + "ttft": null, + "cost": 4.199999999999999e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "crewai", + "crew_agents": 4 + } + }, + { + "id": "f6074dce-c385-4a6b-88da-e4a4920f889e", + "name": "CrewAI - Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 220, + "completion_tokens": 35, + "total_tokens": 255, + "latency": 2.0399998902576044e-05, + "ttft": null, + "cost": 5.4e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "crewai" + } + }, + { + "id": "e94e563d-ad3d-43bf-a130-8a571152a840", + "name": "CrewAI - Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00035519999801181257, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "framework": "crewai", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "62511b67-abf5-4e4a-9134-6c0db0db9a27", + "name": "CrewAI - Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00019799999790848233, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "framework": "crewai", + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "54087bbd-e619-465d-ae41-0a8ab1c02622", + "name": "CrewAI - Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 720, + "completion_tokens": 95, + "total_tokens": 815, + "latency": 2.099999983329326e-05, + "ttft": null, + "cost": 0.00016499999999999997 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "crewai", + "intent": "framework=CrewAI; intent=refund_request; priority=normal" + } + }, + { + "id": "d60ee37c-1b2c-4069-9513-1e42346fd564", + "name": "CrewAI - Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 850, + "completion_tokens": 130, + "total_tokens": 980, + "latency": 1.1899999663000926e-05, + "ttft": null, + "cost": 0.00020549999999999998 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "crewai" + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/langgraph/live_travel_report.json b/benchmarks/reports/langgraph/live_travel_report.json new file mode 100644 index 0000000..a631177 --- /dev/null +++ b/benchmarks/reports/langgraph/live_travel_report.json @@ -0,0 +1,157 @@ +{ + "id": "48c2d49b-bfcc-43b3-ad3d-b0e6b86f9586", + "name": "Benchmark - LangGraph - Live Travel Briefing", + "start_time": "2026-07-15T01:19:27.009570Z", + "end_time": "2026-07-15T01:19:28.503404Z", + "steps": [ + { + "id": "6f17b201-3135-4f5a-8251-4c59d41e94e2", + "name": "LangGraph - Classify Trip Request", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 140, + "completion_tokens": 28, + "total_tokens": 168, + "latency": 5.690000398317352e-05, + "ttft": null, + "cost": 3.78e-05 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "langgraph" + } + }, + { + "id": "828109de-022a-45ca-9f2c-f29b299b618e", + "name": "LangGraph - Geocode Destination", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.6253488000002108, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_geocoding", + "tool_args": { + "city": "Tokyo" + }, + "framework": "langgraph", + "tool_result": { + "name": "Tokyo", + "country": "Japan", + "lat": 35.6895, + "lon": 139.69171 + } + } + }, + { + "id": "95188792-7a53-4e2a-8884-b4035ad6e454", + "name": "LangGraph - Fetch Live Weather", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.5643410999982734, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_forecast", + "tool_args": { + "lat": 35.6895, + "lon": 139.69171 + }, + "framework": "langgraph", + "tool_result": { + "time": "2026-07-15T01:15", + "interval": 900, + "temperature_2m": 29.9, + "weather_code": 2, + "wind_speed_10m": 4.2 + } + } + }, + { + "id": "4f6ebcf6-1950-4d8f-8dac-4e2177df6c09", + "name": "LangGraph - Fetch Live Exchange Rate", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.13648050000483636, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "frankfurter_exchange_rate", + "tool_args": { + "base": "USD", + "target": "JPY" + }, + "framework": "langgraph", + "tool_result": { + "base": "USD", + "date": "2026-07-14", + "rate": 162.22 + } + } + }, + { + "id": "bcccaec6-51ee-434e-8e07-a827682d7d4a", + "name": "LangGraph - Retrieve Destination Facts", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.15412629999627825, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "Tokyo", + "framework": "langgraph", + "chunk_count": 3, + "avg_tokens_per_chunk": 28, + "retrieved_doc_ids": [ + "Tokyo" + ] + } + }, + { + "id": "97e74e8c-7852-4244-b6eb-5043193747f5", + "name": "LangGraph - Synthesize Travel Briefing", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 460, + "completion_tokens": 140, + "total_tokens": 600, + "latency": 0.0001340000017080456, + "ttft": null, + "cost": 0.00015299999999999998 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "langgraph" + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/langgraph/support_refund_report.json b/benchmarks/reports/langgraph/support_refund_report.json new file mode 100644 index 0000000..26cd713 --- /dev/null +++ b/benchmarks/reports/langgraph/support_refund_report.json @@ -0,0 +1,138 @@ +{ + "id": "c43030a5-eec9-4a0d-911f-456612f3149a", + "name": "Benchmark - LangGraph - Support Refund", + "start_time": "2026-07-15T00:01:39.404988Z", + "end_time": "2026-07-15T00:01:39.437796Z", + "steps": [ + { + "id": "2e5839a4-c25c-4e36-977f-31ec0812d783", + "name": "LangGraph - Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 190, + "completion_tokens": 30, + "total_tokens": 220, + "latency": 0.00025860000096145086, + "ttft": null, + "cost": 4.649999999999999e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?" + } + }, + { + "id": "899a6eb8-abfd-4c02-8c8f-1bc4b2e40014", + "name": "LangGraph - Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 240, + "completion_tokens": 40, + "total_tokens": 280, + "latency": 9.900000077323057e-05, + "ttft": null, + "cost": 5.9999999999999995e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?" + } + }, + { + "id": "78c5d4e3-55a6-4bbf-94ec-87fe1483e99c", + "name": "LangGraph - Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.001006700000289129, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "be8d9918-765b-4b39-9ecc-2fdd7002d7b6", + "name": "LangGraph - Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.0007781999993312638, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "86cbe2bd-8377-4eae-baf4-f7bb8b638e8d", + "name": "LangGraph - Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 780, + "completion_tokens": 110, + "total_tokens": 890, + "latency": 7.94000006862916e-05, + "ttft": null, + "cost": 0.000183 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window." + } + }, + { + "id": "1da78f49-2fa0-4fbd-8f4f-7adb238cf0ca", + "name": "LangGraph - Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 920, + "completion_tokens": 150, + "total_tokens": 1070, + "latency": 4.649999755201861e-05, + "ttft": null, + "cost": 0.00022799999999999999 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window." + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/llamaindex/live_travel_report.json b/benchmarks/reports/llamaindex/live_travel_report.json new file mode 100644 index 0000000..2455004 --- /dev/null +++ b/benchmarks/reports/llamaindex/live_travel_report.json @@ -0,0 +1,157 @@ +{ + "id": "db6c8c0e-e4a1-40b2-88ad-0140d09cc32f", + "name": "Benchmark - LlamaIndex - Live Travel Briefing", + "start_time": "2026-07-15T01:19:39.222786Z", + "end_time": "2026-07-15T01:19:40.709978Z", + "steps": [ + { + "id": "42e1063b-b4fc-413a-bab9-4d79ce8bc59d", + "name": "LlamaIndex - Classify Trip Request", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 140, + "completion_tokens": 28, + "total_tokens": 168, + "latency": 5.879999662283808e-05, + "ttft": null, + "cost": 3.78e-05 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "llamaindex" + } + }, + { + "id": "ec4b85a5-dbd2-4377-bcd1-0157a50534ea", + "name": "LlamaIndex - Geocode Destination", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.6174403000040911, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_geocoding", + "tool_args": { + "city": "Tokyo" + }, + "framework": "llamaindex", + "tool_result": { + "name": "Tokyo", + "country": "Japan", + "lat": 35.6895, + "lon": 139.69171 + } + } + }, + { + "id": "c2fb77fd-5d59-4c71-a5c1-199a1faaafa2", + "name": "LlamaIndex - Fetch Live Weather", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.5675237000032212, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_forecast", + "tool_args": { + "lat": 35.6895, + "lon": 139.69171 + }, + "framework": "llamaindex", + "tool_result": { + "time": "2026-07-15T01:15", + "interval": 900, + "temperature_2m": 29.9, + "weather_code": 2, + "wind_speed_10m": 4.2 + } + } + }, + { + "id": "477248a3-cbee-430e-815e-bfc92e5a47d3", + "name": "LlamaIndex - Fetch Live Exchange Rate", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.17697830000543036, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "frankfurter_exchange_rate", + "tool_args": { + "base": "USD", + "target": "JPY" + }, + "framework": "llamaindex", + "tool_result": { + "base": "USD", + "date": "2026-07-14", + "rate": 162.22 + } + } + }, + { + "id": "d0131c67-2e86-4691-b867-1cdf221959ec", + "name": "LlamaIndex - Retrieve Destination Facts", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.1244421000010334, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "Tokyo", + "framework": "llamaindex", + "chunk_count": 3, + "avg_tokens_per_chunk": 28, + "retrieved_doc_ids": [ + "74e34f7d-daa1-41dd-a906-23d394367908" + ] + } + }, + { + "id": "4e3d7ab4-353e-4fd2-a955-66a4abc7034e", + "name": "LlamaIndex - Synthesize Travel Briefing", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 460, + "completion_tokens": 140, + "total_tokens": 600, + "latency": 2.3699998564552516e-05, + "ttft": null, + "cost": 0.00015299999999999998 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "llamaindex" + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/llamaindex/support_refund_report.json b/benchmarks/reports/llamaindex/support_refund_report.json new file mode 100644 index 0000000..1b9f475 --- /dev/null +++ b/benchmarks/reports/llamaindex/support_refund_report.json @@ -0,0 +1,148 @@ +{ + "id": "be557309-d687-43ab-8b36-227b3a6d55f1", + "name": "Benchmark - LlamaIndex - Support Refund", + "start_time": "2026-07-15T00:01:53.015789Z", + "end_time": "2026-07-15T00:01:53.016511Z", + "steps": [ + { + "id": "a905b00d-f0e5-4c21-9d0a-2dbe8b9146eb", + "name": "LlamaIndex - Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 180, + "completion_tokens": 25, + "total_tokens": 205, + "latency": 6.349999966914766e-05, + "ttft": null, + "cost": 4.199999999999999e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "llamaindex", + "framework_documents": 8, + "index_type": "llamaindex_documents" + } + }, + { + "id": "76d95e09-bbbd-43bd-9865-2341cce4f670", + "name": "LlamaIndex - Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 220, + "completion_tokens": 35, + "total_tokens": 255, + "latency": 1.5400000847876072e-05, + "ttft": null, + "cost": 5.4e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "llamaindex" + } + }, + { + "id": "d82049d7-8247-4d18-98b0-101e16e5cd77", + "name": "LlamaIndex - Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00025889999960782006, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "framework": "llamaindex", + "index_type": "Document collection", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "5140b857-7e13-4725-8038-0dfc96839a11", + "name": "LlamaIndex - Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00017049999951268546, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "framework": "llamaindex", + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "2f6fa1a7-4339-4f6a-80a3-1a36366e3a56", + "name": "LlamaIndex - Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 720, + "completion_tokens": 95, + "total_tokens": 815, + "latency": 1.3800003216601908e-05, + "ttft": null, + "cost": 0.00016499999999999997 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "llamaindex", + "intent": "framework=LlamaIndex; intent=refund_request; priority=normal" + } + }, + { + "id": "74abb3f0-9d08-41aa-85d0-b987e1b7036e", + "name": "LlamaIndex - Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 850, + "completion_tokens": 130, + "total_tokens": 980, + "latency": 1.0199997632298619e-05, + "ttft": null, + "cost": 0.00020549999999999998 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "llamaindex" + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/native_python/live_travel_report.json b/benchmarks/reports/native_python/live_travel_report.json new file mode 100644 index 0000000..6fe2abd --- /dev/null +++ b/benchmarks/reports/native_python/live_travel_report.json @@ -0,0 +1,157 @@ +{ + "id": "713313db-3feb-4ac5-93a2-bd224cc093dd", + "name": "Benchmark - Native Python - Live Travel Briefing", + "start_time": "2026-07-15T01:19:23.957668Z", + "end_time": "2026-07-15T01:19:25.550452Z", + "steps": [ + { + "id": "8f08dcf2-0f62-4b6c-9684-112e4d807d07", + "name": "Native Python - Classify Trip Request", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 140, + "completion_tokens": 28, + "total_tokens": 168, + "latency": 7.6299998909235e-05, + "ttft": null, + "cost": 3.78e-05 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "native_python" + } + }, + { + "id": "518ca607-8c4c-44fc-a178-4382e6a0331a", + "name": "Native Python - Geocode Destination", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.6669330000004265, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_geocoding", + "tool_args": { + "city": "Tokyo" + }, + "framework": "native_python", + "tool_result": { + "name": "Tokyo", + "country": "Japan", + "lat": 35.6895, + "lon": 139.69171 + } + } + }, + { + "id": "f4b4c7ea-f135-4381-abec-76fa4a92736c", + "name": "Native Python - Fetch Live Weather", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.578566399999545, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_forecast", + "tool_args": { + "lat": 35.6895, + "lon": 139.69171 + }, + "framework": "native_python", + "tool_result": { + "time": "2026-07-15T01:15", + "interval": 900, + "temperature_2m": 29.9, + "weather_code": 2, + "wind_speed_10m": 4.2 + } + } + }, + { + "id": "0034d879-6803-4011-8e8d-15436448b03c", + "name": "Native Python - Fetch Live Exchange Rate", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.20708709999598796, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "frankfurter_exchange_rate", + "tool_args": { + "base": "USD", + "target": "JPY" + }, + "framework": "native_python", + "tool_result": { + "base": "USD", + "date": "2026-07-14", + "rate": 162.22 + } + } + }, + { + "id": "213f65b5-aa78-4850-bb3e-dcd365736485", + "name": "Native Python - Retrieve Destination Facts", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.1394676999989315, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "Tokyo", + "framework": "native_python", + "chunk_count": 3, + "avg_tokens_per_chunk": 28, + "retrieved_doc_ids": [ + "Tokyo" + ] + } + }, + { + "id": "3697a04e-ca43-49c5-87e1-8a06ab607f53", + "name": "Native Python - Synthesize Travel Briefing", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 460, + "completion_tokens": 140, + "total_tokens": 600, + "latency": 3.429999924264848e-05, + "ttft": null, + "cost": 0.00015299999999999998 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "native_python" + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/native_python/support_refund_report.json b/benchmarks/reports/native_python/support_refund_report.json new file mode 100644 index 0000000..d0e04e3 --- /dev/null +++ b/benchmarks/reports/native_python/support_refund_report.json @@ -0,0 +1,139 @@ +{ + "id": "2d9a1e50-2b4d-4fd9-b7da-240808ac553a", + "name": "Benchmark - Native Python - Support Refund", + "start_time": "2026-07-15T00:01:35.653453Z", + "end_time": "2026-07-15T00:01:35.654269Z", + "steps": [ + { + "id": "df6c2f3b-84d4-4dcf-a5ea-5683bc6c3b28", + "name": "Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 180, + "completion_tokens": 25, + "total_tokens": 205, + "latency": 5.7199998991563916e-05, + "ttft": null, + "cost": 4.199999999999999e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?" + } + }, + { + "id": "52f26680-4cdf-4314-9545-208c56530f60", + "name": "Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 220, + "completion_tokens": 35, + "total_tokens": 255, + "latency": 1.3500000932253897e-05, + "ttft": null, + "cost": 5.4e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?" + } + }, + { + "id": "231b9539-e57c-4752-8177-d79eb763cc30", + "name": "Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00024149999808287248, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "c809e3e4-c7bd-4c91-812b-c536a70f187a", + "name": "Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.0001951000012923032, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "61259de7-c83d-4fae-a202-fc50d1ba017c", + "name": "Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 720, + "completion_tokens": 95, + "total_tokens": 815, + "latency": 2.1899999410379678e-05, + "ttft": null, + "cost": 0.00016499999999999997 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "intent": "intent=refund_request; priority=normal" + } + }, + { + "id": "e906019d-9fde-4a17-bd6d-5cd54eb15b90", + "name": "Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 850, + "completion_tokens": 130, + "total_tokens": 980, + "latency": 9.300001693191007e-06, + "ttft": null, + "cost": 0.00020549999999999998 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window." + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/semantic_kernel/live_travel_report.json b/benchmarks/reports/semantic_kernel/live_travel_report.json new file mode 100644 index 0000000..c6b5d61 --- /dev/null +++ b/benchmarks/reports/semantic_kernel/live_travel_report.json @@ -0,0 +1,158 @@ +{ + "id": "2302e741-a154-4e95-b527-f3400baf35f9", + "name": "Benchmark - Semantic Kernel - Live Travel Briefing", + "start_time": "2026-07-15T01:19:43.197923Z", + "end_time": "2026-07-15T01:19:44.807442Z", + "steps": [ + { + "id": "4f8f1e98-999e-4c7d-85c9-ec126f0ab126", + "name": "Semantic Kernel - Classify Trip Request", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 140, + "completion_tokens": 28, + "total_tokens": 168, + "latency": 8.55999969644472e-05, + "ttft": null, + "cost": 3.78e-05 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "semantic_kernel", + "kernel_type": "Kernel" + } + }, + { + "id": "6912d86b-5dc1-4396-bf79-164a2357c5e3", + "name": "Semantic Kernel - Geocode Destination", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.6470667999965372, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_geocoding", + "tool_args": { + "city": "Tokyo" + }, + "framework": "semantic_kernel", + "tool_result": { + "name": "Tokyo", + "country": "Japan", + "lat": 35.6895, + "lon": 139.69171 + } + } + }, + { + "id": "1d3e07e7-e0e8-4adf-a2f4-078f8f0ea3ce", + "name": "Semantic Kernel - Fetch Live Weather", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.5653453999984777, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "open_meteo_forecast", + "tool_args": { + "lat": 35.6895, + "lon": 139.69171 + }, + "framework": "semantic_kernel", + "tool_result": { + "time": "2026-07-15T01:15", + "interval": 900, + "temperature_2m": 29.9, + "weather_code": 2, + "wind_speed_10m": 4.2 + } + } + }, + { + "id": "d2deda13-f2a1-4d2f-bf6d-4e148366a038", + "name": "Semantic Kernel - Fetch Live Exchange Rate", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.21713720000116155, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "frankfurter_exchange_rate", + "tool_args": { + "base": "USD", + "target": "JPY" + }, + "framework": "semantic_kernel", + "tool_result": { + "base": "USD", + "date": "2026-07-14", + "rate": 162.22 + } + } + }, + { + "id": "68e98181-faba-41f7-8b37-47af0666873c", + "name": "Semantic Kernel - Retrieve Destination Facts", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.17905540000356268, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "Tokyo", + "framework": "semantic_kernel", + "chunk_count": 3, + "avg_tokens_per_chunk": 28, + "retrieved_doc_ids": [ + "Tokyo" + ] + } + }, + { + "id": "85d51909-e185-40f9-b04d-e9aed099e36b", + "name": "Semantic Kernel - Synthesize Travel Briefing", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 460, + "completion_tokens": 140, + "total_tokens": 600, + "latency": 0.00010740000288933516, + "ttft": null, + "cost": 0.00015299999999999998 + }, + "metadata": { + "prompt": "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?", + "framework": "semantic_kernel" + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/reports/semantic_kernel/support_refund_report.json b/benchmarks/reports/semantic_kernel/support_refund_report.json new file mode 100644 index 0000000..ad87812 --- /dev/null +++ b/benchmarks/reports/semantic_kernel/support_refund_report.json @@ -0,0 +1,146 @@ +{ + "id": "fe9fdd0a-2de7-4fc0-a077-5bf8f9c17513", + "name": "Benchmark - Semantic Kernel - Support Refund", + "start_time": "2026-07-15T00:01:57.429218Z", + "end_time": "2026-07-15T00:01:57.430915Z", + "steps": [ + { + "id": "de067e62-f161-4998-aca9-d0980646e03b", + "name": "Semantic Kernel - Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 180, + "completion_tokens": 25, + "total_tokens": 205, + "latency": 0.000112599998828955, + "ttft": null, + "cost": 4.199999999999999e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "semantic_kernel", + "kernel_type": "Kernel" + } + }, + { + "id": "f6db5031-c826-4768-bc51-70f48a958896", + "name": "Semantic Kernel - Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 220, + "completion_tokens": 35, + "total_tokens": 255, + "latency": 2.970000059576705e-05, + "ttft": null, + "cost": 5.4e-05 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "semantic_kernel" + } + }, + { + "id": "3d42c803-2f68-4b3c-be28-d174ee494187", + "name": "Semantic Kernel - Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.0006422000005841255, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "framework": "semantic_kernel", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "d9dc1af4-0434-453c-93b8-5a49943637d4", + "name": "Semantic Kernel - Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00034080000114045106, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "framework": "semantic_kernel", + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "54bc7085-232c-4652-9016-b2cde854b5a5", + "name": "Semantic Kernel - Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 720, + "completion_tokens": 95, + "total_tokens": 815, + "latency": 3.819999983534217e-05, + "ttft": null, + "cost": 0.00016499999999999997 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "semantic_kernel", + "intent": "framework=Semantic Kernel; intent=refund_request; priority=normal" + } + }, + { + "id": "2f73c623-9762-4214-a1b9-e1cd1bae7454", + "name": "Semantic Kernel - Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 850, + "completion_tokens": 130, + "total_tokens": 980, + "latency": 1.979999797185883e-05, + "ttft": null, + "cost": 0.00020549999999999998 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "semantic_kernel" + } + } + ], + "chaos_events": [] +} \ No newline at end of file diff --git a/benchmarks/results/benchmark_cost_chart.png b/benchmarks/results/benchmark_cost_chart.png new file mode 100644 index 0000000..fe1e723 Binary files /dev/null and b/benchmarks/results/benchmark_cost_chart.png differ diff --git a/benchmarks/results/benchmark_results.csv b/benchmarks/results/benchmark_results.csv new file mode 100644 index 0000000..07e1712 --- /dev/null +++ b/benchmarks/results/benchmark_results.csv @@ -0,0 +1,7 @@ +framework,workflow_name,step_count,total_tokens,prompt_tokens,completion_tokens,total_cost_usd,total_latency_sec,tool_calls,retrieved_chunks,highest_token_step,highest_step_tokens,highest_cost_step,highest_step_cost_usd +AutoGen,Benchmark - AutoGen - Support Refund,6,2255,1970,285,0.0004665,0.000928,1,4,AutoGen - Generate Customer Reply,980,AutoGen - Generate Customer Reply,0.0002055 +CrewAI,Benchmark - CrewAI - Support Refund,6,2255,1970,285,0.0004665,0.0006721,1,4,CrewAI - Generate Customer Reply,980,CrewAI - Generate Customer Reply,0.0002055 +LlamaIndex,Benchmark - LlamaIndex - Support Refund,6,2255,1970,285,0.0004665,0.0005323,1,4,LlamaIndex - Generate Customer Reply,980,LlamaIndex - Generate Customer Reply,0.0002055 +Native Python,Benchmark - Native Python - Support Refund,6,2255,1970,285,0.0004665,0.0005385,1,4,Generate Customer Reply,980,Generate Customer Reply,0.0002055 +Semantic Kernel,Benchmark - Semantic Kernel - Support Refund,6,2255,1970,285,0.0004665,0.0011833,1,4,Semantic Kernel - Generate Customer Reply,980,Semantic Kernel - Generate Customer Reply,0.0002055 +LangGraph,Benchmark - LangGraph - Support Refund,6,2460,2130,330,0.0005175,0.0022684,1,4,LangGraph - Generate Customer Reply,1070,LangGraph - Generate Customer Reply,0.000228 diff --git a/benchmarks/results/benchmark_results.json b/benchmarks/results/benchmark_results.json new file mode 100644 index 0000000..b5815d8 --- /dev/null +++ b/benchmarks/results/benchmark_results.json @@ -0,0 +1,104 @@ +[ + { + "framework": "native_python", + "workflow_name": "Benchmark - Native Python - Support Refund", + "step_count": 6, + "total_tokens": 2255, + "prompt_tokens": 1970, + "completion_tokens": 285, + "total_cost": 0.0004665, + "total_latency": 0.0005, + "tool_calls": 1, + "retriever_steps": 1, + "retrieved_chunks": 4, + "memory_tokens": 0, + "highest_token_step": "Generate Customer Reply", + "highest_cost_step": "Generate Customer Reply", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\native_python\\support_refund_report.json" + }, + { + "framework": "langgraph", + "workflow_name": "Benchmark - LangGraph - Support Refund", + "step_count": 6, + "total_tokens": 2460, + "prompt_tokens": 2130, + "completion_tokens": 330, + "total_cost": 0.0005175, + "total_latency": 0.0023, + "tool_calls": 1, + "retriever_steps": 1, + "retrieved_chunks": 4, + "memory_tokens": 0, + "highest_token_step": "LangGraph - Generate Customer Reply", + "highest_cost_step": "LangGraph - Generate Customer Reply", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\langgraph\\support_refund_report.json" + }, + { + "framework": "crewai", + "workflow_name": "Benchmark - CrewAI - Support Refund", + "step_count": 6, + "total_tokens": 2255, + "prompt_tokens": 1970, + "completion_tokens": 285, + "total_cost": 0.0004665, + "total_latency": 0.0007, + "tool_calls": 1, + "retriever_steps": 1, + "retrieved_chunks": 4, + "memory_tokens": 0, + "highest_token_step": "CrewAI - Generate Customer Reply", + "highest_cost_step": "CrewAI - Generate Customer Reply", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\crewai\\support_refund_report.json" + }, + { + "framework": "autogen", + "workflow_name": "Benchmark - AutoGen - Support Refund", + "step_count": 6, + "total_tokens": 2255, + "prompt_tokens": 1970, + "completion_tokens": 285, + "total_cost": 0.0004665, + "total_latency": 0.0009, + "tool_calls": 1, + "retriever_steps": 1, + "retrieved_chunks": 4, + "memory_tokens": 0, + "highest_token_step": "AutoGen - Generate Customer Reply", + "highest_cost_step": "AutoGen - Generate Customer Reply", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\autogen\\support_refund_report.json" + }, + { + "framework": "llamaindex", + "workflow_name": "Benchmark - LlamaIndex - Support Refund", + "step_count": 6, + "total_tokens": 2255, + "prompt_tokens": 1970, + "completion_tokens": 285, + "total_cost": 0.0004665, + "total_latency": 0.0005, + "tool_calls": 1, + "retriever_steps": 1, + "retrieved_chunks": 4, + "memory_tokens": 0, + "highest_token_step": "LlamaIndex - Generate Customer Reply", + "highest_cost_step": "LlamaIndex - Generate Customer Reply", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\llamaindex\\support_refund_report.json" + }, + { + "framework": "semantic_kernel", + "workflow_name": "Benchmark - Semantic Kernel - Support Refund", + "step_count": 6, + "total_tokens": 2255, + "prompt_tokens": 1970, + "completion_tokens": 285, + "total_cost": 0.0004665, + "total_latency": 0.0012, + "tool_calls": 1, + "retriever_steps": 1, + "retrieved_chunks": 4, + "memory_tokens": 0, + "highest_token_step": "Semantic Kernel - Generate Customer Reply", + "highest_cost_step": "Semantic Kernel - Generate Customer Reply", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\semantic_kernel\\support_refund_report.json" + } +] \ No newline at end of file diff --git a/benchmarks/results/benchmark_step_breakdown.csv b/benchmarks/results/benchmark_step_breakdown.csv new file mode 100644 index 0000000..39c2992 --- /dev/null +++ b/benchmarks/results/benchmark_step_breakdown.csv @@ -0,0 +1,37 @@ +framework,workflow_name,step_name,step_type,provider,model,prompt_tokens,completion_tokens,total_tokens,cost_usd,latency_sec,chunk_count,tool_name +Native Python,Benchmark - Native Python - Support Refund,Classify Ticket Intent,planner,openai,gpt-4o-mini,180,25,205,4.199999999999999e-05,5.7199998991563916e-05,, +Native Python,Benchmark - Native Python - Support Refund,Rewrite Query For Retrieval,llm_call,openai,gpt-4o-mini,220,35,255,5.4e-05,1.3500000932253897e-05,, +Native Python,Benchmark - Native Python - Support Refund,Retrieve Refund Policy,retriever,,,0,0,0,0.0,0.00024149999808287248,4.0, +Native Python,Benchmark - Native Python - Support Refund,Lookup Order,tool_call,,,0,0,0,0.0,0.0001951000012923032,,lookup_order +Native Python,Benchmark - Native Python - Support Refund,Refund Eligibility Check,llm_call,openai,gpt-4o-mini,720,95,815,0.00016499999999999997,2.1899999410379678e-05,, +Native Python,Benchmark - Native Python - Support Refund,Generate Customer Reply,final_response,openai,gpt-4o-mini,850,130,980,0.00020549999999999998,9.300001693191007e-06,, +LangGraph,Benchmark - LangGraph - Support Refund,LangGraph - Classify Ticket Intent,planner,openai,gpt-4o-mini,190,30,220,4.649999999999999e-05,0.00025860000096145086,, +LangGraph,Benchmark - LangGraph - Support Refund,LangGraph - Rewrite Query For Retrieval,llm_call,openai,gpt-4o-mini,240,40,280,5.9999999999999995e-05,9.900000077323057e-05,, +LangGraph,Benchmark - LangGraph - Support Refund,LangGraph - Retrieve Refund Policy,retriever,,,0,0,0,0.0,0.001006700000289129,4.0, +LangGraph,Benchmark - LangGraph - Support Refund,LangGraph - Lookup Order,tool_call,,,0,0,0,0.0,0.0007781999993312638,,lookup_order +LangGraph,Benchmark - LangGraph - Support Refund,LangGraph - Refund Eligibility Check,llm_call,openai,gpt-4o-mini,780,110,890,0.000183,7.94000006862916e-05,, +LangGraph,Benchmark - LangGraph - Support Refund,LangGraph - Generate Customer Reply,final_response,openai,gpt-4o-mini,920,150,1070,0.00022799999999999999,4.649999755201861e-05,, +CrewAI,Benchmark - CrewAI - Support Refund,CrewAI - Classify Ticket Intent,planner,openai,gpt-4o-mini,180,25,205,4.199999999999999e-05,6.55999974696897e-05,, +CrewAI,Benchmark - CrewAI - Support Refund,CrewAI - Rewrite Query For Retrieval,llm_call,openai,gpt-4o-mini,220,35,255,5.4e-05,2.0399998902576044e-05,, +CrewAI,Benchmark - CrewAI - Support Refund,CrewAI - Retrieve Refund Policy,retriever,,,0,0,0,0.0,0.00035519999801181257,4.0, +CrewAI,Benchmark - CrewAI - Support Refund,CrewAI - Lookup Order,tool_call,,,0,0,0,0.0,0.00019799999790848233,,lookup_order +CrewAI,Benchmark - CrewAI - Support Refund,CrewAI - Refund Eligibility Check,llm_call,openai,gpt-4o-mini,720,95,815,0.00016499999999999997,2.099999983329326e-05,, +CrewAI,Benchmark - CrewAI - Support Refund,CrewAI - Generate Customer Reply,final_response,openai,gpt-4o-mini,850,130,980,0.00020549999999999998,1.1899999663000926e-05,, +AutoGen,Benchmark - AutoGen - Support Refund,AutoGen - Classify Ticket Intent,planner,openai,gpt-4o-mini,180,25,205,4.199999999999999e-05,7.470000127796084e-05,, +AutoGen,Benchmark - AutoGen - Support Refund,AutoGen - Rewrite Query For Retrieval,llm_call,openai,gpt-4o-mini,220,35,255,5.4e-05,1.8200000340584666e-05,, +AutoGen,Benchmark - AutoGen - Support Refund,AutoGen - Retrieve Refund Policy,retriever,,,0,0,0,0.0,0.0004650000009860378,4.0, +AutoGen,Benchmark - AutoGen - Support Refund,AutoGen - Lookup Order,tool_call,,,0,0,0,0.0,0.00032500000088475645,,lookup_order +AutoGen,Benchmark - AutoGen - Support Refund,AutoGen - Refund Eligibility Check,llm_call,openai,gpt-4o-mini,720,95,815,0.00016499999999999997,3.119999746559188e-05,, +AutoGen,Benchmark - AutoGen - Support Refund,AutoGen - Generate Customer Reply,final_response,openai,gpt-4o-mini,850,130,980,0.00020549999999999998,1.3900000340072438e-05,, +LlamaIndex,Benchmark - LlamaIndex - Support Refund,LlamaIndex - Classify Ticket Intent,planner,openai,gpt-4o-mini,180,25,205,4.199999999999999e-05,6.349999966914766e-05,, +LlamaIndex,Benchmark - LlamaIndex - Support Refund,LlamaIndex - Rewrite Query For Retrieval,llm_call,openai,gpt-4o-mini,220,35,255,5.4e-05,1.5400000847876072e-05,, +LlamaIndex,Benchmark - LlamaIndex - Support Refund,LlamaIndex - Retrieve Refund Policy,retriever,,,0,0,0,0.0,0.00025889999960782006,4.0, +LlamaIndex,Benchmark - LlamaIndex - Support Refund,LlamaIndex - Lookup Order,tool_call,,,0,0,0,0.0,0.00017049999951268546,,lookup_order +LlamaIndex,Benchmark - LlamaIndex - Support Refund,LlamaIndex - Refund Eligibility Check,llm_call,openai,gpt-4o-mini,720,95,815,0.00016499999999999997,1.3800003216601908e-05,, +LlamaIndex,Benchmark - LlamaIndex - Support Refund,LlamaIndex - Generate Customer Reply,final_response,openai,gpt-4o-mini,850,130,980,0.00020549999999999998,1.0199997632298619e-05,, +Semantic Kernel,Benchmark - Semantic Kernel - Support Refund,Semantic Kernel - Classify Ticket Intent,planner,openai,gpt-4o-mini,180,25,205,4.199999999999999e-05,0.000112599998828955,, +Semantic Kernel,Benchmark - Semantic Kernel - Support Refund,Semantic Kernel - Rewrite Query For Retrieval,llm_call,openai,gpt-4o-mini,220,35,255,5.4e-05,2.970000059576705e-05,, +Semantic Kernel,Benchmark - Semantic Kernel - Support Refund,Semantic Kernel - Retrieve Refund Policy,retriever,,,0,0,0,0.0,0.0006422000005841255,4.0, +Semantic Kernel,Benchmark - Semantic Kernel - Support Refund,Semantic Kernel - Lookup Order,tool_call,,,0,0,0,0.0,0.00034080000114045106,,lookup_order +Semantic Kernel,Benchmark - Semantic Kernel - Support Refund,Semantic Kernel - Refund Eligibility Check,llm_call,openai,gpt-4o-mini,720,95,815,0.00016499999999999997,3.819999983534217e-05,, +Semantic Kernel,Benchmark - Semantic Kernel - Support Refund,Semantic Kernel - Generate Customer Reply,final_response,openai,gpt-4o-mini,850,130,980,0.00020549999999999998,1.979999797185883e-05,, diff --git a/benchmarks/results/benchmark_summary.md b/benchmarks/results/benchmark_summary.md new file mode 100644 index 0000000..462f91d --- /dev/null +++ b/benchmarks/results/benchmark_summary.md @@ -0,0 +1,32 @@ +# AgenticLens Framework Benchmark Comparison + +Use case: Practical customer support refund workflow. + +The workflow includes: + +- ticket intent classification +- query rewriting +- refund policy retrieval +- order lookup +- refund eligibility check +- customer reply generation + +## Summary Results + +| Framework | Total Tokens | Prompt Tokens | Completion Tokens | Cost USD | Latency Sec | Steps | Tool Calls | Retrieved Chunks | Highest Token Step | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---| +| AutoGen | 2255 | 1970 | 285 | $0.00046650 | 0.00092800 | 6 | 1 | 4 | AutoGen - Generate Customer Reply | +| CrewAI | 2255 | 1970 | 285 | $0.00046650 | 0.00067210 | 6 | 1 | 4 | CrewAI - Generate Customer Reply | +| LlamaIndex | 2255 | 1970 | 285 | $0.00046650 | 0.00053230 | 6 | 1 | 4 | LlamaIndex - Generate Customer Reply | +| Native Python | 2255 | 1970 | 285 | $0.00046650 | 0.00053850 | 6 | 1 | 4 | Generate Customer Reply | +| Semantic Kernel | 2255 | 1970 | 285 | $0.00046650 | 0.00118330 | 6 | 1 | 4 | Semantic Kernel - Generate Customer Reply | +| LangGraph | 2460 | 2130 | 330 | $0.00051750 | 0.00226840 | 6 | 1 | 4 | LangGraph - Generate Customer Reply | + +## Key Finding + +The final customer reply step is the highest token-consuming step across the benchmark runs. + +## Important Note + +These results are workload-specific. They should not be treated as a universal ranking of frameworks. +The purpose is to show how AgenticLens can normalize and compare token, cost, latency, retrieval, and tool-call metrics across framework implementations. \ No newline at end of file diff --git a/benchmarks/results/benchmark_tokens_chart.png b/benchmarks/results/benchmark_tokens_chart.png new file mode 100644 index 0000000..4110d34 Binary files /dev/null and b/benchmarks/results/benchmark_tokens_chart.png differ diff --git a/benchmarks/results/live_travel_results.csv b/benchmarks/results/live_travel_results.csv new file mode 100644 index 0000000..f82446e --- /dev/null +++ b/benchmarks/results/live_travel_results.csv @@ -0,0 +1,7 @@ +framework,workflow_name,step_count,total_tokens,prompt_tokens,completion_tokens,total_cost,total_latency,tool_calls,retriever_steps,retrieved_chunks,memory_tokens,highest_token_step,highest_cost_step,report_path,highest_latency_step,highest_step_latency +native_python,Benchmark - Native Python - Live Travel Briefing,6,768,600,168,0.0001908,1.5922,3,1,3,0,Native Python - Synthesize Travel Briefing,Native Python - Synthesize Travel Briefing,E:\agenticlens\benchmarks\reports\native_python\live_travel_report.json,Native Python - Geocode Destination,0.6669 +langgraph,Benchmark - LangGraph - Live Travel Briefing,6,768,600,168,0.0001908,1.4805,3,1,3,0,LangGraph - Synthesize Travel Briefing,LangGraph - Synthesize Travel Briefing,E:\agenticlens\benchmarks\reports\langgraph\live_travel_report.json,LangGraph - Geocode Destination,0.6253 +crewai,Benchmark - CrewAI - Live Travel Briefing,6,768,600,168,0.0001908,1.5783,3,1,3,0,CrewAI - Synthesize Travel Briefing,CrewAI - Synthesize Travel Briefing,E:\agenticlens\benchmarks\reports\crewai\live_travel_report.json,CrewAI - Geocode Destination,0.5961 +autogen,Benchmark - AutoGen - Live Travel Briefing,6,768,600,168,0.0001908,1.4647,3,1,3,0,AutoGen - Synthesize Travel Briefing,AutoGen - Synthesize Travel Briefing,E:\agenticlens\benchmarks\reports\autogen\live_travel_report.json,AutoGen - Geocode Destination,0.6201 +llamaindex,Benchmark - LlamaIndex - Live Travel Briefing,6,768,600,168,0.0001908,1.4865,3,1,3,0,LlamaIndex - Synthesize Travel Briefing,LlamaIndex - Synthesize Travel Briefing,E:\agenticlens\benchmarks\reports\llamaindex\live_travel_report.json,LlamaIndex - Geocode Destination,0.6174 +semantic_kernel,Benchmark - Semantic Kernel - Live Travel Briefing,6,768,600,168,0.0001908,1.6088,3,1,3,0,Semantic Kernel - Synthesize Travel Briefing,Semantic Kernel - Synthesize Travel Briefing,E:\agenticlens\benchmarks\reports\semantic_kernel\live_travel_report.json,Semantic Kernel - Geocode Destination,0.6471 diff --git a/benchmarks/results/live_travel_results.json b/benchmarks/results/live_travel_results.json new file mode 100644 index 0000000..e56f62f --- /dev/null +++ b/benchmarks/results/live_travel_results.json @@ -0,0 +1,116 @@ +[ + { + "framework": "native_python", + "workflow_name": "Benchmark - Native Python - Live Travel Briefing", + "step_count": 6, + "total_tokens": 768, + "prompt_tokens": 600, + "completion_tokens": 168, + "total_cost": 0.0001908, + "total_latency": 1.5922, + "tool_calls": 3, + "retriever_steps": 1, + "retrieved_chunks": 3, + "memory_tokens": 0, + "highest_token_step": "Native Python - Synthesize Travel Briefing", + "highest_cost_step": "Native Python - Synthesize Travel Briefing", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\native_python\\live_travel_report.json", + "highest_latency_step": "Native Python - Geocode Destination", + "highest_step_latency": 0.6669 + }, + { + "framework": "langgraph", + "workflow_name": "Benchmark - LangGraph - Live Travel Briefing", + "step_count": 6, + "total_tokens": 768, + "prompt_tokens": 600, + "completion_tokens": 168, + "total_cost": 0.0001908, + "total_latency": 1.4805, + "tool_calls": 3, + "retriever_steps": 1, + "retrieved_chunks": 3, + "memory_tokens": 0, + "highest_token_step": "LangGraph - Synthesize Travel Briefing", + "highest_cost_step": "LangGraph - Synthesize Travel Briefing", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\langgraph\\live_travel_report.json", + "highest_latency_step": "LangGraph - Geocode Destination", + "highest_step_latency": 0.6253 + }, + { + "framework": "crewai", + "workflow_name": "Benchmark - CrewAI - Live Travel Briefing", + "step_count": 6, + "total_tokens": 768, + "prompt_tokens": 600, + "completion_tokens": 168, + "total_cost": 0.0001908, + "total_latency": 1.5783, + "tool_calls": 3, + "retriever_steps": 1, + "retrieved_chunks": 3, + "memory_tokens": 0, + "highest_token_step": "CrewAI - Synthesize Travel Briefing", + "highest_cost_step": "CrewAI - Synthesize Travel Briefing", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\crewai\\live_travel_report.json", + "highest_latency_step": "CrewAI - Geocode Destination", + "highest_step_latency": 0.5961 + }, + { + "framework": "autogen", + "workflow_name": "Benchmark - AutoGen - Live Travel Briefing", + "step_count": 6, + "total_tokens": 768, + "prompt_tokens": 600, + "completion_tokens": 168, + "total_cost": 0.0001908, + "total_latency": 1.4647, + "tool_calls": 3, + "retriever_steps": 1, + "retrieved_chunks": 3, + "memory_tokens": 0, + "highest_token_step": "AutoGen - Synthesize Travel Briefing", + "highest_cost_step": "AutoGen - Synthesize Travel Briefing", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\autogen\\live_travel_report.json", + "highest_latency_step": "AutoGen - Geocode Destination", + "highest_step_latency": 0.6201 + }, + { + "framework": "llamaindex", + "workflow_name": "Benchmark - LlamaIndex - Live Travel Briefing", + "step_count": 6, + "total_tokens": 768, + "prompt_tokens": 600, + "completion_tokens": 168, + "total_cost": 0.0001908, + "total_latency": 1.4865, + "tool_calls": 3, + "retriever_steps": 1, + "retrieved_chunks": 3, + "memory_tokens": 0, + "highest_token_step": "LlamaIndex - Synthesize Travel Briefing", + "highest_cost_step": "LlamaIndex - Synthesize Travel Briefing", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\llamaindex\\live_travel_report.json", + "highest_latency_step": "LlamaIndex - Geocode Destination", + "highest_step_latency": 0.6174 + }, + { + "framework": "semantic_kernel", + "workflow_name": "Benchmark - Semantic Kernel - Live Travel Briefing", + "step_count": 6, + "total_tokens": 768, + "prompt_tokens": 600, + "completion_tokens": 168, + "total_cost": 0.0001908, + "total_latency": 1.6088, + "tool_calls": 3, + "retriever_steps": 1, + "retrieved_chunks": 3, + "memory_tokens": 0, + "highest_token_step": "Semantic Kernel - Synthesize Travel Briefing", + "highest_cost_step": "Semantic Kernel - Synthesize Travel Briefing", + "report_path": "E:\\agenticlens\\benchmarks\\reports\\semantic_kernel\\live_travel_report.json", + "highest_latency_step": "Semantic Kernel - Geocode Destination", + "highest_step_latency": 0.6471 + } +] \ No newline at end of file diff --git a/benchmarks/results/live_travel_summary.md b/benchmarks/results/live_travel_summary.md new file mode 100644 index 0000000..13c5deb --- /dev/null +++ b/benchmarks/results/live_travel_summary.md @@ -0,0 +1,18 @@ +# AgenticLens Live Travel Briefing Benchmark + +Use case: real-time trip briefing (weather, currency, destination facts). + +Every tool/retriever step below calls a real, live, free API (Open-Meteo geocoding + forecast, Frankfurter exchange rates, Wikipedia REST summary) -- no API key required, no mocking. LLM steps use a deterministic fallback unless OPENAI_API_KEY is set, so token counts are stable across frameworks but latency is genuinely live. + +| Framework | Total Tokens | Cost | Latency | Steps | Tool Calls | Retrieved Chunks | Highest Latency Step | +|---|---:|---:|---:|---:|---:|---:|---| +| native_python | 768 | $0.000191 | 1.592s | 6 | 3 | 3 | Native Python - Geocode Destination | +| langgraph | 768 | $0.000191 | 1.480s | 6 | 3 | 3 | LangGraph - Geocode Destination | +| crewai | 768 | $0.000191 | 1.578s | 6 | 3 | 3 | CrewAI - Geocode Destination | +| autogen | 768 | $0.000191 | 1.465s | 6 | 3 | 3 | AutoGen - Geocode Destination | +| llamaindex | 768 | $0.000191 | 1.486s | 6 | 3 | 3 | LlamaIndex - Geocode Destination | +| semantic_kernel | 768 | $0.000191 | 1.609s | 6 | 3 | 3 | Semantic Kernel - Geocode Destination | + +## Interpretation + +Total tokens and cost are near-identical across frameworks because the LLM steps are a deterministic fallback (no OPENAI_API_KEY set). Latency is the meaningful column here: it reflects real network round-trips to four live APIs, not scripted timings, so it varies between runs and frameworks based on real network conditions and per-framework overhead. \ No newline at end of file diff --git a/benchmarks/shared/__init__.py b/benchmarks/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/shared/benchmark_runner.py b/benchmarks/shared/benchmark_runner.py new file mode 100644 index 0000000..90d58b1 --- /dev/null +++ b/benchmarks/shared/benchmark_runner.py @@ -0,0 +1,136 @@ +import csv +import json +import os +import subprocess +from pathlib import Path + +from benchmarks.shared.metrics_collector import summarize_agenticlens_report + +ROOT = Path(__file__).resolve().parents[1] +PROJECT_ROOT = ROOT.parent + +FRAMEWORKS = { + "native_python": ROOT / "frameworks" / "native_python" / "run_native.py", + "langgraph": ROOT / "frameworks" / "langgraph" / "run_langgraph.py", + "crewai": ROOT / "frameworks" / "crewai" / "run_crewai.py", + "autogen": ROOT / "frameworks" / "autogen" / "run_autogen.py", + "llamaindex": ROOT / "frameworks" / "llamaindex" / "run_llamaindex.py", + "semantic_kernel": ROOT / "frameworks" / "semantic_kernel" / "run_semantic_kernel.py", +} + +REPORT_DIR = ROOT / "reports" +RESULTS_DIR = ROOT / "results" + + +def run_framework(framework_name: str, script_path: Path) -> Path | None: + if not script_path.exists(): + print(f"Skipping {framework_name}: file not found: {script_path}") + return None + + framework_report_dir = REPORT_DIR / framework_name + framework_report_dir.mkdir(parents=True, exist_ok=True) + + report_path = framework_report_dir / "support_refund_report.json" + + cmd = [ + "agenticlens", + "profile", + str(script_path), + "--save", + str(report_path), + ] + + # Each framework script does `from benchmarks.shared...`, which only resolves + # if the project root is importable in the subprocess -- add it to PYTHONPATH + # rather than relying on each script to patch sys.path itself. + env = os.environ.copy() + existing_path = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = ( + f"{PROJECT_ROOT}{os.pathsep}{existing_path}" if existing_path else str(PROJECT_ROOT) + ) + + print(f"\nRunning {framework_name} benchmark...") + subprocess.run(cmd, check=True, env=env) + + return report_path + + +def write_csv(results: list[dict], output_path: Path) -> None: + if not results: + return + + fieldnames = list(results[0].keys()) + + with output_path.open("w", newline="", encoding="utf-8") as file: + writer = csv.DictWriter(file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(results) + + +def write_markdown_summary(results: list[dict], output_path: Path) -> None: + lines = [ + "# AgenticLens Practical Benchmark Summary", + "", + "Use case: Customer Support Refund Copilot", + "", + "This benchmark profiles the same practical support workflow across multiple " + "agentic implementations.", + "", + "| Framework | Total Tokens | Cost | Latency | Steps | Tool Calls | " + "Retrieved Chunks | Highest Token Step | Highest Cost Step |", + "|---|---:|---:|---:|---:|---:|---:|---|---|", + ] + + for row in results: + lines.append( + "| {framework} | {total_tokens} | ${total_cost:.6f} | {total_latency:.3f}s | " + "{step_count} | {tool_calls} | {retrieved_chunks} | {highest_token_step} | " + "{highest_cost_step} |".format(**row) + ) + + lines.extend( + [ + "", + "## Interpretation", + "", + "These results are workload-specific. They should not be read as a " + "universal ranking of frameworks.", + "The goal is to show how AgenticLens makes token usage, cost, latency, " + "retrieval behavior, and tool activity visible across workflows.", + ] + ) + + output_path.write_text("\n".join(lines), encoding="utf-8") + + +def main() -> None: + REPORT_DIR.mkdir(parents=True, exist_ok=True) + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + + results = [] + + for framework_name, script_path in FRAMEWORKS.items(): + report_path = run_framework(framework_name, script_path) + + if report_path is None: + continue + + summary = summarize_agenticlens_report(framework_name, report_path) + results.append(summary) + + json_output = RESULTS_DIR / "benchmark_results.json" + csv_output = RESULTS_DIR / "benchmark_results.csv" + md_output = RESULTS_DIR / "benchmark_summary.md" + + json_output.write_text(json.dumps(results, indent=2), encoding="utf-8") + write_csv(results, csv_output) + write_markdown_summary(results, md_output) + + print("\nBenchmark complete.") + print(f"JSON: {json_output}") + print(f"CSV: {csv_output}") + print(f"Markdown: {md_output}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/shared/live_benchmark_runner.py b/benchmarks/shared/live_benchmark_runner.py new file mode 100644 index 0000000..983b600 --- /dev/null +++ b/benchmarks/shared/live_benchmark_runner.py @@ -0,0 +1,147 @@ +"""Runs the live (real-API) travel-briefing workflow across all six frameworks. + +Companion to benchmark_runner.py, which profiles a fully deterministic +workload. This one calls real, live, free APIs (Open-Meteo, Frankfurter, +Wikipedia) for every tool/retriever step, so token counts stay identical +across frameworks (the LLM steps are still a deterministic fallback without +an OPENAI_API_KEY) but latency genuinely differs run to run and framework to +framework -- it reflects real network conditions, not scripted timings. +""" + +import csv +import json +import os +import subprocess +from pathlib import Path + +from benchmarks.shared.metrics_collector import summarize_agenticlens_report + +ROOT = Path(__file__).resolve().parents[1] +PROJECT_ROOT = ROOT.parent + +FRAMEWORKS = { + "native_python": ROOT / "frameworks" / "native_python" / "run_native_live.py", + "langgraph": ROOT / "frameworks" / "langgraph" / "run_langgraph_live.py", + "crewai": ROOT / "frameworks" / "crewai" / "run_crewai_live.py", + "autogen": ROOT / "frameworks" / "autogen" / "run_autogen_live.py", + "llamaindex": ROOT / "frameworks" / "llamaindex" / "run_llamaindex_live.py", + "semantic_kernel": ROOT / "frameworks" / "semantic_kernel" / "run_semantic_kernel_live.py", +} + +REPORT_DIR = ROOT / "reports" +RESULTS_DIR = ROOT / "results" + + +def run_framework(framework_name: str, script_path: Path) -> Path | None: + if not script_path.exists(): + print(f"Skipping {framework_name}: file not found: {script_path}") + return None + + framework_report_dir = REPORT_DIR / framework_name + framework_report_dir.mkdir(parents=True, exist_ok=True) + + report_path = framework_report_dir / "live_travel_report.json" + + cmd = ["agenticlens", "profile", str(script_path), "--save", str(report_path)] + + env = os.environ.copy() + existing_path = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = ( + f"{PROJECT_ROOT}{os.pathsep}{existing_path}" if existing_path else str(PROJECT_ROOT) + ) + + print(f"\nRunning {framework_name} live travel benchmark...") + subprocess.run(cmd, check=True, env=env) + + return report_path + + +def write_csv(results: list[dict], output_path: Path) -> None: + if not results: + return + + fieldnames = list(results[0].keys()) + + with output_path.open("w", newline="", encoding="utf-8") as file: + writer = csv.DictWriter(file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(results) + + +def write_markdown_summary(results: list[dict], output_path: Path) -> None: + lines = [ + "# AgenticLens Live Travel Briefing Benchmark", + "", + "Use case: real-time trip briefing (weather, currency, destination facts).", + "", + "Every tool/retriever step below calls a real, live, free API " + "(Open-Meteo geocoding + forecast, Frankfurter exchange rates, Wikipedia " + "REST summary) -- no API key required, no mocking. LLM steps use a " + "deterministic fallback unless OPENAI_API_KEY is set, so token counts " + "are stable across frameworks but latency is genuinely live.", + "", + "| Framework | Total Tokens | Cost | Latency | Steps | Tool Calls | " + "Retrieved Chunks | Highest Latency Step |", + "|---|---:|---:|---:|---:|---:|---:|---|", + ] + + for row in results: + lines.append( + "| {framework} | {total_tokens} | ${total_cost:.6f} | {total_latency:.3f}s | " + "{step_count} | {tool_calls} | {retrieved_chunks} | " + "{highest_latency_step} |".format(**row) + ) + + lines.extend( + [ + "", + "## Interpretation", + "", + "Total tokens and cost are near-identical across frameworks because the " + "LLM steps are a deterministic fallback (no OPENAI_API_KEY set). Latency " + "is the meaningful column here: it reflects real network round-trips to " + "four live APIs, not scripted timings, so it varies between runs and " + "frameworks based on real network conditions and per-framework overhead.", + ] + ) + + output_path.write_text("\n".join(lines), encoding="utf-8") + + +def main() -> None: + REPORT_DIR.mkdir(parents=True, exist_ok=True) + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + + results = [] + + for framework_name, script_path in FRAMEWORKS.items(): + report_path = run_framework(framework_name, script_path) + + if report_path is None: + continue + + summary = summarize_agenticlens_report(framework_name, report_path) + + report = json.loads(report_path.read_text(encoding="utf-8")) + highest_latency = max(report.get("steps", []), key=lambda s: s["metrics"]["latency"]) + summary["highest_latency_step"] = highest_latency["name"] + summary["highest_step_latency"] = round(highest_latency["metrics"]["latency"], 4) + + results.append(summary) + + json_output = RESULTS_DIR / "live_travel_results.json" + csv_output = RESULTS_DIR / "live_travel_results.csv" + md_output = RESULTS_DIR / "live_travel_summary.md" + + json_output.write_text(json.dumps(results, indent=2), encoding="utf-8") + write_csv(results, csv_output) + write_markdown_summary(results, md_output) + + print("\nLive travel benchmark complete.") + print(f"JSON: {json_output}") + print(f"CSV: {csv_output}") + print(f"Markdown: {md_output}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/shared/live_travel_tasks.py b/benchmarks/shared/live_travel_tasks.py new file mode 100644 index 0000000..1d693d0 --- /dev/null +++ b/benchmarks/shared/live_travel_tasks.py @@ -0,0 +1,132 @@ +"""Shared real-API + LLM task functions for the live cross-framework benchmark. + +Unlike benchmarks/shared/support_tasks.py, the tool/retriever calls here hit +real, live, free, no-auth-required APIs, so latency (and the underlying data) +is genuinely non-deterministic run to run and framework to framework -- this +is what makes it a meaningful latency comparison rather than a fixed-timing +fixture. +""" + +import json +import os +import time +import urllib.request +from typing import Any + +USE_REAL_OPENAI = bool(os.getenv("OPENAI_API_KEY")) +OPENAI_MODEL = "gpt-4o-mini" + +USER_AGENT = "AgenticLens-Benchmark/1.0 (+https://github.com/DeepAgentLabs/agenticlens)" + +TRIP = {"origin": "New York", "destination": "Tokyo", "purpose": "work trip"} + +QUESTION = ( + "I'm flying from New York to Tokyo next week for a work trip. What should I know before I go?" +) + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeMessage: + def __init__(self, content: str): + self.content = content + + +class FakeChoice: + def __init__(self, content: str): + self.message = FakeMessage(content) + + +class FakeResponse: + def __init__(self, content: str, prompt_tokens: int, completion_tokens: int): + self.usage = FakeUsage(prompt_tokens, completion_tokens) + self.choices = [FakeChoice(content)] + + +# Also used to normalize *real* per-framework LLM responses (CrewAI's +# CrewOutput.token_usage, AutoGen's RequestUsage, LlamaIndex's +# additional_kwargs, Semantic Kernel's metadata["usage"], LangChain's +# usage_metadata) into the single shape `StepHandle.record()` understands, +# so every framework's real call still flows through the same AgenticLens +# provider-detection path as the fallback. +LLMResponse = FakeResponse + + +def classify_trip(framework: str) -> tuple[FakeResponse, float]: + start = time.time() + response = FakeResponse( + content=f"{framework}: intent=trip_briefing_request; destination={TRIP['destination']}", + prompt_tokens=140, + completion_tokens=28, + ) + return response, time.time() - start + + +def synthesize_briefing( + framework: str, + place: dict[str, Any], +) -> tuple[FakeResponse, float]: + start = time.time() + response = FakeResponse( + content=( + f"[{framework}] Heads-up for your {place['name']} trip: pack for the current " + "conditions, budget at today's live rate, and skim the destination facts below." + ), + prompt_tokens=460, + completion_tokens=140, + ) + return response, time.time() - start + + +def fetch_json(url: str) -> dict[str, Any]: + """Real HTTP call -- no mocking, no API key required for any of these.""" + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=10) as resp: + return json.load(resp) # type: ignore[no-any-return] + + +def geocode_city(city: str) -> tuple[dict[str, Any], float]: + start = time.time() + data = fetch_json(f"https://geocoding-api.open-meteo.com/v1/search?name={city}&count=1") + result = data["results"][0] + place = { + "name": result["name"], + "country": result.get("country"), + "lat": result["latitude"], + "lon": result["longitude"], + } + return place, time.time() - start + + +def fetch_weather(lat: float, lon: float) -> tuple[dict[str, Any], float]: + start = time.time() + data = fetch_json( + f"https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}" + "¤t=temperature_2m,weather_code,wind_speed_10m" + ) + return data["current"], time.time() - start # type: ignore[no-any-return] + + +def fetch_exchange_rate(base: str, target: str) -> tuple[dict[str, Any], float]: + start = time.time() + data = fetch_json(f"https://api.frankfurter.app/latest?from={base}&to={target}") + fx = {"base": data["base"], "date": data["date"], "rate": data["rates"][target]} + return fx, time.time() - start + + +def fetch_destination_summary(city: str) -> tuple[dict[str, Any], list[str], float]: + start = time.time() + data = fetch_json(f"https://en.wikipedia.org/api/rest_v1/page/summary/{city}") + summary = {"title": data["title"], "extract": data["extract"]} + paragraphs = [p.strip() for p in summary["extract"].split(". ") if p.strip()] + return summary, paragraphs, time.time() - start + + +def estimate_avg_tokens_per_chunk(paragraphs: list[str]) -> int: + if not paragraphs: + return 0 + return round(sum(len(p) for p in paragraphs) / len(paragraphs) / 4) diff --git a/benchmarks/shared/metrics_collector.py b/benchmarks/shared/metrics_collector.py new file mode 100644 index 0000000..07332df --- /dev/null +++ b/benchmarks/shared/metrics_collector.py @@ -0,0 +1,82 @@ +import json +from pathlib import Path +from typing import Any + + +def load_report(report_path: str | Path) -> dict[str, Any]: + path = Path(report_path) + return json.loads(path.read_text(encoding="utf-8")) + + +def summarize_agenticlens_report(framework: str, report_path: str | Path) -> dict[str, Any]: + report = load_report(report_path) + steps = report.get("steps", []) + + total_tokens = 0 + prompt_tokens = 0 + completion_tokens = 0 + total_cost = 0.0 + total_latency = 0.0 + tool_calls = 0 + retriever_steps = 0 + retrieved_chunks = 0 + memory_tokens = 0 + + highest_token_step = None + highest_step_tokens = -1 + + highest_cost_step = None + highest_step_cost = -1.0 + + for step in steps: + metrics = step.get("metrics") or {} + metadata = step.get("metadata") or {} + step_type = step.get("type") + + step_total_tokens = metrics.get("total_tokens") or 0 + step_prompt_tokens = metrics.get("prompt_tokens") or 0 + step_completion_tokens = metrics.get("completion_tokens") or 0 + step_cost = metrics.get("cost") or 0.0 + step_latency = metrics.get("latency") or 0.0 + + total_tokens += step_total_tokens + prompt_tokens += step_prompt_tokens + completion_tokens += step_completion_tokens + total_cost += step_cost + total_latency += step_latency + + if step_type == "tool_call": + tool_calls += 1 + + if step_type == "retriever": + retriever_steps += 1 + retrieved_chunks += metadata.get("chunk_count") or 0 + + if step_type == "memory": + memory_tokens += metadata.get("history_tokens") or step_prompt_tokens + + if step_total_tokens > highest_step_tokens: + highest_step_tokens = step_total_tokens + highest_token_step = step.get("name") + + if step_cost > highest_step_cost: + highest_step_cost = step_cost + highest_cost_step = step.get("name") + + return { + "framework": framework, + "workflow_name": report.get("name"), + "step_count": len(steps), + "total_tokens": total_tokens, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_cost": round(total_cost, 8), + "total_latency": round(total_latency, 4), + "tool_calls": tool_calls, + "retriever_steps": retriever_steps, + "retrieved_chunks": retrieved_chunks, + "memory_tokens": memory_tokens, + "highest_token_step": highest_token_step, + "highest_cost_step": highest_cost_step, + "report_path": str(report_path), + } diff --git a/benchmarks/shared/support_data.py b/benchmarks/shared/support_data.py new file mode 100644 index 0000000..77bb7b8 --- /dev/null +++ b/benchmarks/shared/support_data.py @@ -0,0 +1,75 @@ +import json +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +DATASET_DIR = ROOT / "datasets" + + +def load_json(filename: str) -> list[dict[str, Any]]: + path = DATASET_DIR / filename + return json.loads(path.read_text(encoding="utf-8")) + + +def load_support_cases() -> list[dict[str, Any]]: + return load_json("support_cases.json") + + +def load_policy_docs() -> list[dict[str, Any]]: + return load_json("refund_policy_docs.json") + + +def load_orders() -> list[dict[str, Any]]: + return load_json("orders.json") + + +def lookup_order(order_id: str) -> dict[str, Any]: + orders = load_orders() + + for order in orders: + if order["order_id"] == order_id: + return { + "found": True, + **order, + } + + return { + "found": False, + "order_id": order_id, + } + + +def simple_retrieve(query: str, top_k: int = 6) -> list[dict[str, Any]]: + docs = load_policy_docs() + + clean_query = query.lower().replace("?", "").replace(".", "") + query_words = set(clean_query.split()) + + scored_docs = [] + + for doc in docs: + clean_doc = doc["text"].lower().replace("?", "").replace(".", "") + doc_words = set(clean_doc.split()) + score = len(query_words.intersection(doc_words)) + scored_docs.append((score, doc)) + + scored_docs.sort(reverse=True, key=lambda item: item[0]) + + return [doc for score, doc in scored_docs[:top_k] if score > 0] + + +def estimate_avg_tokens_per_chunk(chunks: list[dict[str, Any]]) -> int: + if not chunks: + return 0 + + total_words = sum(len(chunk["text"].split()) for chunk in chunks) + + # Rough practical approximation: + # 1 word is around 1.3 tokens in many English text workflows. + estimated_tokens = int(total_words * 1.3) + + return max(1, estimated_tokens // len(chunks)) + + +def build_policy_context(chunks: list[dict[str, Any]]) -> str: + return "\n".join(f"- {chunk['text']}" for chunk in chunks) diff --git a/benchmarks/shared/support_tasks.py b/benchmarks/shared/support_tasks.py new file mode 100644 index 0000000..433cadf --- /dev/null +++ b/benchmarks/shared/support_tasks.py @@ -0,0 +1,101 @@ +import time +from typing import Any + +from benchmarks.shared.support_data import ( + build_policy_context, + estimate_avg_tokens_per_chunk, + lookup_order, + simple_retrieve, +) + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeMessage: + def __init__(self, content: str): + self.content = content + + +class FakeChoice: + def __init__(self, content: str): + self.message = FakeMessage(content) + + +class FakeResponse: + def __init__(self, content: str, prompt_tokens: int, completion_tokens: int): + self.usage = FakeUsage(prompt_tokens, completion_tokens) + self.choices = [FakeChoice(content)] + + +def classify_ticket(ticket: str, framework: str) -> FakeResponse: + return FakeResponse( + content=f"framework={framework}; intent=refund_request; priority=normal", + prompt_tokens=180, + completion_tokens=25, + ) + + +def rewrite_query(ticket: str, framework: str) -> FakeResponse: + return FakeResponse( + content=( + "refund eligibility delivered order opened package unused item refund processing time" + ), + prompt_tokens=220, + completion_tokens=35, + ) + + +def retrieve_policy(query: str, top_k: int = 6) -> tuple[list[dict[str, Any]], str, int, float]: + start = time.time() + chunks = simple_retrieve(query, top_k=top_k) + latency = time.time() - start + policy_context = build_policy_context(chunks) + avg_tokens = estimate_avg_tokens_per_chunk(chunks) + return chunks, policy_context, avg_tokens, latency + + +def lookup_order_tool(order_id: str) -> tuple[dict[str, Any], float]: + start = time.time() + order = lookup_order(order_id) + latency = time.time() - start + return order, latency + + +def check_refund_eligibility( + ticket: str, + order: dict[str, Any], + policy_context: str, + framework: str, +) -> FakeResponse: + return FakeResponse( + content=( + f"{framework}: The order is within the 30-day refund window. " + "The item was not used, but the package was opened, so manual review may be required." + ), + prompt_tokens=720, + completion_tokens=95, + ) + + +def generate_customer_reply( + ticket: str, + order: dict[str, Any], + policy_context: str, + decision: str, + framework: str, +) -> FakeResponse: + return FakeResponse( + content=( + f"[{framework}] Your order is within the 30-day refund window. " + "Since the package was opened, the refund may need manual review. " + "Because the item was not used, you may still be eligible. " + "If approved, the refund will return to your original payment method and " + "may take 5 to 10 business days." + ), + prompt_tokens=850, + completion_tokens=130, + ) diff --git a/docs/langchain-integration.md b/docs/langchain-integration.md new file mode 100644 index 0000000..818fa31 --- /dev/null +++ b/docs/langchain-integration.md @@ -0,0 +1,61 @@ +# LangChain Integration + +AgenticLens can auto-instrument a LangChain (or LangGraph) run through its +callback system, instead of requiring a manual `with step(...)` block around +every LLM call, tool call, and retrieval. + +## Installation + +```bash +pip install "agenticlens[langchain]" +``` + +## Usage + +```python +from agenticlens import profile +from agenticlens.adapters.langchain import AgenticLensCallbackHandler + +handler = AgenticLensCallbackHandler() + +with profile("My LangChain App") as workflow: + chain.invoke(inputs, config={"callbacks": [handler]}) + +print(workflow.total_tokens) +print(workflow.total_cost) +``` + +The handler must be used inside a `with profile(...):` block -- it attaches +each step to whichever workflow is active in the current context. + +## What gets tracked + +| LangChain event | AgenticLens step type | Metadata captured | +| --- | --- | --- | +| `on_llm_start` / `on_chat_model_start` → `on_llm_end` | `llm_call` | `prompt`, token usage, `model` (when reported) | +| `on_tool_start` → `on_tool_end` | `tool_call` | `tool_name`, `tool_args` | +| `on_retriever_start` → `on_retriever_end` | `retriever` | `query`, `chunk_count`, `avg_tokens_per_chunk` | + +Token usage is extracted in priority order: + +1. Per-message `usage_metadata` (populated by most current LangChain chat + model integrations) +2. `llm_output["token_usage"]` / `llm_output["usage"]` (older, provider-specific + integrations) +3. If neither is present, token fields stay `0` for that step -- the step + itself, its latency, and its metadata are still recorded. + +`avg_tokens_per_chunk` for retriever steps is estimated from +`len(document.page_content) / 4` -- LangChain retrievers return text, not a +token count, and this is only an approximation used to feed the +excessive-chunks recommender's savings estimate. + +Because the adapter produces the same `Step` shape as the manual `step()` API, +every recommender, exporter, and the CLI work against LangChain-sourced +workflows without any changes. + +## What doesn't get tracked + +Chain-level (`on_chain_*`) events are intentionally ignored. LCEL chains fire +one such event per internal runnable, which would flood the workflow with +steps that don't correspond to a real LLM/tool/retrieval cost. diff --git a/examples/live_multiagent_travel_briefing.py b/examples/live_multiagent_travel_briefing.py new file mode 100644 index 0000000..f743e3c --- /dev/null +++ b/examples/live_multiagent_travel_briefing.py @@ -0,0 +1,240 @@ +"""Live multi-agent demo: real external APIs, profiled end-to-end with AgenticLens. + +Unlike the other examples, every tool/retriever step here makes a real network +call to a live, free, no-auth-required API: + +- Open-Meteo Geocoding -- resolve a city name to coordinates +- Open-Meteo Forecast -- current weather at those coordinates +- Frankfurter -- live currency exchange rate +- Wikipedia REST API -- destination summary, used as retrieved context + +LLM steps use a real OpenAI call when OPENAI_API_KEY is set, and a +deterministic fallback otherwise -- the same pattern as +examples/support_copilot.py. The point is to profile a workflow where latency +and step behavior are genuinely variable (live network calls), not scripted. +""" + +import json +import os +import time +import urllib.request +from typing import Any + +from agenticlens import profile, step +from agenticlens.recommenders import RecommendationEngine + +USE_REAL_OPENAI = bool(os.getenv("OPENAI_API_KEY")) +USER_AGENT = "AgenticLens-Demo/1.0 (+https://github.com/DeepAgentLabs/agenticlens)" + +if USE_REAL_OPENAI: + from openai import OpenAI + + client = OpenAI() + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeMessage: + def __init__(self, content: str): + self.content = content + + +class FakeChoice: + def __init__(self, content: str): + self.message = FakeMessage(content) + + +class FakeResponse: + def __init__(self, content: str, prompt_tokens: int, completion_tokens: int): + self.usage = FakeUsage(prompt_tokens, completion_tokens) + self.choices = [FakeChoice(content)] + + +def fake_llm(task: str) -> FakeResponse: + if task == "classify": + return FakeResponse( + content="intent: trip_briefing_request; origin=New York; destination=Tokyo", + prompt_tokens=140, + completion_tokens=28, + ) + return FakeResponse( + content=( + "Heads-up for your Tokyo trip: pack for the current conditions, budget in " + "JPY at today's live rate, and skim the destination facts below before you go." + ), + prompt_tokens=460, + completion_tokens=140, + ) + + +def call_llm(task: str, prompt: str) -> Any: + if not USE_REAL_OPENAI: + return fake_llm(task) + + return client.chat.completions.create( + model="gpt-4o-mini", + temperature=0, + messages=[{"role": "user", "content": prompt}], + ) + + +def fetch_json(url: str) -> dict[str, Any]: + """Real HTTP call -- no mocking, no API key required for any of these.""" + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(req, timeout=10) as resp: + return json.load(resp) # type: ignore[no-any-return] + + +def geocode_city(city: str) -> dict[str, Any]: + data = fetch_json(f"https://geocoding-api.open-meteo.com/v1/search?name={city}&count=1") + result = data["results"][0] + return { + "name": result["name"], + "country": result.get("country"), + "lat": result["latitude"], + "lon": result["longitude"], + } + + +def fetch_weather(lat: float, lon: float) -> dict[str, Any]: + data = fetch_json( + f"https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}" + "¤t=temperature_2m,weather_code,wind_speed_10m" + ) + return data["current"] # type: ignore[no-any-return] + + +def fetch_exchange_rate(base: str, target: str) -> dict[str, Any]: + data = fetch_json(f"https://api.frankfurter.app/latest?from={base}&to={target}") + return {"base": data["base"], "date": data["date"], "rate": data["rates"][target]} + + +def fetch_destination_summary(city: str) -> dict[str, Any]: + data = fetch_json(f"https://en.wikipedia.org/api/rest_v1/page/summary/{city}") + return {"title": data["title"], "extract": data["extract"]} + + +def main() -> None: + trip = {"traveler_id": "TRV-4471", "origin": "New York", "destination": "Tokyo"} + question = ( + "I'm flying from New York to Tokyo next week for a work trip. " + "What should I know before I go?" + ) + + with profile("Live Travel Briefing - Multi-Agent") as workflow: + with step( + "Planner - Classify Trip Request", + type="planner", + provider="openai" if USE_REAL_OPENAI else None, + model="gpt-4o-mini" if USE_REAL_OPENAI else None, + prompt=question, + traveler_id=trip["traveler_id"], + ) as s: + start = time.time() + resp = call_llm("classify", f"Classify this trip request:\n{question}") + s.record(resp) + s.step.metrics.latency = time.time() - start + intent = resp.choices[0].message.content + + with step( + "Geocode Destination", + type="tool_call", + tool_name="open_meteo_geocoding", + tool_args={"city": trip["destination"]}, + ) as s: + start = time.time() + place = geocode_city(trip["destination"]) + s.step.metrics.latency = time.time() - start + s.step.metadata["tool_result"] = place + + with step( + "Fetch Live Weather", + type="tool_call", + tool_name="open_meteo_forecast", + tool_args={"lat": place["lat"], "lon": place["lon"]}, + ) as s: + start = time.time() + weather = fetch_weather(place["lat"], place["lon"]) + s.step.metrics.latency = time.time() - start + s.step.metadata["tool_result"] = weather + + with step( + "Fetch Live Exchange Rate", + type="tool_call", + tool_name="frankfurter_exchange_rate", + tool_args={"base": "USD", "target": "JPY"}, + ) as s: + start = time.time() + fx = fetch_exchange_rate("USD", "JPY") + s.step.metrics.latency = time.time() - start + s.step.metadata["tool_result"] = fx + + with step( + "Retrieve Destination Facts", + type="retriever", + query=trip["destination"], + ) as s: + start = time.time() + summary = fetch_destination_summary(trip["destination"]) + paragraphs = [p.strip() for p in summary["extract"].split(". ") if p.strip()] + s.step.metrics.latency = time.time() - start + s.step.metadata["chunk_count"] = len(paragraphs) + s.step.metadata["avg_tokens_per_chunk"] = round( + sum(len(p) for p in paragraphs) / max(len(paragraphs), 1) / 4 + ) + s.step.metadata["retrieved_doc_ids"] = [summary["title"]] + + with step( + "Synthesize Travel Briefing", + type="final_response", + provider="openai" if USE_REAL_OPENAI else None, + model="gpt-4o-mini" if USE_REAL_OPENAI else None, + prompt=question, + ) as s: + start = time.time() + synth_prompt = ( + f"Traveler question: {question}\n\n" + f"Intent: {intent}\n" + f"Live weather at {place['name']}: {weather}\n" + f"Live USD->JPY rate: {fx['rate']} (as of {fx['date']})\n" + f"Destination facts: {summary['extract']}\n\n" + "Write a concise, friendly travel briefing using only this data." + ) + resp = call_llm("brief", synth_prompt) + s.record(resp) + s.step.metrics.latency = time.time() - start + briefing = resp.choices[0].message.content + + print("=" * 72) + print("TRAVEL BRIEFING") + print("=" * 72) + print(briefing) + print() + print(f"(live weather: {weather})") + print(f"(live rate: 1 USD = {fx['rate']} JPY as of {fx['date']})") + + print("\nWorkflow summary:") + print(f" Total tokens: {workflow.total_tokens}") + print(f" Total cost: ${workflow.total_cost or 0:.6f}") + wall = (workflow.end_time - workflow.start_time).total_seconds() + print(f" Wall latency: {wall:.3f}s (dominated by real network calls, not tokens)") + + print("\nStep breakdown:") + for st in workflow.steps: + print( + f" - {st.name:<32} {st.type.value:<15} " + f"{st.metrics.total_tokens:>5} tok {st.metrics.latency * 1000:>8.1f} ms" + ) + + recs = RecommendationEngine().run(workflow) + print(f"\nRecommendations: {len(recs)}") + for r in recs: + print(f" - {r.title}: {r.description}") + + +if __name__ == "__main__": + main() diff --git a/examples/multiagent_edge_cases_demo.py b/examples/multiagent_edge_cases_demo.py new file mode 100644 index 0000000..4da3392 --- /dev/null +++ b/examples/multiagent_edge_cases_demo.py @@ -0,0 +1,128 @@ +from agenticlens import profile, step + +SYSTEM_PROMPT = ( + "You are a careful travel support assistant. " + "Use only verified policy, booking, and refund information. " + "Avoid guessing. Explain next steps clearly. " +) * 20 + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int) -> None: + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeResponse: + def __init__(self, prompt_tokens: int, completion_tokens: int) -> None: + self.usage = FakeUsage(prompt_tokens, completion_tokens) + + +def main() -> None: + user_question = ( + "My flight was cancelled. I booked a hotel and airport taxi. " + "Can I get a refund and what should I do next?" + ) + + with profile("Multi-Agent Travel Refund Edge Case Workflow"): + # Edge case 1: large repeated system prompt + with step( + "Planner Agent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=SYSTEM_PROMPT + user_question, + ) as s: + s.record(FakeResponse(prompt_tokens=1200, completion_tokens=180)) + + # Edge case 2: excessive retrieved chunks + # Default recommender limit is usually 8. This sends 14 chunks. + with step( + "Policy Retriever Agent", + type="retriever", + chunk_count=14, + avg_tokens_per_chunk=90, + ): + pass + + # Edge case 3: long memory history + with step( + "Memory Agent", + type="memory", + provider="openai", + model="gpt-4o-mini", + history_tokens=7200, + ) as s: + s.record(FakeResponse(prompt_tokens=7400, completion_tokens=0)) + + # Edge case 4: normal tool call + with step( + "Tool Agent - Lookup Booking", + type="tool_call", + provider="openai", + model="gpt-4o-mini", + tool_name="lookup_booking", + tool_args={"booking_id": "TRV-8842"}, + ) as s: + s.record(FakeResponse(prompt_tokens=220, completion_tokens=60)) + + # Edge case 5: duplicate tool call with same args + with step( + "Tool Agent - Lookup Booking Retry", + type="tool_call", + provider="openai", + model="gpt-4o-mini", + tool_name="lookup_booking", + tool_args={"booking_id": "TRV-8842"}, + ) as s: + s.record(FakeResponse(prompt_tokens=220, completion_tokens=60)) + + # Another tool call, not duplicate because tool and args are different + with step( + "Tool Agent - Check Refund Eligibility", + type="tool_call", + provider="openai", + model="gpt-4o-mini", + tool_name="check_refund_eligibility", + tool_args={ + "booking_id": "TRV-8842", + "reason": "flight_cancelled", + "hotel_used": False, + "taxi_used": False, + }, + ) as s: + s.record(FakeResponse(prompt_tokens=300, completion_tokens=80)) + + # Writer agent adds useful value, but also adds token cost + with step( + "Writer Agent", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt="Write a customer-friendly refund explanation.", + ) as s: + s.record(FakeResponse(prompt_tokens=950, completion_tokens=260)) + + # Reviewer agent adds quality control cost + with step( + "Reviewer Agent", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt="Review the answer for policy accuracy and missing next steps.", + ) as s: + s.record(FakeResponse(prompt_tokens=780, completion_tokens=160)) + + # Edge case 6: repeated system prompt again + with step( + "Final Response Agent", + type="final_response", + provider="anthropic", + model="claude-3-5-sonnet", + prompt=SYSTEM_PROMPT + "Give the final customer-facing answer.", + ) as s: + s.record(FakeResponse(prompt_tokens=1100, completion_tokens=240)) + + +if __name__ == "__main__": + main() diff --git a/examples/support_copilot.py b/examples/support_copilot.py new file mode 100644 index 0000000..3aa3642 --- /dev/null +++ b/examples/support_copilot.py @@ -0,0 +1,350 @@ +import os +import re +import sqlite3 +import time +from typing import Any + +from agenticlens import profile, step + +USE_REAL_OPENAI = bool(os.getenv("OPENAI_API_KEY")) + +if USE_REAL_OPENAI: + from openai import OpenAI + + client = OpenAI() + + +POLICY_DOCS = [ + { + "id": "refund_001", + "text": "Customers can request a refund within 30 days of delivery.", + }, + { + "id": "refund_002", + "text": "Items must be unused and in original packaging to qualify for a standard refund.", + }, + { + "id": "refund_003", + "text": "Opened items may require manual review unless the item is defective.", + }, + { + "id": "refund_004", + "text": "Refunds are processed to the original payment method.", + }, + { + "id": "refund_005", + "text": "Refunds may take 5 to 10 business days after approval.", + }, + { + "id": "shipping_001", + "text": ( + "Delivered orders are eligible for return review if the delivery date " + "is within the return window." + ), + }, +] + + +SYSTEM_PROMPT = """ +You are a customer support copilot. +Use only the provided policy and order information. +Do not invent refund rules. +If manual review is needed, clearly say so. +""" + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeMessage: + def __init__(self, content: str): + self.content = content + + +class FakeChoice: + def __init__(self, content: str): + self.message = FakeMessage(content) + + +class FakeResponse: + def __init__(self, content: str, prompt_tokens: int, completion_tokens: int): + self.usage = FakeUsage(prompt_tokens, completion_tokens) + self.choices = [FakeChoice(content)] + + +def setup_order_db() -> sqlite3.Connection: + conn = sqlite3.connect(":memory:") + conn.execute( + """ + CREATE TABLE orders ( + order_id TEXT PRIMARY KEY, + status TEXT, + delivered_days_ago INTEGER, + package_opened INTEGER, + item_used INTEGER, + payment_method TEXT + ) + """ + ) + conn.execute( + """ + INSERT INTO orders VALUES + ('A123', 'delivered', 12, 1, 0, 'Visa ending 4242') + """ + ) + return conn + + +def extract_order_id(text: str) -> str | None: + match = re.search(r"\b[A-Z]\d{3}\b", text) + return match.group(0) if match else None + + +def retrieve_policy_chunks(query: str, top_k: int = 6) -> list[dict[str, str]]: + query_words = set(query.lower().replace("?", "").replace(".", "").split()) + + scored = [] + for doc in POLICY_DOCS: + doc_words = set(doc["text"].lower().replace(".", "").split()) + score = len(query_words.intersection(doc_words)) + scored.append((score, doc)) + + scored.sort(reverse=True, key=lambda x: x[0]) + return [doc for score, doc in scored[:top_k] if score > 0] + + +def lookup_order(conn: sqlite3.Connection, order_id: str) -> dict[str, Any]: + row = conn.execute( + """ + SELECT order_id, status, delivered_days_ago, package_opened, item_used, payment_method + FROM orders + WHERE order_id = ? + """, + (order_id,), + ).fetchone() + + if not row: + return {"found": False, "order_id": order_id} + + return { + "found": True, + "order_id": row[0], + "status": row[1], + "delivered_days_ago": row[2], + "package_opened": bool(row[3]), + "item_used": bool(row[4]), + "payment_method": row[5], + } + + +def fake_llm(task: str, prompt: str) -> FakeResponse: + if task == "classify": + return FakeResponse( + content="intent: refund_request; urgency: normal", + prompt_tokens=180, + completion_tokens=20, + ) + + if task == "rewrite": + return FakeResponse( + content="refund eligibility delivered order opened package unused item processing time", + prompt_tokens=220, + completion_tokens=30, + ) + + if task == "decision": + return FakeResponse( + content=( + "The order is within the 30-day window and the item is unused. " + "However, because the package was opened, manual review may be required." + ), + prompt_tokens=520, + completion_tokens=70, + ) + + return FakeResponse( + content=( + "Your order A123 was delivered 12 days ago, so it is within the 30-day " + "refund window. Because the package was opened, the refund may need " + "manual review, but since the item was not used, you may still be " + "eligible. If approved, the refund will go back to your original " + "payment method and may take 5 to 10 business days after approval." + ), + prompt_tokens=850, + completion_tokens=120, + ) + + +def call_llm(task: str, prompt: str): + if not USE_REAL_OPENAI: + return fake_llm(task, prompt) + + return client.chat.completions.create( + model="gpt-4o-mini", + temperature=0, + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt}, + ], + ) + + +def main() -> None: + conn = setup_order_db() + + ticket = { + "ticket_id": "TCK-1001", + "tenant_id": "retail-us", + "customer_id": "CUST-789", + "message": ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ), + } + + with profile("Practical Support Copilot - Refund Ticket") as workflow: + with step( + "Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket["message"], + ticket_id=ticket["ticket_id"], + tenant_id=ticket["tenant_id"], + ) as s: + start = time.time() + classify_response = call_llm( + "classify", + f"Classify this support ticket:\n{ticket['message']}", + ) + s.record(classify_response) + s.step.metrics.latency = time.time() - start + intent = classify_response.choices[0].message.content + + with step( + "Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket["message"], + ) as s: + start = time.time() + rewrite_response = call_llm( + "rewrite", + f"Rewrite this ticket as a search query for refund policy retrieval:\n" + f"{ticket['message']}", + ) + s.record(rewrite_response) + s.step.metrics.latency = time.time() - start + search_query = rewrite_response.choices[0].message.content + + with step( + "Retrieve Refund Policy", + type="retriever", + chunk_count=6, + avg_tokens_per_chunk=35, + query=search_query, + ) as s: + start = time.time() + policy_chunks = retrieve_policy_chunks(search_query, top_k=6) + s.step.metrics.latency = time.time() - start + s.step.metadata["chunk_count"] = len(policy_chunks) + s.step.metadata["retrieved_doc_ids"] = [doc["id"] for doc in policy_chunks] + s.step.metadata["retrieved_chunks"] = [doc["text"] for doc in policy_chunks] + + order_id = extract_order_id(ticket["message"]) + + with step( + "Lookup Order System", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + ) as s: + start = time.time() + order = lookup_order(conn, order_id) + s.step.metrics.latency = time.time() - start + s.step.metadata["tool_result"] = order + + with step( + "Refund Eligibility Decision", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=SYSTEM_PROMPT, + ) as s: + start = time.time() + decision_response = call_llm( + "decision", + f""" +Ticket: +{ticket["message"]} + +Intent: +{intent} + +Order: +{order} + +Policy: +{[doc["text"] for doc in policy_chunks]} + +Decide refund eligibility and whether human review is needed. +""", + ) + s.record(decision_response) + s.step.metrics.latency = time.time() - start + decision = decision_response.choices[0].message.content + + with step( + "Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=SYSTEM_PROMPT, + ) as s: + start = time.time() + final_response = call_llm( + "final", + f""" +Customer message: +{ticket["message"]} + +Order: +{order} + +Policy: +{[doc["text"] for doc in policy_chunks]} + +Eligibility decision: +{decision} + +Write the final customer-facing response. +""", + ) + s.record(final_response) + s.step.metrics.latency = time.time() - start + answer = final_response.choices[0].message.content + + print("\nFinal customer reply:\n") + print(answer) + + print("\nWorkflow summary:") + print("Total tokens:", workflow.total_tokens) + print("Total cost:", workflow.total_cost) + + print("\nStep summary:") + for st in workflow.steps: + print( + f"- {st.name}: " + f"{st.metrics.total_tokens or 0} tokens, " + f"${st.metrics.cost or 0:.6f}, " + f"{st.metrics.latency or 0:.3f}s" + ) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index e085c8a..f16a2ea 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -15,6 +15,7 @@ nav: - Export Formats: export-formats.md - RAG Chunk Utility: rag-chunk-utility.md - Workflow Schema Spec: workflow-schema-spec.md + - LangChain Integration: langchain-integration.md - Trace and Comparison: trace-and-comparison.md - Evaluation and Release Gates: evaluation-and-release-gates.md - Multi-Agent Reference Workflows: multi-agent-reference-workflows.md diff --git a/pyproject.toml b/pyproject.toml index 65f14f4..b5e6032 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,9 @@ dev = [ docs = [ "mkdocs-material>=9.5", ] +langchain = [ + "langchain-core>=0.3", +] langgraph = [ "langgraph>=1.0,<2", ] @@ -98,6 +101,12 @@ mypy_path = "src" module = ["jsonschema", "referencing", "referencing.*"] ignore_missing_imports = true +[[tool.mypy.overrides]] +# langchain-core is an optional dependency (the `langchain` extra) whose own +# transitive imports ship stubs newer than our target Python version. +module = ["langchain_core", "langchain_core.*"] +follow_imports = "skip" + [tool.pytest.ini_options] testpaths = ["tests"] pythonpath = ["src"] diff --git a/src/agenticlens/adapters/__init__.py b/src/agenticlens/adapters/__init__.py new file mode 100644 index 0000000..07096a9 --- /dev/null +++ b/src/agenticlens/adapters/__init__.py @@ -0,0 +1,6 @@ +"""Optional auto-instrumentation adapters for third-party agent frameworks. + +Adapters are not imported here so that `agenticlens` itself never requires +the frameworks they integrate with. Import the specific adapter module you +need, e.g. `from agenticlens.adapters.langchain import AgenticLensCallbackHandler`. +""" diff --git a/src/agenticlens/adapters/langchain.py b/src/agenticlens/adapters/langchain.py new file mode 100644 index 0000000..0e10e2c --- /dev/null +++ b/src/agenticlens/adapters/langchain.py @@ -0,0 +1,260 @@ +"""Auto-instrumentation adapter for LangChain / LangGraph via its callback system. + +Requires the optional `langchain-core` dependency: + + pip install "agenticlens[langchain]" + +Usage: + + from agenticlens import profile + from agenticlens.adapters.langchain import AgenticLensCallbackHandler + + with profile("My LangChain App"): + chain.invoke(inputs, config={"callbacks": [AgenticLensCallbackHandler()]}) + +The handler creates one AgenticLens step per LLM call, tool call, and retriever +call, using the run lifecycle to time each one and, for LLM calls, extracting +token usage the way the rest of AgenticLens does from `s.record(...)`. +""" + +from __future__ import annotations + +import threading +import time +from typing import Any +from uuid import UUID + +try: + from langchain_core.callbacks.base import BaseCallbackHandler + from langchain_core.outputs import LLMResult +except ImportError as exc: # pragma: no cover - exercised only without the extra installed + raise ImportError( + "agenticlens.adapters.langchain requires the 'langchain-core' package. " + 'Install it with: pip install "agenticlens[langchain]"' + ) from exc + +from agenticlens.models.enums import StepType +from agenticlens.models.step import Step +from agenticlens.profiler.context import get_active_workflow + +CHARS_PER_TOKEN_ESTIMATE = 4 +"""Rough chars-to-tokens heuristic used only for retrieved-chunk size estimates. + +LangChain retrievers return `Document` text, not a token count, and the +excessive-chunks recommender needs `avg_tokens_per_chunk` to estimate savings. +This is intentionally approximate. +""" + + +def _extract_llm_usage(response: LLMResult) -> tuple[int, int] | None: + """Best-effort prompt/completion token extraction from an `LLMResult`. + + Tries the modern per-message `usage_metadata` first (populated by most + current LangChain chat model integrations), then falls back to the + provider-specific `llm_output` dict used by older integrations. + """ + for generations in response.generations: + for generation in generations: + message = getattr(generation, "message", None) + usage = getattr(message, "usage_metadata", None) if message is not None else None + if usage: + return usage.get("input_tokens", 0), usage.get("output_tokens", 0) + + llm_output = response.llm_output or {} + token_usage = llm_output.get("token_usage") or llm_output.get("usage") + if token_usage: + prompt = token_usage.get("prompt_tokens", token_usage.get("input_tokens")) + completion = token_usage.get("completion_tokens", token_usage.get("output_tokens")) + if prompt is not None and completion is not None: + return int(prompt), int(completion) + + return None + + +class AgenticLensCallbackHandler(BaseCallbackHandler): # type: ignore[misc] + """Turns LangChain callback events into AgenticLens steps automatically. + + Must be used inside a `with profile(...):` block -- each tracked run is + attached to whichever workflow is active in the current context when that + run starts. + + Chain-level (`on_chain_*`) events are intentionally not tracked: LCEL + chains fire one per internal runnable, which would flood the workflow + with steps that don't correspond to a real LLM/tool/retrieval cost. + """ + + def __init__(self, provider: str | None = None) -> None: + super().__init__() + self._provider = provider + self._lock = threading.Lock() + self._runs: dict[UUID, tuple[Step, float]] = {} + + def _start(self, run_id: UUID, step_type: StepType, name: str, **metadata: Any) -> None: + step_model = Step( + name=name, + type=step_type, + provider=self._provider, + metadata={k: v for k, v in metadata.items() if v is not None}, + ) + get_active_workflow().steps.append(step_model) + with self._lock: + self._runs[run_id] = (step_model, time.perf_counter()) + + def _finish(self, run_id: UUID) -> Step | None: + with self._lock: + entry = self._runs.pop(run_id, None) + if entry is None: + return None + step_model, start = entry + step_model.metrics.latency = time.perf_counter() - start + return step_model + + # LLM / chat model events + + def on_llm_start( + self, + serialized: dict[str, Any], + prompts: list[str], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + name = (serialized or {}).get("name") or "LLM Call" + self._start(run_id, StepType.LLM_CALL, name, prompt=prompts[0] if prompts else None) + + def on_chat_model_start( + self, + serialized: dict[str, Any], + messages: list[list[Any]], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + name = (serialized or {}).get("name") or "Chat Model Call" + prompt = None + if messages and messages[0]: + prompt = "\n".join(str(getattr(m, "content", m)) for m in messages[0]) + self._start(run_id, StepType.LLM_CALL, name, prompt=prompt) + + def on_llm_end( + self, + response: LLMResult, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + **kwargs: Any, + ) -> None: + step_model = self._finish(run_id) + if step_model is None: + return + + usage = _extract_llm_usage(response) + if usage is not None: + prompt_tokens, completion_tokens = usage + step_model.metrics.prompt_tokens = prompt_tokens + step_model.metrics.completion_tokens = completion_tokens + step_model.metrics.total_tokens = prompt_tokens + completion_tokens + + if step_model.model is None and response.llm_output: + step_model.model = response.llm_output.get("model_name") + + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + **kwargs: Any, + ) -> None: + self._finish(run_id) + + # Tool events + + def on_tool_start( + self, + serialized: dict[str, Any], + input_str: str, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + inputs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + tool_name = (serialized or {}).get("name") or "Tool Call" + tool_args = inputs if inputs is not None else {"input": input_str} + self._start(run_id, StepType.TOOL_CALL, tool_name, tool_name=tool_name, tool_args=tool_args) + + def on_tool_end( + self, + output: Any, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + self._finish(run_id) + + def on_tool_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + self._finish(run_id) + + # Retriever events + + def on_retriever_start( + self, + serialized: dict[str, Any], + query: str, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + name = (serialized or {}).get("name") or "Retriever" + self._start(run_id, StepType.RETRIEVER, name, query=query) + + def on_retriever_end( + self, + documents: list[Any], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + step_model = self._finish(run_id) + if step_model is None: + return + + step_model.metadata["chunk_count"] = len(documents) + if documents: + avg_chars = sum(len(getattr(d, "page_content", "")) for d in documents) / len(documents) + step_model.metadata["avg_tokens_per_chunk"] = round( + avg_chars / CHARS_PER_TOKEN_ESTIMATE + ) + + def on_retriever_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + self._finish(run_id) diff --git a/tests/test_adapters_langchain.py b/tests/test_adapters_langchain.py new file mode 100644 index 0000000..9f14a70 --- /dev/null +++ b/tests/test_adapters_langchain.py @@ -0,0 +1,145 @@ +from uuid import uuid4 + +import pytest +from langchain_core.documents import Document +from langchain_core.messages import AIMessage +from langchain_core.outputs import ChatGeneration, Generation, LLMResult + +from agenticlens import profile +from agenticlens.adapters.langchain import AgenticLensCallbackHandler +from agenticlens.models.enums import StepType +from agenticlens.recommenders import RecommendationEngine + + +def test_llm_call_records_usage_from_message_metadata() -> None: + handler = AgenticLensCallbackHandler() + run_id = uuid4() + + with profile("Test") as workflow: + handler.on_chat_model_start({"name": "ChatOpenAI"}, [[]], run_id=run_id) + result = LLMResult( + generations=[ + [ + ChatGeneration( + message=AIMessage( + content="hi", + usage_metadata={ + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + }, + ) + ) + ] + ] + ) + handler.on_llm_end(result, run_id=run_id) + + assert len(workflow.steps) == 1 + step = workflow.steps[0] + assert step.type == StepType.LLM_CALL + assert step.metrics.prompt_tokens == 10 + assert step.metrics.completion_tokens == 5 + assert step.metrics.total_tokens == 15 + assert step.metrics.latency >= 0 + + +def test_llm_call_falls_back_to_llm_output_token_usage() -> None: + handler = AgenticLensCallbackHandler() + run_id = uuid4() + + with profile("Test") as workflow: + handler.on_llm_start({"name": "OpenAI"}, ["hello"], run_id=run_id) + result = LLMResult( + generations=[[Generation(text="hi")]], + llm_output={ + "token_usage": {"prompt_tokens": 20, "completion_tokens": 8}, + "model_name": "gpt-4o-mini", + }, + ) + handler.on_llm_end(result, run_id=run_id) + + step = workflow.steps[0] + assert step.metrics.prompt_tokens == 20 + assert step.metrics.completion_tokens == 8 + assert step.metadata["prompt"] == "hello" + assert step.model == "gpt-4o-mini" + + +def test_llm_error_finishes_step_without_tokens() -> None: + handler = AgenticLensCallbackHandler() + run_id = uuid4() + + with profile("Test") as workflow: + handler.on_llm_start({"name": "OpenAI"}, ["hello"], run_id=run_id) + handler.on_llm_error(RuntimeError("boom"), run_id=run_id) + + step = workflow.steps[0] + assert step.metrics.prompt_tokens == 0 + assert step.metrics.latency >= 0 + + +def test_tool_call_records_name_and_args() -> None: + handler = AgenticLensCallbackHandler() + run_id = uuid4() + + with profile("Test") as workflow: + handler.on_tool_start( + {"name": "lookup_order"}, + '{"order_id": "A123"}', + run_id=run_id, + inputs={"order_id": "A123"}, + ) + handler.on_tool_end("found", run_id=run_id) + + step = workflow.steps[0] + assert step.type == StepType.TOOL_CALL + assert step.metadata["tool_name"] == "lookup_order" + assert step.metadata["tool_args"] == {"order_id": "A123"} + + +def test_retriever_records_chunk_count_and_avg_tokens() -> None: + handler = AgenticLensCallbackHandler() + run_id = uuid4() + + with profile("Test") as workflow: + handler.on_retriever_start({"name": "Retriever"}, "refund policy", run_id=run_id) + handler.on_retriever_end( + [Document(page_content="x" * 40), Document(page_content="x" * 20)], + run_id=run_id, + ) + + step = workflow.steps[0] + assert step.type == StepType.RETRIEVER + assert step.metadata["chunk_count"] == 2 + assert step.metadata["avg_tokens_per_chunk"] == 8 # (40 + 20) / 2 / 4 chars-per-token + + +def test_duplicate_tool_calls_detected_from_adapter_metadata() -> None: + handler = AgenticLensCallbackHandler() + + with profile("Test") as workflow: + for _ in range(2): + run_id = uuid4() + handler.on_tool_start( + {"name": "lookup_order"}, + "", + run_id=run_id, + inputs={"order_id": "A123"}, + ) + handler.on_tool_end("found", run_id=run_id) + + recs = RecommendationEngine().run(workflow) + assert any(r.title == "Duplicate tool call" for r in recs) + + +def test_finish_on_unknown_run_id_is_a_noop() -> None: + handler = AgenticLensCallbackHandler() + with profile("Test"): + handler.on_llm_end(LLMResult(generations=[]), run_id=uuid4()) + + +def test_handler_outside_profile_raises() -> None: + handler = AgenticLensCallbackHandler() + with pytest.raises(RuntimeError): + handler.on_llm_start({"name": "OpenAI"}, ["hello"], run_id=uuid4()) diff --git a/uv.lock b/uv.lock index bda2bf3..a778512 100644 --- a/uv.lock +++ b/uv.lock @@ -40,6 +40,9 @@ dev = [ docs = [ { name = "mkdocs-material" }, ] +langchain = [ + { name = "langchain-core" }, +] langgraph = [ { name = "langgraph" }, ] @@ -51,6 +54,7 @@ multi-agent = [ requires-dist = [ { name = "build", marker = "extra == 'dev'", specifier = ">=1.2" }, { name = "jsonschema", specifier = ">=4.23" }, + { name = "langchain-core", marker = "extra == 'langchain'", specifier = ">=0.3" }, { name = "langgraph", marker = "extra == 'langgraph'", specifier = ">=1.0,<2" }, { name = "langgraph", marker = "extra == 'multi-agent'", specifier = ">=1.0,<2" }, { name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9.5" }, @@ -66,7 +70,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0" }, ] -provides-extras = ["dev", "docs", "langgraph", "multi-agent"] +provides-extras = ["dev", "docs", "langchain", "langgraph", "multi-agent"] [[package]] name = "annotated-doc"