From 95f3b5153710a2ab4db1215ef2cbfdc27b64913d Mon Sep 17 00:00:00 2001 From: manemsai Date: Sun, 5 Jul 2026 23:34:33 -0500 Subject: [PATCH 1/9] updated readme file --- README.md | 335 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 324 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 8ed1425..b02e5f4 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,346 @@ # AgenticLens -An open-source profiler for AI agents that analyzes token usage, cost, latency, and optimization opportunities across LLM workflows. +AgenticLens is an open-source profiler for LLM applications, RAG pipelines, and agentic workflows. -> **Status:** early scaffold. Core data models, provider abstraction, and the explicit `profile()`/`step()` instrumentation API are in place. The recommendation engine's heuristic rules are not yet implemented — see [AgenticLens_Spec.md](AgenticLens_Spec.md). +It helps developers understand where tokens, cost, and latency are being spent across each step of an AI workflow. -## Install (development) +> **Status:** MVP. Core profiling, CLI reports, exporters, and heuristic recommendation rules are implemented. + +AgenticLens tracks: + +- Prompt tokens +- Completion tokens +- Total tokens +- Estimated cost +- Latency +- Workflow steps +- Tool calls +- Retrieval metadata +- Optimization opportunities + +## Why AgenticLens? + +LLM applications often become expensive and slow as they grow from simple prompts into RAG pipelines, tool-using agents, and multi-agent workflows. + +AgenticLens gives developers a lightweight way to answer: + +- Which step used the most tokens? +- Which agent was most expensive? +- Did retrieval send too many chunks? +- Is conversation history too large? +- Are we repeating the same system prompt? +- Did the workflow call the same tool twice? +- How much token usage can be reduced? + +## Installation + +### From PyPI + +```bash +pip install agenticlens +``` + +### Development Install ```bash uv sync --extra dev ``` -## Usage +or: + +```bash +pip install -e . +``` + +## Quick Start ```python from agenticlens import profile, step -with profile("Customer Support"): - with step("Planner", type="planner") as s: +with profile("Customer Support") as workflow: + with step( + "Planner", + type="planner", + provider="openai", + model="gpt-4o-mini", + ) as s: response = planner_llm.invoke(prompt) s.record(response) + +print(workflow.total_tokens) +print(workflow.total_cost) +``` + +## Core Concepts + +| Concept | Meaning | +|---|---| +| `profile()` | Starts profiling one workflow | +| `step()` | Tracks one operation inside the workflow | +| `s.record(response)` | Extracts token usage from an LLM response | +| `provider` | LLM provider, such as `openai` or `anthropic` | +| `model` | Model name used for cost estimation | +| `metadata` | Extra step details such as retrieved chunks, tool names, prompts, or history size | + +## Supported Step Types + +AgenticLens supports these step types: + +```text +planner +retriever +memory +tool_call +llm_call +final_response +``` + +## CLI Usage + +Run and profile a Python script: + +```bash +agenticlens profile examples/basic_usage.py +``` + +Save a workflow report: + +```bash +agenticlens profile examples/basic_usage.py --save report.json +``` + +View a saved report: + +```bash +agenticlens report report.json +``` + +Analyze optimization opportunities: + +```bash +agenticlens analyze report.json +``` + +## Example Output + +```text +╔═ Customer Support ═╗ +║ Total Tokens 160 ║ +║ Total Cost $0.00 ║ +║ Latency 0.12 sec ║ +╚═══════════════════╝ + +Step Breakdown + +Planner planner 120 prompt 40 completion +``` + +## RAG Example + +```python +from agenticlens import profile, step + +with profile("RAG Workflow") as workflow: + with step( + "Retrieve Chunks", + type="retriever", + chunk_count=5, + avg_tokens_per_chunk=120, + ): + chunks = retriever.search(query) + + with step( + "Generate Answer", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + ) as s: + response = llm.invoke(query, context=chunks) + s.record(response) +``` + +## Multi-Agent Example + +```python +from agenticlens import profile, step + +with profile("Multi-Agent Support Workflow") as workflow: + with step( + "Planner Agent", + type="planner", + provider="openai", + model="gpt-4o-mini", + ) as s: + response = planner_agent.run(user_query) + s.record(response) + + with step( + "Retriever Agent", + type="retriever", + chunk_count=8, + ): + chunks = retriever.search(user_query) + + with step( + "Tool Agent - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": "A123"}, + ): + order = lookup_order("A123") + + with step( + "Final Response Agent", + type="final_response", + provider="openai", + model="gpt-4o-mini", + ) as s: + response = final_agent.run(user_query, chunks, order) + s.record(response) ``` +## Optimization Recommendations + +AgenticLens can identify common token waste patterns: + +| Recommendation | Meaning | +|---|---| +| Repeated system prompt | Same long prompt appears across multiple steps | +| Excessive retrieved chunks | Retriever sends more chunks than the configured limit | +| Long conversation history | Memory/history exceeds the configured token threshold | +| Duplicate tool call | Same tool is called again with the same arguments | + +Example: + +```bash +agenticlens analyze report.json +``` + +Output: + +```text +Optimization Suggestions + +* Repeated system prompt + -- Step 'Final Response' repeats the same prompt prefix as 'Planner'. (~295 tokens) + +* Excessive retrieved chunks + -- Step 'Retriever' retrieved 12 chunks, 4 more than the configured limit of 8. (~320 tokens) + +Estimated Savings: 32% +``` + +## Exporters + +AgenticLens supports exporting workflow reports. + +```python +from agenticlens.exporters import JSONExporter, CSVExporter + +JSONExporter().export(workflow, "report.json") +CSVExporter().export(workflow, "steps.csv") +``` + +## Examples + +Example scripts are available in: + +```text +examples/basic_usage.py +examples/recommendations_demo.py +examples/rag_customer_support_demo.py +examples/multiagent_support_demo.py +``` + +## Notebooks + +Beginner-friendly notebooks are available in: + +```text +notebooks/agenticlens_workflow_demo_beginner.ipynb +notebooks/agenticlens_multiagent_demo_beginner.ipynb +``` + +The notebooks show: + +- Step-by-step RAG profiling +- Step-by-step multi-agent profiling +- Token usage tables +- Latency charts +- Cost charts +- Saved AgenticLens reports +- Optimization analysis + ## Development +Install development dependencies: + +```bash +uv sync --extra dev +``` + +Run tests: + +```bash +uv run pytest +``` + +Run linting: + +```bash +uv run ruff check . +``` + +Format code: + +```bash +uv run ruff format . +``` + +Run type checks: + +```bash +uv run mypy +``` + +## Test Status + +The current test suite covers: + +- CLI commands +- Profiling API +- Models +- Providers +- Pricing +- Exporters +- Recommendation engine +- Recommendation rules + +Example: + ```bash -uv run pytest # tests -uv run ruff check . # lint -uv run ruff format . # format -uv run mypy # type check +pytest -v +``` + +Expected result: + +```text +42 passed ``` -See [AgenticLens_Spec.md](AgenticLens_Spec.md) for the full project specification and [ROADMAP.md](ROADMAP.md) for what's planned beyond the MVP. +## Project Docs + +See: + +- [AgenticLens_Spec.md](AgenticLens_Spec.md) +- [ROADMAP.md](ROADMAP.md) + +## Positioning + +AgenticLens is a lightweight, developer-first profiler for token usage, cost, latency, and optimization suggestions. + +It is not intended to replace full observability platforms such as LangSmith, Langfuse, Helicone, or Phoenix. It is designed to be simple, local-first, and easy to add to Python-based LLM workflows. + +## License + +MIT \ No newline at end of file From ab608e9c399ef76b101e418aad8933f52bede189 Mon Sep 17 00:00:00 2001 From: manemsai Date: Sun, 5 Jul 2026 23:34:33 -0500 Subject: [PATCH 2/9] updated readme file --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index b3363cd..71a7fff 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,17 @@ Other examples: Some examples call real provider APIs and require provider API keys. +## Notebooks + +Beginner-friendly notebooks are available in: + +- `notebooks/agenticlens_workflow_demo_beginner.ipynb` +- `notebooks/agenticlens_multiagent_demo_beginner.ipynb` + +The notebooks walk through step-by-step RAG and multi-agent profiling, token +usage tables, latency/cost charts, saved AgenticLens reports, and optimization +analysis. + ## Exporting Reports ### Markdown From 3cbef6943a5256f24400e52d40b995496e7269bb Mon Sep 17 00:00:00 2001 From: manemsai Date: Tue, 14 Jul 2026 17:46:18 -0500 Subject: [PATCH 3/9] Add cross-framework benchmark harness and practical support examples Adds a benchmark suite that runs the same refund-ticket workload through AutoGen, CrewAI, LangGraph, LlamaIndex, Semantic Kernel, and native Python, profiling each with AgenticLens to normalize tokens/cost/latency for an apples-to-apples comparison. Also adds two more realistic examples (support_copilot.py, multiagent_edge_cases_demo.py) exercising the full step lifecycle plus recommender edge cases. Co-Authored-By: Claude Sonnet 5 --- .gitignore | 5 + benchmarks/__init__.py | 0 benchmarks/compare_results.py | 250 +++++++++++++ benchmarks/datasets/orders.json | 29 ++ benchmarks/datasets/refund_policy_docs.json | 42 +++ benchmarks/datasets/support_cases.json | 31 ++ benchmarks/frameworks/__init__.py | 0 benchmarks/frameworks/autogen/__init__.py | 0 benchmarks/frameworks/autogen/run_autogen.py | 135 +++++++ benchmarks/frameworks/crewai/__init__.py | 0 benchmarks/frameworks/crewai/run_crewai.py | 182 +++++++++ benchmarks/frameworks/langgraph/__init__.py | 0 .../frameworks/langgraph/run_langgraph.py | 243 ++++++++++++ benchmarks/frameworks/llamaindex/__init__.py | 0 .../frameworks/llamaindex/run_llamaindex.py | 134 +++++++ .../frameworks/native_python/__init__.py | 0 .../frameworks/native_python/run_native.py | 169 +++++++++ .../frameworks/semantic_kernel/__init__.py | 0 .../semantic_kernel/run_semantic_kernel.py | 120 ++++++ .../autogen/support_refund_report.json | 147 ++++++++ .../reports/crewai/support_refund_report.json | 145 ++++++++ .../langgraph/support_refund_report.json | 137 +++++++ .../llamaindex/support_refund_report.json | 147 ++++++++ .../native_python/support_refund_report.json | 138 +++++++ .../support_refund_report.json | 145 ++++++++ benchmarks/results/benchmark_cost_chart.png | Bin 0 -> 26868 bytes benchmarks/results/benchmark_results.csv | 7 + .../results/benchmark_step_breakdown.csv | 37 ++ benchmarks/results/benchmark_summary.md | 32 ++ benchmarks/results/benchmark_tokens_chart.png | Bin 0 -> 25734 bytes benchmarks/shared/__init__.py | 0 benchmarks/shared/benchmark_runner.py | 123 +++++++ benchmarks/shared/metrics_collector.py | 82 +++++ benchmarks/shared/support_data.py | 76 ++++ benchmarks/shared/support_tasks.py | 98 +++++ examples/multiagent_edge_cases_demo.py | 132 +++++++ examples/support_copilot.py | 346 ++++++++++++++++++ 37 files changed, 3132 insertions(+) create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/compare_results.py create mode 100644 benchmarks/datasets/orders.json create mode 100644 benchmarks/datasets/refund_policy_docs.json create mode 100644 benchmarks/datasets/support_cases.json create mode 100644 benchmarks/frameworks/__init__.py create mode 100644 benchmarks/frameworks/autogen/__init__.py create mode 100644 benchmarks/frameworks/autogen/run_autogen.py create mode 100644 benchmarks/frameworks/crewai/__init__.py create mode 100644 benchmarks/frameworks/crewai/run_crewai.py create mode 100644 benchmarks/frameworks/langgraph/__init__.py create mode 100644 benchmarks/frameworks/langgraph/run_langgraph.py create mode 100644 benchmarks/frameworks/llamaindex/__init__.py create mode 100644 benchmarks/frameworks/llamaindex/run_llamaindex.py create mode 100644 benchmarks/frameworks/native_python/__init__.py create mode 100644 benchmarks/frameworks/native_python/run_native.py create mode 100644 benchmarks/frameworks/semantic_kernel/__init__.py create mode 100644 benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py create mode 100644 benchmarks/reports/autogen/support_refund_report.json create mode 100644 benchmarks/reports/crewai/support_refund_report.json create mode 100644 benchmarks/reports/langgraph/support_refund_report.json create mode 100644 benchmarks/reports/llamaindex/support_refund_report.json create mode 100644 benchmarks/reports/native_python/support_refund_report.json create mode 100644 benchmarks/reports/semantic_kernel/support_refund_report.json create mode 100644 benchmarks/results/benchmark_cost_chart.png create mode 100644 benchmarks/results/benchmark_results.csv create mode 100644 benchmarks/results/benchmark_step_breakdown.csv create mode 100644 benchmarks/results/benchmark_summary.md create mode 100644 benchmarks/results/benchmark_tokens_chart.png create mode 100644 benchmarks/shared/__init__.py create mode 100644 benchmarks/shared/benchmark_runner.py create mode 100644 benchmarks/shared/metrics_collector.py create mode 100644 benchmarks/shared/support_data.py create mode 100644 benchmarks/shared/support_tasks.py create mode 100644 examples/multiagent_edge_cases_demo.py create mode 100644 examples/support_copilot.py diff --git a/.gitignore b/.gitignore index d78e6f4..c4c111e 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,11 @@ site/ .env test/ +# Generated workflow reports from running examples/benchmarks locally +/*_report.json +/workflow.json +/report.json + # Local patent/provisional drafting artifacts docs/patents/ tools/build_agenticlens_provisional.py diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/compare_results.py b/benchmarks/compare_results.py new file mode 100644 index 0000000..49b549a --- /dev/null +++ b/benchmarks/compare_results.py @@ -0,0 +1,250 @@ +import json +from pathlib import Path + +import pandas as pd +import matplotlib.pyplot as plt + + +REPORTS = { + "Native Python": "benchmarks/reports/native_python/support_refund_report.json", + "LangGraph": "benchmarks/reports/langgraph/support_refund_report.json", + "CrewAI": "benchmarks/reports/crewai/support_refund_report.json", + "AutoGen": "benchmarks/reports/autogen/support_refund_report.json", + "LlamaIndex": "benchmarks/reports/llamaindex/support_refund_report.json", + "Semantic Kernel": "benchmarks/reports/semantic_kernel/support_refund_report.json", +} + +RESULTS_DIR = Path("benchmarks/results") +RESULTS_DIR.mkdir(parents=True, exist_ok=True) + + +def load_report(path: str | Path) -> dict: + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"Report not found: {path}") + return json.loads(path.read_text(encoding="utf-8")) + + +def summarize_report(framework: str, report: dict) -> dict: + steps = report.get("steps", []) + + total_tokens = 0 + prompt_tokens = 0 + completion_tokens = 0 + total_cost = 0.0 + total_latency = 0.0 + tool_calls = 0 + retrieved_chunks = 0 + + highest_token_step = None + highest_step_tokens = -1 + + highest_cost_step = None + highest_step_cost = -1.0 + + for step in steps: + metrics = step.get("metrics") or {} + metadata = step.get("metadata") or {} + + step_tokens = metrics.get("total_tokens") or 0 + step_prompt_tokens = metrics.get("prompt_tokens") or 0 + step_completion_tokens = metrics.get("completion_tokens") or 0 + step_cost = metrics.get("cost") or 0.0 + step_latency = metrics.get("latency") or 0.0 + + total_tokens += step_tokens + prompt_tokens += step_prompt_tokens + completion_tokens += step_completion_tokens + total_cost += step_cost + total_latency += step_latency + + if step.get("type") == "tool_call": + tool_calls += 1 + + if step.get("type") == "retriever": + retrieved_chunks += metadata.get("chunk_count") or 0 + + if step_tokens > highest_step_tokens: + highest_step_tokens = step_tokens + highest_token_step = step.get("name") + + if step_cost > highest_step_cost: + highest_step_cost = step_cost + highest_cost_step = step.get("name") + + return { + "framework": framework, + "workflow_name": report.get("name"), + "step_count": len(steps), + "total_tokens": total_tokens, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_cost_usd": round(total_cost, 8), + "total_latency_sec": round(total_latency, 8), + "tool_calls": tool_calls, + "retrieved_chunks": retrieved_chunks, + "highest_token_step": highest_token_step, + "highest_step_tokens": highest_step_tokens, + "highest_cost_step": highest_cost_step, + "highest_step_cost_usd": round(highest_step_cost, 8), + } + + +def extract_step_rows(framework: str, report: dict) -> list[dict]: + rows = [] + + for step in report.get("steps", []): + metrics = step.get("metrics") or {} + metadata = step.get("metadata") or {} + + rows.append( + { + "framework": framework, + "workflow_name": report.get("name"), + "step_name": step.get("name"), + "step_type": step.get("type"), + "provider": step.get("provider"), + "model": step.get("model"), + "prompt_tokens": metrics.get("prompt_tokens") or 0, + "completion_tokens": metrics.get("completion_tokens") or 0, + "total_tokens": metrics.get("total_tokens") or 0, + "cost_usd": metrics.get("cost") or 0.0, + "latency_sec": metrics.get("latency") or 0.0, + "chunk_count": metadata.get("chunk_count"), + "tool_name": metadata.get("tool_name"), + } + ) + + return rows + + +def create_markdown_summary(summary_df: pd.DataFrame, output_path: Path) -> None: + lines = [ + "# AgenticLens Framework Benchmark Comparison", + "", + "Use case: Practical customer support refund workflow.", + "", + "The workflow includes:", + "", + "- ticket intent classification", + "- query rewriting", + "- refund policy retrieval", + "- order lookup", + "- refund eligibility check", + "- customer reply generation", + "", + "## Summary Results", + "", + "| Framework | Total Tokens | Prompt Tokens | Completion Tokens | Cost USD | Latency Sec | Steps | Tool Calls | Retrieved Chunks | Highest Token Step |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", + ] + + for _, row in summary_df.iterrows(): + lines.append( + f"| {row['framework']} | " + f"{row['total_tokens']} | " + f"{row['prompt_tokens']} | " + f"{row['completion_tokens']} | " + f"${row['total_cost_usd']:.8f} | " + f"{row['total_latency_sec']:.8f} | " + f"{row['step_count']} | " + f"{row['tool_calls']} | " + f"{row['retrieved_chunks']} | " + f"{row['highest_token_step']} |" + ) + + lines.extend( + [ + "", + "## Key Finding", + "", + "The final customer reply step is the highest token-consuming step across the benchmark runs.", + "", + "## Important Note", + "", + "These results are workload-specific. They should not be treated as a universal ranking of frameworks.", + "The purpose is to show how AgenticLens can normalize and compare token, cost, latency, retrieval, and tool-call metrics across framework implementations.", + ] + ) + + output_path.write_text("\n".join(lines), encoding="utf-8") + + +def plot_total_tokens(summary_df: pd.DataFrame) -> None: + plt.figure(figsize=(10, 5)) + plt.bar(summary_df["framework"], summary_df["total_tokens"]) + plt.title("AgenticLens Benchmark: Total Tokens by Framework") + plt.xlabel("Framework") + plt.ylabel("Total Tokens") + plt.xticks(rotation=30, ha="right") + plt.tight_layout() + + output = RESULTS_DIR / "benchmark_tokens_chart.png" + plt.savefig(output) + plt.close() + + print(f"Saved token chart: {output}") + + +def plot_total_cost(summary_df: pd.DataFrame) -> None: + plt.figure(figsize=(10, 5)) + plt.bar(summary_df["framework"], summary_df["total_cost_usd"]) + plt.title("AgenticLens Benchmark: Estimated Cost by Framework") + plt.xlabel("Framework") + plt.ylabel("Estimated Cost USD") + plt.xticks(rotation=30, ha="right") + plt.tight_layout() + + output = RESULTS_DIR / "benchmark_cost_chart.png" + plt.savefig(output) + plt.close() + + print(f"Saved cost chart: {output}") + + +def main() -> None: + summary_rows = [] + step_rows = [] + + for framework, report_path in REPORTS.items(): + path = Path(report_path) + + if not path.exists(): + print(f"Skipping {framework}: report not found at {report_path}") + continue + + report = load_report(path) + + summary_rows.append(summarize_report(framework, report)) + step_rows.extend(extract_step_rows(framework, report)) + + if not summary_rows: + raise RuntimeError("No reports found. Run AgenticLens profile commands first.") + + summary_df = pd.DataFrame(summary_rows) + step_df = pd.DataFrame(step_rows) + + summary_df = summary_df.sort_values(by=["total_tokens", "framework"]) + + summary_csv = RESULTS_DIR / "benchmark_results.csv" + step_csv = RESULTS_DIR / "benchmark_step_breakdown.csv" + summary_md = RESULTS_DIR / "benchmark_summary.md" + + summary_df.to_csv(summary_csv, index=False) + step_df.to_csv(step_csv, index=False) + create_markdown_summary(summary_df, summary_md) + + plot_total_tokens(summary_df) + plot_total_cost(summary_df) + + print("\nBenchmark comparison complete.") + print(f"Summary CSV: {summary_csv}") + print(f"Step breakdown CSV: {step_csv}") + print(f"Markdown summary: {summary_md}") + + print("\nSummary:") + print(summary_df.to_string(index=False)) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/datasets/orders.json b/benchmarks/datasets/orders.json new file mode 100644 index 0000000..9305a43 --- /dev/null +++ b/benchmarks/datasets/orders.json @@ -0,0 +1,29 @@ +[ + { + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + }, + { + "order_id": "B456", + "status": "delivered", + "delivered_days_ago": 45, + "package_opened": false, + "item_used": false, + "payment_method": "Mastercard ending 1111", + "tracking_updated_hours_ago": 72 + }, + { + "order_id": "C789", + "status": "in_transit", + "delivered_days_ago": null, + "package_opened": false, + "item_used": false, + "payment_method": "Visa ending 9999", + "tracking_updated_hours_ago": 52 + } +] \ No newline at end of file diff --git a/benchmarks/datasets/refund_policy_docs.json b/benchmarks/datasets/refund_policy_docs.json new file mode 100644 index 0000000..d0c1698 --- /dev/null +++ b/benchmarks/datasets/refund_policy_docs.json @@ -0,0 +1,42 @@ +[ + { + "doc_id": "refund_001", + "category": "refund", + "text": "Customers can request a refund within 30 days of delivery." + }, + { + "doc_id": "refund_002", + "category": "refund", + "text": "Items must be unused and in original packaging to qualify for a standard refund." + }, + { + "doc_id": "refund_003", + "category": "refund", + "text": "Opened items may require manual review unless the item is defective." + }, + { + "doc_id": "refund_004", + "category": "payment", + "text": "Refunds are processed to the original payment method." + }, + { + "doc_id": "refund_005", + "category": "payment", + "text": "Refunds may take 5 to 10 business days after approval." + }, + { + "doc_id": "shipping_001", + "category": "shipping", + "text": "Delivered orders are eligible for return review if the delivery date is within the return window." + }, + { + "doc_id": "tracking_001", + "category": "tracking", + "text": "If tracking is not updated for 48 hours, customers should contact support." + }, + { + "doc_id": "cancel_001", + "category": "cancellation", + "text": "Orders can be cancelled before shipment. Shipped orders cannot be cancelled." + } +] \ No newline at end of file diff --git a/benchmarks/datasets/support_cases.json b/benchmarks/datasets/support_cases.json new file mode 100644 index 0000000..30f773d --- /dev/null +++ b/benchmarks/datasets/support_cases.json @@ -0,0 +1,31 @@ +[ + { + "case_id": "refund_001", + "ticket": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "order_id": "A123", + "expected_facts": [ + "within 30 day refund window", + "opened package may require manual review", + "item was not used", + "refund may take 5 to 10 business days" + ] + }, + { + "case_id": "refund_002", + "ticket": "My order B456 arrived 45 days ago. Can I return it?", + "order_id": "B456", + "expected_facts": [ + "outside 30 day refund window", + "refund may not be eligible" + ] + }, + { + "case_id": "tracking_001", + "ticket": "My tracking has not updated for 2 days. What should I do?", + "order_id": "C789", + "expected_facts": [ + "tracking not updated for 48 hours", + "contact support" + ] + } +] \ No newline at end of file diff --git a/benchmarks/frameworks/__init__.py b/benchmarks/frameworks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/autogen/__init__.py b/benchmarks/frameworks/autogen/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/autogen/run_autogen.py b/benchmarks/frameworks/autogen/run_autogen.py new file mode 100644 index 0000000..aae4fdc --- /dev/null +++ b/benchmarks/frameworks/autogen/run_autogen.py @@ -0,0 +1,135 @@ +import time + +from agenticlens import profile, step + +from benchmarks.shared.support_tasks import ( + check_refund_eligibility, + classify_ticket, + generate_customer_reply, + lookup_order_tool, + retrieve_policy, + rewrite_query, +) + + +def main() -> None: + framework = "AutoGen" + + try: + from autogen_agentchat.agents import AssistantAgent + except ImportError as exc: + raise RuntimeError( + "AutoGen AgentChat is not installed. Run: pip install autogen-agentchat autogen-core" + ) from exc + + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + order_id = "A123" + + # Framework-specific agents. + # We instantiate agents for benchmark identity, but do not call a live model client here. + classifier_agent = AssistantAgent( + name="support_intent_classifier", + model_client=None, + ) + refund_agent = AssistantAgent( + name="refund_decision_agent", + model_client=None, + ) + response_agent = AssistantAgent( + name="customer_response_agent", + model_client=None, + ) + + with profile("Benchmark - AutoGen - Support Refund"): + + with step( + "AutoGen - Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="autogen", + agent_name=classifier_agent.name, + ) as s: + start = time.time() + response = classify_ticket(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + intent = response.choices[0].message.content + + with step( + "AutoGen - Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="autogen", + ) as s: + start = time.time() + response = rewrite_query(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + query = response.choices[0].message.content + + with step( + "AutoGen - Retrieve Refund Policy", + type="retriever", + query=query, + framework="autogen", + ) as s: + chunks, policy_context, avg_tokens, latency = retrieve_policy(query, top_k=6) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = avg_tokens + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + + with step( + "AutoGen - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + framework="autogen", + ) as s: + order, latency = lookup_order_tool(order_id) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = order + + with step( + "AutoGen - Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="autogen", + agent_name=refund_agent.name, + ) as s: + start = time.time() + response = check_refund_eligibility(ticket, order, policy_context, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + decision = response.choices[0].message.content + s.step.metadata["intent"] = intent + + with step( + "AutoGen - Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="autogen", + agent_name=response_agent.name, + ) as s: + start = time.time() + response = generate_customer_reply(ticket, order, policy_context, decision, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/frameworks/crewai/__init__.py b/benchmarks/frameworks/crewai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/crewai/run_crewai.py b/benchmarks/frameworks/crewai/run_crewai.py new file mode 100644 index 0000000..0e2601d --- /dev/null +++ b/benchmarks/frameworks/crewai/run_crewai.py @@ -0,0 +1,182 @@ +import time + +from agenticlens import profile, step + +from benchmarks.shared.support_tasks import ( + check_refund_eligibility, + classify_ticket, + generate_customer_reply, + lookup_order_tool, + retrieve_policy, + rewrite_query, +) + + +def main() -> None: + framework = "CrewAI" + + try: + from crewai import Agent, Crew, Process, Task + except ImportError as exc: + raise RuntimeError("CrewAI is not installed. Run: pip install crewai") from exc + + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + order_id = "A123" + + # Framework-specific objects. + # These make this a CrewAI benchmark adapter, while AgenticLens measures each business step. + classifier_agent = Agent( + role="Support Intent Classifier", + goal="Classify customer support tickets", + backstory="You classify customer tickets into support intents.", + verbose=False, + allow_delegation=False, + ) + + policy_agent = Agent( + role="Policy Retrieval Agent", + goal="Find relevant refund and tracking policies", + backstory="You retrieve policy evidence for support decisions.", + verbose=False, + allow_delegation=False, + ) + + refund_agent = Agent( + role="Refund Decision Agent", + goal="Decide refund eligibility using order and policy facts", + backstory="You apply company refund rules to customer orders.", + verbose=False, + allow_delegation=False, + ) + + response_agent = Agent( + role="Customer Response Agent", + goal="Write clear customer-facing replies", + backstory="You write helpful support responses.", + verbose=False, + allow_delegation=False, + ) + + tasks = [ + Task( + description="Classify the refund ticket intent.", + expected_output="Ticket intent and priority.", + agent=classifier_agent, + ), + Task( + description="Retrieve relevant refund policy chunks.", + expected_output="Relevant policy evidence.", + agent=policy_agent, + ), + Task( + description="Check refund eligibility.", + expected_output="Eligibility decision.", + agent=refund_agent, + ), + Task( + description="Generate customer reply.", + expected_output="Customer-facing answer.", + agent=response_agent, + ), + ] + + # We create the CrewAI crew so the benchmark records that this implementation uses CrewAI. + # We do not call crew.kickoff() in this deterministic benchmark because that would call a live LLM. + crew = Crew( + agents=[classifier_agent, policy_agent, refund_agent, response_agent], + tasks=tasks, + process=Process.sequential, + verbose=False, + ) + + with profile("Benchmark - CrewAI - Support Refund"): + + with step( + "CrewAI - Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="crewai", + crew_agents=len(crew.agents), + ) as s: + start = time.time() + response = classify_ticket(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + intent = response.choices[0].message.content + + with step( + "CrewAI - Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="crewai", + ) as s: + start = time.time() + response = rewrite_query(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + query = response.choices[0].message.content + + with step( + "CrewAI - Retrieve Refund Policy", + type="retriever", + query=query, + framework="crewai", + ) as s: + chunks, policy_context, avg_tokens, latency = retrieve_policy(query, top_k=6) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = avg_tokens + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + + with step( + "CrewAI - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + framework="crewai", + ) as s: + order, latency = lookup_order_tool(order_id) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = order + + with step( + "CrewAI - Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="crewai", + ) as s: + start = time.time() + response = check_refund_eligibility(ticket, order, policy_context, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + decision = response.choices[0].message.content + s.step.metadata["intent"] = intent + + with step( + "CrewAI - Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="crewai", + ) as s: + start = time.time() + response = generate_customer_reply(ticket, order, policy_context, decision, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/frameworks/langgraph/__init__.py b/benchmarks/frameworks/langgraph/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/langgraph/run_langgraph.py b/benchmarks/frameworks/langgraph/run_langgraph.py new file mode 100644 index 0000000..efa399d --- /dev/null +++ b/benchmarks/frameworks/langgraph/run_langgraph.py @@ -0,0 +1,243 @@ +import time +from typing import TypedDict, Any + +from agenticlens import profile, step + +from benchmarks.shared.support_data import ( + build_policy_context, + estimate_avg_tokens_per_chunk, + lookup_order, + simple_retrieve, +) + + +class SupportState(TypedDict, total=False): + ticket: str + order_id: str + intent: str + query: str + chunks: list[dict[str, Any]] + policy_context: str + order: dict[str, Any] + decision: str + final_answer: str + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeMessage: + def __init__(self, content: str): + self.content = content + + +class FakeChoice: + def __init__(self, content: str): + self.message = FakeMessage(content) + + +class FakeResponse: + def __init__(self, content: str, prompt_tokens: int, completion_tokens: int): + self.usage = FakeUsage(prompt_tokens, completion_tokens) + self.choices = [FakeChoice(content)] + + +def classify_ticket_llm(ticket: str) -> FakeResponse: + return FakeResponse( + content="intent=refund_request; priority=normal", + prompt_tokens=190, + completion_tokens=30, + ) + + +def rewrite_query_llm(ticket: str) -> FakeResponse: + return FakeResponse( + content="refund eligibility delivered order opened package unused item refund processing time", + prompt_tokens=240, + completion_tokens=40, + ) + + +def refund_decision_llm(ticket: str, order: dict, policy_context: str) -> FakeResponse: + return FakeResponse( + content=( + "The order is within the 30-day refund window. " + "The item was not used, but the package was opened, so manual review may be required." + ), + prompt_tokens=780, + completion_tokens=110, + ) + + +def final_response_llm(ticket: str, order: dict, policy_context: str, decision: str) -> FakeResponse: + return FakeResponse( + content=( + "Your order is within the 30-day refund window. Since the package was opened, " + "the refund may need manual review. Because the item was not used, you may still be eligible. " + "If approved, the refund will return to your original payment method and may take 5 to 10 business days." + ), + prompt_tokens=920, + completion_tokens=150, + ) + + +def classify_ticket_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=state["ticket"], + ) as s: + start = time.time() + response = classify_ticket_llm(state["ticket"]) + s.record(response) + s.step.metrics.latency = time.time() - start + + state["intent"] = response.choices[0].message.content + return state + + +def rewrite_query_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=state["ticket"], + ) as s: + start = time.time() + response = rewrite_query_llm(state["ticket"]) + s.record(response) + s.step.metrics.latency = time.time() - start + + state["query"] = response.choices[0].message.content + return state + + +def retrieve_policy_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Retrieve Refund Policy", + type="retriever", + query=state["query"], + ) as s: + start = time.time() + chunks = simple_retrieve(state["query"], top_k=6) + s.step.metrics.latency = time.time() - start + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = estimate_avg_tokens_per_chunk(chunks) + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + + state["chunks"] = chunks + state["policy_context"] = build_policy_context(chunks) + return state + + +def lookup_order_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": state["order_id"]}, + ) as s: + start = time.time() + order = lookup_order(state["order_id"]) + s.step.metrics.latency = time.time() - start + s.step.metadata["tool_result"] = order + + state["order"] = order + return state + + +def refund_decision_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=state["policy_context"], + ) as s: + start = time.time() + response = refund_decision_llm( + state["ticket"], + state["order"], + state["policy_context"], + ) + s.record(response) + s.step.metrics.latency = time.time() - start + + state["decision"] = response.choices[0].message.content + return state + + +def final_response_node(state: SupportState) -> SupportState: + with step( + "LangGraph - Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=state["policy_context"], + ) as s: + start = time.time() + response = final_response_llm( + state["ticket"], + state["order"], + state["policy_context"], + state["decision"], + ) + s.record(response) + s.step.metrics.latency = time.time() - start + + state["final_answer"] = response.choices[0].message.content + return state + + +def main() -> None: + try: + from langgraph.graph import StateGraph, END + except ImportError as exc: + raise RuntimeError( + "LangGraph is not installed. Run: pip install langgraph" + ) from exc + + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + + initial_state: SupportState = { + "ticket": ticket, + "order_id": "A123", + } + + graph = StateGraph(SupportState) + + graph.add_node("classify_ticket", classify_ticket_node) + graph.add_node("rewrite_query", rewrite_query_node) + graph.add_node("retrieve_policy", retrieve_policy_node) + graph.add_node("lookup_order", lookup_order_node) + graph.add_node("refund_decision", refund_decision_node) + graph.add_node("final_response", final_response_node) + + graph.set_entry_point("classify_ticket") + graph.add_edge("classify_ticket", "rewrite_query") + graph.add_edge("rewrite_query", "retrieve_policy") + graph.add_edge("retrieve_policy", "lookup_order") + graph.add_edge("lookup_order", "refund_decision") + graph.add_edge("refund_decision", "final_response") + graph.add_edge("final_response", END) + + app = graph.compile() + + with profile("Benchmark - LangGraph - Support Refund"): + final_state = app.invoke(initial_state) + + print(final_state["final_answer"]) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/frameworks/llamaindex/__init__.py b/benchmarks/frameworks/llamaindex/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/llamaindex/run_llamaindex.py b/benchmarks/frameworks/llamaindex/run_llamaindex.py new file mode 100644 index 0000000..4fbabaf --- /dev/null +++ b/benchmarks/frameworks/llamaindex/run_llamaindex.py @@ -0,0 +1,134 @@ +import time + +from agenticlens import profile, step + +from benchmarks.shared.support_data import load_policy_docs +from benchmarks.shared.support_tasks import ( + check_refund_eligibility, + classify_ticket, + generate_customer_reply, + lookup_order_tool, + retrieve_policy, + rewrite_query, +) + + +def main() -> None: + framework = "LlamaIndex" + + try: + from llama_index.core import Document, VectorStoreIndex + except ImportError as exc: + raise RuntimeError("LlamaIndex is not installed. Run: pip install llama-index") from exc + + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + order_id = "A123" + + # Framework-specific indexing object. + # This builds a LlamaIndex document collection, but the deterministic benchmark uses shared retrieval + # so results stay comparable with other framework runs. + policy_docs = load_policy_docs() + documents = [ + Document(text=doc["text"], metadata={"doc_id": doc["doc_id"], "category": doc["category"]}) + for doc in policy_docs + ] + + # Do not build a real embedding index in the deterministic run because it may require model configuration. + # Keep this object as the framework-specific document representation. + index_metadata = { + "framework_documents": len(documents), + "index_type": "llamaindex_documents", + } + + with profile("Benchmark - LlamaIndex - Support Refund"): + + with step( + "LlamaIndex - Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="llamaindex", + **index_metadata, + ) as s: + start = time.time() + response = classify_ticket(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + intent = response.choices[0].message.content + + with step( + "LlamaIndex - Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="llamaindex", + ) as s: + start = time.time() + response = rewrite_query(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + query = response.choices[0].message.content + + with step( + "LlamaIndex - Retrieve Refund Policy", + type="retriever", + query=query, + framework="llamaindex", + index_type="Document collection", + ) as s: + chunks, policy_context, avg_tokens, latency = retrieve_policy(query, top_k=6) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = avg_tokens + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + + with step( + "LlamaIndex - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + framework="llamaindex", + ) as s: + order, latency = lookup_order_tool(order_id) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = order + + with step( + "LlamaIndex - Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="llamaindex", + ) as s: + start = time.time() + response = check_refund_eligibility(ticket, order, policy_context, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + decision = response.choices[0].message.content + s.step.metadata["intent"] = intent + + with step( + "LlamaIndex - Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="llamaindex", + ) as s: + start = time.time() + response = generate_customer_reply(ticket, order, policy_context, decision, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/frameworks/native_python/__init__.py b/benchmarks/frameworks/native_python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/native_python/run_native.py b/benchmarks/frameworks/native_python/run_native.py new file mode 100644 index 0000000..6f9201f --- /dev/null +++ b/benchmarks/frameworks/native_python/run_native.py @@ -0,0 +1,169 @@ +import time +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(PROJECT_ROOT)) + +from agenticlens import profile, step + +from benchmarks.shared.support_data import ( + build_policy_context, + estimate_avg_tokens_per_chunk, + lookup_order, + simple_retrieve, +) + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeMessage: + def __init__(self, content: str): + self.content = content + + +class FakeChoice: + def __init__(self, content: str): + self.message = FakeMessage(content) + + +class FakeResponse: + def __init__(self, content: str, prompt_tokens: int, completion_tokens: int): + self.usage = FakeUsage(prompt_tokens, completion_tokens) + self.choices = [FakeChoice(content)] + + +def classify_ticket(ticket: str) -> FakeResponse: + return FakeResponse( + content="intent=refund_request; priority=normal", + prompt_tokens=180, + completion_tokens=25, + ) + + +def rewrite_query(ticket: str) -> FakeResponse: + return FakeResponse( + content="refund eligibility delivered order opened package unused item refund processing time", + prompt_tokens=220, + completion_tokens=35, + ) + + +def check_refund_eligibility(ticket: str, order: dict, policy_context: str) -> FakeResponse: + return FakeResponse( + content=( + "The order is within the 30-day refund window. " + "The item was not used, but the package was opened, so manual review may be required." + ), + prompt_tokens=720, + completion_tokens=95, + ) + + +def generate_customer_reply(ticket: str, order: dict, policy_context: str, decision: str) -> FakeResponse: + return FakeResponse( + content=( + "Your order is within the 30-day refund window. Since the package was opened, " + "the refund may need manual review. Because the item was not used, you may still be eligible. " + "If approved, the refund will return to your original payment method and may take 5 to 10 business days." + ), + prompt_tokens=850, + completion_tokens=130, + ) + + +def main() -> None: + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + order_id = "A123" + + with profile("Benchmark - Native Python - Support Refund"): + + with step( + "Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + ) as s: + start = time.time() + response = classify_ticket(ticket) + s.record(response) + s.step.metrics.latency = time.time() - start + intent = response.choices[0].message.content + + with step( + "Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + ) as s: + start = time.time() + response = rewrite_query(ticket) + s.record(response) + s.step.metrics.latency = time.time() - start + query = response.choices[0].message.content + + with step( + "Retrieve Refund Policy", + type="retriever", + query=query, + ) as s: + start = time.time() + chunks = simple_retrieve(query, top_k=6) + s.step.metrics.latency = time.time() - start + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = estimate_avg_tokens_per_chunk(chunks) + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + policy_context = build_policy_context(chunks) + + with step( + "Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + ) as s: + start = time.time() + order = lookup_order(order_id) + s.step.metrics.latency = time.time() - start + s.step.metadata["tool_result"] = order + + with step( + "Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + ) as s: + start = time.time() + response = check_refund_eligibility(ticket, order, policy_context) + s.record(response) + s.step.metrics.latency = time.time() - start + decision = response.choices[0].message.content + s.step.metadata["intent"] = intent + + with step( + "Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + ) as s: + start = time.time() + response = generate_customer_reply(ticket, order, policy_context, decision) + s.record(response) + s.step.metrics.latency = time.time() - start + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/frameworks/semantic_kernel/__init__.py b/benchmarks/frameworks/semantic_kernel/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py b/benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py new file mode 100644 index 0000000..590ab69 --- /dev/null +++ b/benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py @@ -0,0 +1,120 @@ +import time + +from agenticlens import profile, step + +from benchmarks.shared.support_tasks import ( + check_refund_eligibility, + classify_ticket, + generate_customer_reply, + lookup_order_tool, + retrieve_policy, + rewrite_query, +) + + +def main() -> None: + framework = "Semantic Kernel" + + try: + import semantic_kernel as sk + except ImportError as exc: + raise RuntimeError("Semantic Kernel is not installed. Run: pip install semantic-kernel") from exc + + ticket = ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ) + order_id = "A123" + + # Framework-specific kernel object. + # This confirms the implementation is using the Semantic Kernel runtime surface. + kernel = sk.Kernel() + + with profile("Benchmark - Semantic Kernel - Support Refund"): + + with step( + "Semantic Kernel - Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="semantic_kernel", + kernel_type=type(kernel).__name__, + ) as s: + start = time.time() + response = classify_ticket(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + intent = response.choices[0].message.content + + with step( + "Semantic Kernel - Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket, + framework="semantic_kernel", + ) as s: + start = time.time() + response = rewrite_query(ticket, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + query = response.choices[0].message.content + + with step( + "Semantic Kernel - Retrieve Refund Policy", + type="retriever", + query=query, + framework="semantic_kernel", + ) as s: + chunks, policy_context, avg_tokens, latency = retrieve_policy(query, top_k=6) + s.step.metrics.latency = latency + s.step.metadata["chunk_count"] = len(chunks) + s.step.metadata["avg_tokens_per_chunk"] = avg_tokens + s.step.metadata["retrieved_doc_ids"] = [chunk["doc_id"] for chunk in chunks] + + with step( + "Semantic Kernel - Lookup Order", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + framework="semantic_kernel", + ) as s: + order, latency = lookup_order_tool(order_id) + s.step.metrics.latency = latency + s.step.metadata["tool_result"] = order + + with step( + "Semantic Kernel - Refund Eligibility Check", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="semantic_kernel", + ) as s: + start = time.time() + response = check_refund_eligibility(ticket, order, policy_context, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + decision = response.choices[0].message.content + s.step.metadata["intent"] = intent + + with step( + "Semantic Kernel - Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=policy_context, + framework="semantic_kernel", + ) as s: + start = time.time() + response = generate_customer_reply(ticket, order, policy_context, decision, framework) + s.record(response) + s.step.metrics.latency = time.time() - start + + print(response.choices[0].message.content) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/reports/autogen/support_refund_report.json b/benchmarks/reports/autogen/support_refund_report.json new file mode 100644 index 0000000..c4947e4 --- /dev/null +++ b/benchmarks/reports/autogen/support_refund_report.json @@ -0,0 +1,147 @@ +{ + "id": "ed77978d-1a40-41c5-91c0-65aeadcf7466", + "name": "Benchmark - AutoGen - Support Refund", + "start_time": "2026-07-14T21:36:33.420007Z", + "end_time": "2026-07-14T21:36:33.420742Z", + "steps": [ + { + "id": "783d0f9d-8531-4503-9120-a87d66ea8fac", + "name": "AutoGen - Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 180, + "completion_tokens": 25, + "total_tokens": 205, + "latency": 0.0000518000015290454, + "ttft": null, + "cost": 0.00004199999999999999 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "autogen", + "agent_name": "support_intent_classifier" + } + }, + { + "id": "5c2614b0-baeb-4cf9-916e-ce959c8b15e1", + "name": "AutoGen - Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 220, + "completion_tokens": 35, + "total_tokens": 255, + "latency": 0.0000121000011858996, + "ttft": null, + "cost": 0.000054 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "autogen" + } + }, + { + "id": "36da4dab-60bb-49fe-bc56-c98cf6dd3849", + "name": "AutoGen - Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00026449999859323725, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "framework": "autogen", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "fe25246c-7805-45f5-bdf0-eba6beabd356", + "name": "AutoGen - Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00016279999908874743, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "framework": "autogen", + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "9c3c0f3a-299a-4170-994e-d669a7258563", + "name": "AutoGen - Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 720, + "completion_tokens": 95, + "total_tokens": 815, + "latency": 0.000015300000086426735, + "ttft": null, + "cost": 0.00016499999999999997 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "autogen", + "agent_name": "refund_decision_agent", + "intent": "framework=AutoGen; intent=refund_request; priority=normal" + } + }, + { + "id": "74e5b606-8ef8-444b-9094-268658dc68ac", + "name": "AutoGen - Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 850, + "completion_tokens": 130, + "total_tokens": 980, + "latency": 8.799997885944322e-6, + "ttft": null, + "cost": 0.00020549999999999998 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "autogen", + "agent_name": "customer_response_agent" + } + } + ] +} \ No newline at end of file diff --git a/benchmarks/reports/crewai/support_refund_report.json b/benchmarks/reports/crewai/support_refund_report.json new file mode 100644 index 0000000..6e46a90 --- /dev/null +++ b/benchmarks/reports/crewai/support_refund_report.json @@ -0,0 +1,145 @@ +{ + "id": "7f04e0a9-01c7-4833-ada0-c53ddc0da5fe", + "name": "Benchmark - CrewAI - Support Refund", + "start_time": "2026-07-14T21:36:31.420353Z", + "end_time": "2026-07-14T21:36:31.421058Z", + "steps": [ + { + "id": "0388abf3-38df-4207-87f1-e156b86a6680", + "name": "CrewAI - Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 180, + "completion_tokens": 25, + "total_tokens": 205, + "latency": 0.00005239999882178381, + "ttft": null, + "cost": 0.00004199999999999999 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "crewai", + "crew_agents": 4 + } + }, + { + "id": "505c1d82-9071-4320-b08e-6d19218344e5", + "name": "CrewAI - Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 220, + "completion_tokens": 35, + "total_tokens": 255, + "latency": 0.000012599997717188671, + "ttft": null, + "cost": 0.000054 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "crewai" + } + }, + { + "id": "43677484-150f-4b57-99eb-c38b4aa2ec86", + "name": "CrewAI - Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.0002473999993526377, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "framework": "crewai", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "5f59edaa-7280-46a3-a5a1-22ff5253d701", + "name": "CrewAI - Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.0001565000020491425, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "framework": "crewai", + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "a63dafd2-480b-4bcf-80f1-8b32283eaeb8", + "name": "CrewAI - Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 720, + "completion_tokens": 95, + "total_tokens": 815, + "latency": 0.000017400001524947584, + "ttft": null, + "cost": 0.00016499999999999997 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "crewai", + "intent": "framework=CrewAI; intent=refund_request; priority=normal" + } + }, + { + "id": "b396cc25-fe38-4fd2-b474-9669e2643e9b", + "name": "CrewAI - Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 850, + "completion_tokens": 130, + "total_tokens": 980, + "latency": 9.399998816661537e-6, + "ttft": null, + "cost": 0.00020549999999999998 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "crewai" + } + } + ] +} \ No newline at end of file diff --git a/benchmarks/reports/langgraph/support_refund_report.json b/benchmarks/reports/langgraph/support_refund_report.json new file mode 100644 index 0000000..f2cc6ea --- /dev/null +++ b/benchmarks/reports/langgraph/support_refund_report.json @@ -0,0 +1,137 @@ +{ + "id": "76a810aa-b143-4ef6-ab3e-ae678ad7bf80", + "name": "Benchmark - LangGraph - Support Refund", + "start_time": "2026-07-14T21:39:51.399227Z", + "end_time": "2026-07-14T21:39:51.409770Z", + "steps": [ + { + "id": "e3563f4a-1747-466e-b8b9-6d504da39267", + "name": "LangGraph - Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 190, + "completion_tokens": 30, + "total_tokens": 220, + "latency": 0.00010509999992791563, + "ttft": null, + "cost": 0.00004649999999999999 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?" + } + }, + { + "id": "fc297d97-a69d-4f60-bdd8-7e7d61fd6bc1", + "name": "LangGraph - Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 240, + "completion_tokens": 40, + "total_tokens": 280, + "latency": 0.00003239999932702631, + "ttft": null, + "cost": 0.000059999999999999995 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?" + } + }, + { + "id": "5da0b776-104c-4299-84e2-8bdca5914386", + "name": "LangGraph - Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.0004374000018287916, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "0f1a9f08-4508-4119-a8a8-e14af0901f09", + "name": "LangGraph - Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00030819999665254727, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "fc871ee8-2d17-4f32-9ad0-5cef325ed213", + "name": "LangGraph - Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 780, + "completion_tokens": 110, + "total_tokens": 890, + "latency": 0.00003389999983482994, + "ttft": null, + "cost": 0.000183 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window." + } + }, + { + "id": "b614121a-21ca-490f-937d-3e6c86628f09", + "name": "LangGraph - Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 920, + "completion_tokens": 150, + "total_tokens": 1070, + "latency": 0.000018700000509852543, + "ttft": null, + "cost": 0.00022799999999999999 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window." + } + } + ] +} \ No newline at end of file diff --git a/benchmarks/reports/llamaindex/support_refund_report.json b/benchmarks/reports/llamaindex/support_refund_report.json new file mode 100644 index 0000000..d475877 --- /dev/null +++ b/benchmarks/reports/llamaindex/support_refund_report.json @@ -0,0 +1,147 @@ +{ + "id": "485b74e2-156e-4d5f-b83b-582e29071353", + "name": "Benchmark - LlamaIndex - Support Refund", + "start_time": "2026-07-14T21:36:36.532376Z", + "end_time": "2026-07-14T21:36:36.533155Z", + "steps": [ + { + "id": "57477c47-60e0-47df-8422-3c94f4585f08", + "name": "LlamaIndex - Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 180, + "completion_tokens": 25, + "total_tokens": 205, + "latency": 0.0000703999976394698, + "ttft": null, + "cost": 0.00004199999999999999 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "llamaindex", + "framework_documents": 8, + "index_type": "llamaindex_documents" + } + }, + { + "id": "6b2b2a60-c5ea-4210-9ffe-c65761599b9d", + "name": "LlamaIndex - Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 220, + "completion_tokens": 35, + "total_tokens": 255, + "latency": 0.0000168999977177009, + "ttft": null, + "cost": 0.000054 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "llamaindex" + } + }, + { + "id": "0e4156e9-8568-43a8-a6b3-6743c93c5597", + "name": "LlamaIndex - Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.0002535999992687721, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "framework": "llamaindex", + "index_type": "Document collection", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "416c75f4-551e-4885-85eb-237cd175ce8e", + "name": "LlamaIndex - Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00018619999900693074, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "framework": "llamaindex", + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "99e8a871-c2c5-4b61-8ad8-6bd8cb7501b1", + "name": "LlamaIndex - Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 720, + "completion_tokens": 95, + "total_tokens": 815, + "latency": 0.000014399996871361509, + "ttft": null, + "cost": 0.00016499999999999997 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "llamaindex", + "intent": "framework=LlamaIndex; intent=refund_request; priority=normal" + } + }, + { + "id": "d222d8ae-8092-41c3-9374-ce8789a97631", + "name": "LlamaIndex - Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 850, + "completion_tokens": 130, + "total_tokens": 980, + "latency": 0.000012500000593718141, + "ttft": null, + "cost": 0.00020549999999999998 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "llamaindex" + } + } + ] +} \ No newline at end of file diff --git a/benchmarks/reports/native_python/support_refund_report.json b/benchmarks/reports/native_python/support_refund_report.json new file mode 100644 index 0000000..227d1c3 --- /dev/null +++ b/benchmarks/reports/native_python/support_refund_report.json @@ -0,0 +1,138 @@ +{ + "id": "627d5ccb-9eb7-4237-b214-197e04ccf38f", + "name": "Benchmark - Native Python - Support Refund", + "start_time": "2026-07-14T21:24:10.816128Z", + "end_time": "2026-07-14T21:24:10.817377Z", + "steps": [ + { + "id": "579a7d59-67d2-4091-9217-7cef0cf20701", + "name": "Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 180, + "completion_tokens": 25, + "total_tokens": 205, + "latency": 0.00009779999891179614, + "ttft": null, + "cost": 0.00004199999999999999 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?" + } + }, + { + "id": "0712b159-2ce1-41a3-94fa-8190f0d91d6c", + "name": "Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 220, + "completion_tokens": 35, + "total_tokens": 255, + "latency": 0.000030399998649954796, + "ttft": null, + "cost": 0.000054 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?" + } + }, + { + "id": "46cb3d96-b2bf-4594-8a4b-20b18d94056b", + "name": "Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00039590000233147293, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "94d99203-43f2-40af-a6b3-fcae079102b6", + "name": "Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.0002775000029942021, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "7e78e3d3-4f64-4b62-b16b-8de37eaebd82", + "name": "Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 720, + "completion_tokens": 95, + "total_tokens": 815, + "latency": 0.000023199998395284638, + "ttft": null, + "cost": 0.00016499999999999997 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "intent": "intent=refund_request; priority=normal" + } + }, + { + "id": "8781e05d-00a8-4165-a66c-011107270655", + "name": "Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 850, + "completion_tokens": 130, + "total_tokens": 980, + "latency": 0.000015800000255694613, + "ttft": null, + "cost": 0.00020549999999999998 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window." + } + } + ] +} \ No newline at end of file diff --git a/benchmarks/reports/semantic_kernel/support_refund_report.json b/benchmarks/reports/semantic_kernel/support_refund_report.json new file mode 100644 index 0000000..d67dc50 --- /dev/null +++ b/benchmarks/reports/semantic_kernel/support_refund_report.json @@ -0,0 +1,145 @@ +{ + "id": "fb13d0d6-1f6f-4d6a-ab4d-5c668d0cea5c", + "name": "Benchmark - Semantic Kernel - Support Refund", + "start_time": "2026-07-14T21:36:43.468619Z", + "end_time": "2026-07-14T21:36:43.469494Z", + "steps": [ + { + "id": "93b69a84-c49b-4b76-a1df-adcb25fdb0c8", + "name": "Semantic Kernel - Classify Ticket Intent", + "type": "planner", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 180, + "completion_tokens": 25, + "total_tokens": 205, + "latency": 0.0000700999989931006, + "ttft": null, + "cost": 0.00004199999999999999 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "semantic_kernel", + "kernel_type": "Kernel" + } + }, + { + "id": "1d4a1a12-241d-4f27-9f27-59ffd9f2b0a7", + "name": "Semantic Kernel - Rewrite Query For Retrieval", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 220, + "completion_tokens": 35, + "total_tokens": 255, + "latency": 0.0000180000024556648, + "ttft": null, + "cost": 0.000054 + }, + "metadata": { + "prompt": "My order A123 was delivered 12 days ago. I opened the package but did not use the item. Can I get a refund, and how long will it take?", + "framework": "semantic_kernel" + } + }, + { + "id": "3c9fcf52-d2ba-4074-af02-0f50764874f2", + "name": "Semantic Kernel - Retrieve Refund Policy", + "type": "retriever", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.0003157999999530148, + "ttft": null, + "cost": null + }, + "metadata": { + "query": "refund eligibility delivered order opened package unused item refund processing time", + "framework": "semantic_kernel", + "chunk_count": 4, + "avg_tokens_per_chunk": 16, + "retrieved_doc_ids": [ + "refund_002", + "refund_003", + "refund_001", + "shipping_001" + ] + } + }, + { + "id": "34544144-fe85-4115-906b-9542f115e3eb", + "name": "Semantic Kernel - Lookup Order", + "type": "tool_call", + "provider": null, + "model": null, + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.00019090000205324031, + "ttft": null, + "cost": null + }, + "metadata": { + "tool_name": "lookup_order", + "tool_args": { + "order_id": "A123" + }, + "framework": "semantic_kernel", + "tool_result": { + "found": true, + "order_id": "A123", + "status": "delivered", + "delivered_days_ago": 12, + "package_opened": true, + "item_used": false, + "payment_method": "Visa ending 4242", + "tracking_updated_hours_ago": 24 + } + } + }, + { + "id": "35081202-09fc-4d6a-a41b-e484a9c98c15", + "name": "Semantic Kernel - Refund Eligibility Check", + "type": "llm_call", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 720, + "completion_tokens": 95, + "total_tokens": 815, + "latency": 0.000020200001017656177, + "ttft": null, + "cost": 0.00016499999999999997 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "semantic_kernel", + "intent": "framework=Semantic Kernel; intent=refund_request; priority=normal" + } + }, + { + "id": "7a1c0cc4-db14-4a72-ac8b-18d2dcd2c968", + "name": "Semantic Kernel - Generate Customer Reply", + "type": "final_response", + "provider": "openai", + "model": "gpt-4o-mini", + "metrics": { + "prompt_tokens": 850, + "completion_tokens": 130, + "total_tokens": 980, + "latency": 0.000011500000255182385, + "ttft": null, + "cost": 0.00020549999999999998 + }, + "metadata": { + "prompt": "- Items must be unused and in original packaging to qualify for a standard refund.\n- Opened items may require manual review unless the item is defective.\n- Customers can request a refund within 30 days of delivery.\n- Delivered orders are eligible for return review if the delivery date is within the return window.", + "framework": "semantic_kernel" + } + } + ] +} \ No newline at end of file diff --git a/benchmarks/results/benchmark_cost_chart.png b/benchmarks/results/benchmark_cost_chart.png new file mode 100644 index 0000000000000000000000000000000000000000..fe1e7233168785de5d6e5902f73624efb9cb4d91 GIT binary patch literal 26868 zcmeFZcTiO8);HLo2;xyNAWAk12#92`2?~NplAKX;&asJNLJ=^KbC8@tGBlzBO3pc> zWN46_npwL&_r2$SH8u18@y=H@HDlGOgV?>pv)8lM`h~S$Dac74C#NAtp-{)~ONlF@ zP={tvD55p8Bk-G?>!fV(KS4(cbw?!|V@H?A_C_e#$Bwp^Hjb8N`b^G7_6}w?)?DnI zoa{GmFqt|!+Byhwa9I7s#aOT~eGtBR=R2%wBOSESs8s5LJanbPA z#e$OWni~tjF{|E{)dvR=2Te{Ky;TmlXrWb%FoTjyOEi~*@f5#<@F;G2cl(||;N&^D z0u+i;k@?Q)twOCOC6`VbwGDV18P*ZPRr z9ClgUO!2lsRlY*ei}s}{Ccmy)cExW`->qWJXVLtW<~H_()ORPkJ5!CX%wd}MM@_y# z{esI+?+X0Rd~-A3sO8SaqD3H`z^OT%Zz@=hgQ&nNiV{c8B*eeK5wVJ&!CvysZO2riSrGRIj(_38r)fg|husAs2P=^l^w>Us09 zs%G8XUY+J+^xn%~%$K(2wHh!qZVG>VvBn?>ZA@36U*e75uCIjan2K>3;XF>mO}3sh z&N3QyVAWq>B(_^gPQ_vvS7=ZlcygcKaqsW3KudLnRTWVRk`_7bDVG$Ue~Vr-N2fn zoUVA)b7yUX%d|c5c{8!p^(tfirHb9?yD$FYEE%%**O4Dp3uD)5dUA-AU2D_4b^2gH zaLKxS_Nsd&w^_%xO%Gi40hWFwe!9&iUH754NlWxmsX^Ho!6F^KuRLq&GP%hz?oMqwHa5>fc#LF?6F5yzRg zvjat$=+xONsvGKUYYY9IDL+W(E6Qj*NhMyN$sZM~eNENuOLEL3h#rmUiWOSNu?j94 z)V(>sSSECCGNZ(L*j%Kd(rtBMcgJhp=F-kv8os+1il<#6vSeDK1r|3qe}0f3@V-{Q zzBD{|;#!Bv*4R^uBOY}*`@5Tin;EU@qc2<7Fv6SagB32hp7cD$rE<~yj}?=pjcj%X zc`$3H6*)99Iy+V62l!p9r=NsvV!pz|Gpe2}^|MHL&6DjI_+5Hb! z1s*rf9pr~b1K7bVC|`tlyP#G>7!50ReWhYNL4Op%%OIN0CBu0wcEcV%n!mcjMJ z1kmycN(M1Fz4+^VV&)Z-sip{`R<}-BK7r;azMvJ)A^S$Q7T9>Sd6qM63C@gb<%18FmW$AmCs;in(R~Q|g*dxnql(p69RV+io4( z({bJl4oGyAokG9Voy@OiSY_LV1I7qAO!f7aPRF{R@*HiA5t<%yn~sr#OcvcLyxSxI z0im-A*X;0ug3+tFbvh>#=l$7-XoJ69_$-I+5Cb)%*H%JjuAYd=0e-(_syQm={gnrT zRII9%6Vo4YZV(vGjBy^ne>CdOV|SMvFdGYIK`SAwYJ$~!zx{FfKR#R=bXt#rEmZp> ziZ8bivp(lD-Roth=aCvbQMoUt&dqJT&lTp~8Acpl`-j%nW;+LuiS#Bv2wRZQ z_;6KrfxN4eOx_fU&Y`#&FMXk()q6R(|IGg)p+wT zR-g51+e%F)$7KJ-aBGZ?j?~o+?7Xi^BX)Z#N_}%xGqzQyM7Ihg}pf43!C1$chlx(FywNQ%CKi%ND$D* zc<-5wIxVPVYj&4dHb?R}>yZ4!ONX&r(Q-|=|FtpOS)Vg`B&PND>YLMQ=IUv>RpmJB zE}c2q@6^>?tO3vAZ6(pHUxK|Af=+XWVG=fUs`0tgu_E5ss?j7#3R=^O(LyhiGmaMC zv91$GD`Wy#&FNfsW>Y;|I}3~+>}okIkJb20s~vhplkUB{Sd!_*0FR*M2`1nyn`Ymr zENVpbwUCdYEc2xFV~>ef*E|}7_3_^?1%*RSyBUpR1m)&ui5^3JKU! zu(CrmNePpu*d!k(yr`v`u6RXAwJX{2PBMe_C>A|}Tdcrg3(Z2LAIkYb_ztXX?X1s} z-sr`LvT1D;1n<`c(2@@Inj~?XA8gvsXO|4DOhA?6obS!8p33OJlfd@C_HGS4krEP-kUg1f483tzMsD_F%4) z&W5z*^0{0#iZ{S1j+$NDR78^ATV!wfsceyK7O~FVmnV)7aN1qDzujMVIj25Bc=xv3 z?>@)J-miE(6LsY!I=Gdu?ab>y&f8e|hPO^+^xxS`@aEg$%g$d*4E9j@sCGfB7g|Cq z!J@$IErzF;?!76U!20#L!@bCC|NgbOOvcx(b5}T+m?6wS7I*h+d?KTT=90*bd-bpI zi)N)yM2>qZ?v%Sn4>|Sf@S=5{ju_(iww)ua1#$ku)!qmDz3dLNyIq{Q<5YNVF24~Z zJ438~74g)qzYle=sViM+5qA}313mBCEHmv4c9}Dl+d*8)Dz*V;>x{={k9g5&ozqz} zqLIArG~ZK19bMkn{k+OLvQeG7ltyc|L&D9mJYN3$8PN=D_I8eqoRAU!Iv(@e9}nEy zr$73O9{BL5Y$iJKOm!So&2Y{1=Eg;Eo4u#6$URoqldV;6v6oo)P!&qqoAX?CXZ<># z8$eSctQPR&jeqHFJbK1`&RnuK1imnOcp>>#-E+r>u>g2VlheBJ3wB30e7`{lXjmj z{ho3_uG%AZ@}8nND63pGr#h>Tbsu+~iZEGNKxN3s-;?(|$e=yd=@R$WK(?mb*L3ro zUC?QgX$;#Yzuh5cl1W(K5;U%isGR2b+F)|L&b+Bk7!9Yv4tCgRY11>g^(&9vg!tg2 zGI^5eyqP1qedvYx8suvo|e%`Tj;~>v?d7Rex&`}))^A=;- z{f{FtGt{r_lJ8hfFmhk6!_GUXJp5Q%8j9m7cAET6$6}whuQ>UPJb{@qm|n>FTPc=( z$Y6EABDepdB4CTS2wtnm`z){QWS&hF2He*Ux*KDl&%%AKvmm^*>XW&$pFd>c+O44h z5t64%ZZ|lqqxwd2iIqrv*r{4NH{&Nv1xMxLgoj&3D0}PtOZ{d?Yp=z5=r3V+tzik6 zw8U0Jum{iREOzkg6_X5?+JjD)+>EAa$xu$$KjcU<(cr6Khc?omwQ(uKOm;KPbWFP? zPqJcKcS2fD&n@=agp+4l_vMXe;Es!ST!3J;oF%@C)@oOx*jJ*e#_w1_auRbNl|2+7 zOcKSeey}we1jLcX}#peT5f~guFKM?=m{H7(w&>R zi(bQ!pIUd-_vULqpl80`EuhN0m=V6D_{&|zLhVsb!pWTz6Okja3hk-ModS9h6g4T6 z;&*fO%?}64RsCw1X|qe4tXs6}#Rb#!aKC8~snS{VoI9s1k6DWl7H>VRCfGVx`f%NI zy#u*@yOFb^6Bm({VqR|B($s0II~&iMZTscfQEdLR#le!2D#=I=9p+4$p!7A#bzO0J z9+jXEiof(cRwM=o6Z1RBe}>RxPQRj%O-}K*NaoQE*mykl`N@W-c?MlF+B~kiB60iO zq;%)i*)Ou4#Iv6B6AxRH{O`)vb2~5eS$HvKYZetKZC^K1re332ollW5ubeMZXvoK` za-UTc^YunQfrUn=Vop%qm=WmiOuIp1q$SSS8L1rXo1NeG^(93~0BIDTO>s@6;=$xy z>K&wmLl;xMk%;@v!I`4r-^iK3AT-?D_8~kajr?EFp|8xrgm&3ovTweQH`2n=+u(e4NC3z^1#k4MRhU>&caB!riN? zqZ!OO^4EIX{AdV1ch$r((o2S@vhLaZdScL~Toz6*fXuwF&C!LtsoRxGX;R|OEZPEt zuHpzF?cK~ZkXV;@T)3fC+*Fd|YqOq{nL1Qm`~=_Q65q;<@yT)Y+^?_#C>>)h8^9Gg zu>_d>Q@yX&(+fFkp7ZP74mc4!y{PHQ_(M{mjy2h3?gsN{>SmElH}8FSN~&kflndRN zVY1l^?MW|>Hz+e*!*KXd<*3MaL4UtkStfn0`}sSJ-kDGJFEN42!E(D)o(yZ@rVDB8 z9o?)+FZ`et85FPqushqRNVi`7D2fTj6iTuAZ5$!Emg!)pRF2P9em9-z-e?jrN%DIZS%HLA&ib&C zYOIjurLD9BkMhu&;+?Zoje2kOI_rBE zs|}QtGQ6Y-%ks|L*~~^M1f}!!SLb}`HVt&Snq_aIunTPJWy9l&HQJA4aD+E<4yNX7E$hhy#Ql^t$t9&#BiV%4Vlt-H#v+ z62X^R(Y0;n)*a?aJ9AxuX(l)M3?}C5+BVB0Rq1LP(N7fS>g^^!6%W0}s;y9b8?Igt zCm-BaLgjq1$H=75&XM$TCVEfeNw#vH(?~eUn4R;vh3bkw%=9TyxdHuc7lpboeWck= zR)PejEwrxdz0q${%%>M{66GVh7JtakRa&Im(F)&fPlT#TetbuorslRltYIitLTUd2zc+cRV-T$#il#nh3;;HO0z?|pzz7@h%JVH{qQe_D_%;QOuO9i z$HH($Z%bH)+dTpm_9M=iQ%$T1`m)F~%sEve(J1ucGtrs)+z1=Sw6`!&)cCVC?jW+{ zk|+hydUS=JCNL#;@{b|^?878vH}P!ojgtY^Lc)iE*jlr~*WnMdM#iy1-yrjjANd&S ze+$d@pC^(!KSv<;d_F3t;Vgxh4b7+h>aIq&KRtnb&WDsT{t)uR7!j~M$RAe}ks#p* z22r2CKjK>OdKZ|HrjPeOEF3fjGaG0YnHvGVju^A@&2Itjs0p^ zgM|6TkrBRoN*rM=DGnpB%W!L8v&M=SS!H%9ne{sh{e@F0 zQP$+@IlIy#K{(zTP07T>M44jCei?w2&A`9O4wpO0ZY~YyZeiCd&F`;N6Jl(v|Ds*0 zTui}1YdELDXZz$Mr9rhOQIE1OdG2oTkQ_U^Hf%FoUIef&uxxY0T^10p{!sn+*OyUZ zPmk#Q8YVocixH`qw&Ag@zI+3s#vleXMq|u_ZSb%K{R%}(vh(6Vis%z!W1toDvAJIR zJN=wwOI83k^K%G+>L(LR-o`T}1jN@AWCSv}0YI;>H{lMh@lLnI6dLpy0C1!kJdN`v zob7WVQTgja9-EP&H;f`9ux54agL#ji#<)}h)W)u-$iedi?#y*+T>C-Bif|9*aos64gq&kmtX|DPwsk@flSV-&H7UuXjaZb^D5p1hJhp5JtY$VOA85Ui(^H;@T$zgKCwl09{28VSv+0$7$k#HF z+&{APsz;9W13QybMKA2pXE{{bj$h=Ld$XICzgQo{xH;UVMM1a)6Qv){L<9!1}bVH#o`!lQoa~bjC zIPE~9rM1l(ENU~95Sh*tS7{MH<>jDFU71H&pc!ND9DG&h=F*eO&IUKZ+bg z>fY_{r|cZndj;9iUWjahs6#-QUP}Q%CIxB*)=7AOO{q}7mK>;}Mt&Oz8svK@slQ_9 zf3rPId?tHA&Bp`cYIUKoH4^fCcrf#Ux0Wxa0H`k~SxDY7Gn$`FA7`j!hF*^&1Q(ON zYYwSSFz*P^80w5c>5{wVkizJ_TSX2$qI>^*r2p>?Pxtr8jFhbL zFVAJ$6D6K*uMw{JS$6#48CJC%Za~P~@QlgTbwP^c0!2{w z4@+0ejr|eA8nz8ePZO|Fi&jwnuDWk5Y~(}Sk(k&J0*OrN=#rTCWB{KGJfmiyyN#3v zb%3Pg1^2rMsV_18Bj5-o1Z+0er7c&hA(1ky33De0# zYrtnlYS~or^wc+ou$mOxjK;Qw5zfSvb%kH_0HhO{owZp5&)Z0u8}&YrooOUoH)YSM z+ee1&=nCC8mojq*$r5$SEwYH1g#7R%2(s|UPyOhLHemJ{qtSnk=nF^ey**9%$IpkD zm)dnoZEITmz1C(P(tf69R=muqOW2)0dGE$pmLMQ+Qrun^6%{Lmkc$MKE)sTdA9EkV zcR$1ay?ksnL_2;bFUe%8W_xT-H%FNOPj6YDVnJREv``fEJ6G3?2qz@7fU5zXTOUg2 zY{dac9}xpZ7K2?I(DXzQ9Xb*Y+xgl*KZ~&K9VxQ&71h9|C;_iJx(_uw5|qU&|NNLg z6{|5KP#(Z`DA6b~4_Op*u^Pn=4>@+q9`!s=IQJJ#-$QHAUWCiV2u1+!NYF%wJXV`i zKmowkn>0rn0qGt2&y}|s#a3;DcqFs2(BBM-gavkgL+it>Uk7njR=AO>6s;2Le;-|8 zN=%K)J(>Sps;X~&yinljN!iX^Hb9@ib@o+mP59UJYyJ6mWw@0-!=~_dIV)z>^HAJEAhT4UZ(Ku2N3WgKt75DUZ;cG zvhU`~S|A`vNd| zl=rWIIP-~N4S4KuXg~Fe_mIln^MG(Ml*PV1PnU3GHBYDFyz%$jK$6V=nx&rqg$|A3 zP8&RIKwody1Y5od_Gt5~)7K^{y$RpDeE(f3WWDa6-`{=x*%ZkpqmO z)l$`_F4AvwrYl|hz-OboT1RVh*%eCkYPXtR1l%Peenm(!xP5C`N2&pTwFu#;r+8{T zzrVdOm0dD&3xN+VLv}9_JB7&p+K2O?iV8T*DM!~QRKq?fHo<`A&ih@hrYuD^Dw0Jd zqi;F@K`qk|nZ}U*oxNN%${o|wH40PM-m6u6?K62sA}8^YIt`0KA-wfDkPkuhvLI+% z{c_&VHkzyp@Lyd;FoEMjU;B~o9Ut$XR)SKo3XY0vrf~I1((Bf??BqjmYi7{=Gfr<3 zkez7adb~xG8vs6bEnVoa_=6-z3r&J_)CR@9Ppn(kyTDf`6*+bwdoFCPB6@I+1E{$5 zJ6KS6Eo|89vI*gJDx0)Q{MD%`M8>n+Mwr%m%lhY9!La)+i5huA(Z#-2c1X8TS`2b` z=Qz)XHuKV)O6%JTce8L7Tg&4}`oL;EAtuS$5+E$+MR|q#cs{^6frB|9#_1G80!7-v zRUkI&W(A-FS!C^*k~ioZuAeAGFyib!n{H(vzRI%ymQt=R`h&11X0`1uMQv#i#5xx^ z68@SIyCy}*U6-XP$}{3}b*lz8_*tdmcq~H8#$V7b+(MtFMrIe@6b?0n+9KbqN z^Ppx#4iU8Df!wwVM~s5v5CwpK(5!{Kw{+@?i<$NM33@^8PTm@I$>$?1s#%D0!we*J z1MrO)(r}wT3Uk4Y>_-ao5mwZfCLR*Pt})mV@`ZEixk` zHS16gJqCqmvJ%s0uIn|zkr7FJY8O{sV(@*`ZJK^?a0E2gRlwldoW~gmF)7Y*DwSnX zjr{Xqk%b9R)E=_vrbsRAlD@Q;yP8Pn9C^tTv^q79KbiD>C4q&Qnu@XsTwl?zeH{S; zDp$h4($oW6AFq`O39ajFycn6rv0(-h@~#JU}uC zL#G&l+9)HM{D)zBOC~XaU|I0F=%>;u$(ONY2?ZL|$Jh*f`tiwTRQQoJflETYB}30Z z6)>Lo_G-$X1DjVn0Z(axm2MM4EplbDe(+ECD0*=tz9N#(#`;ey%XAn$L%2Go@&DU) zkEu+}P9EB~=w38r!aKTuw^gC>C{GZivDH8bSilZ52C$-!JjXC#KN>0lb5DT@L4GTi z7&#GT-KjJ2e3pG}o11M3cbkKmm736-Lz7xP87izwpur#sHXJC1E0*$q+B^d&3|Oyy zj04E;y2aW=2H)#xQ_J9^BjIFl-a1rVJ-~G*lqtbSpe1_jCS7(xfA`!hH zP|Ss{J`1)=Qv_E%X!95TqoCyI2qD%VE^wJ}dG0t?WI=O}1X#+Q=j8vW2E>ktpuP>= zw(oW@VbR2&#RrJF(DU1V-dpRGukq3({Pv4x8}V9_WY7R}kNtFuRfcNzoYu%EGo?bP zRFA6u?$(%1i7?Q;EFJ)na5k?JddFfPI>Ohbd^u+zMtC8h3Yit+ML*lCI3A3@Wc2-K z^;xED0iMcf4Cmgn?U8NYQ-!i~KBj-L7Y;<*36n+uF3Vq&u{9X}#$LHNiiu3Gee zwG#!MxlXtd^glO|uyND5-2-r;wQZKLf(s%AJYp%ZjQ`Q=KzqB3X`S!Rq<-T`h%u&# z!&ZL@*8EAq4B^JI+|%2L(+!W`wE;ruxlcV7>enOe8NxL@zG8}=0Wix8EKYIFHl!Z!~eWhQqi19e!dD>_!WW{LR|p zpX6e8fp}{|QYm!1CeV5^Wc`gQZjfY??nBsKh1|4(L#$J4&JW|gKx_W@8Pg;tw1JC1E0w@&GU>x(z`#l&xasNopL8Z3 z98WR>g@PC9E)dIGkJ6vK!08)0F@o$wtcM$yS;u9hYdK9tMYX&I&-dv%xM<?d@Bwsl_($j~gb$y*WNOgX?H&uA$s=@3neDi*fb)WS z8deEZ1=hjSO-Ri(GbxW*ul0OhCRM3xTfkK3SMwnbsl{`Km3``3W zACMbl!FOZ%YX@mi^}M#Ph~WrLqYqpj;_8dI?N;z3k|%RfRxP9AfGualb@KSGHgF!C zNFj9t14#2BirbGMLr2FWbT^P61PF^bix9_`!_V(0k*(I5_VAJec#B-Uoe8%jE-=S? z!WXi+-pUFzZv=_+VuX4Tf1@03 z<=d;%#bLT)Sj>1;n!;#*!U)Ob&n+hdKCG=yy51k**ZrJN+`>l2)x*P+=x_=eC!r-xx#n-PKsA#EE%SgGfC1>HBYtp3GhQ$k+|fz@}AIZ?Lj zVmYDs4$Z>Spe8XKx1#47CY8_x->jX40>bV|IeO+Pf^&nx9 zFFx>998yt`wuk*bF?*Axs|PSy`%FZSg^EE{}h;L-c~uM$Hmy ztDamv8$jK3siFv&C>HfOIR0lr&Oa5l?8|dSpB)D8anrM-r<-0fdYKZ8t)QbdzD9OCyxR8R1#)_#>I7*dtHEb1q+12%C!*bKv%WKx zggQ$39yKhY5U5L}gKi-=2TX_%8X2?JO;SXS)>NK5ZC-5BBAOeZr-In?t$SAvSMfde;o_0GIZ-z?a?TtQz6tI#S zL(4f8qNbaVoYJ~Jfv`G0muJMF)Mov7f5krnI&4`dxPW*a;m2PNbJ>o4{&%8=65{sz zCnuB0epOP&7~)Sf+{+Jq(2TM;IlM#>je1th6jC4t~Xoe~g@6Y=)C6Gup` zK7L^Uxxry~^56pD7jG5LydvD*{}Xn6cjtz(SZNKYx(?vXh)LK)6lwVUG;;I zi?u_qj!Oh|3`dNj9%VTIST?-C9PEF5u+E)NqHd#ncK`TOUO6fjMt0FW_; z%7v(Wx4^N<30C9>Z3topp^@AFEL*`mqA_Zh+Fm#*LdYDVAD6X#kQCRMaF?{R*h-6* z2_YLmGp&RCBXo2CQqP-noL>;TDcF|Eu88~XL3 z(hLaO9w;s$L9(qw2!gSElqYne8wX|fzsS{NxlH6n4G5h2x8D9j6NGXgwk8CbE3c?a zved~^EoP+@?!ZX`o#2oiEC@i)cogC2k75YXD7pn_&lxcX4>1@^ks*ambq|CMU+IQ~ ztEJFV1ri0TSSta=l8JzJ2<7fEm;?2_w#Hxj{CW_^9#C{0Nn7xh+ulmO=lt#UE~OMG$i1JSED=7>#0e5iC%6rxjJ*I_ zMIkBS42$yjrf|-$XINF6N?M>fBPP$Tg@>4pG##toPRnoAC679^qMf5LqgcRf2jjZ_eb$4qeH{S!#1_g?#=1$xZWe z_XKR=-Kt|!J~5YQuFX!@ue`6ctMoAd;~Ay--r1VW`i3HD)2!(-2FHW#I)h`023lUp z@Py>%y3+qXc^<Y61t;35*)$F3da^BE1DTH5T(UxB6;N=D2@e|9uMZG$tt+E(;oni4n)mwtxRqlATUAKX?7tU(TthW zgV+MYeg;gE(6NpU`6^77km;gTCn~%HC&LXHEePz!uP_nh`n*X;SNkPKqNJ4^wZGiLM{uMS=GU`>> zxUW<<_{q#c7P)0YxL%%TaC94gPQ5TR0fJ+Z*>{A&h6x%@7@AniE(AY8A@Cj+16P@Q z|0J(|(iv$I@F%o3Bl;qEWh|MZISuN3oRIM-{Ql-B;j>RlW`*eZZO4qhoYN>{tpc1gzP7&R2?q&a|~sdthLoIuC9r@4^$kSf&Di;BCiwM#Yw zs%?fIT?X+oRS8KQ31mQR(RIgof<-`&LVfow7Plw9fX3|Zy}b#SUQ zmD)`rZX;Q!&CM{za;Hjb8AURdTlxESKWtE=Aec62{OjD!&y_H-P)to&b@C&Oz!4OBzzZJydG=7=kcom=uN^aYZ$z^KedDUl zNaernH)KwpGYDVQQS$Es(OgVCQmzz2#H2 zB!X#Y!6*q7FcG6__ytkPf-p8MUPQmwA}Rv%iYUD?qXn)1&F}3JM1m{WGGB#mC}zR|{c*sow0u{C?UJLChXq(IVc$;_l=eeK5OI2)&$-5!8QMa4XFP zv#PbkS}=GliG4wG;gHRzX34I7n}JuXn4(fImp^~rtKFIrC`uKnUjAh__DNBu1>{<(Q_<=*Vf}fhXF`3l#85qYV90j_Potlc$!_X zKpG=G;c6pCbvP_gIHbUVGseBNHkFU-6QI516?YH#2QFBX`+DGQ`9()=hL(y#|miT8BdBBy-Bt>k^S9T?)`VEyaG_xSm@!k#;Qk6om# zsGDzbfpowHrg0?kkCt2aJkRm|QBKBqct(nsTg?dwX&G48g+4689N6(^7&1#UcWvn( zFiV3K!ICHd6wLk=LBGGqzo}eSP}v`5ys3kly(7hac>=&Y?-d!dG#Wk@$N~k-O&Gdy zGg$26ntPMClc>Eqxg`bbXS5S`VG)a&pS3@Ax5^!-ByZ|cPy6&0#5#djqet&(Y>LnR zh)i4o#jJp+Oi`x$dfE%R{j<#3!_^(t#GJ>I5K-d#z4J9|Q6EPWKy>LtRf1RuXp^hB zP^^eu+kC52w&wOl|D4B}Q0I|xF8>E}IN&S$g;N{?1J9G-@a0bP#@_zQHl&X)p2Ylm za=0G^_Cn}%eF(bD2WF+KCa~Nac$iI&UK*|#{KUY=CWFxMN?#$qsq^-ss&A=Y{0`+b z8pgSVz<0FDmI<@YotbJJselZ51BTJox~}6SQM&vf!0NGZeF^K3d=SFoq_H!D7?E0; z5i6WYTf7)yL=cJ*GE}jay$R$eKkQh!-}^6Ilyh~5XNSrPad?DnZM}eJPC>{^^tAZF z;x@D;9~|{g3c^^kEiwz03LB^ej^okD^Nq!2X$TAfPSM9{c@JB10ZTVJ3fiIrFn?hm zci*&Wl-edBfJ=aH*zp4t)b4FGVB|EQ;+`c2;O+TVpRKgH2Zy$?xY2NWKfGlv0 z#hx<>FQb8wxV*w}iTPWyom|n;u9B1JJPe2wR!{S3J{SSwE5`qT(stU)dt3*qza3N$ z9MZ(}WM5a!3a7t&!~D9cY%q{itdD5vS&^ykc7{t`NOK`{_7CtVzlsIy=eY^0E0pEPF^xKgZKlIAR1T4J>@ionzK}En=L}P zK8LrrcNYpm@Vti7I7y4A% z_zU(3Yg25810DKP{j)(I?N6o;2h*7ZW4lg1{3vzO1R!z?Jkv+C`=3sbSX~-~I=8r8 zyI*BDTba1(uhpj-AOthjjYa{tk)2OghRnx}WM->$y2>@|2UFQBc(o zie9nWJ_u$f`i!ARE>4&RQtiTk#bv7GuyS;GLu?ut1I~GseSl5nsXNHLZ^qZVFCcHyTbJ zS7{i-K%P8Q{P1kLS6o-$)9<425O*NAF9vDIq<<8AwGLk4;+y_;g9!VFSa+=`%6++t z1X)d|Cj@@aOduUZ3v3b3Dix&8RmDm3sEFoZ4ljiZufLg9H!#oo&JFaJ(6^e!1Tdck zBPfA3Z9Y-z2nB5w<^oJj<|sO5vau$xB4g)dKUnFX_rr+kV-yZ4;^ zttg*DW3T-6OC0-Ku$0f!rj7tXu`Bd(wec&3h^58HHg^UMsD8hcVboDKDH?p>CE5W# zg+O`k|ZRXJhyrvoltoza}m3TKVYklE!t<;!)ZK$>}h#PxkBfUXOZ7 zZ2RsxivB{f|Kpfv-0WSWpwP}_X_s}VI>Jt$x5A}~tb4u7pcClo$f6-(Igu7>5KBN*%qCc2}<6 zK|#G^bK++-M!F!7nq168+#VleSIO zdZ<<3Y8=f27)Q5uuLyQ~@;!;6Y34YmJbFF2`OCvpu{D&8hR@FaouBRNuU=GLUrm8* zfSs^TY`G%aou+x+`clE%O$m{bZ@gAFFCHQieE&@DW@MeTnIL%zpp*%BnQT94($1~w znXae=W@*RvfbZie2^JY>oxRd-+ir)PPtHYxyFpiaYu`nkN?|!5vymoQG9n&idWdZ{*R~fPaWrB? zWw>iiVx7#gTXJuAQ$39c1QxLi{Wh0lz}(xX+f*8W5oG8xO}cO#|D*zQt;>IjciD_# z0y>g@24@}0^g0Qs-sjm*KMq#RctLBYGyqw*+s6sg-)Q=b(%kIDXoj_AVP5$BuZmoh zcD1D7%d`vXUrp9Zr{bNTx+Y&`ZRAKHenJ!bOQiVR_+1_nP6`*Pqa{*y^mvMsG(2YPO>ROGAetSII_J>Y*t z@;D=O;cTOV#Tu9%t-@v}G}wX!nc4KP(Kgrbq}?jG#oFbNyz=um{A+*1_ww)_iui-* zIF^PaMqY7-m@Lat?()g;vSVN+R3aHPRAVcvkYX93%B=57`92|VeSme z&?kS&Ofy^y-T-UVK%J5J2`XZb#iCx~7oV?F^`mhFAeo@f^}W<=Q83lC0@3mecLP2q z-|;5LhMcDvTz+soQ}I&Vfm8m2&gEH``{xrgvL{FL7Ub*Cu7_ScO(f`BM<17JmK)5T zA^L#rCG<`O)Pn8IR4QU&)$pQV$dl3o#*vQg>n7Lb^JJEL(%kP#fat-NRJgHMuC#$PfG z*3a5b4KdDOT1hk-gb;td>go&Tuz0LI88I38Eflp zez=GKH`wR&dQN8q8y+wh?oAim+fuY5@BZ@Ym2ptJTYYv^Qk^Kh0kn4aC#uBl!@i)z zZdvr)aDdl=L;$lL0US#>#O1&*Fu=tNAf_c?rVsV>Vj#y)uLe;PUaFI?|9l%dfo5=) z$$*7I7Ft-(AQ$k&AwUy>S44wF@Magp^^SNG zkev`)BlZ`_!sXjw^Jq?cn6$7Dj72yM_Cx?27JO6%^ZMcN7@{spyk7qGJON1m$WvFO z!(nK25Vs9YJRdTukg1l-3$Fp-0|N6FyupMIF$eaSJ7o^10HE0@weQZ+8N}afjztfC zl#S{E!L!B?%IY&5`KxSTypY!bAx@x@O(!q2o-B>ef`Gp{5t~}I5IA5r*m~`{s=d7t zab>gs=+EED5~vs@U^mX?Swcs;u{G0QnA%@#6}3HjfHw)7{ud{+{N>mbI1Voi7+ijk zV0}UKcS;xCx=g8qe1$_}hGKMWN#^p)VzG{A(c4Mujbqz@72GLx{&r42NhM96D6e@z}q~qjF3iW zoynU2`H|CyokRucWoL2eipd3&0U^3CzPAB<6B6cUM{NE|-^jjQih9A7mh#+}gG6oV z+(W9jYjMsGL;6*SN`M0+3b9F2`*m;2emYup{EBy^MJcqWM2)&$+eW)W%3u|p2F6Bq z0{y%=G@ST#x1T_}JOw3uq-Ypktrb@FJ-@s#=E>d3^y3!_l z^8Dmt6lk#NcYH-XPR370dN(Qa*N;N9nWx@X%sjNb+?>Z~UE#v{hM|u9BwyQxjn1yv@|1@rzPE02U&&%Qs8`LV z9XA8^u9)-Qb0Q}XWZKKQEXRPSx=irYPz2MSS?EP0KSk|RM{CYGYJUKT)Y%1+a5NC+ zC5Q?xkBE0W|%cO;_&~;_Pr~E9L4LGuH0*t%~eOJXSyacN1g(p;^CLjhDaX}1v z0B3b1ry~$)jDZh<>)AGJ$Bt4%SW49n>^|x|ZK}cU zO9|wq>;WYYemaVlgqFz((YC zXU>0>;d39M%aOJvg-Oy|#v#t+``ag<7`{oF>Fu0^WTy9d%RYdor?wNEUEi)zC)7WG z?Pi9b0|fn5_Z&02zuNDE;^43W3}T^+^HVhnjnDY{-?WQR%?FyXI6VF!*qpOH*nKC) zU}SDN?P<~%?e;1F=Nlf07=H4+AXNFGWPM2N>u~fP7&ZGz*j5$)i%|wNhS2L@t6={nf*&j)}c!0bK&AlEt zmzm5!y3+x>%6IL6e>TvOEEI!r?ayw%Lex%~<-QBT;P3G(mB)sb34>&>KM<^2Pwir=Z=%I&o9v&uNP0-V&XvpHDYC5JMEru! zaK!g#BTQoRi%7xNcT7lEn@67e#U5B;ICFCC;v#!NF>W`8yXMSs*kA7*oXi2{=~d;X z6nVK^`?(dUOxvOG0)hU(W2*ZfN;~ZG&AVCL@nTz1;NhymcHiFIv3hviA8bT@^^Ww2 zWHHD&jjz_zbb*yYtm4PVV=rC2Y`zs_sC67(w^o4NMPB>VNe||L76H$lPd3{?B3QkS zuZzvpzhbbB`eqF3hjRwa`m=I}(9C-2LbT&t7xN>X_1O8LTVSyiZX^4tdQX+69Fui| zQP>ifdGC2ggi7v&bk)p&t_BU6Jh1Ne5RZ!$xvM`QWf^4VR5-~f7v2dg#TPb~?#*j3 z{YE@PfmeEM-kDSQ&BdCkr{0npcJRx23J$bcL<(duh>VfqQ4D6w5WFs`pGZm{#;jXe zYIUKDl1#-&&A$Mzs?vb81c(b~Jr*;^yb2&b683f^LKuUaaMg_1?;!v>!c;O)rn2yE zJ`c3&+^q7|7N}aD^CR%ivlQreAXf@+3_bmxU?8TljTro|0xK|8KA%%q^>MDh5@xX$ z^5Mml{pxX-pg#aEUGRQsuaq==vuOGyIO}RRd{_=;0`T0R5kF7;9-0HK$L@eeu>D5j z(;tuq25Q?%vP2r_HjzN!_gUyP(_8}fycj*z@Rv@3;VbmZ>kbbJMThL_=|OV<&KI#r zBSUt^(3~WA(ULhFh9H*HjJ&r9f<*r8Gx8=im>ULQVbb|NBr=0h3VA74ItWeRoGQIY z@+DD)df@ZK()mjka}Or@`i1_+kF zXt!I37v)8Qq83FbVE=4Ohe^4sH1)Bn`OGS#Zt~{}7!*u_SI zUd|JzH)cCLOq@`UZ-L5Cko_DQ(tNRUuf5l2*~}5(J$=Xurosih@wemXC~NfRpX|Wy zlLeB@iXs%lqTKi;`B)rC0__{-+wQ=GTHcMvaA-pxTA|Ou{GOU*%DhSCIFUC^?<@}b zhNAP@FD(|)ugr+gjJGCuJpXrO#pa$Ch*)%{CHnFkn8= z)omEq5t$#`LEUG_#f;eI`W-Sxy(wqsqgmXyL1A0C4K7H%7qtQz(<>li&CjRFu3pr+ zlCg8ybc7lt#&}*BaYRMme6&rrsL~F`Mk0?nl&th1F%bo`FGDbmWxIzd92~e}ma(#2 zWs?0gKrc4*aWz!7zUl!6PUcsbP!IQM+9aE4W*BsC|E5n@;(HqEh|x4cm0wP!XO8lR z5GCU@C^GC%)o_z2x=H(ST{*3L$_Wbu!8;&&*P6U2Tz9EqCa;h=$jHp z@k4N|I_3<8S*Mvb1&40hqk4lojGm?FcKOs8dyi#!4Bofni4kSo&f0_gm z=l52f!VQUNTN3;mq2VBi!>Z8#2IxvFW$^D*813u^v)!hNYUvW-M}9(Ral<~WxU{r0b0L4wo1INLPdmo_gx5(FS;DHm;1xW)gR?nER zL4Q|*4R~Q!$3w*=i~1*mye#Iq@cpEjN3AI62HLwVm{yI|l}wwOT^WPdVqFBngDn4w z$bk`s!Vj*Y-yUD%!cfMX)giLFqz-@(Q3&bMr^jkg9j6uEUUg=ekWI*`PlkHLCYrwkM$4z00a5NYQ(4icr zO)9lY7jF2>mXx8^k{+eB4g1#2h{=Qm;KQLRt104qj@Sugq-pay~U0=V?K+7K>enAI5COvO-|V( zrV4T`LXx|rV@Fx1PM%ZIg~ZzAFXB;J!)1YoI2X@TO5Fz4)c=?VLxoX=FZ)g@uEeN! z*IGvf>z1iI8AkWoi}FGS%l_wAxmFt&RYp#wda^+%$D=7g6J&wKXVli;U#HGpuu}H? zxXL&2Ld4qS#U3l8Nhym5pM+8%D|R)E8cSeScf0YD{oIw<%8H##c1m4yS+} zFS$(}Wyhg<6L73uf(q0XTiK@N#$~$i_?69Sn0;n~Z%AAmD1?-u*K#T#>}K)F0wEeg z1-nwO?dsqlFl_&9YDOP7E5@YpHFA0*ZHd)gxR3lA*EmxaY_4f@5Vq_$&%eOv)*I}fWj751P|b@Rr&>n;|n}7wmA@lwY5wO6J0t zQ^Q!Wocrrfun4f97QxUWv?@1P;?wE6d1Q@%^I+N{`^9Gn=oWu}3V@m}hUF`<9gY*s zID#L20;qCJsWl~%acGTxlEh`cmN&}v|VC+6JmM+0SQ z&C*roZOgYP2xgqBE62KbF2W|V$Kswv(+6Z+l3mT`nEGt&6v)c`h}riWf+K|pQ(vmM zd~8`y3K#yC6nunem{sqV7hUM|0jR&Oo49bqOAm%S@2|mu)ARRgMxeEUq1g;AG4Lq0 zBA;>Lo|@DvFb=p9x$rZN)}CEbH^QpUz;KKRajdCXN#~#M$pG3Z?iIzVrDxpZ9m7cN zM)|%G1;JvJdcS{|q;GZu8WVZ-0j+z{SeFaInq8%(w8pLZC-33B!P4^)P@OSS#O-!a z)BC3Qk*;+M@ptV;h8MzCRIm0+f4dA~eDWM_xx^1c`4xunSFar*8R<*&Bz2oL{z-n# zzg67P^r_!G%V7_BTvGj4StG^yZ~Xi-7h@N`$q;euO1`~zFD%M}89DPjKQM+&&1M#C zj?G)$3qd;Lil#+RjKC>`iL&#CTCHC!uf%~W6-I6*h5NxQlOAkoAC3<|{`jRG(s2OOFF51-J;nqA^2UqoqVi-Tm0d3*XQhSEIi;9{Q5AE4?pUn5QuBQrv!dyd zOmpCfIL1y7mK265Mf!LWl+|R>W{m^!OS#g))ss;B^jLeej--7Vfx5Ott0J^6_f{H? z4sOqWbLt>9T_{1VFyPmkr2jTdq&VW@P-IOcRl|>bjQb3m+lF>BEa! zHSnUayo)HJT}$3Ngo*sQjsp{_yYQOG#UNMP3${@qe4-rb{;?n)3|b*aRLzh0AwW8W zto$dTlI!EB0#>UXeOE$tohyZQ?#N#aW!?+EPf)4bqX=v)VWyUcK36_k831E&Hq(n* zHr>y7w|UQro`ob8=T4bUcOFbVx9(@~TuSvoDeelyE_>vr9s3<{kW1f0qW2#6d@Xw0 z^t>uT8Car(Zf2r5dUZ9_E zm}W`-g04)vDiI^Ju%4S0%13U*tM5_(`-k5!V z{Vz!Auy)nDeIN5bL5j0!{QsDP|Lun@X!axmX&g*NH=o@^%Q4hbys2ys2XLU{aVqX9 z>uJE|y&&>GQE)I2ep`I?tsOGFY5@GJ5?35h$b}J*hSBsfC;9qJbDW_aI2@_bKS)t zMoB%mRCBk=zY>(fg*7y>EQXeJwm69EIC%e;V9}zrLxi)%C`wBA&Lbxcm>)RgnHRZD zqrK$!%O!6*Asqg-Irg2r1!yXW0&fJqRradzA>%mkh`c}P4s{nAUVQ#>GP7e6cD)X? zezU=MIlEQ~7j_W6f2mqz^5yA63JDxugG%7g+(1KdKF~Q8v9okwdmwk3*jF5eGfao+ z3z|16zX8lmygv=zd^}XiaY#@BPf`M0MkB|lTCn$h#5~MU@#sHh+kO$C--^2dO=4kx{n>@FId-!N5)Sb_l<5e8W3SpZ0yLJs!pns*nbE{W-D z0Fxf!SgQBT0Z69{0f-~`%!LX^tQ-0}?P(Y8tO6~~Ozh!$co}XL)Sc}Bm^=~(Mj_#> zj2NZwIPg>7*xHvz8b)=2eEuugjur2wwEXkwa3d`EKyT~FfpuP#779SXE2pYHDPC)t z8T`rPIxpX}Bk32E$Mo6(u$#JKXwg1=J!{}Jao;OUjg)2o1h?`mA5*aBh;^l-jm?gh ze3g^XGoZBmQD~%IR?QoDm}4ts#@k?0kJ8cXI+VL^f!{j$yh8zKXPlF}Xdc`HVf_o3EKzE|%Fv#*6m!GrQb zi$TuED_hjf0gva-ms;JbKKXEhIqB9f2e+BKqb<8v0>X*rm;ycS0Bk-j#m5|ZCTeMW z$l>=9yp9O#({;I=o1;Em*-@Q>-Chy^-PVF@{yc0j*Y{TRt$4o#CJnewoKl71GXkhD zW}mi9+I-HwMG#^cgl2cx7o`Ee z?uwm&C}l0fXkxrBZFa)oemB!S0Jibx_V29(8)HO^W=EF$t+G945M@iwbXyI#JIVSc zD1mPyG;ZeC+Jxh(^BLB<6FagXJ#bP65~UQ|e=fu61l^>AbjgY^m={Fz=GdNlQI4h) zkZo>8E@@Hkjzv#m!>ES*kVG&|8&x^<<{tqh%FU)!LkRn_PfWAU2O!HAw&uf<1)7^U zgvs(vz!6ckWojd&=2%r*H$=f!DF`3QIisrq-M$ zj=AQKd<=M4{-Lcjx$#j&%AL70C=##kW~n>WnMpf5P?EZao_7$|?j)#kGpRiLx*t$Z zcs}&vgDkfQV7+KV^O8D2aqyUhsk~{$>&@6-u%RGie<~d^}^PG7ZiwcDX$~>fHj%1!wq0IBNOr^{$ z!V=4}-s95!d!FC-ZSVKj_q^}3ecSiiwtLIXvetE-*Kr=le(d{xoZ(s;%Cs~rG-PCC zv?|vXb;!slXUWLO*AMK6pA=l8=7oPGJd}()bX}}Hyl>yLB2&NZ;p*t(;b?c~w3pRA zcRLqnAwB^CzAKkc+j@Aox=Zl$JN?Hi_+0MU@W(b|T;N>}x?aEOPDVy|8~KksTQi zDbX)RJYbY(v{QX)El(d~{#JC(YI#j^Q^Wbi*h8uh)y3a7*Q9*CgyPS?yek;LnY3BM z+2VGp89twkELbu2vRf+q-uDN{CHDShmkU0!_d9C(G)m-~N%DQL0Py8Q_Fw-J6~&PK zv}%?2ejW1@y7g2Xy`CK&5g`aaQx(K1>x@q4xwZ0^$2CQ=a(;Cm^PUJ$nXeWL*!Ca{ z^IOzO6^T+AevdUYFG^(%{(3d|$mu)6F5{yqizSZ3JwE#Hj7!^=$|u-+njQOdV*|!) zI+6rbIBxrncN-It@A_(thTf=z0`z~X47P?0DdsWT&5hGj*((Mx;M|AOe4$4k4B z61$qZaw1_fx)*EC zWVkrAno~aJl}OB6DDqu+SFZN_tdip`qoO-Xtn@N#Z9)Q>wtPnKKlT-vVefV%4Zl@+ z*_4?TMBEr8U&tObWX9MyobugXpKGTT9ld3ap1XiGnoOOR4BGWCMsL_#7~4KNe6m1n zzZo`7plJAmmC#8c>yn1W8sD{od7}1IqaUu;rsWl{gGITyL&mkpwnwzAcTRoIZyqe0 zMZLbH{YfoZ0ySb;N7^N1`fvUAN?b=T^yk?G?rzz**RCh6)-Ivjj-I;~N5jND)XC$$ zJesNWL`||mKZKe4uwvLu@>9T z3@6vkCox>Q30EJ|#LuJ#jh_;~cfM&xr|IE1NiRnyo=)b^>84Gc!nHo_Kua6ia4Lt)I7Fepg*V zvlF)L-EduiUy>{xW_zafb?12fwMZ80_5@TNR@8a)ghMjL65-p63qk<|JSzQ!;k(W2 z&HYK@?g>)9zlX3TsoPnN3&cVljB1HxF3D1A>*vdh4g>iY4GT=`oZyy8i}#jXw$&s& zW^eJCR?s_{lmt0^eHo>>?eAQ-Wlb2_sC|aH@p@!t3y*aP*t}UESZkI^?2^U4x$&CM z%daz47H5)47|hXa-j$gp4&XYKIRY4GHqTw$Ubg)j9VK4vwN&*;lPq{foQ!_@6`xsd z@g*C$SNwLg?5=Ggze(BUk*Zq1O|KW6n#(nWjiSga~| z#cMJ$=m$H%Xd=Vlov}CsxW`3}BwLG3{1P@9R$ct&hr4cNj>C5b#qa$L3piC9Rpx}1 zFe!KT#N#G#QurZDJj!LfW_Z&_H%p@wiwfNFnVId*z}Lo$x}*f~7b@b4$EjbJnH1*S zYIuMTkb_P4)k+QtiA#xFQGsV4|m{&0sdYr0u#AL~A7<}H3fHO3Vl z&#&rt9&GBa4wcjpcme0G)6EbyU%nCZ)B<`TBqtx z^g^siP#-z5&IT#Y!V)>_omzCE}K?#)p;%PzvR^Pf^6wEJUB1&7(dt$ zLM=&6$6nH@ooB#H80#kS1#HqV^DX+Z6);=)ceCS4Qk$%oMoRI%+Zzka3ljYGJ0Hxl z)DuNIURwm(Mbk1|R0}IhNwFv_RE}oH`)(wJa41Y<#S7VtY;Hq-C|{V1%fR8*=X#g! zu{~KIo~_;3Y~~0WvWy%Lu?Qme2g;!HAtC+2AFSORG4y=uJy(9lX`}?dwShJ(AH8?u zqqjB|o{MU=#e!e?dDVGR5 zH$S|kjo9G|R(W|b(oEK;QsWMh(j)m4tnpYS+=55)ygFrgVJTl{LBj?=Iv9J3&;7*rB2Vfu|}-NZ_748 z4PN~1Qnxi7f$N7_Y(1!ddWvL9MZ*|2@a#KBngbD%z1TwCKZb{ zgVI;b^_G@Z*U)f~H-K z07k7jwoIO@WY8owaJnrX@8Zhjq_1gczYC9o;uTM_c}^1_<~6s~0IBrS*v`lMlp^}Z zIn@G~*G;9@M=RZD+Mp;n>;8#do=^2+>@&Cu)xD>kTZTPDIyRM54LYLZfAR;P~@SB-4c&Meps2sqlic5vunDl z)v_423HMHEFEM4)S--adrntPaVOgqOwC)Pg&Nf`9Go;)wbvT4h#&&mmfxp;q(|O`I zL<07^AWKU+#UoWsd{#g)RPyd|3o}Es$4XsJ=RhifC8XXEJ$j##dSt26ShapLi>R-_ z8@RKoJRwL+r!CN~opEjJ)tphzRB}&KVd-_#6#_#I?>+qD&BV-t!Gai9(SEZ=0n6=t zlw(tBovtbqop}K+=8<9r*PO9(;mrlR12-`IIMA7FLF%meqO0AB7PwoR$4_jqKK)}%KS)uP(3m4<1chYp_079ii0-BnKbBDIrzf@a}(G4?2N+VH^*Nsu}3krm*&zI2pgDe zb}x*VIhi~yl$0qqJi;z@;yo^4bIiM}NI~BD%2>f_%LqSlQ=Kiy0~?E1Us7%$+~@we zP>m^?r@7|IfqKyBc^IvhTK;OQp3ljKfM05dU<|+&&oj&t(k7+#?u;nwYX_}NHcXce znirE7;pqa#?H!-Y71t%;VpuHt$xfqd(T=EsiquHLR(u`W}&uQe@G(~f`9NI^2Bj!-Aem)^y8q#$_L2YL-q5?hQaS%N;4hrX$I>K^=d5Kf6|q%ZMZiq=L<)x+u|{VDSAF9Ytcr?y6r@<*OGA}RWB4z z+uDY~b3PpX_&j-iT6#d9#kkZ&w`16ufOivI0Rw19;{J+M^_7{7vne|HMgk@kn&ed! zZcOfal))4pY@B_)1=sjRpLxr2Iw_cLV&cZ>f^NLJTvWT<^0PzAXfO#VnmRzZb44BP5{Vg6>H$n0A@I@`vC zGb{HE8waBikJ8OrR~kXj4?(2U0`?`h+P%a-FDM3MHj2SAR5S zY}~USdhkGm&89DJ-a^}gkjee_u^{aPNl0&`j6w}nkxeblcRx|`{1?o)FIC)b>cd&d zOg#j?jEtkh9ngf0x`N40P$LRrIjJ{nxuD4z(# z9JVlt+@==2 z93rwWswWI#iI2H$>kT!XT%YAYuE<=!d5KaX{xaqM!(js^Y&0|p>*kkAs{!nkW$9uw zaP@U2P#_~$1MP=;-&0~j7UF&!sqQLLtS#JLWXzu1$aA|op;=qsKuhkHevM4M7sGCa zvYjPhQhYG7%~lp#l&>jNogPOy&F2_U8mdh&unm?>fhI7+TN(MXL+$m{lihQF-Y zs9A8scyqS2 zzxK+cJd&DQ(@`a`y42jxy4^>3XO7QH?=-|cVcFmJvH*7|ctsJIj>}v|yMLylm%^Vu zSc%InHzf}qaQD+@EXgK&sc|Q^Df{Z}d8@uZpC(h1r`r>;3>T@6%=dR4e64Y%^m3+e z=+ss6IE{72FS*TqLG0^sm&Ejg3mD`!hO!^y`VB2g$qlcN-rU$_-guLvEx%UpzI( zCD|8#bQk#~P0W05Nnac{ic(HYhso0_{}OG=#Eqz*I zU})|OOVDAmgDN>VZuaa1I%kwSncOERS7fpYEh?6hC@# ze~66Z&=uYHx8!w8?e%rb9B;b!s*A>Nt<6y9_tpmm0UFzk6LA{(!zYbiKVUQiS3kYl z%u&A9*$=GP&z9I$wIs3C+`+vyI{gFP#cy6?Tjo4w58Qw?wAnVuI+^?aVp^LFp-X^f z8@s=&z_`>tK1VmJY1_!9X7v%$>u)U-);o6o;Y{_vV_fd+)bg6&h2PDL8oB4-umyQQ zVwJvtj+|Wi5b_4vcoe5b(v>{J!XrkT&~RBpQL=_oFCa7UXLU*)cofG;UT9g~bFDxR zT9NCyw|HR{WYQPwiTFgK350y`FR0sB;7wE2@ACskaP$4G`?xZoR}!TC@NufK+*G48 z9g;X<;D5r`{*uAx88xYAp_N5J-+YBmY9*LUKR3puvMJRCTh{J4{GsuMe&JqNI4#Gl_@X3B@`zZkH+ZVg>0 zdAT%@8)eUu-m1p7)U0(#;z@J8S=f3m-OP2W-d&HeT3)_|FLx(D-oFS`Pu-3C6jWDN z?DrP`IbXbC$QqDT6H(G6pg`4t+vTgSqp)Xs*4yX!Mqp7}TU#V@ELCS@ELZ~Gk^zF1BWYnT}-WqddILRAVmPF0}djtRae* zgwR_H2%jNi4ghDWMng>}OT!5$5=Pg*fshZy$qEmQW&-qMLtLFe8d~wciNV7n4Og}T zZ|5Z`QVYz7b#oM3re>;4=G|{EGOb!))iuv!%L{4OOKX6iER*nWdU~=QXx{++g}qCB zM*rrNr;1-Fx?P`)?;<>_vbFRl)2f3NYo!i@3HdjR6YTqQ)FQWZG@|Ar6ZI_5|D>(RwO&{feZ&h z&F5A@ohoQB1EbJ^Px^Oc1DIk%fVG zLMVp;VJ02}VaNdjuw6>ZD-8-v?UpO1j#R6aBTb>mGg(l1zW!&D&3N;sErN+ht-!R> zzm;1r`&P)Tex5-*B&}hqv1)Ho&-p%q4CUyuLEu8~{+{V{LD*e{0en`lCt}Hc*+}D& z+obEb0J;Glw>s6lGyxoG94H9!kJ-^*%cFF9`ln>zm;d?$#r?AazJ#T6bzpNH`Ky4E z`D|L_(~b8a@Mxt44_A3QTptYoHe77uWO<)NwYNpyAL-J+?EEF{sK_rV%ciGZz{k!k zPQa$8%FkO^c1kzqyhurXL>TBmycuKH-gVKJk#8f^VG2;JLO=%hR=8PC+$;aI{r&S} z$^luA*=M*it#=FzktVWUi*dLQKyWVjV~|fH^AVOK0X?ObXYj5qPQbE_K~w6et4W>z zmKq{V0CZ+~ZyGxR91&}e=baZ4p7t*xrv@L|``g>hX_|>5?IpI|p9~BjmpeUu_!B8Z z7uDi!`BncyvItaxqBYFpdwXes?CiY+6q70ZES?Pua*ze~3FLhZFIC~W!NSy~Ek5&_ z3NECiRC~RfgM5}_UR&it)B-RAUvgilX6<@2X%DmzV=v$A*$e`es0|_b0|`r_#V!*v zx>X)_FGXF(gEtgP|Ebc5MJTA{MTe! zIWekP<)~8v44fM4W~B9A&2|u7ofA+F)<@VR7<04!mdDEvFUtjO{d%pgrx!yc0jrn^ zkR}sRsz|%UX%O2E_xJ8aCjBd&(E0la>#+aGDO*yikWJKHt^vZTts_}d9aw&a4dK0) zhGL#=qyYy$A{J#yA^}EI2WEbW1PTn2u4w+_4X++gn{aDp%LEn18j_Rsd|wWNK)kkc z_48)s_aN8lbc%IETI4k;i%XIAOE9lR#~-_(I`j*u>C8D$AhJ^s`JXKKJ>v&~S-1uv zpeVx2{Fg^7)6s}k0FdtSwZD2fy$1O)SN-%WQtm~j*EUe75hpDqvF z_}E=Q66eWDi132sOhJ%rToYd{cm)9f5(Ski8ZVYG?F1lCgi03JcmWwh(}BOMsj_dD zC8*AA>e@??!57e{^aD7AZJ*t&@hK8_oeV~BUFA%QAH_iT&U`K-?)V#qjnTXZH4AO= zLI~vW(!e9?=MuIY`@oY;(yQhlw0K>I91Vn#I;>A2!429=y59VoLad=Yvr&-`yr|}i z;JTtgOTjETb-v_`j+Y6j@?B}5R>!EuF2=b23%s63?I#vBvkMWoH-wl08y7Ug-j75m zGpjSphL1g6>i2R8r&XtXP^$CB`>vbLJS)kqp0rZNb z2H@klOe@}lJAkGlVC(mt59SHKw7K2Nu12bi6OaQ|r07-&0c}e;xExA!*CEDB^luFJLMlw}%%Lu``EG-4 zpR4=d(e(7M0m^gsGwV#gd{c13^K}99Wuq${3ttrH*3&ON+KX1X8pd!*&-obN>3Ew( z|DU_$&Jb(uung?ESprd0L*~vwiJ~rLx^HtnCceG#x&!F&MtpX)Mv9bO;Ggf*pqnm< z75+`lFBas3mP!ZPpj_AkQ?MtTPy7Z2En(0iXv}}3-_Q-61O|viM8Xj7Qfdf@VB~V_ zDr|**$?3MJz6HNoRS7Sn=&g@ry6IP`dTV^$C#KeM6BGMKR8f zGydD%obM0W1u8rNs!`m0e;)N^gdyJACE%uZ#B%qMT6hZM%E!^ItTEI~@dmhnHs*9{t-#6{O%0 zHJwDvGSZFS>~D)SFEp3-t31_iQeeKuqa9q>y73P}crhnG_{XLEHC9%R1B9HC%BH=| zCMUI#=HR>n-CY7Cw4sG_%2C2lg;CIr2uuWQn*Cd5U|$DS5PQ`X(Pb)^4e*w4{OgnY=R*8O~fJcsVb}ynoTI!GP{V-TL;)@wzrnV&UUdN&L z4Une+uvxK)INBnxvE`rIuo!w-4)ICkSk%{nuocvXm>}K&;Da@9Qx_e(3%a`1>%Z^l zqv3|@%X!IVT#}8b-NO}b8B0ooB&7&Ofn-VVBmbH62SVy@HBhWU6vvxZ-UD%mBw*PT z2lDabd9Yc**MS>n7UT_8tDDlA_2#5+3fgz?eS)j_nI}oGog79=9c;iu^WCH@fZ){& z!i68}-rt2alhs1?z~c}%3p8HVpbtC#o^E&G=RWNH`=^Q!-04EVKhI{wGg>}X+OLe5 z7YvS;U)GRPtRXRq{4(p!9|NF^588XrXL4(4JoNRDvlsV}B-we_;6TtacnV%ZxM@i- z?Y-(Y*ZI8!U04w{C6?~Pz4yaUI@c-IA;UiWar^J(8J#)U#VYE2D&q0p{hYq@(2@Nw zkMk^y91V(J;pUmYuk;yYOXHA63;&o<-1|WRGuQIoH~-H@uV;NT^7RQ@omrZxfzq0L z*D7aJT|g!c2B>uGvAw@OI|Na5?%LA>W3Tr9pT6Nl$R2o;)4|EpdHwiuuoyZ6By+oe z|MV~z3*Hj?mMjY7wWsqT6fA1drAR0cgP`5M|$YC0b_N_TRP~SpeXIOxUw{Qg-$w_vKM>Q2p73oJYlk zfE&`1rbXV7>^9e3@@?n~)WHd|L;}aahqn%qhiz%;FZ=hxRg5DR0)ioB zH~pzMt@8MmZ&5$4_s)0&o%#=C@fssXN1YPCvtyx?sb&|l2EKqF|NhG=lXSBNU%+7s zMe2JWqnPw8Ql3+6Z>2w;Q~F_$Z*;pk9U8ci7;gQR-Fjduv}FDU>fWk>nNB^HTOZ(s zjoR9DyE^2*B~mPpVOjZJ2nPF{3Ry-&*U%nz^jsSl-&o)Kug`Yl%O+`Xe{YK&-6iTY za@Vgtq!c<5$MG8Re_w_aN$=$o&4~`C+h9AR|Bq!f^Nt_@GE)cEr%k9+%91@`U3(1_ z@&w2+TMyQt3=CQQ^Y)bMz?85wyQyt1kEtWw$7DlDJLo%0POvG9r#dCQ7Jc-iD?!l` z@>(=124J;UY2ZGUOq3f)+1eRxP{1HvTD9Haue}Qfl9T^*w$JYBj6!d!27t2-;UNKq zEGm&4;+>!HP_1i9t?qH7H%n}vFQwDrf!B!#x@K5zJ0~k;57s~rivp|I;jMq@wo2bF zGx(GbuN~6I@lYt@!9CPer4h|8t%expy3!TYUtUyidwNo+4IpMaP=5+_e;Xqe>jV@g z-QwFJ{u=tM=9Uy`f|ZLbV1%WuUEm@8TVsqZz|tb*GA>1sO8-|SW=h*!9A1l92<4vC zAK8KKeDRN0P!%N32hR%jxVw=j3-B3de9e%B4E}oj1sM?-O`(IoKzzl!fb)^^RTcmY zZ|uifv#TolQ@`M<$y+^v$kPK9(?36U)~5=wT%P|2hYi0l z4K83@hb$cryuiN8Bmf6$@LUwC_DHrRo(JC2jVpU^dB84s7pi_@=s>1AtI+LF`y{<9 zk>aug=)uc3`yaYM<*7V}z@5;87-+u}AXRVC6y;QyLYO#qE$;6~;0gHrh;}+N(OFP* z+$UqW#2$DtXhx1 z56u1tkDsMnhXujjFlFS@`C@+T&flK-hwclWV|$xOPVDU8y!DY1bUEZ!R>=NEt_PF- zeKj4z*2CBTc?q%#WK#d%SeIbNiw8#lgbbHD7=HY4_@rPQqNt8k0sa;NlY$Vq-mV}{ zz}3}%2@tDXD*I=_aa0g5;)Gz}r!K>QcTUi!rAm0V2s@4RA-Fd4`t!3Q**M@t%%}~W&iO{Q=_Qzff|8H*5GEg zhOLFr-et^y#HSJE0*?n+GgJ>2;X@S`z*YtPM`wk&j=j0G(&WBca3EoD`uF)^GHKwQ z_G&j``GWk%BIG`;R()ANR}YVao-_l~__sAfs`4pGuQqRhhBjdqdy97Z*o9$6?;sFS zilx7+dNXg>JKS8+bF4D2aQ!Y;$nX+s7xr&^esnA(5G>X1KfZ(qTd|5bUO^xQw9BP~ zaK#RJH9*>rl}9ZLK`#Q-SMt9)sH-aXq-UYg6^05Z_|BwU1kz}kWfxNaxeW^(fe4v4 zarqcbj*uOh(DUNUz+904XmEIS-mv&@@;-Rx+r}Q@br~E)2F`;Xwhn^@D8#Y*0VLo}+Ye`28SLghaQ z@YB)%dIvu|r2iTs>HoY767c_Tr!0>m{&T1)A z&mKB{`3ge8gWsTdYl5_E10F#~@JQQhB#AvAO8iSpTsg{ypo3n}mA)VieP5x)gg=0| zX(;E`qZMx9wR?S3@BxM*@@>#I+d%tNhsqkz3Q*k!V2%}V4UZMaM39@3`>AsKY#;D< z?XWx4#%p{jUoC8dvT9j70p!Y3qvzs~0AUlLMpG+avUKcFV(>yt7vaSJU4z-Vc7|(F zbg(KQYAVj*D10uk5K9xVg{dzCo&641w#UwZy_R$2H;B+*J{wSwET+AEMO`U699Y z9-U55R8nf3IzDypou2(^Z39KH+4*G9b1sTj9A6Z@RWE;a(ufidWQ+6+O-G1O=@j=J zN{`yx06;|QcYZ&y>q@)tvW4gurS|=IYE#jpA%0^>7gUIisL*!R2#(5Y3=DB-2&BZb zp5GURx>c`|P=J{kRYX4*vjc;F{FU1ehHL%GYj?q(5f6l6FW*0Hkdh-^6{7NW1O$SL zJI2bKI58CPEh|Daj&3s_b)whAU=%!Zx0u2_&=Kp3V!bifd0$)J~Fa5@e-c7 zNEh#8L#C*JI8|!FsJS=$0!dPBQJtLZ^hcy_dhG&zl@(ce`dmeEfNg;W_Minzjy}Hi)&JYekBo)ZJ&ykbp}8ERr*IKYT=`% zVJ0K;_@ILOpA#S1+V^GO zMj9({!0Y-EVR%Roh%I3Rl5zp1hS&pq20t;9OGw#4@tDTEHWx&uNVee0+Yr;haXwR9 z8PbkBGN^JMKi#0f*$f6?p-ve>zEh1j)W?rYu(QfJ+a$JNDkI>H6KBm0S}~UWz(5@G zJBz6FMdZw`l@u{hl2AZI3n9rs`uN<>EkGlaqBL$AHwWCN5jRl0InevxlNEx&K|gf( z38+=WZK<*oDa?5l&_+=&IkB{WY$iTldIh9nO^Kl0nv;Te!=&vx5>R<&1_&t#)p)I* zMC?qG36LlfJ<9%!*F>Z`s0JosL@IJYDcS>165zW?;sPDX_xnTUV@{Ia%|O1N1{Sxt zISGL@`%__kx7}sAF{9H{*qDD7#wvb zVf^coZ?;~}i}rzTtpw;15|ADap2ko*gE8{~@bfAIp>yP=Ou%;VFOV(EBV}4BC`T_R zBiuJK0D#p09oRi(^6$YmjW49YgM>Mv#y+( zaBtFL;_OEfBs}Ni0F-PGbbIy-b~(Kbc~}o>&X+i=j6R+I(u(hoSRIcx0Cr0NmUzUw zT4%rV(-XIagb>e)AD@S#4qc8;!qG~q$#s*rQ%|r$%lq|J-6!7P4skI{mZM`4{#pZD z&KjU%ct!t81mwh#0C_@mhdWHfxO4`@c&6)FX_Ezk>WdxsP!vHOcNiK^bg7%;k;=LA z094|Vc*)(86p3PCHHzC!d$*8d@GfEPBPHFw zQ5@8v4m0nuU;Y50!{7H`cSH<5L(2OPpO^$;@#AhiaH=yvlmct_ZR&38t9B&S0gRt3GmLCym68|b`9`u66DTnCc98o#bFS!yx0jVf{dedfHIW?jH)<{CFQxW z@gmNh;yaBrGf;*)V6-RsPGf{Ps4OMT?24}->53r}Lb-Z5ccQrf4SO0CnlDw?0>jw> zbRe@U#}X$HCtk+D7#uJ;TzC>>owY6nI%H7l10l4-54QZ8?E(?{UHp&`Y+}V>jq`QL z+!0XbdBobvt?9(2RV0<-%b?Oqz!vHNCqBjU8Vo#$0SI`2D?{s`yvZv{5Yl1>U(okJD;oGs7uKt$81MQ>2x zk@3lqmuXNeC4rWH;|>*krc;*W0pdGu6y!Gim+LSw_AJaKDpb(-_gzjH2@;|A&RM-i z8w~EO-^q)kApe#Xb&|dyF|68opo0HwFRXbIkQ>|hXlM|`pjdw3YzO-h?SWjlMzG9m1(=|uOE>ya5Fz;exU293*%|&-(Sy(*y`|y z1Xk?Kofh((*Yo?Gf%Uv)4Ew93*%#)`D354`U6xpN8n3B@mf@gZ!0yhLNY_2hI|GFl zE|j^55@3EW2r7bi3VNPv2Gkji7$@Z)UEvWd)*hdxV16rgvFN(IF6;{@kw!@A$Cm0^ z-goVOQMu>yp zd6ll7kyEbGszT_&BnWEJ0fsS124avZOKOh-bfmRCs;4!bvY4&R^T$i^2-S?L!a(>_ z4cm?s>4m%CjPlmWSeS1B=Kz#gZmDBFM2IepLi5_igPjl_{tEUn?AfPly>a7Z-k&hz zUAsgAmJ}$4LT zy@eLC*aM*}*Mg&-=~fv4Tq_V8*aUT@T-Hb;{>bu%o=a$AM8wy702P9_DL`H=s!azg ziiiDRffK7g3ry5`%=djOj|v2~hgWQh|Q1;~6nLF|J%+cM?MSZOW3H!Iz381Aj+-`jyr-xHKX=*>IOPMCAcMZH_ z&e5W~C_cqf4-XE-;}jmDNi@|t&kWUs>sp)+>S?}4bLw^7kTnI;v_#ejCaK=%;KI~c zrqdX`M5SGU*MU>iA<|PiKQ=aM^o8Uz;ZVzh8 z|J0Hz$ge{ngJqS~EZ?Xp)9`{swew_Mqbzh@^q0vunh8R2CF>dd{I0~mbxwi4>-GWZZZi#P+lI3^WCo`b4y^!JK2`5N;rBjb%E2mOU zGjr>G{q~wa^_@*R@fv48i{aC+Zx(s2Hgn$pbSSJn?uz9pthw{#1^2J9RF-=&-!Y8V zitM6)23Frfo~@fWW93}CJ)egcy}@>bk2YK@BPvQ>Vj$b=z8`xiOt42i>piIHN>_ze zng}E~`Dy8V7h|&h%|V%KU{`+c!r8}Y;r^GHmll|_gsig8IDyZ{mAj)~HyzG1N;7$? zOe?;9``kNe!5kZR04e1CgPwQtjrTS5@>l5(d)^EE`cp&mt@$NPY%cQ+S_ztwQkQ*$ zg2?E+`HSRgJX(?#T$= zNrN-&rSuvJ!r|Vs51c>^tOQ)KcrX$z-5<=5bR^r{owxVIphB63ee`k{$Sq3o~&HWryHym_{L5r5QrBb-F zx%7R0_!Tq4ltBK)Df=jD(e|*f+UGwqeRJDAH8mNG_tvf>O%uY_0Zs`GG+Kfz2oiXwtAfAc|Pc?Fh zc5$zI)QFq%p)M@UctbQn*B;nVxjV zB|*gK;S0-_5qqc%av^E+=Cgk?7WqlH(V)$}k4_(v3{%@TTX8WVL|fklC2~zSCkLa& zCF%UEVAY5w+vCmR{vWyKIrdpB_01}pKf8xCm{wuF@o8;vSl9LXlCipJ&^J!(`BO0%i>=1K8O6&0o~jDOrzZ@Wj)`I#;;@F zrl@G2W&>heW8*jH$Gt|IQKp~>B;>i=6;66af#R3HV^UAOzHzf zRA`fIyIHy25C=-COm zm+rGyi+sg3M*m!aDqb%{?j{4kk#$}}&FoxGM6@#y-cWATnca6iUd;ZZn`Q(mMN^3) zMTKF=qJIN3jL+ZWACSD^1Y6)Ad6azdXo zq-W4Gk%cZ+Wwsu-Dp4+)re5=o7aVU5qn=l8C$q! zq+dcrcl}p4H4pau+w?70XfxA~N@QIuKJ;24II6YhTcnO>pYd$jr9AEr8ZY$QMgBb2 zlyyuvd+50m;ZaYm$A>qk&2rM6H?)_STT--Ga%7Vav^^U(+7RSUGKFW%3ZNDP-?dT6TI$b#FzHi2C0qFcX)X4k#ZpH;*7$66H2An=+dBUQB(3*oj6a& zRnj{hEe!gSy;h$`RM;trrV7Y{zHdk&l$X>Oavr5j7X8ewomX?Q_Dp1kP^fJ6)Uow1$`X&w$|^nAc+vkUzesG`O}Cqk=sHKH#O-%i z+e-;>AYXM9{vcDBA2L8$W5(I}v^+v;gkO(W68r*QB&~EgHbdczxM! z*WKeMQ0sHv>V#+W)3YZq*G?x-R6KM&>Z=zm*L*CQfuPIejmgVv&|)*>rkL3VGyjsO zJ^y$6qiN?@Hq@x1X$`isGjmq6a$_Y_2FPzr$@Uk!&32s`?>_M6NAUIzDBtKt&#}49 z>lTj}!Z}mkTecB5Ie2v&R7iFXeeRgKKE0iG+CJgkms^j{bwbbQ#2@?M-s?>4p_E73 z!xAlIx~j#2oewl~k5=}PWfPTZf40hzGtUc`&GK@FdCO9(e!EOnm(xlUs9-B?mTAOd+m>*vte|W@!|g5>v{L?>L}?D zPvpdk@gWvo&4jUda{A%Ek5ks8+1VZ7(Vk|!r1dfLkQaCr%2P}N2%z?;eR=ngd3GqB zYkf3DlQyR(#%P!DAycow4i)sxciFr&KUl0njeN4w80+d|1H1c{a$52{L?vccZUdPt2Ieg4&UY%;$hU6A#=%BFtDyj#PoA2E`Gu{v6H# z$^n;=3NR&{|!gS^k@GMm3J^(A`NNNNLg8o-#m zk_BZoC7+Vfpz)iE$0MOIZZ&?v8C9TN&Ox$|6Cc?Xf@jOhd z{YBU6?cCz4O_hEzcAfLjwCEFPtk-y$B<0&vxiMg}^nO2Wea}WK(@c+;dpmu4{8=(> zUJ7**i&`@sYWt7q;kgr-v)^`w6v@)BM)>W!r1LgRoedsVzPdK>L{HSe-?M6o`Bt;q z#41Ac0miuQ7d%Z-kTpcWu2pO&C8t*6VmB1RvV$D`lOXEiJW}T5@EBI3xGCSL=u<(+ zS2rL)df`|ecO0DE5s(hvk!g6IB{>gNd3lZAr}={UIbfmOzQ4uXkfnOp0zA6vCrxY7 zD1yd57$(R=j;3+6NBmh!GH=fa3Ls`zb;$t0m6UM9Q2FcAQ*Lde{ibY%f}5_7cD zDO`;Ab*n4~k%%_Y7eKfb08&&K_NP!Jt7w_!wkhnNk_G}Wc1u+fO+ZjI=bKjcjCu~5 zc^$5bQVPqTtN}&I%Q34d06F_26S73vxC-z_mNlL5pwwXiTOC-bJyzaK8FjdC^65(Y z+Q~?cLEmqPoA5Q_D;l_@6@0{W**h1ZR)<`h*}UUmAtsR1!k(TI?-|duYyn#3xK`pE zkY&QaWz{T|^b|!*aDJ(Nw$b>xS2#9IGWK~>lmlEwGT1ts%A>en9lvbU74@x0X{sWw z_mrIJrC7Z=Ou~a}mXc<>z@{r~x$kTu{*b=4VeBp( z@A0KP)wbnUzw#CgzAbu2q6Xi0A%`D-_<(UfTv7kX80h&B zI{r^{{szNuTc&lqqWl6)-Wj7ziii)H%~$l@keQc7C85Ym3NxTiprWOIhaoNs@bS3c zsb^Gt@WuIz+$URL61BN1^y$?We5nplosyC9d{TdLtfaTINtzufH{=UB&8o#=r|&`cp*sKr z1S2=k>>MHkb5W1sVT}c-=8bhC&ZD#bP~bFRR?+`tXqaQmY5fgPQIlp&%5NYyM@~lX zJkei6`=Ujd(EnR|eJO=O9~U`JH3wo6sheTrbL zgXhn$&riJ8Oc4HtznA;yA)0-(vR&U#&i!-lY!u0K6Iz!a_YoKDrEQErGc3waS~{V5 zVjYqmrFskEB3yHtO{Hhv5NCn^LAlg!ys{g5pYFnD|En!bGW?uBsRXjMy+@9-am?Ro-}U%u*X+AZrL}kY$X)rsBU2RC!^Q&!Vn|&1M&lH5w}NOQ2~Fs=K!#VaulO`O^gwv&LAJrxX9aPl=iOdKViZ z|1NI8^y*OCI|xDPThbeM`m?n^dlQk=TWf@?S&Kq|gAEk!ve#qDK_{LoO*IjP2p%=s z7d2d6Dl#AR58YDt{T01^{JCeV=Ufj5!YxW>n=x(yRB4>9ishRN_?f_+s9 z-~@4)xBAViI~|otbOicJ2(+(#0lZ<3`(}^c0iUUe^wXD?WNy;d*~sA=?*6Sn@7Wys zRRKo6X=pq;*-c_p(HI6hMT2JrofEtrm=aGW9;fa%4|!KEz&Tst>Y|=@Tc8HoYZmCA zE&vqL!4b6c+_sQBT;aYJh<8hF7{%~oBKl((xjax7$Q#Oao}q+)#M`8(yybmlm|cXl z-5pT4{K@ezHD&BJFOd2?m_mDTi9uJ6_#EubR|*67%COxTgh*ufnY<4|DVd5K=%!BH z((oc$Rc%eMxgUGzr~&U!{-Ini?h?|S&kr{Jrh%4&r^kg z883f0F5DO!g<|=T#b~A8%n7HmiGsOK1R0V8;|sV1XRdDxx5rDCj{ky_$Od;G<=Eyt zGJwj}cKqr<;CWU3L63m#NJ=TrB9jBpT!Fb+1bysj`6;=iMTo&~APuUE<3(ZI8{0aE zcZBGd<=o|JQ*|&fmi&||VlBjM-N=_Fa?ry?iZdey@m~;4n_zEF? z03uAx#zue1GG77TjyU8LPZ@~(;5X(YsW7E9Ym&*X<6R5q!40!}p20}()B=p~`qLKM zVFhPS+C7%8u$yBM%$Ud~r`-Q%04esxKyUg?SWg;*m@c=E0EWjTL~uh_H<43E$0D5!mep?tNId>^qD_1;?VG=}J0I4|(q;s!&9m+0%#>edP zzF){4C=9q%5rLRi7gv=cC6`a!!4yyU&Lo#j-KQZv|NO43ApS9*a{n$dng@~ z%m1U9GYy1tZ^QV52I+9ri%N%_qEL~wkW!~zvL!k8v|x}^Nn~v~b!16R*(ox%kjT-aF(HLe*PNW(}T4pMW_xkJo{=OgIFMZVW%=0|=|GuyLcU?|5l5s^}2a`7JSsC)- z>VI10obTT*j*M_`>^)1&;#N)p5NZ$9m`d+(bvm>wLB*`=@&5RSSHd^25sj7KUnA1is%_?N{Gr5VvLk(@vkslvDAhj$+)8^%7@#q88nVg;LC zCm2;i{D$HBR8uQ^ml}mLk(g2|H%2o*tsm&9yW;f#n+N||nK%;y@Mp+bl4bpg!|alF zy~dk?|E%L}&L7vZ*)#yFYkINkmEWhOcIYiaZO}*D-U_nH#|d8<_t01R>T0p`GE-NF zx1r2D9pbD3G}fSN2)A@>d#o{zfrUGiI_~nI|8VxzuWd8uKGh^v(2r*D-A9^3K+jW#A!r;D;Vy!md&y`5iiPsv2RhL z`^;IXqLet44y%frTIbY_=KeXy;qRSSbcCC~au(Z^tx$AE5a>aRsNAP^!t+Z4 z-*uPe%!P+dWm6b%B8`rsC`a!|+vqCvaNNdP>@a(csX;_w^tx4e+kPa|p)dCXDn!qB zI%r8_+1`3R>FQr{D~eXf=r4cfX>75QP|aUL>kR$lV(PTM?f^i6u4iFFD9ebkmR3_?oqy?+ zPIl%&wmLF*VlsTK8LoPO+IgdSO%ticmJU2NFB3o1`UoTH+!q2{ujWt3-alVu6DShx z-W2J}vC~_F-1D}pn91$Z;>%#%&78ZWZ|8w8paix7v+?gMUJxjAjUfrc6W!Txdyirvzf?8Ih!4z8z06&FRyD^( zYuK0ZWCCw)XO>UB&E82esa-8B%C1+Qc-kySRjD!PJ z92Y(Ne7sXwI+WTz@})J1PA<`OuL{VJWe%V`8Kdbkc-@v&)#+96*+uJ7lt{{mpWn^H znG<&4Az%zaMO?sh@ubEA$D*CBa)}gmU#Dkp0Guo{vx&LW-&H8(vem0A>?vQk6z=(2 zmBbAf*+ZPt1CSxSWR#YAzt)-EYEXRqIClE_=tVvjm!{HwHve?)EjGedr-GrG$0GdZ zWfTEfJqiMom=8OWU-lh8&|;+$f`fN&UKt;8UCSnz{9)};W5jasymJ^w9DE$DNpSBw zR+jI-@@Kma74g?lY9num>9@}08qfhx;uzsJ2`1n-JDPNtF?ikaW{_3Z>vsgWBBOtQ zt5HG?U(Q>m4!jPJ=b*k_aniveP{?Q3jTdCI_)7BW5)JHygxz0V*p~_D$Sg?dE)VR_ z@O7F|lXq-&jQE(nxwv9@n#mz%vE^kom)UHMzyfZ4Va52*aqfZ7 znR7zBh`xP1SyPi^=!;I{rui#n_YtIfoJTJh*LtlR?SmD)6bhVpoj4_JJ!KJ}xzX!y zc)tAHpqK69*anFJ0{3Kog75U}CEedE9v;SfNCs+VjRduB@doMngm~W4fOUK;_~~#C zPUU@PZ~3eWg z(q0%^rD}gcx$GD~H0U?v0~pIs4qUMIYcI);CR~@*{(200EkIE;DJkmxNrhlg!_qlg z9Z#3kft9hxna{IygF$89Rd`*vk^{J#irPUc>Ld_CTx%9#9vLto#ha5*;CDY!>T2XG zpmsrhFhK6jaxf+OID2xe=u(vWFN+F*BA)TkR;q$egLry~IhCM22L4>b@Wi z_WJ9nj6AK*t(TViptavSQv130ey=)pX4lluN?_doYKi-w7EhI0mk5t3{cZ&L~!Ou49m_0>RFCv$u{_7M@is_c< zMcbf3hBpIMpYxt@-CIJHye`ioZI@Dp3_W zAE;%ALE00G77!a(^&kC2>lqeelaWFa0!S)66J^b~8YXvE>vhk2EyQig0mK(e!lQ_3 z3A##}YZnX=AG^I0dMNk9SqQu0Qm$fE0{|OD0gf>F_U^0=4dPj@4*LQwp+rb>QRc~1 zJO-$+4~_dPURSbP-WqcP?6lzS1s^4-a|(p-JvXg2)7;GCr_c=3pka5~BRyI%KgA8X zi?V*gNT?buO&6C)h#wZFv+mU7Z*qA%AoExQN!lV`$WQO;f`~hqe}YVv5?VO-^&^3wm$P18wnc?kaZP4`}EFeYj?#aJK90$U9!mIEz*K^>g#a6U-fL zR%5LYC1f;400$kf7yqz7Go*2td$U=xU)ijLA_>4hG`?jV(;I-syrZi+-d>c8BrI2b zw_Bd~v(Wm8?|w$XC>IuXqzqZivDFQ}4m^&nw8!4&-9h`V(xlL8EExQO6v_{oq64Vv zzt7GoE_(&Xd{uO1gV8`8#+9uWzxcr$rrxWKL)Y-ru&RfZk5K2}+uQ4X>Wh`K; zM>x`y#k#MP8n$AT#V)&xhiH!`4i(VTgEjL_Suxi$R)y$$)MVbvYK*44R5zw1!HlT@ z`SDrcJ=x0T8DXmjT^Zj!@S~p+A z@s-d6QcZ< Path | None: + if not script_path.exists(): + print(f"Skipping {framework_name}: file not found: {script_path}") + return None + + framework_report_dir = REPORT_DIR / framework_name + framework_report_dir.mkdir(parents=True, exist_ok=True) + + report_path = framework_report_dir / "support_refund_report.json" + + cmd = [ + "agenticlens", + "profile", + str(script_path), + "--save", + str(report_path), + ] + + print(f"\nRunning {framework_name} benchmark...") + subprocess.run(cmd, check=True) + + return report_path + + +def write_csv(results: list[dict], output_path: Path) -> None: + if not results: + return + + fieldnames = list(results[0].keys()) + + with output_path.open("w", newline="", encoding="utf-8") as file: + writer = csv.DictWriter(file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(results) + + +def write_markdown_summary(results: list[dict], output_path: Path) -> None: + lines = [ + "# AgenticLens Practical Benchmark Summary", + "", + "Use case: Customer Support Refund Copilot", + "", + "This benchmark profiles the same practical support workflow across multiple agentic implementations.", + "", + "| Framework | Total Tokens | Cost | Latency | Steps | Tool Calls | Retrieved Chunks | Highest Token Step | Highest Cost Step |", + "|---|---:|---:|---:|---:|---:|---:|---|---|", + ] + + for row in results: + lines.append( + "| {framework} | {total_tokens} | ${total_cost:.6f} | {total_latency:.3f}s | " + "{step_count} | {tool_calls} | {retrieved_chunks} | {highest_token_step} | {highest_cost_step} |".format( + **row + ) + ) + + lines.extend( + [ + "", + "## Interpretation", + "", + "These results are workload-specific. They should not be read as a universal ranking of frameworks.", + "The goal is to show how AgenticLens makes token usage, cost, latency, retrieval behavior, and tool activity visible across workflows.", + ] + ) + + output_path.write_text("\n".join(lines), encoding="utf-8") + + +def main() -> None: + REPORT_DIR.mkdir(parents=True, exist_ok=True) + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + + results = [] + + for framework_name, script_path in FRAMEWORKS.items(): + report_path = run_framework(framework_name, script_path) + + if report_path is None: + continue + + summary = summarize_agenticlens_report(framework_name, report_path) + results.append(summary) + + json_output = RESULTS_DIR / "benchmark_results.json" + csv_output = RESULTS_DIR / "benchmark_results.csv" + md_output = RESULTS_DIR / "benchmark_summary.md" + + json_output.write_text(json.dumps(results, indent=2), encoding="utf-8") + write_csv(results, csv_output) + write_markdown_summary(results, md_output) + + print("\nBenchmark complete.") + print(f"JSON: {json_output}") + print(f"CSV: {csv_output}") + print(f"Markdown: {md_output}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/shared/metrics_collector.py b/benchmarks/shared/metrics_collector.py new file mode 100644 index 0000000..9ea0568 --- /dev/null +++ b/benchmarks/shared/metrics_collector.py @@ -0,0 +1,82 @@ +import json +from pathlib import Path +from typing import Any + + +def load_report(report_path: str | Path) -> dict[str, Any]: + path = Path(report_path) + return json.loads(path.read_text(encoding="utf-8")) + + +def summarize_agenticlens_report(framework: str, report_path: str | Path) -> dict[str, Any]: + report = load_report(report_path) + steps = report.get("steps", []) + + total_tokens = 0 + prompt_tokens = 0 + completion_tokens = 0 + total_cost = 0.0 + total_latency = 0.0 + tool_calls = 0 + retriever_steps = 0 + retrieved_chunks = 0 + memory_tokens = 0 + + highest_token_step = None + highest_step_tokens = -1 + + highest_cost_step = None + highest_step_cost = -1.0 + + for step in steps: + metrics = step.get("metrics") or {} + metadata = step.get("metadata") or {} + step_type = step.get("type") + + step_total_tokens = metrics.get("total_tokens") or 0 + step_prompt_tokens = metrics.get("prompt_tokens") or 0 + step_completion_tokens = metrics.get("completion_tokens") or 0 + step_cost = metrics.get("cost") or 0.0 + step_latency = metrics.get("latency") or 0.0 + + total_tokens += step_total_tokens + prompt_tokens += step_prompt_tokens + completion_tokens += step_completion_tokens + total_cost += step_cost + total_latency += step_latency + + if step_type == "tool_call": + tool_calls += 1 + + if step_type == "retriever": + retriever_steps += 1 + retrieved_chunks += metadata.get("chunk_count") or 0 + + if step_type == "memory": + memory_tokens += metadata.get("history_tokens") or step_prompt_tokens + + if step_total_tokens > highest_step_tokens: + highest_step_tokens = step_total_tokens + highest_token_step = step.get("name") + + if step_cost > highest_step_cost: + highest_step_cost = step_cost + highest_cost_step = step.get("name") + + return { + "framework": framework, + "workflow_name": report.get("name"), + "step_count": len(steps), + "total_tokens": total_tokens, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_cost": round(total_cost, 8), + "total_latency": round(total_latency, 4), + "tool_calls": tool_calls, + "retriever_steps": retriever_steps, + "retrieved_chunks": retrieved_chunks, + "memory_tokens": memory_tokens, + "highest_token_step": highest_token_step, + "highest_cost_step": highest_cost_step, + "report_path": str(report_path), + } \ No newline at end of file diff --git a/benchmarks/shared/support_data.py b/benchmarks/shared/support_data.py new file mode 100644 index 0000000..2de4a7f --- /dev/null +++ b/benchmarks/shared/support_data.py @@ -0,0 +1,76 @@ +import json +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +DATASET_DIR = ROOT / "datasets" + + +def load_json(filename: str) -> list[dict[str, Any]]: + path = DATASET_DIR / filename + return json.loads(path.read_text(encoding="utf-8")) + + +def load_support_cases() -> list[dict[str, Any]]: + return load_json("support_cases.json") + + +def load_policy_docs() -> list[dict[str, Any]]: + return load_json("refund_policy_docs.json") + + +def load_orders() -> list[dict[str, Any]]: + return load_json("orders.json") + + +def lookup_order(order_id: str) -> dict[str, Any]: + orders = load_orders() + + for order in orders: + if order["order_id"] == order_id: + return { + "found": True, + **order, + } + + return { + "found": False, + "order_id": order_id, + } + + +def simple_retrieve(query: str, top_k: int = 6) -> list[dict[str, Any]]: + docs = load_policy_docs() + + clean_query = query.lower().replace("?", "").replace(".", "") + query_words = set(clean_query.split()) + + scored_docs = [] + + for doc in docs: + clean_doc = doc["text"].lower().replace("?", "").replace(".", "") + doc_words = set(clean_doc.split()) + score = len(query_words.intersection(doc_words)) + scored_docs.append((score, doc)) + + scored_docs.sort(reverse=True, key=lambda item: item[0]) + + return [doc for score, doc in scored_docs[:top_k] if score > 0] + + +def estimate_avg_tokens_per_chunk(chunks: list[dict[str, Any]]) -> int: + if not chunks: + return 0 + + total_words = sum(len(chunk["text"].split()) for chunk in chunks) + + # Rough practical approximation: + # 1 word is around 1.3 tokens in many English text workflows. + estimated_tokens = int(total_words * 1.3) + + return max(1, estimated_tokens // len(chunks)) + + +def build_policy_context(chunks: list[dict[str, Any]]) -> str: + return "\n".join(f"- {chunk['text']}" for chunk in chunks) \ No newline at end of file diff --git a/benchmarks/shared/support_tasks.py b/benchmarks/shared/support_tasks.py new file mode 100644 index 0000000..2bbba00 --- /dev/null +++ b/benchmarks/shared/support_tasks.py @@ -0,0 +1,98 @@ +import time +from typing import Any + +from benchmarks.shared.support_data import ( + build_policy_context, + estimate_avg_tokens_per_chunk, + lookup_order, + simple_retrieve, +) + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeMessage: + def __init__(self, content: str): + self.content = content + + +class FakeChoice: + def __init__(self, content: str): + self.message = FakeMessage(content) + + +class FakeResponse: + def __init__(self, content: str, prompt_tokens: int, completion_tokens: int): + self.usage = FakeUsage(prompt_tokens, completion_tokens) + self.choices = [FakeChoice(content)] + + +def classify_ticket(ticket: str, framework: str) -> FakeResponse: + return FakeResponse( + content=f"framework={framework}; intent=refund_request; priority=normal", + prompt_tokens=180, + completion_tokens=25, + ) + + +def rewrite_query(ticket: str, framework: str) -> FakeResponse: + return FakeResponse( + content="refund eligibility delivered order opened package unused item refund processing time", + prompt_tokens=220, + completion_tokens=35, + ) + + +def retrieve_policy(query: str, top_k: int = 6) -> tuple[list[dict[str, Any]], str, int, float]: + start = time.time() + chunks = simple_retrieve(query, top_k=top_k) + latency = time.time() - start + policy_context = build_policy_context(chunks) + avg_tokens = estimate_avg_tokens_per_chunk(chunks) + return chunks, policy_context, avg_tokens, latency + + +def lookup_order_tool(order_id: str) -> tuple[dict[str, Any], float]: + start = time.time() + order = lookup_order(order_id) + latency = time.time() - start + return order, latency + + +def check_refund_eligibility( + ticket: str, + order: dict[str, Any], + policy_context: str, + framework: str, +) -> FakeResponse: + return FakeResponse( + content=( + f"{framework}: The order is within the 30-day refund window. " + "The item was not used, but the package was opened, so manual review may be required." + ), + prompt_tokens=720, + completion_tokens=95, + ) + + +def generate_customer_reply( + ticket: str, + order: dict[str, Any], + policy_context: str, + decision: str, + framework: str, +) -> FakeResponse: + return FakeResponse( + content=( + f"[{framework}] Your order is within the 30-day refund window. " + "Since the package was opened, the refund may need manual review. " + "Because the item was not used, you may still be eligible. " + "If approved, the refund will return to your original payment method and may take 5 to 10 business days." + ), + prompt_tokens=850, + completion_tokens=130, + ) \ No newline at end of file diff --git a/examples/multiagent_edge_cases_demo.py b/examples/multiagent_edge_cases_demo.py new file mode 100644 index 0000000..2902d35 --- /dev/null +++ b/examples/multiagent_edge_cases_demo.py @@ -0,0 +1,132 @@ + + +from agenticlens import profile, step + + +SYSTEM_PROMPT = ( + "You are a careful travel support assistant. " + "Use only verified policy, booking, and refund information. " + "Avoid guessing. Explain next steps clearly. " +) * 20 + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int) -> None: + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeResponse: + def __init__(self, prompt_tokens: int, completion_tokens: int) -> None: + self.usage = FakeUsage(prompt_tokens, completion_tokens) + + +def main() -> None: + user_question = ( + "My flight was cancelled. I booked a hotel and airport taxi. " + "Can I get a refund and what should I do next?" + ) + + with profile("Multi-Agent Travel Refund Edge Case Workflow"): + + # Edge case 1: large repeated system prompt + with step( + "Planner Agent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=SYSTEM_PROMPT + user_question, + ) as s: + s.record(FakeResponse(prompt_tokens=1200, completion_tokens=180)) + + # Edge case 2: excessive retrieved chunks + # Default recommender limit is usually 8. This sends 14 chunks. + with step( + "Policy Retriever Agent", + type="retriever", + chunk_count=14, + avg_tokens_per_chunk=90, + ): + pass + + # Edge case 3: long memory history + with step( + "Memory Agent", + type="memory", + provider="openai", + model="gpt-4o-mini", + history_tokens=7200, + ) as s: + s.record(FakeResponse(prompt_tokens=7400, completion_tokens=0)) + + # Edge case 4: normal tool call + with step( + "Tool Agent - Lookup Booking", + type="tool_call", + provider="openai", + model="gpt-4o-mini", + tool_name="lookup_booking", + tool_args={"booking_id": "TRV-8842"}, + ) as s: + s.record(FakeResponse(prompt_tokens=220, completion_tokens=60)) + + # Edge case 5: duplicate tool call with same args + with step( + "Tool Agent - Lookup Booking Retry", + type="tool_call", + provider="openai", + model="gpt-4o-mini", + tool_name="lookup_booking", + tool_args={"booking_id": "TRV-8842"}, + ) as s: + s.record(FakeResponse(prompt_tokens=220, completion_tokens=60)) + + # Another tool call, not duplicate because tool and args are different + with step( + "Tool Agent - Check Refund Eligibility", + type="tool_call", + provider="openai", + model="gpt-4o-mini", + tool_name="check_refund_eligibility", + tool_args={ + "booking_id": "TRV-8842", + "reason": "flight_cancelled", + "hotel_used": False, + "taxi_used": False, + }, + ) as s: + s.record(FakeResponse(prompt_tokens=300, completion_tokens=80)) + + # Writer agent adds useful value, but also adds token cost + with step( + "Writer Agent", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt="Write a customer-friendly refund explanation.", + ) as s: + s.record(FakeResponse(prompt_tokens=950, completion_tokens=260)) + + # Reviewer agent adds quality control cost + with step( + "Reviewer Agent", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt="Review the answer for policy accuracy and missing next steps.", + ) as s: + s.record(FakeResponse(prompt_tokens=780, completion_tokens=160)) + + # Edge case 6: repeated system prompt again + with step( + "Final Response Agent", + type="final_response", + provider="anthropic", + model="claude-3-5-sonnet", + prompt=SYSTEM_PROMPT + "Give the final customer-facing answer.", + ) as s: + s.record(FakeResponse(prompt_tokens=1100, completion_tokens=240)) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/support_copilot.py b/examples/support_copilot.py new file mode 100644 index 0000000..e3c9c33 --- /dev/null +++ b/examples/support_copilot.py @@ -0,0 +1,346 @@ +import os +import re +import sqlite3 +import time +from pathlib import Path +from typing import Any + +from agenticlens import profile, step + +USE_REAL_OPENAI = bool(os.getenv("OPENAI_API_KEY")) + +if USE_REAL_OPENAI: + from openai import OpenAI + client = OpenAI() + + +POLICY_DOCS = [ + { + "id": "refund_001", + "text": "Customers can request a refund within 30 days of delivery.", + }, + { + "id": "refund_002", + "text": "Items must be unused and in original packaging to qualify for a standard refund.", + }, + { + "id": "refund_003", + "text": "Opened items may require manual review unless the item is defective.", + }, + { + "id": "refund_004", + "text": "Refunds are processed to the original payment method.", + }, + { + "id": "refund_005", + "text": "Refunds may take 5 to 10 business days after approval.", + }, + { + "id": "shipping_001", + "text": "Delivered orders are eligible for return review if the delivery date is within the return window.", + }, +] + + +SYSTEM_PROMPT = """ +You are a customer support copilot. +Use only the provided policy and order information. +Do not invent refund rules. +If manual review is needed, clearly say so. +""" + + +class FakeUsage: + def __init__(self, prompt_tokens: int, completion_tokens: int): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + + +class FakeMessage: + def __init__(self, content: str): + self.content = content + + +class FakeChoice: + def __init__(self, content: str): + self.message = FakeMessage(content) + + +class FakeResponse: + def __init__(self, content: str, prompt_tokens: int, completion_tokens: int): + self.usage = FakeUsage(prompt_tokens, completion_tokens) + self.choices = [FakeChoice(content)] + + +def setup_order_db() -> sqlite3.Connection: + conn = sqlite3.connect(":memory:") + conn.execute( + """ + CREATE TABLE orders ( + order_id TEXT PRIMARY KEY, + status TEXT, + delivered_days_ago INTEGER, + package_opened INTEGER, + item_used INTEGER, + payment_method TEXT + ) + """ + ) + conn.execute( + """ + INSERT INTO orders VALUES + ('A123', 'delivered', 12, 1, 0, 'Visa ending 4242') + """ + ) + return conn + + +def extract_order_id(text: str) -> str | None: + match = re.search(r"\b[A-Z]\d{3}\b", text) + return match.group(0) if match else None + + +def retrieve_policy_chunks(query: str, top_k: int = 6) -> list[dict[str, str]]: + query_words = set(query.lower().replace("?", "").replace(".", "").split()) + + scored = [] + for doc in POLICY_DOCS: + doc_words = set(doc["text"].lower().replace(".", "").split()) + score = len(query_words.intersection(doc_words)) + scored.append((score, doc)) + + scored.sort(reverse=True, key=lambda x: x[0]) + return [doc for score, doc in scored[:top_k] if score > 0] + + +def lookup_order(conn: sqlite3.Connection, order_id: str) -> dict[str, Any]: + row = conn.execute( + """ + SELECT order_id, status, delivered_days_ago, package_opened, item_used, payment_method + FROM orders + WHERE order_id = ? + """, + (order_id,), + ).fetchone() + + if not row: + return {"found": False, "order_id": order_id} + + return { + "found": True, + "order_id": row[0], + "status": row[1], + "delivered_days_ago": row[2], + "package_opened": bool(row[3]), + "item_used": bool(row[4]), + "payment_method": row[5], + } + + +def fake_llm(task: str, prompt: str) -> FakeResponse: + if task == "classify": + return FakeResponse( + content="intent: refund_request; urgency: normal", + prompt_tokens=180, + completion_tokens=20, + ) + + if task == "rewrite": + return FakeResponse( + content="refund eligibility delivered order opened package unused item processing time", + prompt_tokens=220, + completion_tokens=30, + ) + + if task == "decision": + return FakeResponse( + content=( + "The order is within the 30-day window and the item is unused. " + "However, because the package was opened, manual review may be required." + ), + prompt_tokens=520, + completion_tokens=70, + ) + + return FakeResponse( + content=( + "Your order A123 was delivered 12 days ago, so it is within the 30-day refund window. " + "Because the package was opened, the refund may need manual review, but since the item was not used, " + "you may still be eligible. If approved, the refund will go back to your original payment method and " + "may take 5 to 10 business days after approval." + ), + prompt_tokens=850, + completion_tokens=120, + ) + + +def call_llm(task: str, prompt: str): + if not USE_REAL_OPENAI: + return fake_llm(task, prompt) + + return client.chat.completions.create( + model="gpt-4o-mini", + temperature=0, + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt}, + ], + ) + + +def main() -> None: + conn = setup_order_db() + + ticket = { + "ticket_id": "TCK-1001", + "tenant_id": "retail-us", + "customer_id": "CUST-789", + "message": ( + "My order A123 was delivered 12 days ago. " + "I opened the package but did not use the item. " + "Can I get a refund, and how long will it take?" + ), + } + + with profile("Practical Support Copilot - Refund Ticket") as workflow: + + with step( + "Classify Ticket Intent", + type="planner", + provider="openai", + model="gpt-4o-mini", + prompt=ticket["message"], + ticket_id=ticket["ticket_id"], + tenant_id=ticket["tenant_id"], + ) as s: + start = time.time() + classify_response = call_llm( + "classify", + f"Classify this support ticket:\n{ticket['message']}", + ) + s.record(classify_response) + s.step.metrics.latency = time.time() - start + intent = classify_response.choices[0].message.content + + with step( + "Rewrite Query For Retrieval", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=ticket["message"], + ) as s: + start = time.time() + rewrite_response = call_llm( + "rewrite", + f"Rewrite this ticket as a search query for refund policy retrieval:\n{ticket['message']}", + ) + s.record(rewrite_response) + s.step.metrics.latency = time.time() - start + search_query = rewrite_response.choices[0].message.content + + with step( + "Retrieve Refund Policy", + type="retriever", + chunk_count=6, + avg_tokens_per_chunk=35, + query=search_query, + ) as s: + start = time.time() + policy_chunks = retrieve_policy_chunks(search_query, top_k=6) + s.step.metrics.latency = time.time() - start + s.step.metadata["chunk_count"] = len(policy_chunks) + s.step.metadata["retrieved_doc_ids"] = [doc["id"] for doc in policy_chunks] + s.step.metadata["retrieved_chunks"] = [doc["text"] for doc in policy_chunks] + + order_id = extract_order_id(ticket["message"]) + + with step( + "Lookup Order System", + type="tool_call", + tool_name="lookup_order", + tool_args={"order_id": order_id}, + ) as s: + start = time.time() + order = lookup_order(conn, order_id) + s.step.metrics.latency = time.time() - start + s.step.metadata["tool_result"] = order + + with step( + "Refund Eligibility Decision", + type="llm_call", + provider="openai", + model="gpt-4o-mini", + prompt=SYSTEM_PROMPT, + ) as s: + start = time.time() + decision_response = call_llm( + "decision", + f""" +Ticket: +{ticket["message"]} + +Intent: +{intent} + +Order: +{order} + +Policy: +{[doc["text"] for doc in policy_chunks]} + +Decide refund eligibility and whether human review is needed. +""", + ) + s.record(decision_response) + s.step.metrics.latency = time.time() - start + decision = decision_response.choices[0].message.content + + with step( + "Generate Customer Reply", + type="final_response", + provider="openai", + model="gpt-4o-mini", + prompt=SYSTEM_PROMPT, + ) as s: + start = time.time() + final_response = call_llm( + "final", + f""" +Customer message: +{ticket["message"]} + +Order: +{order} + +Policy: +{[doc["text"] for doc in policy_chunks]} + +Eligibility decision: +{decision} + +Write the final customer-facing response. +""", + ) + s.record(final_response) + s.step.metrics.latency = time.time() - start + answer = final_response.choices[0].message.content + + print("\nFinal customer reply:\n") + print(answer) + + print("\nWorkflow summary:") + print("Total tokens:", workflow.total_tokens) + print("Total cost:", workflow.total_cost) + + print("\nStep summary:") + for st in workflow.steps: + print( + f"- {st.name}: " + f"{st.metrics.total_tokens or 0} tokens, " + f"${st.metrics.cost or 0:.6f}, " + f"{st.metrics.latency or 0:.3f}s" + ) + + +if __name__ == "__main__": + main() \ No newline at end of file From 6b82ebfb8a1a8e53defabe1258a713a89337fe68 Mon Sep 17 00:00:00 2001 From: manemsai Date: Tue, 14 Jul 2026 17:46:36 -0500 Subject: [PATCH 4/9] Link framework benchmark comparison from README Surfaces the new benchmarks/ harness so the apples-to-apples token/cost/latency comparison across agent frameworks is discoverable from the main README instead of buried in a results folder. Co-Authored-By: Claude Sonnet 5 --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 71a7fff..efe3f93 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,15 @@ Other examples: Some examples call real provider APIs and require provider API keys. +## Framework Benchmarks + +`benchmarks/` runs the same practical refund-ticket workload through +AutoGen, CrewAI, LangGraph, LlamaIndex, Semantic Kernel, and native Python, +profiling each with AgenticLens to normalize tokens, cost, latency, tool +calls, and retrieved chunks across implementations. See +[benchmarks/results/benchmark_summary.md](benchmarks/results/benchmark_summary.md) +for the current comparison table. + ## Notebooks Beginner-friendly notebooks are available in: From 420eac79a3dcc706e9290c89f19a54ea5bf100c5 Mon Sep 17 00:00:00 2001 From: manemsai Date: Tue, 14 Jul 2026 18:55:53 -0500 Subject: [PATCH 5/9] Fix lint and formatting in benchmark harness and example scripts Applies ruff format, wraps long string literals under the 100-char line limit, drops an unused import, and adds noqa for the intentional sys.path manipulation in the native-Python benchmark runner. These files were added without a lint pass in the prior commit. Co-Authored-By: Claude Sonnet 5 --- benchmarks/compare_results.py | 18 ++++++++------ benchmarks/frameworks/autogen/run_autogen.py | 4 +--- benchmarks/frameworks/crewai/run_crewai.py | 7 +++--- .../frameworks/langgraph/run_langgraph.py | 24 ++++++++++--------- .../frameworks/llamaindex/run_llamaindex.py | 15 ++++++------ .../frameworks/native_python/run_native.py | 23 ++++++++++-------- .../semantic_kernel/run_semantic_kernel.py | 8 +++---- benchmarks/shared/benchmark_runner.py | 20 +++++++++------- benchmarks/shared/metrics_collector.py | 2 +- benchmarks/shared/support_data.py | 3 +-- benchmarks/shared/support_tasks.py | 9 ++++--- examples/multiagent_edge_cases_demo.py | 6 +---- examples/support_copilot.py | 22 ++++++++++------- 13 files changed, 85 insertions(+), 76 deletions(-) diff --git a/benchmarks/compare_results.py b/benchmarks/compare_results.py index 49b549a..bd2ce78 100644 --- a/benchmarks/compare_results.py +++ b/benchmarks/compare_results.py @@ -1,9 +1,8 @@ import json from pathlib import Path -import pandas as pd import matplotlib.pyplot as plt - +import pandas as pd REPORTS = { "Native Python": "benchmarks/reports/native_python/support_refund_report.json", @@ -135,7 +134,8 @@ def create_markdown_summary(summary_df: pd.DataFrame, output_path: Path) -> None "", "## Summary Results", "", - "| Framework | Total Tokens | Prompt Tokens | Completion Tokens | Cost USD | Latency Sec | Steps | Tool Calls | Retrieved Chunks | Highest Token Step |", + "| Framework | Total Tokens | Prompt Tokens | Completion Tokens | Cost USD | " + "Latency Sec | Steps | Tool Calls | Retrieved Chunks | Highest Token Step |", "|---|---:|---:|---:|---:|---:|---:|---:|---:|---|", ] @@ -158,12 +158,16 @@ def create_markdown_summary(summary_df: pd.DataFrame, output_path: Path) -> None "", "## Key Finding", "", - "The final customer reply step is the highest token-consuming step across the benchmark runs.", + "The final customer reply step is the highest token-consuming step across " + "the benchmark runs.", "", "## Important Note", "", - "These results are workload-specific. They should not be treated as a universal ranking of frameworks.", - "The purpose is to show how AgenticLens can normalize and compare token, cost, latency, retrieval, and tool-call metrics across framework implementations.", + "These results are workload-specific. They should not be treated as a " + "universal ranking of frameworks.", + "The purpose is to show how AgenticLens can normalize and compare token, " + "cost, latency, retrieval, and tool-call metrics across framework " + "implementations.", ] ) @@ -247,4 +251,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/benchmarks/frameworks/autogen/run_autogen.py b/benchmarks/frameworks/autogen/run_autogen.py index aae4fdc..02cb6f8 100644 --- a/benchmarks/frameworks/autogen/run_autogen.py +++ b/benchmarks/frameworks/autogen/run_autogen.py @@ -1,7 +1,6 @@ import time from agenticlens import profile, step - from benchmarks.shared.support_tasks import ( check_refund_eligibility, classify_ticket, @@ -45,7 +44,6 @@ def main() -> None: ) with profile("Benchmark - AutoGen - Support Refund"): - with step( "AutoGen - Classify Ticket Intent", type="planner", @@ -132,4 +130,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/benchmarks/frameworks/crewai/run_crewai.py b/benchmarks/frameworks/crewai/run_crewai.py index 0e2601d..a4bc48d 100644 --- a/benchmarks/frameworks/crewai/run_crewai.py +++ b/benchmarks/frameworks/crewai/run_crewai.py @@ -1,7 +1,6 @@ import time from agenticlens import profile, step - from benchmarks.shared.support_tasks import ( check_refund_eligibility, classify_ticket, @@ -85,7 +84,8 @@ def main() -> None: ] # We create the CrewAI crew so the benchmark records that this implementation uses CrewAI. - # We do not call crew.kickoff() in this deterministic benchmark because that would call a live LLM. + # We do not call crew.kickoff() in this deterministic benchmark because that would + # call a live LLM. crew = Crew( agents=[classifier_agent, policy_agent, refund_agent, response_agent], tasks=tasks, @@ -94,7 +94,6 @@ def main() -> None: ) with profile("Benchmark - CrewAI - Support Refund"): - with step( "CrewAI - Classify Ticket Intent", type="planner", @@ -179,4 +178,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/benchmarks/frameworks/langgraph/run_langgraph.py b/benchmarks/frameworks/langgraph/run_langgraph.py index efa399d..1b00f07 100644 --- a/benchmarks/frameworks/langgraph/run_langgraph.py +++ b/benchmarks/frameworks/langgraph/run_langgraph.py @@ -1,8 +1,7 @@ import time -from typing import TypedDict, Any +from typing import Any, TypedDict from agenticlens import profile, step - from benchmarks.shared.support_data import ( build_policy_context, estimate_avg_tokens_per_chunk, @@ -55,7 +54,9 @@ def classify_ticket_llm(ticket: str) -> FakeResponse: def rewrite_query_llm(ticket: str) -> FakeResponse: return FakeResponse( - content="refund eligibility delivered order opened package unused item refund processing time", + content=( + "refund eligibility delivered order opened package unused item refund processing time" + ), prompt_tokens=240, completion_tokens=40, ) @@ -72,12 +73,15 @@ def refund_decision_llm(ticket: str, order: dict, policy_context: str) -> FakeRe ) -def final_response_llm(ticket: str, order: dict, policy_context: str, decision: str) -> FakeResponse: +def final_response_llm( + ticket: str, order: dict, policy_context: str, decision: str +) -> FakeResponse: return FakeResponse( content=( "Your order is within the 30-day refund window. Since the package was opened, " - "the refund may need manual review. Because the item was not used, you may still be eligible. " - "If approved, the refund will return to your original payment method and may take 5 to 10 business days." + "the refund may need manual review. Because the item was not used, you may " + "still be eligible. If approved, the refund will return to your original " + "payment method and may take 5 to 10 business days." ), prompt_tokens=920, completion_tokens=150, @@ -197,11 +201,9 @@ def final_response_node(state: SupportState) -> SupportState: def main() -> None: try: - from langgraph.graph import StateGraph, END + from langgraph.graph import END, StateGraph except ImportError as exc: - raise RuntimeError( - "LangGraph is not installed. Run: pip install langgraph" - ) from exc + raise RuntimeError("LangGraph is not installed. Run: pip install langgraph") from exc ticket = ( "My order A123 was delivered 12 days ago. " @@ -240,4 +242,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/benchmarks/frameworks/llamaindex/run_llamaindex.py b/benchmarks/frameworks/llamaindex/run_llamaindex.py index 4fbabaf..8a8f265 100644 --- a/benchmarks/frameworks/llamaindex/run_llamaindex.py +++ b/benchmarks/frameworks/llamaindex/run_llamaindex.py @@ -1,7 +1,6 @@ import time from agenticlens import profile, step - from benchmarks.shared.support_data import load_policy_docs from benchmarks.shared.support_tasks import ( check_refund_eligibility, @@ -17,7 +16,7 @@ def main() -> None: framework = "LlamaIndex" try: - from llama_index.core import Document, VectorStoreIndex + from llama_index.core import Document except ImportError as exc: raise RuntimeError("LlamaIndex is not installed. Run: pip install llama-index") from exc @@ -29,23 +28,23 @@ def main() -> None: order_id = "A123" # Framework-specific indexing object. - # This builds a LlamaIndex document collection, but the deterministic benchmark uses shared retrieval - # so results stay comparable with other framework runs. + # This builds a LlamaIndex document collection, but the deterministic benchmark + # uses shared retrieval so results stay comparable with other framework runs. policy_docs = load_policy_docs() documents = [ Document(text=doc["text"], metadata={"doc_id": doc["doc_id"], "category": doc["category"]}) for doc in policy_docs ] - # Do not build a real embedding index in the deterministic run because it may require model configuration. - # Keep this object as the framework-specific document representation. + # Do not build a real embedding index in the deterministic run because it may + # require model configuration. Keep this object as the framework-specific + # document representation. index_metadata = { "framework_documents": len(documents), "index_type": "llamaindex_documents", } with profile("Benchmark - LlamaIndex - Support Refund"): - with step( "LlamaIndex - Classify Ticket Intent", type="planner", @@ -131,4 +130,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/benchmarks/frameworks/native_python/run_native.py b/benchmarks/frameworks/native_python/run_native.py index 6f9201f..b106738 100644 --- a/benchmarks/frameworks/native_python/run_native.py +++ b/benchmarks/frameworks/native_python/run_native.py @@ -1,13 +1,12 @@ -import time import sys +import time from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(PROJECT_ROOT)) -from agenticlens import profile, step - -from benchmarks.shared.support_data import ( +from agenticlens import profile, step # noqa: E402 +from benchmarks.shared.support_data import ( # noqa: E402 build_policy_context, estimate_avg_tokens_per_chunk, lookup_order, @@ -47,7 +46,9 @@ def classify_ticket(ticket: str) -> FakeResponse: def rewrite_query(ticket: str) -> FakeResponse: return FakeResponse( - content="refund eligibility delivered order opened package unused item refund processing time", + content=( + "refund eligibility delivered order opened package unused item refund processing time" + ), prompt_tokens=220, completion_tokens=35, ) @@ -64,12 +65,15 @@ def check_refund_eligibility(ticket: str, order: dict, policy_context: str) -> F ) -def generate_customer_reply(ticket: str, order: dict, policy_context: str, decision: str) -> FakeResponse: +def generate_customer_reply( + ticket: str, order: dict, policy_context: str, decision: str +) -> FakeResponse: return FakeResponse( content=( "Your order is within the 30-day refund window. Since the package was opened, " - "the refund may need manual review. Because the item was not used, you may still be eligible. " - "If approved, the refund will return to your original payment method and may take 5 to 10 business days." + "the refund may need manual review. Because the item was not used, you may " + "still be eligible. If approved, the refund will return to your original " + "payment method and may take 5 to 10 business days." ), prompt_tokens=850, completion_tokens=130, @@ -85,7 +89,6 @@ def main() -> None: order_id = "A123" with profile("Benchmark - Native Python - Support Refund"): - with step( "Classify Ticket Intent", type="planner", @@ -166,4 +169,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py b/benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py index 590ab69..39ecfac 100644 --- a/benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py +++ b/benchmarks/frameworks/semantic_kernel/run_semantic_kernel.py @@ -1,7 +1,6 @@ import time from agenticlens import profile, step - from benchmarks.shared.support_tasks import ( check_refund_eligibility, classify_ticket, @@ -18,7 +17,9 @@ def main() -> None: try: import semantic_kernel as sk except ImportError as exc: - raise RuntimeError("Semantic Kernel is not installed. Run: pip install semantic-kernel") from exc + raise RuntimeError( + "Semantic Kernel is not installed. Run: pip install semantic-kernel" + ) from exc ticket = ( "My order A123 was delivered 12 days ago. " @@ -32,7 +33,6 @@ def main() -> None: kernel = sk.Kernel() with profile("Benchmark - Semantic Kernel - Support Refund"): - with step( "Semantic Kernel - Classify Ticket Intent", type="planner", @@ -117,4 +117,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/benchmarks/shared/benchmark_runner.py b/benchmarks/shared/benchmark_runner.py index acea32f..ded65c0 100644 --- a/benchmarks/shared/benchmark_runner.py +++ b/benchmarks/shared/benchmark_runner.py @@ -5,7 +5,6 @@ from benchmarks.shared.metrics_collector import summarize_agenticlens_report - ROOT = Path(__file__).resolve().parents[1] FRAMEWORKS = { @@ -63,18 +62,19 @@ def write_markdown_summary(results: list[dict], output_path: Path) -> None: "", "Use case: Customer Support Refund Copilot", "", - "This benchmark profiles the same practical support workflow across multiple agentic implementations.", + "This benchmark profiles the same practical support workflow across multiple " + "agentic implementations.", "", - "| Framework | Total Tokens | Cost | Latency | Steps | Tool Calls | Retrieved Chunks | Highest Token Step | Highest Cost Step |", + "| Framework | Total Tokens | Cost | Latency | Steps | Tool Calls | " + "Retrieved Chunks | Highest Token Step | Highest Cost Step |", "|---|---:|---:|---:|---:|---:|---:|---|---|", ] for row in results: lines.append( "| {framework} | {total_tokens} | ${total_cost:.6f} | {total_latency:.3f}s | " - "{step_count} | {tool_calls} | {retrieved_chunks} | {highest_token_step} | {highest_cost_step} |".format( - **row - ) + "{step_count} | {tool_calls} | {retrieved_chunks} | {highest_token_step} | " + "{highest_cost_step} |".format(**row) ) lines.extend( @@ -82,8 +82,10 @@ def write_markdown_summary(results: list[dict], output_path: Path) -> None: "", "## Interpretation", "", - "These results are workload-specific. They should not be read as a universal ranking of frameworks.", - "The goal is to show how AgenticLens makes token usage, cost, latency, retrieval behavior, and tool activity visible across workflows.", + "These results are workload-specific. They should not be read as a " + "universal ranking of frameworks.", + "The goal is to show how AgenticLens makes token usage, cost, latency, " + "retrieval behavior, and tool activity visible across workflows.", ] ) @@ -120,4 +122,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/benchmarks/shared/metrics_collector.py b/benchmarks/shared/metrics_collector.py index 9ea0568..07332df 100644 --- a/benchmarks/shared/metrics_collector.py +++ b/benchmarks/shared/metrics_collector.py @@ -79,4 +79,4 @@ def summarize_agenticlens_report(framework: str, report_path: str | Path) -> dic "highest_token_step": highest_token_step, "highest_cost_step": highest_cost_step, "report_path": str(report_path), - } \ No newline at end of file + } diff --git a/benchmarks/shared/support_data.py b/benchmarks/shared/support_data.py index 2de4a7f..77bb7b8 100644 --- a/benchmarks/shared/support_data.py +++ b/benchmarks/shared/support_data.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Any - ROOT = Path(__file__).resolve().parents[1] DATASET_DIR = ROOT / "datasets" @@ -73,4 +72,4 @@ def estimate_avg_tokens_per_chunk(chunks: list[dict[str, Any]]) -> int: def build_policy_context(chunks: list[dict[str, Any]]) -> str: - return "\n".join(f"- {chunk['text']}" for chunk in chunks) \ No newline at end of file + return "\n".join(f"- {chunk['text']}" for chunk in chunks) diff --git a/benchmarks/shared/support_tasks.py b/benchmarks/shared/support_tasks.py index 2bbba00..433cadf 100644 --- a/benchmarks/shared/support_tasks.py +++ b/benchmarks/shared/support_tasks.py @@ -41,7 +41,9 @@ def classify_ticket(ticket: str, framework: str) -> FakeResponse: def rewrite_query(ticket: str, framework: str) -> FakeResponse: return FakeResponse( - content="refund eligibility delivered order opened package unused item refund processing time", + content=( + "refund eligibility delivered order opened package unused item refund processing time" + ), prompt_tokens=220, completion_tokens=35, ) @@ -91,8 +93,9 @@ def generate_customer_reply( f"[{framework}] Your order is within the 30-day refund window. " "Since the package was opened, the refund may need manual review. " "Because the item was not used, you may still be eligible. " - "If approved, the refund will return to your original payment method and may take 5 to 10 business days." + "If approved, the refund will return to your original payment method and " + "may take 5 to 10 business days." ), prompt_tokens=850, completion_tokens=130, - ) \ No newline at end of file + ) diff --git a/examples/multiagent_edge_cases_demo.py b/examples/multiagent_edge_cases_demo.py index 2902d35..4da3392 100644 --- a/examples/multiagent_edge_cases_demo.py +++ b/examples/multiagent_edge_cases_demo.py @@ -1,8 +1,5 @@ - - from agenticlens import profile, step - SYSTEM_PROMPT = ( "You are a careful travel support assistant. " "Use only verified policy, booking, and refund information. " @@ -28,7 +25,6 @@ def main() -> None: ) with profile("Multi-Agent Travel Refund Edge Case Workflow"): - # Edge case 1: large repeated system prompt with step( "Planner Agent", @@ -129,4 +125,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/examples/support_copilot.py b/examples/support_copilot.py index e3c9c33..3aa3642 100644 --- a/examples/support_copilot.py +++ b/examples/support_copilot.py @@ -2,7 +2,6 @@ import re import sqlite3 import time -from pathlib import Path from typing import Any from agenticlens import profile, step @@ -11,6 +10,7 @@ if USE_REAL_OPENAI: from openai import OpenAI + client = OpenAI() @@ -37,7 +37,10 @@ }, { "id": "shipping_001", - "text": "Delivered orders are eligible for return review if the delivery date is within the return window.", + "text": ( + "Delivered orders are eligible for return review if the delivery date " + "is within the return window." + ), }, ] @@ -164,10 +167,11 @@ def fake_llm(task: str, prompt: str) -> FakeResponse: return FakeResponse( content=( - "Your order A123 was delivered 12 days ago, so it is within the 30-day refund window. " - "Because the package was opened, the refund may need manual review, but since the item was not used, " - "you may still be eligible. If approved, the refund will go back to your original payment method and " - "may take 5 to 10 business days after approval." + "Your order A123 was delivered 12 days ago, so it is within the 30-day " + "refund window. Because the package was opened, the refund may need " + "manual review, but since the item was not used, you may still be " + "eligible. If approved, the refund will go back to your original " + "payment method and may take 5 to 10 business days after approval." ), prompt_tokens=850, completion_tokens=120, @@ -203,7 +207,6 @@ def main() -> None: } with profile("Practical Support Copilot - Refund Ticket") as workflow: - with step( "Classify Ticket Intent", type="planner", @@ -232,7 +235,8 @@ def main() -> None: start = time.time() rewrite_response = call_llm( "rewrite", - f"Rewrite this ticket as a search query for refund policy retrieval:\n{ticket['message']}", + f"Rewrite this ticket as a search query for refund policy retrieval:\n" + f"{ticket['message']}", ) s.record(rewrite_response) s.step.metrics.latency = time.time() - start @@ -343,4 +347,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() From f7e2a7f68058b836269396ed7e2ef550bbc73264 Mon Sep 17 00:00:00 2001 From: manemsai Date: Tue, 14 Jul 2026 18:56:07 -0500 Subject: [PATCH 6/9] Add LangChain/LangGraph auto-instrumentation adapter Adds agenticlens.adapters.langchain.AgenticLensCallbackHandler, an optional (pip install "agenticlens[langchain]") callback handler that turns LangChain's own on_llm_*/on_tool_*/on_retriever_* events into AgenticLens steps automatically, so LLM calls, tool calls, and retrieval don't need manual step() wrapping. Since it produces the same Step shape as the manual API, every existing recommender, exporter, and CLI command works against LangChain-sourced workflows unmodified. This was the top item on the roadmap's near-term integrations list (README, ROADMAP). Adds langchain-core as the optional `langchain` extra, wires it into CI so the adapter is actually tested, and adds a mypy override to stop following into langchain-core's internals (its transitive numpy stub can be newer than our target Python version). Documented in docs/langchain-integration.md and linked from the README and mkdocs nav. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 16 ++ README.md | 27 ++- ROADMAP.md | 5 +- docs/langchain-integration.md | 61 ++++++ mkdocs.yml | 1 + pyproject.toml | 10 + src/agenticlens/adapters/__init__.py | 6 + src/agenticlens/adapters/langchain.py | 260 ++++++++++++++++++++++++++ tests/test_adapters_langchain.py | 145 ++++++++++++++ 10 files changed, 530 insertions(+), 3 deletions(-) create mode 100644 docs/langchain-integration.md create mode 100644 src/agenticlens/adapters/__init__.py create mode 100644 src/agenticlens/adapters/langchain.py create mode 100644 tests/test_adapters_langchain.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 437adb2..ea27802 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: run: uv python install ${{ matrix.python-version }} - name: Install dependencies - run: uv sync --extra dev --python ${{ matrix.python-version }} + run: uv sync --extra dev --extra langchain --python ${{ matrix.python-version }} - name: Lint (ruff check) run: uv run ruff check . diff --git a/CHANGELOG.md b/CHANGELOG.md index 6198f69..75741eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ All notable changes to this project will be documented here. This project follows [Semantic Versioning](https://semver.org/). +## Unreleased + +### Added + +- `agenticlens.adapters.langchain.AgenticLensCallbackHandler`, an optional + (`pip install "agenticlens[langchain]"`) LangChain/LangGraph callback + handler that auto-instruments LLM, tool, and retriever calls as AgenticLens + steps without manual `step()` blocks. Documented in + `docs/langchain-integration.md`. +- `benchmarks/`, a cross-framework benchmark harness that profiles the same + refund-ticket workload through AutoGen, CrewAI, LangGraph, LlamaIndex, + Semantic Kernel, and native Python for an apples-to-apples token/cost/latency + comparison. Linked from the README. +- `examples/support_copilot.py` and `examples/multiagent_edge_cases_demo.py`, + additional practical and edge-case profiling examples. + ## 0.2.0 - 2026-07-13 ### Added diff --git a/README.md b/README.md index efe3f93..9bc332e 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,7 @@ RAG chunk utility. | Costing | Local pricing table plus user pricing overrides | | Recommendations | Repeated prompts, excessive chunks, low-utility chunks, long history, duplicate tool calls | | Budget impact | Dollar-per-run and monthly savings projections | +| Integrations | Auto-instrumentation adapter for LangChain / LangGraph via callbacks | | CLI | `profile`, `report`, and `analyze` commands | | Export | JSON, CSV, Markdown, and Jira workflow export | | Tooling | pytest, Ruff, mypy, GitHub Actions | @@ -317,6 +318,29 @@ Set credentials via environment variables for safety — see For sample output previews of all formats, see [docs/export-formats.md](docs/export-formats.md). +## LangChain Integration + +Auto-instrument a LangChain (or LangGraph) run via its callback system instead +of wrapping every call in `step()`: + +```bash +pip install "agenticlens[langchain]" +``` + +```python +from agenticlens import profile +from agenticlens.adapters.langchain import AgenticLensCallbackHandler + +with profile("My LangChain App") as workflow: + chain.invoke(inputs, config={"callbacks": [AgenticLensCallbackHandler()]}) +``` + +LLM calls, tool calls, and retriever calls are tracked automatically as +`llm_call`, `tool_call`, and `retriever` steps, with token usage extracted the +same way `s.record(...)` does. See +[docs/langchain-integration.md](docs/langchain-integration.md) for details on +what is and isn't tracked. + ## CLI Reference Profile a Python script: @@ -392,7 +416,8 @@ Near-term priorities: - model-tier mismatch detection - prompt caching opportunity detection -- integrations for LangChain, LangGraph, LiteLLM, and OpenAI Agents SDK +- integrations for LiteLLM and OpenAI Agents SDK (LangChain / LangGraph done — + see [docs/langchain-integration.md](docs/langchain-integration.md)) - OpenTelemetry and OpenInference trace import - optional prompt compression handoff diff --git a/ROADMAP.md b/ROADMAP.md index 62d8651..0ce28f0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -209,8 +209,11 @@ Goal: make the core model easy to adopt across real agent stacks. Planned work: +- [x] LangChain / LangGraph adapter (`agenticlens.adapters.langchain`, + optional `langchain` extra) — auto-instruments LLM/tool/retriever calls via + callbacks - [ ] Add integrations for: - LangGraph, LiteLLM, OpenAI Agents SDK, LangChain, CrewAI + LiteLLM, OpenAI Agents SDK, CrewAI - [ ] Add OpenTelemetry and OpenInference import paths - [ ] Broaden provider support: Gemini, Ollama, vLLM, LiteLLM, Azure OpenAI diff --git a/docs/langchain-integration.md b/docs/langchain-integration.md new file mode 100644 index 0000000..818fa31 --- /dev/null +++ b/docs/langchain-integration.md @@ -0,0 +1,61 @@ +# LangChain Integration + +AgenticLens can auto-instrument a LangChain (or LangGraph) run through its +callback system, instead of requiring a manual `with step(...)` block around +every LLM call, tool call, and retrieval. + +## Installation + +```bash +pip install "agenticlens[langchain]" +``` + +## Usage + +```python +from agenticlens import profile +from agenticlens.adapters.langchain import AgenticLensCallbackHandler + +handler = AgenticLensCallbackHandler() + +with profile("My LangChain App") as workflow: + chain.invoke(inputs, config={"callbacks": [handler]}) + +print(workflow.total_tokens) +print(workflow.total_cost) +``` + +The handler must be used inside a `with profile(...):` block -- it attaches +each step to whichever workflow is active in the current context. + +## What gets tracked + +| LangChain event | AgenticLens step type | Metadata captured | +| --- | --- | --- | +| `on_llm_start` / `on_chat_model_start` → `on_llm_end` | `llm_call` | `prompt`, token usage, `model` (when reported) | +| `on_tool_start` → `on_tool_end` | `tool_call` | `tool_name`, `tool_args` | +| `on_retriever_start` → `on_retriever_end` | `retriever` | `query`, `chunk_count`, `avg_tokens_per_chunk` | + +Token usage is extracted in priority order: + +1. Per-message `usage_metadata` (populated by most current LangChain chat + model integrations) +2. `llm_output["token_usage"]` / `llm_output["usage"]` (older, provider-specific + integrations) +3. If neither is present, token fields stay `0` for that step -- the step + itself, its latency, and its metadata are still recorded. + +`avg_tokens_per_chunk` for retriever steps is estimated from +`len(document.page_content) / 4` -- LangChain retrievers return text, not a +token count, and this is only an approximation used to feed the +excessive-chunks recommender's savings estimate. + +Because the adapter produces the same `Step` shape as the manual `step()` API, +every recommender, exporter, and the CLI work against LangChain-sourced +workflows without any changes. + +## What doesn't get tracked + +Chain-level (`on_chain_*`) events are intentionally ignored. LCEL chains fire +one such event per internal runnable, which would flood the workflow with +steps that don't correspond to a real LLM/tool/retrieval cost. diff --git a/mkdocs.yml b/mkdocs.yml index 9702968..4c4c8fb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,6 +14,7 @@ nav: - Export Formats: export-formats.md - RAG Chunk Utility: rag-chunk-utility.md - Workflow Schema Spec: workflow-schema-spec.md + - LangChain Integration: langchain-integration.md repo_url: https://github.com/DeepAgentLabs/agenticlens repo_name: DeepAgentLabs/agenticlens diff --git a/pyproject.toml b/pyproject.toml index 3c8b150..de56376 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ dev = [ docs = [ "mkdocs-material>=9.5", ] +langchain = [ + "langchain-core>=0.3", +] [project.scripts] agenticlens = "agenticlens.cli.main:app" @@ -79,6 +82,13 @@ strict = true packages = ["agenticlens"] mypy_path = "src" +[[tool.mypy.overrides]] +# langchain-core is an optional dependency (the `langchain` extra) whose own +# transitive imports (e.g. numpy) ship stubs newer than our target Python +# version. Skip following into it rather than type-checking its internals. +module = ["langchain_core", "langchain_core.*"] +follow_imports = "skip" + [tool.pytest.ini_options] testpaths = ["tests"] pythonpath = ["src"] diff --git a/src/agenticlens/adapters/__init__.py b/src/agenticlens/adapters/__init__.py new file mode 100644 index 0000000..07096a9 --- /dev/null +++ b/src/agenticlens/adapters/__init__.py @@ -0,0 +1,6 @@ +"""Optional auto-instrumentation adapters for third-party agent frameworks. + +Adapters are not imported here so that `agenticlens` itself never requires +the frameworks they integrate with. Import the specific adapter module you +need, e.g. `from agenticlens.adapters.langchain import AgenticLensCallbackHandler`. +""" diff --git a/src/agenticlens/adapters/langchain.py b/src/agenticlens/adapters/langchain.py new file mode 100644 index 0000000..0e10e2c --- /dev/null +++ b/src/agenticlens/adapters/langchain.py @@ -0,0 +1,260 @@ +"""Auto-instrumentation adapter for LangChain / LangGraph via its callback system. + +Requires the optional `langchain-core` dependency: + + pip install "agenticlens[langchain]" + +Usage: + + from agenticlens import profile + from agenticlens.adapters.langchain import AgenticLensCallbackHandler + + with profile("My LangChain App"): + chain.invoke(inputs, config={"callbacks": [AgenticLensCallbackHandler()]}) + +The handler creates one AgenticLens step per LLM call, tool call, and retriever +call, using the run lifecycle to time each one and, for LLM calls, extracting +token usage the way the rest of AgenticLens does from `s.record(...)`. +""" + +from __future__ import annotations + +import threading +import time +from typing import Any +from uuid import UUID + +try: + from langchain_core.callbacks.base import BaseCallbackHandler + from langchain_core.outputs import LLMResult +except ImportError as exc: # pragma: no cover - exercised only without the extra installed + raise ImportError( + "agenticlens.adapters.langchain requires the 'langchain-core' package. " + 'Install it with: pip install "agenticlens[langchain]"' + ) from exc + +from agenticlens.models.enums import StepType +from agenticlens.models.step import Step +from agenticlens.profiler.context import get_active_workflow + +CHARS_PER_TOKEN_ESTIMATE = 4 +"""Rough chars-to-tokens heuristic used only for retrieved-chunk size estimates. + +LangChain retrievers return `Document` text, not a token count, and the +excessive-chunks recommender needs `avg_tokens_per_chunk` to estimate savings. +This is intentionally approximate. +""" + + +def _extract_llm_usage(response: LLMResult) -> tuple[int, int] | None: + """Best-effort prompt/completion token extraction from an `LLMResult`. + + Tries the modern per-message `usage_metadata` first (populated by most + current LangChain chat model integrations), then falls back to the + provider-specific `llm_output` dict used by older integrations. + """ + for generations in response.generations: + for generation in generations: + message = getattr(generation, "message", None) + usage = getattr(message, "usage_metadata", None) if message is not None else None + if usage: + return usage.get("input_tokens", 0), usage.get("output_tokens", 0) + + llm_output = response.llm_output or {} + token_usage = llm_output.get("token_usage") or llm_output.get("usage") + if token_usage: + prompt = token_usage.get("prompt_tokens", token_usage.get("input_tokens")) + completion = token_usage.get("completion_tokens", token_usage.get("output_tokens")) + if prompt is not None and completion is not None: + return int(prompt), int(completion) + + return None + + +class AgenticLensCallbackHandler(BaseCallbackHandler): # type: ignore[misc] + """Turns LangChain callback events into AgenticLens steps automatically. + + Must be used inside a `with profile(...):` block -- each tracked run is + attached to whichever workflow is active in the current context when that + run starts. + + Chain-level (`on_chain_*`) events are intentionally not tracked: LCEL + chains fire one per internal runnable, which would flood the workflow + with steps that don't correspond to a real LLM/tool/retrieval cost. + """ + + def __init__(self, provider: str | None = None) -> None: + super().__init__() + self._provider = provider + self._lock = threading.Lock() + self._runs: dict[UUID, tuple[Step, float]] = {} + + def _start(self, run_id: UUID, step_type: StepType, name: str, **metadata: Any) -> None: + step_model = Step( + name=name, + type=step_type, + provider=self._provider, + metadata={k: v for k, v in metadata.items() if v is not None}, + ) + get_active_workflow().steps.append(step_model) + with self._lock: + self._runs[run_id] = (step_model, time.perf_counter()) + + def _finish(self, run_id: UUID) -> Step | None: + with self._lock: + entry = self._runs.pop(run_id, None) + if entry is None: + return None + step_model, start = entry + step_model.metrics.latency = time.perf_counter() - start + return step_model + + # LLM / chat model events + + def on_llm_start( + self, + serialized: dict[str, Any], + prompts: list[str], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + name = (serialized or {}).get("name") or "LLM Call" + self._start(run_id, StepType.LLM_CALL, name, prompt=prompts[0] if prompts else None) + + def on_chat_model_start( + self, + serialized: dict[str, Any], + messages: list[list[Any]], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + name = (serialized or {}).get("name") or "Chat Model Call" + prompt = None + if messages and messages[0]: + prompt = "\n".join(str(getattr(m, "content", m)) for m in messages[0]) + self._start(run_id, StepType.LLM_CALL, name, prompt=prompt) + + def on_llm_end( + self, + response: LLMResult, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + **kwargs: Any, + ) -> None: + step_model = self._finish(run_id) + if step_model is None: + return + + usage = _extract_llm_usage(response) + if usage is not None: + prompt_tokens, completion_tokens = usage + step_model.metrics.prompt_tokens = prompt_tokens + step_model.metrics.completion_tokens = completion_tokens + step_model.metrics.total_tokens = prompt_tokens + completion_tokens + + if step_model.model is None and response.llm_output: + step_model.model = response.llm_output.get("model_name") + + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + **kwargs: Any, + ) -> None: + self._finish(run_id) + + # Tool events + + def on_tool_start( + self, + serialized: dict[str, Any], + input_str: str, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + inputs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + tool_name = (serialized or {}).get("name") or "Tool Call" + tool_args = inputs if inputs is not None else {"input": input_str} + self._start(run_id, StepType.TOOL_CALL, tool_name, tool_name=tool_name, tool_args=tool_args) + + def on_tool_end( + self, + output: Any, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + self._finish(run_id) + + def on_tool_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + self._finish(run_id) + + # Retriever events + + def on_retriever_start( + self, + serialized: dict[str, Any], + query: str, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + name = (serialized or {}).get("name") or "Retriever" + self._start(run_id, StepType.RETRIEVER, name, query=query) + + def on_retriever_end( + self, + documents: list[Any], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + step_model = self._finish(run_id) + if step_model is None: + return + + step_model.metadata["chunk_count"] = len(documents) + if documents: + avg_chars = sum(len(getattr(d, "page_content", "")) for d in documents) / len(documents) + step_model.metadata["avg_tokens_per_chunk"] = round( + avg_chars / CHARS_PER_TOKEN_ESTIMATE + ) + + def on_retriever_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> None: + self._finish(run_id) diff --git a/tests/test_adapters_langchain.py b/tests/test_adapters_langchain.py new file mode 100644 index 0000000..9f14a70 --- /dev/null +++ b/tests/test_adapters_langchain.py @@ -0,0 +1,145 @@ +from uuid import uuid4 + +import pytest +from langchain_core.documents import Document +from langchain_core.messages import AIMessage +from langchain_core.outputs import ChatGeneration, Generation, LLMResult + +from agenticlens import profile +from agenticlens.adapters.langchain import AgenticLensCallbackHandler +from agenticlens.models.enums import StepType +from agenticlens.recommenders import RecommendationEngine + + +def test_llm_call_records_usage_from_message_metadata() -> None: + handler = AgenticLensCallbackHandler() + run_id = uuid4() + + with profile("Test") as workflow: + handler.on_chat_model_start({"name": "ChatOpenAI"}, [[]], run_id=run_id) + result = LLMResult( + generations=[ + [ + ChatGeneration( + message=AIMessage( + content="hi", + usage_metadata={ + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + }, + ) + ) + ] + ] + ) + handler.on_llm_end(result, run_id=run_id) + + assert len(workflow.steps) == 1 + step = workflow.steps[0] + assert step.type == StepType.LLM_CALL + assert step.metrics.prompt_tokens == 10 + assert step.metrics.completion_tokens == 5 + assert step.metrics.total_tokens == 15 + assert step.metrics.latency >= 0 + + +def test_llm_call_falls_back_to_llm_output_token_usage() -> None: + handler = AgenticLensCallbackHandler() + run_id = uuid4() + + with profile("Test") as workflow: + handler.on_llm_start({"name": "OpenAI"}, ["hello"], run_id=run_id) + result = LLMResult( + generations=[[Generation(text="hi")]], + llm_output={ + "token_usage": {"prompt_tokens": 20, "completion_tokens": 8}, + "model_name": "gpt-4o-mini", + }, + ) + handler.on_llm_end(result, run_id=run_id) + + step = workflow.steps[0] + assert step.metrics.prompt_tokens == 20 + assert step.metrics.completion_tokens == 8 + assert step.metadata["prompt"] == "hello" + assert step.model == "gpt-4o-mini" + + +def test_llm_error_finishes_step_without_tokens() -> None: + handler = AgenticLensCallbackHandler() + run_id = uuid4() + + with profile("Test") as workflow: + handler.on_llm_start({"name": "OpenAI"}, ["hello"], run_id=run_id) + handler.on_llm_error(RuntimeError("boom"), run_id=run_id) + + step = workflow.steps[0] + assert step.metrics.prompt_tokens == 0 + assert step.metrics.latency >= 0 + + +def test_tool_call_records_name_and_args() -> None: + handler = AgenticLensCallbackHandler() + run_id = uuid4() + + with profile("Test") as workflow: + handler.on_tool_start( + {"name": "lookup_order"}, + '{"order_id": "A123"}', + run_id=run_id, + inputs={"order_id": "A123"}, + ) + handler.on_tool_end("found", run_id=run_id) + + step = workflow.steps[0] + assert step.type == StepType.TOOL_CALL + assert step.metadata["tool_name"] == "lookup_order" + assert step.metadata["tool_args"] == {"order_id": "A123"} + + +def test_retriever_records_chunk_count_and_avg_tokens() -> None: + handler = AgenticLensCallbackHandler() + run_id = uuid4() + + with profile("Test") as workflow: + handler.on_retriever_start({"name": "Retriever"}, "refund policy", run_id=run_id) + handler.on_retriever_end( + [Document(page_content="x" * 40), Document(page_content="x" * 20)], + run_id=run_id, + ) + + step = workflow.steps[0] + assert step.type == StepType.RETRIEVER + assert step.metadata["chunk_count"] == 2 + assert step.metadata["avg_tokens_per_chunk"] == 8 # (40 + 20) / 2 / 4 chars-per-token + + +def test_duplicate_tool_calls_detected_from_adapter_metadata() -> None: + handler = AgenticLensCallbackHandler() + + with profile("Test") as workflow: + for _ in range(2): + run_id = uuid4() + handler.on_tool_start( + {"name": "lookup_order"}, + "", + run_id=run_id, + inputs={"order_id": "A123"}, + ) + handler.on_tool_end("found", run_id=run_id) + + recs = RecommendationEngine().run(workflow) + assert any(r.title == "Duplicate tool call" for r in recs) + + +def test_finish_on_unknown_run_id_is_a_noop() -> None: + handler = AgenticLensCallbackHandler() + with profile("Test"): + handler.on_llm_end(LLMResult(generations=[]), run_id=uuid4()) + + +def test_handler_outside_profile_raises() -> None: + handler = AgenticLensCallbackHandler() + with pytest.raises(RuntimeError): + handler.on_llm_start({"name": "OpenAI"}, ["hello"], run_id=uuid4()) From 61271a983c0fb79ac2b18a105cfbd198b9c76632 Mon Sep 17 00:00:00 2001 From: manemsai Date: Tue, 14 Jul 2026 19:14:42 -0500 Subject: [PATCH 7/9] Fix benchmark runner to propagate PYTHONPATH to subprocesses benchmark_runner.py invokes each framework script via `agenticlens profile