From 7058d8d9425550b5689b628effb05d40e7a22b2a Mon Sep 17 00:00:00 2001 From: mehkhan Date: Sun, 24 May 2026 15:28:38 +0530 Subject: [PATCH 1/3] Add multi-model benchmark: Gemini 2.5 Pro, 3 Flash, 3.5 Flash --- Makefile | 8 +- README.md | 63 ++++-- examples/compare_models.py | 245 ++++++++++++++++++++++++ src/behavioral_memory/cli.py | 33 +++- src/behavioral_memory/planner/engine.py | 28 ++- 5 files changed, 343 insertions(+), 34 deletions(-) create mode 100644 examples/compare_models.py diff --git a/Makefile b/Makefile index d05fc04..4d6dd47 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install dev lint format typecheck test test-unit test-e2e demo benchmark benchmark-pg validate clean +.PHONY: help install dev lint format typecheck test test-unit test-e2e demo benchmark benchmark-pg benchmark-multi validate clean help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}' @@ -37,6 +37,12 @@ benchmark: ## Run live benchmark with in-memory store (requires GOOGLE_API_KEY) benchmark-pg: ## Run live benchmark with pgvector (requires GOOGLE_API_KEY + PostgreSQL) uv run python examples/run_live_benchmark.py --postgres +benchmark-multi: ## Run multi-model benchmark comparison (requires GOOGLE_API_KEY) + uv run python examples/run_live_benchmark.py --model gemini-2.5-pro --output results_25pro.json + uv run python examples/run_live_benchmark.py --model gemini-3-flash-preview --output results_3flash.json + uv run python examples/run_live_benchmark.py --model gemini-3.5-flash --output results_35flash.json + uv run python examples/compare_models.py --results results_25pro.json results_3flash.json results_35flash.json --markdown + ablation: ## Run gatekeeper ablation study uv run python examples/gatekeeper_ablation.py --verbose diff --git a/README.md b/README.md index 346ec3e..44c7f7f 100644 --- a/README.md +++ b/README.md @@ -210,29 +210,46 @@ steps = postprocess_plan(raw_output) # returns list[ToolCall] ## Key Results -On a 30-task benchmark with 7 MCP tools (Gemini 2.5 Pro, temperature 0): - -| Metric | Zero-Shot | Static Few-Shot | **With Behavioral Memory** | -|--------|-----------|----------------|---------------------------| -| Tool Selection (TSA) | 63.3% | 70.0% | **83.3%** | -| Parameter Validity (PV) | 72.2% | 79.6% | **84.0%** | -| Plan Correctness (PCR) | 33.3% | 50.0% | **63.3%** | -| Sequence Accuracy (ESA) | 63.3% | 70.0% | **83.3%** | - -McNemar's test: **p = 0.004** vs zero-shot. Plan correctness nearly doubled. +### Multi-Model Comparison + +30-task benchmark, 7 MCP tools, temperature 0, embeddings: `gemini-embedding-001`. Behavioral memory improves plan correctness across all three models tested — the gain is consistent regardless of model family or size. + +| Metric | Strategy | **Gemini 2.5 Pro** | **Gemini 3 Flash Preview** | **Gemini 3.5 Flash** | +|--------|----------|----------------|------------------------|------------------| +| Tool Selection (TSA) | Zero-Shot | 63.3% | 60.0% | 73.3% | +| | Static Few-Shot | 70.0% | 80.0% | 76.7% | +| | **Dynamic (Proposed)** | **93.3%** | **76.7%** | **83.3%** | +| Parameter Validity (PV) | Zero-Shot | 60.6% | 70.2% | 71.1% | +| | Static Few-Shot | 68.9% | 77.2% | 74.7% | +| | **Dynamic (Proposed)** | **85.5%** | **74.6%** | **80.9%** | +| Plan Correctness (PCR) | Zero-Shot | 50.0% | 50.0% | 60.0% | +| | Static Few-Shot | 63.3% | 73.3% | 70.0% | +| | **Dynamic (Proposed)** | **80.0%** | **76.7%** | **80.0%** | +| Sequence Accuracy (ESA) | Zero-Shot | 63.3% | 60.0% | 73.3% | +| | Static Few-Shot | 70.0% | 80.0% | 76.7% | +| | **Dynamic (Proposed)** | **93.3%** | **76.7%** | **83.3%** | + +**McNemar's test (Zero-Shot vs Dynamic):** + +| Model | p-value | Significant? | +|-------|---------|-------------| +| Gemini 2.5 Pro | p = 0.0225 | Yes | +| Gemini 3 Flash Preview | p = 0.0215 | Yes | +| Gemini 3.5 Flash | p = 0.0703 | No | + +Dynamic retrieval reaches **76.7–80.0% PCR** across all models. The improvement is statistically significant (p < 0.05) for two of three models; Gemini 3.5 Flash starts with a higher zero-shot baseline (60.0%), narrowing the gap.
-Reproduced live run (May 2026) +Original paper results (Gemini 2.5 Pro, single-model) -| Metric | Paper | Live Run (pgvector) | -|--------|-------|---------------------| -| TSA | 83.3% | 86.7% | -| PV | 84.0% | 82.2% | -| PCR | 63.3% | 80.0% | -| ESA | 83.3% | 86.7% | -| McNemar p | 0.004 | 0.039 | +| Metric | Zero-Shot | Static Few-Shot | With Behavioral Memory | +|--------|-----------|----------------|------------------------| +| TSA | 63.3% | 70.0% | 83.3% | +| PV | 72.2% | 79.6% | 84.0% | +| PCR | 33.3% | 50.0% | 63.3% | +| ESA | 63.3% | 70.0% | 83.3% | -All results within 95% bootstrap confidence intervals. +McNemar's test: p = 0.004 vs zero-shot.
@@ -263,12 +280,18 @@ cd behavioral-memory pip install -e ".[agent,eval]" export GOOGLE_API_KEY=your-key -# Run the 30-task benchmark +# Run the 30-task benchmark (default: gemini-2.5-pro) python examples/run_live_benchmark.py # Quick test (5 tasks) python examples/run_live_benchmark.py --limit 5 +# Multi-model comparison +python examples/run_live_benchmark.py --model gemini-2.5-pro --output results_25pro.json +python examples/run_live_benchmark.py --model gemini-3-flash-preview --output results_3flash.json +python examples/run_live_benchmark.py --model gemini-3.5-flash --output results_35flash.json +python examples/compare_models.py --results results_25pro.json results_3flash.json results_35flash.json --markdown + # Exact paper reproduction (with pgvector) pip install -e ".[postgres]" docker compose up -d # or: podman-compose up -d diff --git a/examples/compare_models.py b/examples/compare_models.py new file mode 100644 index 0000000..bd38d96 --- /dev/null +++ b/examples/compare_models.py @@ -0,0 +1,245 @@ +"""Compare benchmark results across multiple LLM models. + +Loads JSON results files from run_live_benchmark.py and produces: + 1. A Rich side-by-side table in the terminal + 2. A markdown-formatted table (copy-paste into README) + 3. Cross-model McNemar's tests on PCR + 4. A merged multi_model_results.json for archival + +Usage: + python examples/compare_models.py \\ + --results results_25pro.json results_3flash.json results_35flash.json + + python examples/compare_models.py \\ + --results results_25pro.json results_3flash.json \\ + --output multi_model_comparison.json +""" + +from __future__ import annotations + +import argparse +import json +import sys +from itertools import combinations +from pathlib import Path + +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +from behavioral_memory.evaluation.statistics import mcnemar_test + +console = Console() + +METRIC_LABELS = { + "tsa": "Tool Selection (TSA)", + "pv": "Parameter Validity (PV)", + "pcr": "Plan Correctness (PCR)", + "esa": "Sequence Accuracy (ESA)", +} + +STRATEGY_KEYS = [ + ("zero_shot", "Zero-Shot"), + ("static_few_shot", "Static Few-Shot"), + ("dynamic_retrieval", "Dynamic (Proposed)"), +] + + +def load_results(paths: list[str]) -> list[dict]: + results = [] + for p in paths: + path = Path(p) + if not path.exists(): + console.print(f"[red]File not found: {p}[/red]") + sys.exit(1) + with open(path) as f: + data = json.load(f) + if "model" not in data: + data["model"] = path.stem + results.append(data) + return results + + +def fmt_pct(value: float) -> str: + return f"{value:.1%}" + + +def print_rich_table(all_results: list[dict]) -> None: + """Print a Rich table comparing all models side-by-side.""" + table = Table( + title=f"Multi-Model Benchmark Comparison (N={all_results[0]['n_tasks']})", + show_lines=True, + ) + table.add_column("Metric", style="bold") + table.add_column("Strategy") + for r in all_results: + table.add_column(r["model"], justify="right") + + for metric_key, metric_label in METRIC_LABELS.items(): + for strat_key, strat_label in STRATEGY_KEYS: + style = "bold green" if strat_key == "dynamic_retrieval" else "" + row = [metric_label if strat_key == STRATEGY_KEYS[0][0] else "", strat_label] + for r in all_results: + agg = r[strat_key]["aggregate"][metric_key] + mean = agg["mean"] if isinstance(agg, dict) else agg + row.append(fmt_pct(mean)) + table.add_row(*row, style=style) + + console.print() + console.print(table) + + +def print_mcnemar_comparisons(all_results: list[dict]) -> None: + """Print McNemar's test results for each model and cross-model.""" + console.print("\n[bold]McNemar's Test — Zero-Shot vs Dynamic (per model):[/bold]") + + mcnemar_table = Table(title="Statistical Significance") + mcnemar_table.add_column("Model", style="bold") + mcnemar_table.add_column("p-value", justify="right") + mcnemar_table.add_column("Significant (p<0.05)?", justify="center") + + for r in all_results: + zs_pcr = [t["metrics"]["pcr"] for t in r["zero_shot"]["per_task"]] + dyn_pcr = [t["metrics"]["pcr"] for t in r["dynamic_retrieval"]["per_task"]] + result = mcnemar_test(zs_pcr, dyn_pcr) + p = result["p_value"] + sig = "[green]Yes[/green]" if p < 0.05 else "[yellow]No[/yellow]" + mcnemar_table.add_row(r["model"], f"{p:.4f}", sig) + + console.print(mcnemar_table) + + +def print_difficulty_breakdown(all_results: list[dict]) -> None: + """Print PCR breakdown by difficulty across models.""" + from behavioral_memory.evaluation.benchmark import BenchmarkRunner + + diff_table = Table(title="Plan Correctness (PCR) by Difficulty — Dynamic Retrieval", show_lines=True) + diff_table.add_column("Difficulty", style="bold") + diff_table.add_column("n", justify="right") + for r in all_results: + diff_table.add_column(r["model"], justify="right", style="bold green") + + for diff in ["simple", "moderate", "challenging"]: + row: list[str] = [diff] + n_str = "" + for r in all_results: + by_diff = BenchmarkRunner.results_by_difficulty(r["dynamic_retrieval"]) + d = by_diff.get(diff, {}) + n_str = str(d.get("n", 0)) + row.append(fmt_pct(d.get("pcr", 0))) + row.insert(1, n_str) + diff_table.add_row(*row) + + console.print() + console.print(diff_table) + + +def generate_markdown(all_results: list[dict]) -> str: + """Generate a markdown table for the README.""" + lines = [] + models = [r["model"] for r in all_results] + n = all_results[0]["n_tasks"] + + lines.append( + f"On a {n}-task benchmark with 7 MCP tools (temperature 0, " + f"embeddings: `gemini-embedding-001`):" + ) + lines.append("") + + header = "| Metric | Strategy | " + " | ".join(f"**{m}**" for m in models) + " |" + sep = "|--------|----------|" + "|".join("-" * (len(m) + 6) for m in models) + "|" + lines.append(header) + lines.append(sep) + + for metric_key, metric_label in METRIC_LABELS.items(): + for i, (strat_key, strat_label) in enumerate(STRATEGY_KEYS): + label = metric_label if i == 0 else "" + is_dynamic = strat_key == "dynamic_retrieval" + cells = [] + for r in all_results: + agg = r[strat_key]["aggregate"][metric_key] + mean = agg["mean"] if isinstance(agg, dict) else agg + val = fmt_pct(mean) + cells.append(f"**{val}**" if is_dynamic else val) + + strat_display = f"**{strat_label}**" if is_dynamic else strat_label + row = f"| {label} | {strat_display} | " + " | ".join(cells) + " |" + lines.append(row) + + lines.append("") + lines.append("**McNemar's test (Zero-Shot vs Dynamic):**") + lines.append("") + lines.append("| Model | p-value | Significant? |") + lines.append("|-------|---------|-------------|") + + for r in all_results: + zs_pcr = [t["metrics"]["pcr"] for t in r["zero_shot"]["per_task"]] + dyn_pcr = [t["metrics"]["pcr"] for t in r["dynamic_retrieval"]["per_task"]] + result = mcnemar_test(zs_pcr, dyn_pcr) + p = result["p_value"] + sig = "Yes" if p < 0.05 else "No" + lines.append(f"| {r['model']} | p = {p:.4f} | {sig} |") + + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compare benchmark results across models") + parser.add_argument( + "--results", + nargs="+", + required=True, + help="Paths to benchmark result JSON files (from run_live_benchmark.py)", + ) + parser.add_argument( + "--output", + default="multi_model_results.json", + help="Output path for merged comparison JSON", + ) + parser.add_argument( + "--markdown", + action="store_true", + help="Print markdown-formatted table for README", + ) + args = parser.parse_args() + + all_results = load_results(args.results) + + console.print( + Panel.fit( + "[bold]Multi-Model Benchmark Comparison[/bold]\n\n" + f"Models: {', '.join(r['model'] for r in all_results)}\n" + f"Tasks: {all_results[0]['n_tasks']}", + title="Comparison", + ) + ) + + print_rich_table(all_results) + print_mcnemar_comparisons(all_results) + print_difficulty_breakdown(all_results) + + md = generate_markdown(all_results) + + if args.markdown: + console.print("\n[bold]Markdown for README:[/bold]\n") + console.print(md) + + merged = { + "models": [r["model"] for r in all_results], + "n_tasks": all_results[0]["n_tasks"], + "per_model": {r["model"]: r for r in all_results}, + "markdown_table": md, + } + + with open(args.output, "w") as f: + json.dump(merged, f, indent=2, default=str) + console.print(f"\n[dim]Merged results saved to {args.output}[/dim]") + + console.print("\n[bold]Copy the markdown table into README.md:[/bold]") + console.print(f" python examples/compare_models.py --results {' '.join(args.results)} --markdown") + + +if __name__ == "__main__": + main() diff --git a/src/behavioral_memory/cli.py b/src/behavioral_memory/cli.py index c7b4e66..a6ef742 100644 --- a/src/behavioral_memory/cli.py +++ b/src/behavioral_memory/cli.py @@ -215,21 +215,34 @@ def _find_relevant_traces(query: str, traces: list[ExecutionTrace], top_k: int = def _print_paper_results_table() -> None: - """Print the paper's published results for reference.""" - table = Table(title="Paper Results — 30-Task Benchmark (Gemini 2.5 Pro)") + """Print multi-model benchmark results (Dynamic Retrieval / Proposed).""" + table = Table(title="Benchmark — Dynamic Retrieval (Proposed) across Models") table.add_column("Metric", style="bold") - table.add_column("Zero-Shot", justify="right") - table.add_column("Static Few-Shot", justify="right") - table.add_column("Dynamic Retrieval", justify="right", style="bold green") + table.add_column("Gemini 2.5 Pro", justify="right") + table.add_column("Gemini 3 Flash", justify="right") + table.add_column("Gemini 3.5 Flash", justify="right") - table.add_row("Tool Selection (TSA)", "63.3%", "70.0%", "83.3%") - table.add_row("Parameter Validity (PV)", "72.2%", "79.6%", "84.0%") - table.add_row("Plan Correctness (PCR)", "33.3%", "50.0%", "63.3%") - table.add_row("Sequence Accuracy (ESA)", "63.3%", "70.0%", "83.3%") - table.add_row("McNemar p-value vs ZS", "—", "—", "p = 0.004") + table.add_row("Tool Selection (TSA)", "93.3%", "76.7%", "83.3%") + table.add_row("Parameter Validity (PV)", "85.5%", "74.6%", "80.9%") + table.add_row("Plan Correctness (PCR)", "80.0%", "76.7%", "80.0%") + table.add_row("Sequence Accuracy (ESA)", "93.3%", "76.7%", "83.3%") + table.add_row("McNemar p vs Zero-Shot", "p = 0.023", "p = 0.022", "p = 0.070") console.print(table) + zs_table = Table(title="Zero-Shot Baseline (for comparison)") + zs_table.add_column("Metric", style="bold") + zs_table.add_column("Gemini 2.5 Pro", justify="right") + zs_table.add_column("Gemini 3 Flash", justify="right") + zs_table.add_column("Gemini 3.5 Flash", justify="right") + + zs_table.add_row("TSA", "63.3%", "60.0%", "73.3%") + zs_table.add_row("PV", "60.6%", "70.2%", "71.1%") + zs_table.add_row("PCR", "50.0%", "50.0%", "60.0%") + zs_table.add_row("ESA", "63.3%", "60.0%", "73.3%") + + console.print(zs_table) + # --- Memory commands --- diff --git a/src/behavioral_memory/planner/engine.py b/src/behavioral_memory/planner/engine.py index 93ae9d2..38a7084 100644 --- a/src/behavioral_memory/planner/engine.py +++ b/src/behavioral_memory/planner/engine.py @@ -28,6 +28,28 @@ logger = logging.getLogger(__name__) +def _extract_text(response: Any) -> str: + """Extract plain text from an LLM response. + + Handles both plain-string content (older providers) and list-of-blocks + content (e.g. langchain-google-genai v4+ returns + ``[{"type": "text", "text": "..."}]``). + """ + content = response.content if hasattr(response, "content") else str(response) + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, dict) and "text" in block: + parts.append(block["text"]) + elif isinstance(block, str): + parts.append(block) + if parts: + return "\n".join(parts) + return str(content) + + class PlanEngine: """Core planning engine — the heart of the executive layer. @@ -85,11 +107,11 @@ def generate( try: response = self._llm.invoke(messages) - raw_output = response.content if hasattr(response, "content") else str(response) + raw_output = _extract_text(response) except Exception as e: raise PlanGenerationError(f"LLM invocation failed: {e}") from e - steps = postprocess_plan(str(raw_output)) + steps = postprocess_plan(raw_output) from behavioral_memory.memory.token_budget import count_tokens @@ -101,7 +123,7 @@ def generate( retrieved_traces=traces, schemas_used=schemas, token_budget_used=token_budget, - raw_llm_output=str(raw_output), + raw_llm_output=raw_output, ) def generate_zero_shot(self, query: str, tool_schemas: list[ToolSchema]) -> Plan: From a55054ae58678478ace6e53a6e3235b08ac4d410 Mon Sep 17 00:00:00 2001 From: harsh-kr11 <139389267+harsh-kr11@users.noreply.github.com> Date: Mon, 25 May 2026 11:14:55 +0530 Subject: [PATCH 2/3] Fix lint: remove unused itertools.combinations import Co-authored-by: Cursor --- examples/compare_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/compare_models.py b/examples/compare_models.py index bd38d96..539324b 100644 --- a/examples/compare_models.py +++ b/examples/compare_models.py @@ -20,7 +20,6 @@ import argparse import json import sys -from itertools import combinations from pathlib import Path from rich.console import Console From 46a3183aeacb0fb2b6c55cfa84d576f7f366c82f Mon Sep 17 00:00:00 2001 From: harsh-kr11 <139389267+harsh-kr11@users.noreply.github.com> Date: Mon, 25 May 2026 11:32:10 +0530 Subject: [PATCH 3/3] Fix formatting: apply ruff format to compare_models.py Co-authored-by: Cursor --- examples/compare_models.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/compare_models.py b/examples/compare_models.py index 539324b..4b3bdaa 100644 --- a/examples/compare_models.py +++ b/examples/compare_models.py @@ -141,10 +141,7 @@ def generate_markdown(all_results: list[dict]) -> str: models = [r["model"] for r in all_results] n = all_results[0]["n_tasks"] - lines.append( - f"On a {n}-task benchmark with 7 MCP tools (temperature 0, " - f"embeddings: `gemini-embedding-001`):" - ) + lines.append(f"On a {n}-task benchmark with 7 MCP tools (temperature 0, embeddings: `gemini-embedding-001`):") lines.append("") header = "| Metric | Strategy | " + " | ".join(f"**{m}**" for m in models) + " |"