diff --git a/README.md b/README.md index e1dca46..829db45 100644 --- a/README.md +++ b/README.md @@ -195,12 +195,14 @@ tools/locomo/run-regression-gate.sh 5 tools/locomo/run-category-gate.sh 5 tools/locomo/run-full-gate.sh 5 tools/locomo/refresh-failure-samples.sh 5 80 +./.venv/bin/python tools/locomo/evaluate-single-knob.py ``` - Scripts write artifacts to `/tmp` by default (or `BASE_DIR` when provided). - `run-regression-gate.sh` default threshold is `hit@5 >= 0.55` (second arg can override). - `run-category-gate.sh` also checks default category floors for `multi-hop` and `open-domain`. - `run-full-gate.sh` runs baseline threshold checks, drift checks, and the required 4-command matrix in one command. +- `evaluate-single-knob.py` enforces an auto-stop rule: only recommend a change when both `overall` and `multi-hop` improve; otherwise output `NO-OP`. - CI gate workflow `.github/workflows/locomo-full-gate.yml` runs `REBUILD_PYTHON=0 tools/locomo/run-full-gate.sh 5` on PRs and pushes to `main` (plus manual dispatch), and uploads gate artifacts. - If LoCoMo scores unexpectedly remain old, rebuild local Python binding: `./.venv/bin/python -m pip install -e python` diff --git a/docs/superpowers/plans/2026-03-19-v2-r14-locomo-single-knob-gate-implementation-plan.md b/docs/superpowers/plans/2026-03-19-v2-r14-locomo-single-knob-gate-implementation-plan.md new file mode 100644 index 0000000..1843ba4 --- /dev/null +++ b/docs/superpowers/plans/2026-03-19-v2-r14-locomo-single-knob-gate-implementation-plan.md @@ -0,0 +1,49 @@ +# V2-R14 LoCoMo Single-Knob Gate Implementation Plan + +> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a minimal evaluator script that enforces NO-OP unless `overall` and `multi-hop` both improve. + +**Architecture:** Keep retrieval engine untouched; add one tooling script and docs updates only. + +**Tech Stack:** Python script in `tools/locomo` + markdown docs. + +--- + +### Task 1: Add Evaluator Script + +**Files:** +- Create: `tools/locomo/evaluate-single-knob.py` + +- [ ] Implement baseline and candidate evaluation for K=5 LoCoMo retrieval. +- [ ] Hardcode pass rule (`overall` and `multi-hop` both improve). +- [ ] Emit clear JSON decision payload (`APPLY` or `NO-OP`). + +### Task 2: Update Docs + +**Files:** +- Modify: `tools/locomo/README.md` +- Modify: `README.md` + +- [ ] Document command usage. +- [ ] Document auto-stop decision rule. + +### Task 3: Verification + +**Files:** +- No source edits + +- [ ] `./.venv/bin/python tools/locomo/evaluate-single-knob.py` +- [ ] `cargo test --workspace` +- [ ] `./.venv/bin/pytest python/tests -q` +- [ ] `corepack pnpm --dir packages/node test` +- [ ] `corepack pnpm --dir packages/mcp test` +- [ ] `REBUILD_PYTHON=0 tools/locomo/run-full-gate.sh 5` + +### Task 4: Finish + +**Files:** +- Create: `docs/superpowers/releases/2026-03-19-v2-r14-locomo-single-knob-gate-results.md` + +- [ ] Record decision output and verification evidence. +- [ ] Commit and merge. diff --git a/docs/superpowers/plans/2026-03-19-v2-r15-locomo-closure-implementation-plan.md b/docs/superpowers/plans/2026-03-19-v2-r15-locomo-closure-implementation-plan.md new file mode 100644 index 0000000..51ca365 --- /dev/null +++ b/docs/superpowers/plans/2026-03-19-v2-r15-locomo-closure-implementation-plan.md @@ -0,0 +1,37 @@ +# V2-R15 LoCoMo Closure Implementation Plan + +> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Finalize LoCoMo V2 with evidence-backed closure and a stable tag. + +**Architecture:** Docs and release-management only; no retrieval code changes. + +**Tech Stack:** markdown docs + git tag + existing validation commands. + +--- + +### Task 1: Closure Documentation + +**Files:** +- Create: `docs/superpowers/releases/2026-03-19-v2-r15-locomo-closure-results.md` + +- [ ] Record closure criteria and evidence references. + +### Task 2: Final Verification Evidence + +**Files:** +- No source edits + +- [ ] `cargo test --workspace` +- [ ] `./.venv/bin/pytest python/tests -q` +- [ ] `corepack pnpm --dir packages/node test` +- [ ] `corepack pnpm --dir packages/mcp test` +- [ ] `REBUILD_PYTHON=0 tools/locomo/run-full-gate.sh 5` + +### Task 3: Stable Tag + +**Files:** +- No source edits + +- [ ] Create stable tag from merged `main`. +- [ ] Push stable tag to remote. diff --git a/docs/superpowers/releases/2026-03-19-v2-r14-locomo-single-knob-gate-results.md b/docs/superpowers/releases/2026-03-19-v2-r14-locomo-single-knob-gate-results.md new file mode 100644 index 0000000..ce38b91 --- /dev/null +++ b/docs/superpowers/releases/2026-03-19-v2-r14-locomo-single-knob-gate-results.md @@ -0,0 +1,51 @@ +# V2-R14 LoCoMo Single-Knob Gate Results + +## Summary + +This round adds a minimal auto-stop evaluator and keeps retrieval logic unchanged. + +Added: + +- `tools/locomo/evaluate-single-knob.py` +- docs for running the gate before any new single-knob retrieval tweak + +## Gate Output + +Command: + +```bash +./.venv/bin/python tools/locomo/evaluate-single-knob.py --output /tmp/localmemos-v2-r14-single-knob-result.json +``` + +Decision: + +- action: `NO-OP` +- selected candidate: `null` +- rule: apply only when both `overall` and `multi-hop` strictly improve + +Candidate summary: + +- `games`: overall `0.5822` (no gain), multi-hop `0.4291` +- `kind`: unchanged +- `names`: regresses overall and multi-hop + +## Verification + +Passed in this round: + +1. `cargo test --workspace` +2. `./.venv/bin/pytest python/tests -q` +3. `corepack pnpm --dir packages/node test` +4. `corepack pnpm --dir packages/mcp test` +5. `REBUILD_PYTHON=0 tools/locomo/run-full-gate.sh 5` + +LoCoMo K=5 remains: + +- overall: `0.5822` +- multi-hop: `0.4255` +- open-domain: `0.3370` + +## Outcome + +- Auto-stop gate is now repository-owned and deterministic. +- V2 iteration can stop automatically when evidence does not justify another tweak. diff --git a/docs/superpowers/releases/2026-03-19-v2-r15-locomo-closure-results.md b/docs/superpowers/releases/2026-03-19-v2-r15-locomo-closure-results.md new file mode 100644 index 0000000..4f790cb --- /dev/null +++ b/docs/superpowers/releases/2026-03-19-v2-r15-locomo-closure-results.md @@ -0,0 +1,18 @@ +# V2-R15 LoCoMo Closure Results + +## Closure Status + +- target: LoCoMo V2 retrieval track closure +- status: pending final merge/tag in this branch + +## Closure Checklist + +- [x] V2-R14 auto-stop gate implemented +- [x] required 4-command matrix passed in round evidence +- [x] `run-full-gate.sh` passed with threshold + drift checks +- [x] closure docs created +- [ ] stable tag created on merged `main` + +## Notes + +No additional retrieval tuning was introduced in V2-R15. diff --git a/docs/superpowers/specs/2026-03-19-v2-r14-locomo-single-knob-gate-design.md b/docs/superpowers/specs/2026-03-19-v2-r14-locomo-single-knob-gate-design.md new file mode 100644 index 0000000..cfa698a --- /dev/null +++ b/docs/superpowers/specs/2026-03-19-v2-r14-locomo-single-knob-gate-design.md @@ -0,0 +1,68 @@ +# V2-R14 LoCoMo Single-Knob Auto-Stop Gate Design + +## Goal + +Turn the anti-overdesign rule into a small repository-owned tool: + +- evaluate a fixed small set of single-knob candidates +- only allow a code-change recommendation when both `overall` and `multi-hop` improve +- otherwise output `NO-OP` + +## Baseline + +Current K=5 baseline: + +- overall: `0.5822` +- multi-hop: `0.4255` +- open-domain: `0.3370` + +## Options Considered + +1. Keep manual spreadsheet/eyeballing +- Low implementation cost but decision drift risk. + +2. Add one small deterministic evaluator script (chosen) +- Minimal maintenance, explicit stop condition, no retrieval logic changes. + +3. Build a larger hyperparameter search framework +- Overkill for current stage. + +## Chosen Design + +1. Add `tools/locomo/evaluate-single-knob.py`: +- runs baseline + candidate simulations on LoCoMo K=5 retrieval +- default candidates: `games`, `kind`, `names` +- hardcoded pass rule: candidate must satisfy + - `candidate.overall > baseline.overall` + - `candidate.multi-hop > baseline.multi-hop` +- output includes: + - baseline metrics + - candidate metrics + - decision: `APPLY` or `NO-OP` + - selected candidate when applicable + +2. Keep behavior read-only for repository code: +- script only reports decision; it does not modify retrieval code. + +3. Document usage in LoCoMo README + root README. + +## Test Strategy + +1. run the new script once: +- `./.venv/bin/python tools/locomo/evaluate-single-knob.py` + +2. required validation matrix: +- `cargo test --workspace` +- `./.venv/bin/pytest python/tests -q` +- `corepack pnpm --dir packages/node test` +- `corepack pnpm --dir packages/mcp test` + +3. retrieval gate sanity: +- `REBUILD_PYTHON=0 tools/locomo/run-full-gate.sh 5` + +## Success Criteria + +1. evaluator script exists and runs end-to-end +2. decision rule is deterministic and hardcoded +3. docs clearly state the NO-OP gate behavior +4. full verification passes diff --git a/docs/superpowers/specs/2026-03-19-v2-r15-locomo-closure-design.md b/docs/superpowers/specs/2026-03-19-v2-r15-locomo-closure-design.md new file mode 100644 index 0000000..cb0fbb1 --- /dev/null +++ b/docs/superpowers/specs/2026-03-19-v2-r15-locomo-closure-design.md @@ -0,0 +1,30 @@ +# V2-R15 LoCoMo Closure Design + +## Goal + +Close the LoCoMo V2 track without introducing further retrieval behavior changes. + +## Closure Criteria + +1. V2-R14 auto-stop gate is merged and usable. +2. Required 4-command validation matrix is green. +3. `run-full-gate.sh` is green with thresholds and drift checks passing. +4. Closure evidence is written in a release document. +5. Stable tag is created from merged `main`. + +## Scope + +In-scope: + +- closure docs +- final verification evidence capture +- stable tag creation + +Out-of-scope: + +- new retrieval tuning +- new ranking/query rewrite rules + +## Success Condition + +If all closure criteria are met, mark LoCoMo V2 as closed and shift to maintenance-only mode. diff --git a/tools/locomo/README.md b/tools/locomo/README.md index dc3d02f..960d320 100644 --- a/tools/locomo/README.md +++ b/tools/locomo/README.md @@ -16,6 +16,7 @@ Repository-owned LoCoMo scripts for retrieval-focused V2 validation. - `run-full-gate.sh`: one-shot baseline + threshold checks + drift check + required 4-command matrix - `export-failure-samples.py`: exports prioritized retrieval misses - `refresh-failure-samples.sh`: refreshes `failure-samples-k5.json` from current retriever +- `evaluate-single-knob.py`: evaluates fixed single-knob candidates and outputs `APPLY`/`NO-OP` ## Usage @@ -29,6 +30,7 @@ tools/locomo/run-regression-gate.sh 5 tools/locomo/run-category-gate.sh 5 tools/locomo/run-full-gate.sh 5 tools/locomo/refresh-failure-samples.sh 5 80 +./.venv/bin/python tools/locomo/evaluate-single-knob.py ``` All scripts write artifacts to `/tmp` by default (or `BASE_DIR` if provided). @@ -67,3 +69,16 @@ Triggers: - manual `workflow_dispatch` The workflow uploads gate logs (and result JSON when present) as artifacts. + +## Single-Knob Auto-Stop Gate + +Run: + +```bash +./.venv/bin/python tools/locomo/evaluate-single-knob.py +``` + +Decision rule is hardcoded: + +- `APPLY` only when a candidate improves both `overall` and `multi-hop` +- otherwise `NO-OP` diff --git a/tools/locomo/evaluate-single-knob.py b/tools/locomo/evaluate-single-knob.py new file mode 100755 index 0000000..dc839c5 --- /dev/null +++ b/tools/locomo/evaluate-single-knob.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +import subprocess +import tempfile +from collections import defaultdict +from pathlib import Path +from typing import Dict, List, Optional + +from memory_sdk.client import MemoryClient + +CATEGORY_NAME = { + 1: "multi-hop", + 2: "temporal", + 3: "open-domain", + 4: "single-hop", + 5: "adversarial", +} + +DEFAULT_CANDIDATES = ["games", "kind", "names"] + + +def safe_query(text: str) -> str: + text = re.sub(r"[^\w\s]", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate fixed single-knob candidate stopwords and emit APPLY/NO-OP decision." + ) + parser.add_argument("--k", type=int, default=5, help="Top-K retrieval cutoff") + parser.add_argument( + "--candidates", + default=",".join(DEFAULT_CANDIDATES), + help="Comma-separated candidate tokens", + ) + parser.add_argument( + "--base-dir", + default=None, + help="Optional working directory for dataset/db artifacts (default: mktemp in /tmp)", + ) + parser.add_argument( + "--output", + default=None, + help="Optional output JSON path", + ) + return parser.parse_args() + + +def evaluate( + samples: List[dict], + client: MemoryClient, + k: int, + drop_token: Optional[str], +) -> Dict[str, float]: + stats = defaultdict(lambda: {"n": 0, "hit": 0}) + all_n = 0 + all_hit = 0 + + for sample in samples: + scope = sample["sample_id"] + for qa in sample["qa"]: + evidence = qa.get("evidence") or [] + if not evidence: + continue + + query = safe_query(str(qa.get("question", ""))) + if drop_token: + query = " ".join(t for t in query.split() if t.lower() != drop_token) + if not query: + continue + + cat = int(qa["category"]) + stats[cat]["n"] += 1 + all_n += 1 + + recall = client.recall(namespace="benchmark", scope_id=scope, text_query=query) + top_attrs = [f.get("attribute", "") for f in recall.get("facts", [])[:k]] + if any(ev in top_attrs for ev in evidence): + stats[cat]["hit"] += 1 + all_hit += 1 + + result: Dict[str, float] = { + "overall": round(all_hit / all_n, 4) if all_n else 0.0, + } + for c, v in sorted(stats.items()): + key = CATEGORY_NAME[c] + result[key] = round(v["hit"] / v["n"], 4) if v["n"] else 0.0 + return result + + +def main() -> int: + args = parse_args() + + candidates = [c.strip().lower() for c in args.candidates.split(",") if c.strip()] + if not candidates: + raise SystemExit("No candidate token provided") + + base_dir = Path(args.base_dir) if args.base_dir else Path(tempfile.mkdtemp(prefix="localmemos-single-knob.")) + base_dir.mkdir(parents=True, exist_ok=True) + + locomo_repo = base_dir / "locomo" + if not (locomo_repo / ".git").exists(): + subprocess.run( + ["git", "clone", "--depth", "1", "https://github.com/snap-research/locomo", str(locomo_repo)], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + data_path = locomo_repo / "data" / "locomo10.json" + samples = json.loads(data_path.read_text(encoding="utf-8")) + + db_path = base_dir / "single_knob_eval.db" + client = MemoryClient(str(db_path)) + + ingested_turns = 0 + for sample in samples: + scope = sample["sample_id"] + for key, session in sample["conversation"].items(): + if not re.fullmatch(r"session_\d+", key): + continue + for turn in session: + dia_id = turn.get("dia_id") + text = turn.get("text") + if not dia_id or not text: + continue + client.upsert_fact( + namespace="benchmark", + scope_id=scope, + entity="dialog", + attribute=str(dia_id), + value=text, + source_kind="locomo_ingest", + source_ref="locomo10", + evidence_summary=turn.get("speaker"), + ) + ingested_turns += 1 + + baseline = evaluate(samples, client, args.k, None) + + candidate_results = {} + passing = [] + + for token in candidates: + metrics = evaluate(samples, client, args.k, token) + delta_overall = round(metrics.get("overall", 0.0) - baseline.get("overall", 0.0), 4) + delta_multi = round(metrics.get("multi-hop", 0.0) - baseline.get("multi-hop", 0.0), 4) + + candidate_results[token] = { + "metrics": metrics, + "delta": { + "overall": delta_overall, + "multi-hop": delta_multi, + }, + "passes_gate": delta_overall > 0 and delta_multi > 0, + } + + if delta_overall > 0 and delta_multi > 0: + passing.append((token, delta_overall, delta_multi, metrics)) + + passing.sort(key=lambda x: (x[1], x[2]), reverse=True) + + if passing: + selected_token = passing[0][0] + decision = "APPLY" + else: + selected_token = None + decision = "NO-OP" + + result = { + "meta": { + "k": args.k, + "candidates": candidates, + "ingested_turns": ingested_turns, + "base_dir": str(base_dir), + "db_path": str(db_path), + "source_data": str(data_path), + }, + "baseline": baseline, + "candidates": candidate_results, + "decision": { + "action": decision, + "selected_candidate": selected_token, + "rule": "APPLY only if overall and multi-hop both strictly improve; otherwise NO-OP", + }, + } + + if args.output: + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())