bench/swe: empirical bench harness for tuning PasClaw agent settings

bench/swe/.gitignore

@@ -0,0 +1,7 @@
+results/*.json
+results/run-*
+results/PASCLAW_HOME-*
+harness/__pycache__/
+harness/*.pyc
+provider-stub.log
+.fixture-cache/

bench/swe/ablation.json

@@ -0,0 +1,207 @@
+[
+  {
+    "id": "baseline",
+    "profile": "baseline"
+  },
+  {
+    "id": "stock",
+    "profile": "stock"
+  },
+  {
+    "id": "max-build",
+    "profile": "max-build"
+  },
+  {
+    "id": "low-token",
+    "profile": "low-token"
+  },
+  {
+    "id": "security",
+    "profile": "security"
+  },
+  {
+    "id": "all-on",
+    "profile": "all-on"
+  },
+  {
+    "id": "stock+condenser",
+    "profile": "stock",
+    "config_overrides": {
+      "condense_reversible": true
+    }
+  },
+  {
+    "id": "stock+output-cap-16k",
+    "profile": "stock",
+    "config_overrides": {
+      "tool_output_cap": 16384
+    }
+  },
+  {
+    "id": "stock+orient-task-aware",
+    "profile": "stock",
+    "config_overrides": {
+      "orient_task_aware": true
+    }
+  },
+  {
+    "id": "stock+checkpoints",
+    "profile": "stock",
+    "config_overrides": {
+      "checkpoints_enabled": true,
+      "checkpoints_keep_last": 32
+    }
+  },
+  {
+    "id": "stock+stats",
+    "profile": "stock",
+    "config_overrides": {
+      "stats_collection_enabled": true
+    }
+  },
+  {
+    "id": "stock+cache-1h",
+    "profile": "stock",
+    "config_overrides": {
+      "prompt_cache": {
+        "enabled": true,
+        "ttl": "1h"
+      }
+    }
+  },
+  {
+    "id": "stock+skill-self-manage",
+    "profile": "stock",
+    "config_overrides": {
+      "self_improving_skills": {
+        "self_manage": true
+      }
+    }
+  },
+  {
+    "id": "stock+skill-progressive",
+    "profile": "stock",
+    "config_overrides": {
+      "self_improving_skills": {
+        "progressive_disclosure": true
+      }
+    }
+  },
+  {
+    "id": "stock+skill-distiller",
+    "profile": "stock",
+    "config_overrides": {
+      "self_improving_skills": {
+        "distiller": {
+          "enabled": true,
+          "min_tool_calls": 5
+        }
+      }
+    }
+  },
+  {
+    "id": "stock+auto-router",
+    "profile": "stock",
+    "config_overrides": {
+      "auto_router": {
+        "enabled": true
+      }
+    }
+  },
+  {
+    "id": "stock-no-tools",
+    "profile": "stock",
+    "no_tools": true
+  },
+  {
+    "id": "stock-no-mcp",
+    "profile": "stock",
+    "no_mcp": true
+  },
+  {
+    "id": "lean-stock",
+    "_note": "Stock + every zero-prompt-cost behavioral toggle from max-build. Same prompt bytes as stock (12822) but with checkpoints, stats, 1h cache, distiller, orient-task-aware, auto-router.",
+    "profile": "stock",
+    "config_overrides": {
+      "orient_task_aware": true,
+      "checkpoints_enabled": true,
+      "checkpoints_keep_last": 32,
+      "stats_collection_enabled": true,
+      "prompt_cache": {
+        "enabled": true,
+        "ttl": "1h"
+      },
+      "auto_router": {
+        "enabled": true
+      },
+      "self_improving_skills": {
+        "distiller": {
+          "enabled": true,
+          "min_tool_calls": 5
+        }
+      }
+    }
+  },
+  {
+    "id": "lean-build",
+    "_note": "lean-stock + condenser + 16KB output cap (brings tool_output_get, +552B/+1 tool). ~13374 bytes/turn vs max-build's 15717 -- skips skills_* (+2343 bytes) which only help if you're authoring skills mid-session.",
+    "profile": "stock",
+    "config_overrides": {
+      "condense_reversible": true,
+      "tool_output_cap": 16384,
+      "orient_task_aware": true,
+      "checkpoints_enabled": true,
+      "checkpoints_keep_last": 32,
+      "stats_collection_enabled": true,
+      "prompt_cache": {
+        "enabled": true,
+        "ttl": "1h"
+      },
+      "auto_router": {
+        "enabled": true
+      },
+      "self_improving_skills": {
+        "distiller": {
+          "enabled": true,
+          "min_tool_calls": 5
+        }
+      }
+    }
+  },
+  {
+    "id": "lean-edit",
+    "_note": "lean-stock minus web_fetch and vault. 22.7% smaller than stock.",
+    "profile": "lean-stock",
+    "config_overrides": {
+      "web_fetch_enabled": false,
+      "vault_tools_enabled": false
+    }
+  },
+  {
+    "id": "lean-build-plus-skills-disclosure",
+    "_note": "lean-build + progressive_disclosure (skills_list/view). The cheapest skill-feature add-on (+852B/+2 tools).",
+    "profile": "stock",
+    "config_overrides": {
+      "condense_reversible": true,
+      "tool_output_cap": 16384,
+      "orient_task_aware": true,
+      "checkpoints_enabled": true,
+      "checkpoints_keep_last": 32,
+      "stats_collection_enabled": true,
+      "prompt_cache": {
+        "enabled": true,
+        "ttl": "1h"
+      },
+      "auto_router": {
+        "enabled": true
+      },
+      "self_improving_skills": {
+        "progressive_disclosure": true,
+        "distiller": {
+          "enabled": true,
+          "min_tool_calls": 5
+        }
+      }
+    }
+  }
+]

bench/swe/fixture/01-snippet-window-magic-number/manifest.json

@@ -0,0 +1,11 @@
+{
+  "name": "01-snippet-window-magic-number",
+  "category": "trivial-localisation",
+  "shape_of": "PR #309 (memory/kb: widen FTS5 snippet window 24 -> 60)",
+  "prompt": "Two SQL queries in src/index.pas use a hardcoded snippet width of 24. The 24 is too narrow -- profiling shows the answer line is often clipped. Lift 24 into a named const FTS5_SNIPPET_TOKENS = 60 in the unit's interface section (so other units can reuse it), and replace both occurrences in the SQL strings with IntToStr(FTS5_SNIPPET_TOKENS). Do not touch any other behavior. When you're done, just say done.",
+  "scope_paths": ["src/"],
+  "oracle": {
+    "cmd": "$FIXTURE_DIR/oracle/test.sh",
+    "_doc": "Returns 0 iff (a) FTS5_SNIPPET_TOKENS = 60 is declared in src/index.pas's interface section, AND (b) both SQL string literals reference IntToStr(FTS5_SNIPPET_TOKENS) instead of the literal 24."
+  }
+}

bench/swe/fixture/01-snippet-window-magic-number/mock/default.jsonl

@@ -0,0 +1,3 @@
+{"id":"chatcmpl-mock-1","object":"chat.completion","created":1700000001,"model":"stub","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":"I'll read the file first to see the current state.","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fs_read","arguments":"{\"path\":\"src/index.pas\",\"plain\":true}"}}]}}],"usage":{"prompt_tokens":1200,"completion_tokens":35}}
+{"id":"chatcmpl-mock-2","object":"chat.completion","created":1700000002,"model":"stub","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":"Lifting 24 to a named const in the interface section and updating both SQL strings.","tool_calls":[{"id":"call_2","type":"function","function":{"name":"fs_write","arguments":"{\"path\":\"src/index.pas\",\"content\":\"unit Index;\\n\\n{$IFDEF FPC}{$MODE DELPHI}{$ENDIF}\\n{$H+}\\n\\ninterface\\n\\nuses\\n  SysUtils;\\n\\nconst\\n  FTS5_SNIPPET_TOKENS = 60;\\n\\nfunction BuildSearchSQL: string;\\nfunction BuildKBSearchSQL: string;\\n\\nimplementation\\n\\nfunction BuildSearchSQL: string;\\nbegin\\n  Result :=\\n    'SELECT path, snippet(memory_fts, 1, ''<'', ''>'', ''...'', ' +\\n    IntToStr(FTS5_SNIPPET_TOKENS) + '), bm25(memory_fts) ' +\\n    'FROM memory_fts WHERE memory_fts MATCH :q ' +\\n    'ORDER BY bm25(memory_fts) LIMIT :k';\\nend;\\n\\nfunction BuildKBSearchSQL: string;\\nbegin\\n  Result :=\\n    'SELECT path, chunk_no, snippet(kb_fts, 2, ''<'', ''>'', ''...'', ' +\\n    IntToStr(FTS5_SNIPPET_TOKENS) + '), bm25(kb_fts) ' +\\n    'FROM kb_fts WHERE kb_fts MATCH :q ORDER BY bm25(kb_fts) LIMIT :k';\\nend;\\n\\nend.\\n\"}"}}]}}],"usage":{"prompt_tokens":1500,"completion_tokens":420}}
+{"id":"chatcmpl-mock-3","object":"chat.completion","created":1700000003,"model":"stub","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"done"}}],"usage":{"prompt_tokens":1900,"completion_tokens":5}}

bench/swe/fixture/01-snippet-window-magic-number/oracle/test.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Oracle for fixture 01: confirm the magic number has been lifted to a
+# named const and both SQL strings now reference it.
+#
+# Exit 0 = passed, non-zero = failed. The harness only inspects the exit
+# code -- stdout/stderr are captured into result.json for debugging.
+
+set -u
+src="src/index.pas"
+
+if [ ! -f "$src" ]; then
+  echo "FAIL: $src missing (agent deleted or moved the file)"
+  exit 2
+fi
+
+# (a) FTS5_SNIPPET_TOKENS = 60 declared in the interface section.
+# We check the const declaration appears BEFORE the `implementation`
+# keyword. Using awk to extract just the interface portion.
+interface_block=$(awk '
+  /^[[:space:]]*implementation[[:space:]]*$/ { exit }
+  { print }
+' "$src")
+
+if ! echo "$interface_block" | grep -qE 'FTS5_SNIPPET_TOKENS[[:space:]]*=[[:space:]]*60[[:space:]]*;'; then
+  echo "FAIL: FTS5_SNIPPET_TOKENS = 60 not found in interface section"
+  exit 1
+fi
+
+# (b) Neither SQL string contains the literal 24 anymore.
+if grep -qE "''…''[^)]*,[[:space:]]*24\)|'\\.\\.\\.''[^)]*,[[:space:]]*24\)" "$src"; then
+  echo "FAIL: literal 24 still present in a snippet() SQL call"
+  exit 1
+fi
+# Also accept the explicit ASCII three-dot form the fixture ships with.
+if grep -qE "''\\.\\.\\.''[^)]*,[[:space:]]*24\\)" "$src"; then
+  echo "FAIL: literal 24 still present in a snippet() SQL call"
+  exit 1
+fi
+
+# (c) Both SQL strings reference IntToStr(FTS5_SNIPPET_TOKENS).
+ref_count=$(grep -cE 'IntToStr\([[:space:]]*FTS5_SNIPPET_TOKENS[[:space:]]*\)' "$src" || true)
+if [ "$ref_count" -lt 2 ]; then
+  echo "FAIL: expected 2 references to IntToStr(FTS5_SNIPPET_TOKENS), found $ref_count"
+  exit 1
+fi
+
+echo "PASS"
+exit 0

bench/swe/fixture/01-snippet-window-magic-number/pre-fix/src/index.pas

@@ -0,0 +1,31 @@
+unit Index;
+
+{$IFDEF FPC}{$MODE DELPHI}{$ENDIF}
+{$H+}
+
+interface
+
+uses
+  SysUtils;
+
+function BuildSearchSQL: string;
+function BuildKBSearchSQL: string;
+
+implementation
+
+function BuildSearchSQL: string;
+begin
+  Result :=
+    'SELECT path, snippet(memory_fts, 1, ''<'', ''>'', ''...'', 24), bm25(memory_fts) ' +
+    'FROM memory_fts WHERE memory_fts MATCH :q ' +
+    'ORDER BY bm25(memory_fts) LIMIT :k';
+end;
+
+function BuildKBSearchSQL: string;
+begin
+  Result :=
+    'SELECT path, chunk_no, snippet(kb_fts, 2, ''<'', ''>'', ''...'', 24), bm25(kb_fts) ' +
+    'FROM kb_fts WHERE kb_fts MATCH :q ORDER BY bm25(kb_fts) LIMIT :k';
+end;
+
+end.

bench/swe/fixture/02-windows-shell-quoting/manifest.json

@@ -0,0 +1,10 @@
+{
+  "name": "02-windows-shell-quoting",
+  "category": "function-level-fix",
+  "shape_of": "PR #307 (windows: fix cmd.exe quoting bugs in shell_exec + execute_code)",
+  "prompt": "src/cmd_quote.pas has a QuoteForCmd function that should wrap a command argument in double quotes and escape internal double quotes by doubling them. It currently just wraps in double quotes and doesn't escape internals at all. Fix QuoteForCmd so that every internal `\"` becomes `\"\"`. Also: an empty input must return `\"\"` (two quote chars, not the empty string). Do not rename the function, do not change its signature, do not add new units. Say done when finished.",
+  "scope_paths": ["src/"],
+  "oracle": {
+    "cmd": "$FIXTURE_DIR/oracle/test.sh"
+  }
+}

bench/swe/fixture/02-windows-shell-quoting/mock/default.jsonl

@@ -0,0 +1,3 @@
+{"id":"chatcmpl-mock-1","object":"chat.completion","created":1700000001,"model":"stub","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":"Reading the current state of cmd_quote.pas.","tool_calls":[{"id":"call_1","type":"function","function":{"name":"fs_read","arguments":"{\"path\":\"src/cmd_quote.pas\",\"plain\":true}"}}]}}],"usage":{"prompt_tokens":1100,"completion_tokens":28}}
+{"id":"chatcmpl-mock-2","object":"chat.completion","created":1700000002,"model":"stub","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":"The function just wraps in quotes without escaping internals. Replacing with the doubling form using StringReplace.","tool_calls":[{"id":"call_2","type":"function","function":{"name":"fs_write","arguments":"{\"path\":\"src/cmd_quote.pas\",\"content\":\"unit cmd_quote;\\n\\n{$IFDEF FPC}{$MODE DELPHI}{$ENDIF}\\n{$H+}\\n\\ninterface\\n\\nuses\\n  SysUtils;\\n\\nfunction QuoteForCmd(const Arg: string): string;\\n\\nimplementation\\n\\nfunction QuoteForCmd(const Arg: string): string;\\nbegin\\n  Result := '\\\"' + StringReplace(Arg, '\\\"', '\\\"\\\"', [rfReplaceAll]) + '\\\"';\\nend;\\n\\nend.\\n\"}"}}]}}],"usage":{"prompt_tokens":1450,"completion_tokens":210}}
+{"id":"chatcmpl-mock-3","object":"chat.completion","created":1700000003,"model":"stub","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"done"}}],"usage":{"prompt_tokens":1700,"completion_tokens":5}}

bench/swe/fixture/02-windows-shell-quoting/oracle/test.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Oracle for fixture 02: confirm QuoteForCmd doubles internal quotes and
+# returns "" (two chars) for an empty input.
+#
+# We can't compile/run Pascal portably in CI -- so the oracle is static:
+# we look for the structural changes the fix requires.
+
+set -u
+src="src/cmd_quote.pas"
+
+if [ ! -f "$src" ]; then
+  echo "FAIL: $src missing"
+  exit 2
+fi
+
+# Extract the function body (between begin..end; that follows
+# "function QuoteForCmd").
+body=$(awk '
+  /^function QuoteForCmd/ { in_fn=1 }
+  in_fn { print }
+  in_fn && /^end;/ { exit }
+' "$src")
+
+if [ -z "$body" ]; then
+  echo "FAIL: could not locate QuoteForCmd implementation"
+  exit 1
+fi
+
+# (a) The body must mention StringReplace or equivalent doubling -- any
+# fix that doesn't transform the input character-by-character is too
+# narrow for the contract. We accept either StringReplace, a manual loop,
+# or AdjustLineBreaks-style transforms.
+if ! echo "$body" | grep -qE "StringReplace|ReplaceStr|for .*:=|while .*\b1\b"; then
+  echo "FAIL: QuoteForCmd body doesn't transform the input -- still a thin wrap"
+  echo "--- body ---"; echo "$body"
+  exit 1
+fi
+
+# (b) Reject the obvious "just wrap in quotes" form: Result := '"' + Arg + '"'
+# with no other statements before it.
+if echo "$body" | grep -qE "Result[[:space:]]*:=[[:space:]]*'\"'[[:space:]]*\\+[[:space:]]*Arg[[:space:]]*\\+[[:space:]]*'\"'[[:space:]]*;"; then
+  # Only fail if that's the ONLY assignment in the body.
+  count=$(echo "$body" | grep -cE "Result[[:space:]]*:=")
+  if [ "$count" = "1" ]; then
+    echo "FAIL: body only wraps in quotes; internal quotes are not escaped"
+    exit 1
+  fi
+fi
+
+# (c) Some handling of the empty case must be present -- either an explicit
+# Length(Arg) = 0 / Arg = '' check returning '""', OR the transform must
+# naturally produce '""' for an empty input (which '"' + '' + '"' does).
+# So (c) is informational only -- we don't fail on its absence.
+
+echo "PASS"
+exit 0