agent effectiveness: project facts, edit mini-diff, read ranges + dedup, find_files (+ real-task bench)

Makefile

@@ -767,6 +767,12 @@ test-progress-ledger: | $(BUILDDIR)
 	$(FPC) $(FPCFLAGS) src/tests/progress_ledger_tests.pas -o$(BUILDDIR)/progress_ledger_tests
 	@$(BUILDDIR)/progress_ledger_tests
 
+# Deterministic ## Project prompt section (stack/build/test/git from file reads).
+test-project-facts: | $(BUILDDIR)
+	@mkdir -p $(BUILDDIR)/lib
+	$(FPC) $(FPCFLAGS) src/tests/project_facts_tests.pas -o$(BUILDDIR)/project_facts_tests
+	@$(BUILDDIR)/project_facts_tests
+
 # Deterministic agent-loop harness bench (bench/agentloop) -- pure FPC, no
 # external runtime. Spawns the built binary, plays the scripted model over
 # the relay, asserts on what the loop did. Not part of `make test` (binds a
@@ -846,4 +852,4 @@ test-fs-grep-tier5-6: | $(BUILDDIR)
 	$(FPC) $(FPCFLAGS) src/tests/fs_grep_tier5_6_tests.pas -o$(BUILDDIR)/fs_grep_tier5_6_tests
 	@$(BUILDDIR)/fs_grep_tier5_6_tests
 
-test: smoke test-hashline test-toolview test-anthropic-server-tools test-openai-server-tools test-tool-choice test-responses-tool-choice test-println-helper test-utf8-codepage-tag test-gemini-schema-strip test-markdown-render test-json-utf8-roundtrip test-model-discovery test-cron-tool test-provider-catalog test-output-cache test-working-state test-ansi-width test-shell-filters test-learn test-export test-execute-code test-session-stats test-auto-router test-gateway-stats-buckets test-session-list-filter test-session-endpoints test-config-secret-merge test-skills-install test-self-improving-skills test-loop-shaping-defaults test-max-iter-notice test-max-iter-notice test-plan-build-mode test-config-profile test-tool-rpc test-session-search test-subagent-bg test-subagent-default test-stream-reliability test-mcp-server test-mcp-hub-projection test-checkpoints test-condense-json test-goals-runner test-kb-index test-kb-pdf test-agents-md test-checkpoints-zpaq test-orient-preamble test-component-config test-autoroute-apply test-fallback-models test-memory-distill test-memory-facts test-memory-autodistill test-checkpoints-redo test-build-roundtrip test-promptware test-orient test-condense-reversible test-heartbeat test-shell-backend test-env-inject test-run-timeout test-shell-output-decode test-fs-grep-tier1-4 test-fs-grep-tier5-6 test-fs-tool-naming test-workspace-paths test-progress-ledger test-otel test-logger-level-quiet test-gateway-token test-fs-secret-gate test-config-env-subst test-delphi-build
+test: smoke test-hashline test-toolview test-anthropic-server-tools test-openai-server-tools test-tool-choice test-responses-tool-choice test-println-helper test-utf8-codepage-tag test-gemini-schema-strip test-markdown-render test-json-utf8-roundtrip test-model-discovery test-cron-tool test-provider-catalog test-output-cache test-working-state test-ansi-width test-shell-filters test-learn test-export test-execute-code test-session-stats test-auto-router test-gateway-stats-buckets test-session-list-filter test-session-endpoints test-config-secret-merge test-skills-install test-self-improving-skills test-loop-shaping-defaults test-max-iter-notice test-max-iter-notice test-plan-build-mode test-config-profile test-tool-rpc test-session-search test-subagent-bg test-subagent-default test-stream-reliability test-mcp-server test-mcp-hub-projection test-checkpoints test-condense-json test-goals-runner test-kb-index test-kb-pdf test-agents-md test-checkpoints-zpaq test-orient-preamble test-component-config test-autoroute-apply test-fallback-models test-memory-distill test-memory-facts test-memory-autodistill test-checkpoints-redo test-build-roundtrip test-promptware test-orient test-condense-reversible test-heartbeat test-shell-backend test-env-inject test-run-timeout test-shell-output-decode test-fs-grep-tier1-4 test-fs-grep-tier5-6 test-fs-tool-naming test-workspace-paths test-progress-ledger test-project-facts test-otel test-logger-level-quiet test-gateway-token test-fs-secret-gate test-config-env-subst test-delphi-build

bench/agentloop/agentloop_bench.pas

@@ -93,6 +93,8 @@ function Has(const Hay, Needle: string): Boolean;
   GHandler:  THandler;
   GEnvs:     array of string;   { raw envelopes, reset per scenario }
   GMaxBody:  Integer;           { max envelope bytes, reset per scenario }
+  GTotalBody: Int64;            { sum of envelope bytes, reset per scenario }
+  GStep:     Integer;           { policy-model state for ScenarioRealTask }
 
 procedure ResetScenario(H: THandler);
 begin
@@ -101,6 +103,8 @@ procedure ResetScenario(H: THandler);
     GHandler := H;
     SetLength(GEnvs, 0);
     GMaxBody := 0;
+    GTotalBody := 0;
+    GStep := 0;
   finally
     GLock.Release;
   end;
@@ -197,6 +201,7 @@ procedure DispatchEnvelope(const Data: string);
     GEnvs[High(GEnvs)] := Data;
     N := Length(GEnvs);
     if Length(Data) > GMaxBody then GMaxBody := Length(Data);
+    Inc(GTotalBody, Length(Data));
     H := GHandler;
   finally
     GLock.Release;
@@ -446,26 +451,52 @@ function H_BuildSite(N: Integer; const EnvJSON: string): string;
     1: Result := RoundOf([
          OneCall('write_file', ArgsPathContent('bench-demo.html', '<h1>bench</h1>')),
          OneCall('todo_write', ArgsObj1('checklist', '- [x] write page'#10'- [ ] verify page'))]);
-    2: Result := RoundOf([OneCall('read_file', ArgsPathPlain('bench-demo.html'))]);
+    2: Result := RoundOf([OneCall('edit_file',
+         '{"path":"bench-demo.html","old_text":"<h1>bench</h1>","new_text":"<h1>bench v2</h1>"}')]);
+  else if N = 3 then
+    Result := RoundOf([OneCall('find_files', '{"pattern":"bench-*.html"}')])
   else
     Result := StopOf('page written and verified.');
   end;
 end;
 
 procedure ScenarioBuildSite;
 var
-  Answer, SP2: string;
+  Answer, SP1, SP2: string;
+  S: TStringList;
 begin
   WriteLn;
   WriteLn('== scenario: build-site (goal anchor + ledger fold + deliverable) ==');
+  { B1 fixture: a recognisable project in the workspace -- the ## Project
+    facts (stack, make test, git branch) must reach iteration 1's system
+    prompt with zero exploration calls spent discovering them. }
+  S := TStringList.Create;
+  try
+    S.Text := 'all: build'#10#9'echo hi'#10'build:'#10#9'echo b'#10'test: build'#10#9'echo t'#10;
+    S.SaveToFile(GHomeDir + '/workspace/Makefile');
+  finally
+    S.Free;
+  end;
+  ForceDirectories(GHomeDir + '/workspace/.git');
+  S := TStringList.Create;
+  try
+    S.Text := 'ref: refs/heads/bench-main'#10;
+    S.SaveToFile(GHomeDir + '/workspace/.git/HEAD');
+  finally
+    S.Free;
+  end;
   ResetScenario(H_BuildSite);
   Answer := Chat('[' + MsgObjJSON('user',
     'build a small landing page named bench-demo.html for the demo project') + ']',
     'bench-build');
 
-  Check(EnvCount = 3, Format('loop finished in 3 provider calls (got %d)', [EnvCount]));
-  Check(not Has(EnvSystemPrompt(EnvAt(0)), '[progress ledger'),
+  Check(EnvCount = 4, Format('loop finished in 4 provider calls (got %d)', [EnvCount]));
+  SP1 := EnvSystemPrompt(EnvAt(0));
+  Check(not Has(SP1, '[progress ledger'),
     'iteration 1 system prompt is pristine (prefix-cache preserved)');
+  Check(Has(SP1, '## Project'), 'project facts reach iteration 1');
+  Check(Has(SP1, 'Test:  make test'), 'facts name the verify command (make test)');
+  Check(Has(SP1, 'Git branch: bench-main'), 'facts carry the git branch');
   SP2 := EnvSystemPrompt(EnvAt(1));
   Check(Has(SP2, '[progress ledger'), 'iteration 2 carries the progress ledger');
   Check(Has(SP2, 'bench-demo.html'), 'ledger lists the written file');
@@ -474,6 +505,16 @@ procedure ScenarioBuildSite;
   Check(FileExists(GHomeDir + '/workspace/bench-demo.html'),
     'deliverable exists in the workspace');
   Check(Has(Answer, 'page written'), 'final answer surfaced');
+  { B2: the edit's tool result carries the mini-diff snippet, so the model
+    verifies placement without a full-file re-read. }
+  Check(Has(EnvLastMessage(EnvAt(2)), 'now reads (lines'),
+    'edit_file result carries the mini-diff context snippet');
+  Check(Has(EnvLastMessage(EnvAt(2)), 'bench v2'),
+    'snippet shows the changed content');
+  { D1: one find_files call locates the deliverable by name -- no
+    list_dir ladder. }
+  Check(Has(EnvLastMessage(EnvAt(3)), 'bench-demo.html'),
+    'find_files locates the file by glob in one call');
   Metric('build-site.provider_calls', EnvCount);
 end;
 
@@ -596,6 +637,169 @@ procedure ScenarioResumeAfterCap;
   Check(Has(Turn2, 'finished'), 'turn 2 delivered');
 end;
 
+function H_RepeatRead(N: Integer; const EnvJSON: string): string;
+begin
+  if N <= 3 then
+    Result := RoundOf([OneCall('read_file', ArgsPathPlain('notes.txt'))])
+  else
+    Result := StopOf('reviewed the notes.');
+end;
+
+procedure ScenarioRepeatRead;
+{ C3: re-reading an unchanged file must not re-inject the full body --
+  the second and third reads dedup to a one-line stub, so history growth
+  across the repeat reads is stub-sized, not file-sized. The 8 KB fixture
+  sits below the C1 cap on purpose: what keeps the bodies flat here is
+  the dedup, not the byte cap. }
+var
+  S: TStringList;
+  Body: string;
+  i, Grow: Integer;
+begin
+  WriteLn;
+  WriteLn('== scenario: repeat-read (per-turn dedup keeps history flat) ==');
+  Body := '';
+  for i := 1 to 130 do
+    Body := Body + Format('note %3d: remember the thing about the thing', [i]) + #10;
+  S := TStringList.Create;
+  try
+    S.Text := Body;
+    S.SaveToFile(GHomeDir + '/workspace/notes.txt');
+  finally
+    S.Free;
+  end;
+
+  ResetScenario(H_RepeatRead);
+  Chat('[' + MsgObjJSON('user',
+    'review notes.txt very carefully, twice if you must') + ']', 'bench-repeat');
+  Check(EnvCount = 4, Format('four provider calls (got %d)', [EnvCount]));
+  Check(Has(EnvAt(2), 'unchanged since the earlier read'),
+    'second read deduped to a stub in history');
+  Grow := Length(EnvAt(3)) - Length(EnvAt(1));
+  Check(Grow < 2500,
+    Format('history growth across 2 repeat reads is stub-sized (%d bytes; the file is ~7 KB)', [Grow]));
+  Metric('repeatread.growth_bytes_over_2_rereads', Grow);
+end;
+
+function H_RealTask(N: Integer; const EnvJSON: string): string;
+{ A deterministic POLICY, not a fixed script: the same code runs against
+  both binaries and adapts to what the harness offers -- exactly how a
+  real model reads its tool schemas. Where the after-binary provides
+  find_files / start_line / the mini-diff, the policy uses them; where
+  the before-binary doesn't, it falls back to the list_dir ladder and
+  full-file verify re-reads the original failing transcript showed. The
+  60 KB "site dump" is generated locally (shell printf loop) instead of
+  a live fetch so the run is byte-deterministic -- it mirrors the real
+  transcript's fetch -> save -> parse loop. }
+var
+  LastMsg: string;
+begin
+  LastMsg := EnvLastMessage(EnvJSON);
+  case GStep of
+    0: begin GStep := 1;
+         Result := RoundOf([OneCall('list_dir', ArgsObj1('path', '.'))]); end;
+    1: begin GStep := 2;   { "fetch the reference site" -> save a ~68 KB dump.
+         yes|head, not a shell loop: $((...)) trips the deny-substring
+         guard for command substitution. }
+         Result := RoundOf([OneCall('shell_exec', ArgsObj1('command',
+           'yes "pasclaw dev site: tool-calling agent in object pascal, gateway, mcp, skills, sessions, memory pad" | head -800 > site-dump.html; wc -c site-dump.html'))]); end;
+    2: begin GStep := 3;
+         Result := RoundOf([OneCall('read_file', ArgsPathPlain('site-dump.html'))]); end;
+    3: begin GStep := 4;   { "grab the README for feature listings" }
+         Result := RoundOf([OneCall('shell_exec', ArgsObj1('command',
+           'yes "README feature line: agent loop, tools, sessions, gateway" | head -100 > readme-ref.md; wc -c readme-ref.md'))]); end;
+    4: begin GStep := 5;
+         Result := RoundOf([OneCall('read_file', ArgsPathPlain('readme-ref.md'))]); end;
+    5: begin GStep := 6;   { "double-check the dump" -- the classic re-read }
+         Result := RoundOf([OneCall('read_file', ArgsPathPlain('site-dump.html'))]); end;
+    6: begin GStep := 7;
+         Result := RoundOf([OneCall('write_file', ArgsPathContent('index.html',
+           '<!DOCTYPE html>'#10'<html><head><title>PasClaw</title></head>'#10 +
+           '<body>'#10'<h1>PasClaw</h1>'#10'<p>AI agent in Object Pascal.</p>'#10 +
+           '</body></html>'))]); end;
+    7: begin           { locate the deliverable: glob if the harness has it }
+         if Pos('find_files', EnvJSON) > 0 then
+         begin
+           GStep := 9;
+           Result := RoundOf([OneCall('find_files', '{"pattern":"index.*"}')]);
+         end
+         else
+         begin
+           GStep := 8;
+           Result := RoundOf([OneCall('list_dir', ArgsObj1('path', '.'))]);
+         end;
+       end;
+    8: begin GStep := 9;   { ladder's second rung (no glob tool) }
+         Result := RoundOf([OneCall('list_dir', ArgsObj1('path', '.'))]); end;
+    9: begin GStep := 10;  { verify the write: surgical if ranges exist }
+         if Pos('start_line', EnvJSON) > 0 then
+           Result := RoundOf([OneCall('read_file',
+             '{"path":"index.html","start_line":1,"end_line":6}')])
+         else
+           Result := RoundOf([OneCall('read_file', ArgsPathPlain('index.html'))]);
+       end;
+    10: begin GStep := 11;
+         Result := RoundOf([OneCall('edit_file',
+           '{"path":"index.html","old_text":"<title>PasClaw</title>","new_text":"<title>PasClaw v2 -- better than before</title>"}')]); end;
+    11: begin            { trust the mini-diff; re-read blind edits }
+         if Pos('now reads', LastMsg) > 0 then
+         begin
+           GStep := 13;
+           Result := StopOf('done: index.html built in the workspace with the v2 title.');
+         end
+         else
+         begin
+           GStep := 12;
+           Result := RoundOf([OneCall('read_file', ArgsPathPlain('index.html'))]);
+         end;
+       end;
+  else
+    Result := StopOf('done: index.html built in the workspace with the v2 title.');
+  end;
+end;
+
+procedure ScenarioRealTask;
+{ The original failing prompt, end to end, under a policy model. }
+var
+  Answer, Deliv: string;
+  S: TStringList;
+begin
+  WriteLn;
+  WriteLn('== scenario: real-task (the pasclaw.dev build prompt, end to end) ==');
+  ResetScenario(H_RealTask);
+  Answer := Chat('[' + MsgObjJSON('user',
+    'build a better version of https://pasclaw.dev/ -- ' +
+    'https://github.com/fmxexpress/pasclaw/ if you need more feature ' +
+    'listings. you can build it using HTML and HTMX if you need to or just ' +
+    'vanilla javascript. it could be a single inline file or multiple ' +
+    'files. make sure you use the workspace dir to do your work') + ']',
+    'bench-realtask');
+
+  Deliv := '';
+  if FileExists(GHomeDir + '/workspace/index.html') then
+  begin
+    S := TStringList.Create;
+    try
+      S.LoadFromFile(GHomeDir + '/workspace/index.html');
+      Deliv := S.Text;
+    finally
+      S.Free;
+    end;
+  end;
+  { Guard against silently measuring the wrong task: the dump must have
+    been written and its body must have reached the model. }
+  Check(FileExists(GHomeDir + '/workspace/site-dump.html'),
+    'the reference dump was written by the shell step');
+  Check(Has(EnvAt(3), 'pasclaw dev site:'),
+    'the dump body reached the model in the read result');
+  Check(Deliv <> '', 'deliverable index.html exists in the workspace');
+  Check(Has(Deliv, 'v2 -- better than before'), 'edit landed (v2 title)');
+  Check(Has(Answer, 'done:'), 'turn finished with a final answer');
+  Metric('realtask.provider_calls', EnvCount);
+  Metric('realtask.total_request_bytes', GTotalBody);
+  Metric('realtask.max_request_body_bytes', GMaxBody);
+end;
+
 { ---- gateway lifecycle ---------------------------------------------------- }
 var
   GW: TProcess;
@@ -707,6 +911,8 @@ function WaitHealthy: Boolean;
     ScenarioMalformedRecovery;
     ScenarioResumeAfterCap;
     ScenarioFatRead;
+    ScenarioRepeatRead;
+    ScenarioRealTask;
 
     WriteLn;
     WriteLn('== metrics ==');