diff --git a/skills/github-project-ingestion/SKILL.md b/skills/github-project-ingestion/SKILL.md index 59b8c77e36..a4e2a40de2 100644 --- a/skills/github-project-ingestion/SKILL.md +++ b/skills/github-project-ingestion/SKILL.md @@ -251,8 +251,16 @@ full intended `bodyMarkdown`, never a diff. Copy the exact body and Before constructing final page bodies, freeze the complete ordered page inventory and stable `total_pages`; the inventory may contain at most 32 pages. Once the -inventory is frozen, read and construct each page in order, then immediately call -`brain_stage_ingestion_proposal_page` in its own agent turn with the exact +inventory is frozen, use `search`, `query`, `list_pages`, and `resolve_slugs` +results to work through it without preloading full page bodies. Never request +more than one `get_page` in the same assistant turn or tool batch. For an update, +call exactly one `get_page` only when ready to construct and stage that entry. After an update +target's `get_page` returns, the very next assistant turn must call +`brain_stage_ingestion_proposal_page` for that same update, as the only tool call +in that turn. Do not call `get_page` for another target, or make any other large +read, between that baseline read and its staging call. + +Call `brain_stage_ingestion_proposal_page` with the exact `artifact_id`, `source_id`, `admission_scope`, one-based `sequence`, stable `total_pages`, and page object. Stage only one page per turn. Preserve the returned `{sequence, slug, digest}`; later turns may rely on that durable digest diff --git a/skills/gmail-thread-ingestion/SKILL.md b/skills/gmail-thread-ingestion/SKILL.md index 43e84336ef..14afd39a0a 100644 --- a/skills/gmail-thread-ingestion/SKILL.md +++ b/skills/gmail-thread-ingestion/SKILL.md @@ -214,8 +214,20 @@ timeline entries as `{pageSlug,date,text,ref,refLabel?}` with a strict material. Before staging, freeze the complete ordered page inventory and stable -`total_pages`; it may contain at most 32 pages. Stage only one page per turn by -calling `brain_stage_ingestion_proposal_page` with the exact `artifact_id`, +`total_pages`; it may contain at most 32 pages. Use `search`, `query`, +`list_pages`, and `resolve_slugs` results to work through it without preloading +full page bodies. Never request more than one `get_page` in the same assistant +turn or tool batch. +For an update, call exactly one `get_page` only when ready to construct and stage +that entry. Copy its exact body and `content_hash` into `baseMarkdown` and +`expectedContentHash`. After an update target's `get_page` returns, the very next +assistant turn must call `brain_stage_ingestion_proposal_page` for that same +update, as the only tool call in that turn. Do not call `get_page` for another +target, or make any other large read, between that baseline read and its staging +call. + +Stage only one page per turn by calling `brain_stage_ingestion_proposal_page` +with the exact `artifact_id`, `source_id`, `admission_scope`, one-based `sequence`, stable `total_pages`, and `page` object. Then call `brain_finalize_ingestion_proposal` in its own turn with the same exact `artifact_id`, `source_id`, `admission_scope`, and diff --git a/skills/granola-meeting-ingestion/SKILL.md b/skills/granola-meeting-ingestion/SKILL.md index 04efe6e6ab..663f62b18e 100644 --- a/skills/granola-meeting-ingestion/SKILL.md +++ b/skills/granola-meeting-ingestion/SKILL.md @@ -237,8 +237,16 @@ from the `get_page` read used to draft each update into `baseMarkdown` and Before constructing final page bodies, freeze the complete ordered page inventory and stable `total_pages`; the inventory may contain at most 32 pages. Once the -inventory is frozen, read and construct each page in order, then immediately call -`brain_stage_ingestion_proposal_page` in its own agent turn with the exact +inventory is frozen, use `search`, `query`, `list_pages`, and `resolve_slugs` +results to work through it without preloading full page bodies. Never request +more than one `get_page` in the same assistant turn or tool batch. For an update, +call exactly one `get_page` only when ready to construct and stage that entry. After an update +target's `get_page` returns, the very next assistant turn must call +`brain_stage_ingestion_proposal_page` for that same update, as the only tool call +in that turn. Do not call `get_page` for another target, or make any other large +read, between that baseline read and its staging call. + +Call `brain_stage_ingestion_proposal_page` with the exact `artifact_id`, `source_id`, `admission_scope`, one-based `sequence`, stable `total_pages`, and page object. Stage only one page per turn. Preserve the returned `{sequence, slug, digest}`; later turns may rely on that durable digest diff --git a/src/core/ai/tool-loop-context.ts b/src/core/ai/tool-loop-context.ts index 4213a5ac0b..de4c78b7a3 100644 --- a/src/core/ai/tool-loop-context.ts +++ b/src/core/ai/tool-loop-context.ts @@ -60,6 +60,12 @@ interface ToolRound { evidence: ToolEvidence[]; } +interface ScoredToolRound { + round: ToolRound; + exactResultCount: number; + exactResultBytes: number; +} + interface WorkingContextProjectionSource { kind: 'tool_input' | 'tool_result'; toolName: string; @@ -239,18 +245,161 @@ function buildTaskAnchor(messages: ChatMessage[]): ChatMessage { return { role: 'user', content: combined }; } -/** Find the largest payload representation whose complete round fits. */ +/** Find the fitting projection that retains the most exact read results. */ function compactRoundToFit( round: ToolRound, availableBytes: number, options: ToolLoopContextOptions, ): ToolRound | null { if (availableBytes <= 0) return null; + let best: ScoredToolRound | null = null; for (const perPayload of PAYLOAD_LIMITS) { const compacted = compactRound(round, perPayload, options); - if (jsonBytes([compacted.assistant, compacted.result]) <= availableBytes) return compacted; + if (jsonBytes([compacted.assistant, compacted.result]) <= availableBytes) { + const candidate = restoreExactNonMutatingResults( + round, + compacted, + availableBytes, + options, + ); + if ( + best === null + || candidate.exactResultCount > best.exactResultCount + || ( + candidate.exactResultCount === best.exactResultCount + && candidate.exactResultBytes > best.exactResultBytes + ) + ) { + best = candidate; + } + } + } + return best?.round ?? null; +} + +/** Spend remaining round budget on complete successful read results. */ +function restoreExactNonMutatingResults( + original: ToolRound, + compacted: ToolRound, + availableBytes: number, + options: ToolLoopContextOptions, +): ScoredToolRound { + const originalResults = new Map( + toolResultBlocks(original.result).map(block => [block.toolCallId, block]), + ); + const compactedResults = new Map( + toolResultBlocks(compacted.result).map(block => [block.toolCallId, block]), + ); + const candidates = original.evidence.flatMap(evidence => { + const originalResult = originalResults.get(evidence.toolCallId); + const compactedResult = compactedResults.get(evidence.toolCallId); + if ( + evidence.failed + || isMutationSensitive(evidence.toolName, options.mutatingToolNames) + || !originalResult + || !compactedResult + || originalResult.toolName !== evidence.toolName + ) { + return []; + } + const exactJson = safeJson(originalResult.output); + const compactedJson = safeJson(compactedResult.output); + return [{ + toolCallId: evidence.toolCallId, + output: originalResult.output, + exactJson, + compactedJson, + exactBytes: utf8Bytes(exactJson), + additionalBytes: utf8Bytes(exactJson) - utf8Bytes(compactedJson), + }]; + }); + + // JSON serialization is additive at each result's `output` value. Select + // the best exact-output subset by byte delta, then rebuild only once. + let remainingBytes = availableBytes - jsonBytes([compacted.assistant, compacted.result]); + let exactResultCount = 0; + let exactResultBytes = 0; + const restoredOutputs = new Map(); + const selectable = []; + for (const candidate of candidates) { + if (candidate.exactJson === candidate.compactedJson) { + exactResultCount++; + exactResultBytes += candidate.exactBytes; + } else if (candidate.additionalBytes <= 0) { + restoredOutputs.set(candidate.toolCallId, candidate.output); + remainingBytes -= candidate.additionalBytes; + exactResultCount++; + exactResultBytes += candidate.exactBytes; + } else { + selectable.push(candidate); + } + } + + // One state per reachable byte delta is sufficient: for equal cost, a + // higher count (then more exact bytes) dominates every later extension. + const selections = new Map([[0, { count: 0, exactBytes: 0, mask: 0n }]]); + for (const [index, candidate] of selectable.entries()) { + for (const [cost, selection] of [...selections]) { + const nextCost = cost + candidate.additionalBytes; + if (nextCost > remainingBytes) continue; + const next = { + count: selection.count + 1, + exactBytes: selection.exactBytes + candidate.exactBytes, + mask: selection.mask | (1n << BigInt(index)), + }; + const current = selections.get(nextCost); + if ( + !current + || next.count > current.count + || (next.count === current.count && next.exactBytes > current.exactBytes) + ) { + selections.set(nextCost, next); + } + } + } + + let bestSelection = selections.get(0)!; + for (const selection of selections.values()) { + if ( + selection.count > bestSelection.count + || ( + selection.count === bestSelection.count + && selection.exactBytes > bestSelection.exactBytes + ) + ) { + bestSelection = selection; + } + } + exactResultCount += bestSelection.count; + exactResultBytes += bestSelection.exactBytes; + for (const [index, candidate] of selectable.entries()) { + if ((bestSelection.mask & (1n << BigInt(index))) !== 0n) { + restoredOutputs.set(candidate.toolCallId, candidate.output); + } } - return null; + + if (restoredOutputs.size === 0) { + return { round: compacted, exactResultCount, exactResultBytes }; + } + return { + round: { + ...compacted, + result: { + ...compacted.result, + content: mapBlocks(compacted.result, block => ( + block.type === 'tool-result' && restoredOutputs.has(block.toolCallId) + ? { ...block, output: restoredOutputs.get(block.toolCallId) } + : block + )), + }, + }, + exactResultCount, + exactResultBytes, + }; } /** Bound historical tool inputs/results while keeping provider call IDs paired. */ @@ -259,6 +408,9 @@ function compactRound( perPayloadBytes: number, options: ToolLoopContextOptions, ): ToolRound { + const evidenceById = new Map( + round.evidence.map(evidence => [evidence.toolCallId, evidence]), + ); return { ...round, assistant: { @@ -283,13 +435,21 @@ function compactRound( ...round.result, content: mapBlocks(round.result, block => { if (block.type !== 'tool-result') return block; + const evidence = evidenceById.get(block.toolCallId); + const toolName = evidence?.toolName ?? block.toolName; return { ...block, - output: boundValue(block.output, perPayloadBytes, { - kind: 'tool_result', - toolName: block.toolName, - preserveStructuralIdentity: false, - }), + output: boundValue( + block.output, + // The assistant call owns tool identity. A mismatched result name + // must not disguise a mutation as an exact restorable read. + evidence && evidence.toolName !== block.toolName ? 0 : perPayloadBytes, + { + kind: 'tool_result', + toolName, + preserveStructuralIdentity: false, + }, + ), }; }), }, diff --git a/test/ai/tool-loop-context-exact-results.test.ts b/test/ai/tool-loop-context-exact-results.test.ts new file mode 100644 index 0000000000..fd0dcc6be9 --- /dev/null +++ b/test/ai/tool-loop-context-exact-results.test.ts @@ -0,0 +1,253 @@ +import { describe, expect, it } from 'bun:test'; +import { + compactToolLoopMessages, +} from '../../src/core/ai/tool-loop-context.ts'; +import type { ChatBlock, ChatMessage } from '../../src/core/ai/gateway.ts'; + +interface RoundEntry { + id: string; + callName: string; + input: unknown; + output: unknown; + resultName?: string; + isError?: boolean; +} + +/** Build one provider-valid parallel tool round after a task message. */ +function buildRound(task: string, entries: RoundEntry[]): ChatMessage[] { + return [ + { role: 'user', content: task }, + { + role: 'assistant', + content: entries.map(entry => ({ + type: 'tool-call' as const, + toolCallId: entry.id, + toolName: entry.callName, + input: entry.input, + })), + }, + { + role: 'user', + content: entries.map(entry => ({ + type: 'tool-result' as const, + toolCallId: entry.id, + toolName: entry.resultName ?? entry.callName, + output: entry.output, + ...(entry.isError ? { isError: true } : {}), + })), + }, + ]; +} + +/** Read one result output from a compacted provider projection. */ +function resultOutput(messages: ChatMessage[], toolCallId: string): unknown { + for (const message of messages) { + if (typeof message.content === 'string') continue; + const result = message.content.find(block => ( + block.type === 'tool-result' && block.toolCallId === toolCallId + )); + if (result?.type === 'tool-result') return result.output; + } + throw new Error(`Missing tool result ${toolCallId}`); +} + +describe('tool-loop exact non-mutating result retention', () => { + it('retains an exact successful read from a projected parallel round when one fits', () => { + const smallOutput = { + content_hash: 'a'.repeat(64), + body: `EXACT_SMALL_${'s'.repeat(5_000)}`, + }; + const queryOutput = { hits: `EXACT_QUERY_${'q'.repeat(11_800)}` }; + const messages = buildRound( + `Complete ingestion task\n${'t'.repeat(116_000)}`, + [ + { id: 'read-small', callName: 'get_page', input: { slug: 'people/small' }, output: smallOutput }, + { + id: 'read-small-two', + callName: 'get_page', + input: { slug: 'people/small-two' }, + output: { content_hash: 'b'.repeat(64), body: `EXACT_SMALL_TWO_${'n'.repeat(8_700)}` }, + }, + { + id: 'read-medium', + callName: 'get_page', + input: { slug: 'people/medium' }, + output: { content_hash: 'c'.repeat(64), body: `EXACT_MEDIUM_${'m'.repeat(14_000)}` }, + }, + { + id: 'read-large', + callName: 'get_page', + input: { slug: 'people/large' }, + output: { content_hash: 'd'.repeat(64), body: `EXACT_LARGE_${'l'.repeat(45_000)}` }, + }, + { + id: 'query-related', + callName: 'query', + input: { query: 'related context' }, + output: queryOutput, + }, + ], + ); + const durableSnapshot = structuredClone(messages); + + const compacted = compactToolLoopMessages(messages, 130_000, { + mutatingToolNames: new Set(), + }); + const serialized = JSON.stringify(compacted); + + expect(Buffer.byteLength(serialized, 'utf8')).toBeLessThanOrEqual(130_000); + expect(resultOutput(compacted, 'query-related')).toEqual(queryOutput); + expect(serialized).toContain('working_context_projection'); + expect(serialized).not.toContain('EXACT_LARGE_'); + expect(messages).toEqual(durableSnapshot); + + const assistant = compacted.at(-2)!; + const result = compacted.at(-1)!; + const callIds = new Set( + typeof assistant.content === 'string' + ? [] + : assistant.content + .filter(block => block.type === 'tool-call') + .map(block => block.toolCallId), + ); + const resultIds = new Set( + typeof result.content === 'string' + ? [] + : result.content + .filter(block => block.type === 'tool-result') + .map(block => block.toolCallId), + ); + expect(resultIds).toEqual(callIds); + }); + + it('keeps a successful singleton read exactly equal under the same byte budget', () => { + const singletonOutput = { + content_hash: 'e'.repeat(64), + body: `EXACT_SINGLETON_${'p'.repeat(8_000)}`, + }; + const messages = buildRound( + `Complete ingestion task\n${'t'.repeat(116_000)}`, + [{ + id: 'read-singleton', + callName: 'get_page', + input: { slug: 'people/singleton' }, + output: singletonOutput, + }], + ); + + const compacted = compactToolLoopMessages(messages, 130_000, { + mutatingToolNames: new Set(), + }); + + expect(resultOutput(compacted, 'read-singleton')).toEqual(singletonOutput); + expect(JSON.stringify(compacted)).not.toContain('working_context_projection'); + }); + + it('keeps every read projected when no exact result fits beside the task', () => { + const messages = buildRound( + `Complete ingestion task\n${'t'.repeat(126_000)}`, + [ + { + id: 'read-one', + callName: 'get_page', + input: { slug: 'people/one' }, + output: { content_hash: 'f'.repeat(64), body: `NO_FIT_ONE_${'x'.repeat(8_000)}` }, + }, + { + id: 'read-two', + callName: 'get_page', + input: { slug: 'people/two' }, + output: { content_hash: 'g'.repeat(64), body: `NO_FIT_TWO_${'y'.repeat(8_000)}` }, + }, + ], + ); + const durableSnapshot = structuredClone(messages); + + const compacted = compactToolLoopMessages(messages, 130_000, { + mutatingToolNames: new Set(), + }); + const serialized = JSON.stringify(compacted); + + expect(Buffer.byteLength(serialized, 'utf8')).toBeLessThanOrEqual(130_000); + expect(serialized).toContain('working_context_projection'); + expect(serialized).not.toContain('NO_FIT_ONE_'); + expect(serialized).not.toContain('NO_FIT_TWO_'); + expect(messages).toEqual(durableSnapshot); + }); + + it('projects a large input when a lower tier can retain the exact read result', () => { + const exactOutput = { + content_hash: 'h'.repeat(64), + body: `TIER_PRIORITY_RESULT_${'r'.repeat(13_000)}`, + }; + const messages = buildRound('Read the exact page.', [{ + id: 'tier-priority', + callName: 'get_page', + input: { slug: 'people/tier-priority', detail: 'i'.repeat(11_000) }, + output: exactOutput, + }]); + + const compacted = compactToolLoopMessages(messages, 16_000, { + mutatingToolNames: new Set(), + }); + + expect(resultOutput(compacted, 'tier-priority')).toEqual(exactOutput); + const assistant = compacted[1]!; + const input = typeof assistant.content === 'string' + ? null + : (assistant.content[0] as Extract).input; + expect(JSON.stringify(input)).toContain('working_context_projection'); + }); + + it('prefers the largest exact read when only one projected result fits', () => { + const smallerOutput = { + content_hash: 'i'.repeat(64), + body: `SMALLER_EXACT_${'s'.repeat(13_000)}`, + }; + const largerOutput = { + content_hash: 'j'.repeat(64), + body: `LARGER_EXACT_${'l'.repeat(20_000)}`, + }; + const messages = buildRound('Retain the most complete read evidence.', [ + { + id: 'smaller-read', + callName: 'get_page', + input: { slug: 'people/smaller' }, + output: smallerOutput, + }, + { + id: 'larger-read', + callName: 'get_page', + input: { slug: 'people/larger' }, + output: largerOutput, + }, + ]); + + const compacted = compactToolLoopMessages(messages, 22_000, { + mutatingToolNames: new Set(), + }); + + expect(resultOutput(compacted, 'larger-read')).toEqual(largerOutput); + expect(resultOutput(compacted, 'smaller-read')).not.toEqual(smallerOutput); + }); + + it('never restores a mutation result whose block is mislabeled as a read', () => { + const mutationOutput = { body: `MUTATION_OUTPUT_${'w'.repeat(13_000)}` }; + const messages = buildRound('Write the page once.', [{ + id: 'mislabeled-mutation', + callName: 'put_page', + resultName: 'get_page', + input: { slug: 'wiki/target', content: 'i'.repeat(13_000) }, + output: mutationOutput, + }]); + + const compacted = compactToolLoopMessages(messages, 16_000, { + mutatingToolNames: new Set(['put_page']), + }); + const serialized = JSON.stringify(compacted); + + expect(resultOutput(compacted, 'mislabeled-mutation')).not.toEqual(mutationOutput); + expect(serialized).toContain('working_context_projection'); + expect(serialized).not.toContain('MUTATION_OUTPUT_'); + }); +}); diff --git a/test/github-project-ingestion-skill.test.ts b/test/github-project-ingestion-skill.test.ts index bc91a0d68d..7f0e1178c3 100644 --- a/test/github-project-ingestion-skill.test.ts +++ b/test/github-project-ingestion-skill.test.ts @@ -233,6 +233,19 @@ describe('github-project-ingestion skill', () => { expect(skill).toContain('Omit both fields for a create'); }); + test('stages each update immediately after its sole exact baseline read', () => { + expect(skill).toMatch( + /Never request\s+more than one\s+`get_page` in the same assistant\s+turn or tool batch/, + ); + expect(skill).toMatch( + /After an update\s+target's `get_page` returns, the very next\s+assistant turn must call\s+`brain_stage_ingestion_proposal_page` for that same\s+update/, + ); + expect(skill).toMatch(/as the only tool call\s+in that turn/); + expect(skill).toMatch( + /Do not call `get_page` for another\s+target, or make any other large\s+read, between that baseline read and its staging\s+call/, + ); + }); + test('carries bounded timeline and link mutations in the scoped proposal', () => { expect(skill).toContain( 'proposedTimelineEntries: ', diff --git a/test/gmail-thread-ingestion-skill.test.ts b/test/gmail-thread-ingestion-skill.test.ts index 3bd2427210..d163e99f7b 100644 --- a/test/gmail-thread-ingestion-skill.test.ts +++ b/test/gmail-thread-ingestion-skill.test.ts @@ -169,6 +169,19 @@ describe('gmail-thread-ingestion skill', () => { expect(skill).toContain('`capturePageSlug` is never adjusted'); }); + test('stages each update immediately after its sole exact baseline read', () => { + expect(skill).toMatch( + /Never request\s+more than one\s+`get_page` in the same assistant\s+turn or tool batch/, + ); + expect(skill).toMatch( + /After an update\s+target's `get_page` returns, the very next\s+assistant turn must call\s+`brain_stage_ingestion_proposal_page` for that same\s+update/, + ); + expect(skill).toMatch(/as the only tool call\s+in that turn/); + expect(skill).toMatch( + /Do not call `get_page` for another\s+target, or make any other large\s+read, between that baseline read and its staging\s+call/, + ); + }); + test('stages a normal-mode partial exclusion before any corpus write', () => { expect(skill).toMatch(/newly\s+discovered partial exclusion[\s\S]*staged_proposal/); expect(skill).toMatch(/before any\s+corpus mutation/); diff --git a/test/granola-meeting-ingestion-skill.test.ts b/test/granola-meeting-ingestion-skill.test.ts index de750153cf..6fcda59bc7 100644 --- a/test/granola-meeting-ingestion-skill.test.ts +++ b/test/granola-meeting-ingestion-skill.test.ts @@ -126,6 +126,19 @@ describe('granola-meeting-ingestion skill', () => { expect(skill).toContain('Omit both fields for a create'); }); + test('stages each update immediately after its sole exact baseline read', () => { + expect(skill).toMatch( + /Never request\s+more than one\s+`get_page` in the same assistant\s+turn or tool batch/, + ); + expect(skill).toMatch( + /After an update\s+target's `get_page` returns, the very next\s+assistant turn must call\s+`brain_stage_ingestion_proposal_page` for that same\s+update/, + ); + expect(skill).toMatch(/as the only tool call\s+in that turn/); + expect(skill).toMatch( + /Do not call `get_page` for another\s+target, or make any other large\s+read, between that baseline read and its staging\s+call/, + ); + }); + test('carries bounded timeline and link mutations in the scoped proposal', () => { expect(skill).toContain( 'proposedTimelineEntries: ',