From df22dcdb6ca3648eaf206c40d0ac595d7e2afdf2 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 15:02:02 -0700 Subject: [PATCH 01/18] lore-b74.5.1: stage ingestion proposals durably --- skills/github-project-ingestion/SKILL.md | 106 ++- skills/gmail-thread-ingestion/SKILL.md | 41 +- skills/granola-meeting-ingestion/SKILL.md | 113 +-- src/core/ai/gateway.ts | 5 + src/core/migrate.ts | 87 +++ src/core/minions/agent-job-proposals.ts | 683 +++++++++++++++++++ src/core/minions/handlers/subagent.ts | 7 + src/core/minions/tools/brain-allowlist.ts | 6 + src/core/operations.ts | 130 +++- src/core/pglite-schema.ts | 40 ++ src/core/schema-embedded.ts | 42 ++ src/schema.sql | 42 ++ test/agent-job-proposals.test.ts | 302 ++++++++ test/e2e/subagent-gateway-path.test.ts | 84 +++ test/github-project-ingestion-skill.test.ts | 48 +- test/gmail-thread-ingestion-skill.test.ts | 25 + test/granola-meeting-ingestion-skill.test.ts | 48 +- test/migration-v135.test.ts | 8 +- test/subagent-handler.test.ts | 52 ++ test/submit-agent.test.ts | 129 ++++ 20 files changed, 1903 insertions(+), 95 deletions(-) create mode 100644 src/core/minions/agent-job-proposals.ts create mode 100644 test/agent-job-proposals.test.ts diff --git a/skills/github-project-ingestion/SKILL.md b/skills/github-project-ingestion/SKILL.md index ea5b43d501..43100200a2 100644 --- a/skills/github-project-ingestion/SKILL.md +++ b/skills/github-project-ingestion/SKILL.md @@ -14,6 +14,8 @@ tools: - resolve_slugs - get_links - get_backlinks + - stage_ingestion_proposal_page + - finalize_ingestion_proposal - put_page - add_link - add_timeline_entry @@ -49,7 +51,7 @@ a source-bound remote Minion. - Independently confirm that the complete artifact satisfies the supplied resolver text and revision before writing. Resolver ambiguity returns a classed `needs_attention` receipt without mutation. Partial disqualification - returns a complete `scoped_proposal` without mutation. + returns a complete `staged_proposal` manifest without corpus mutation. - An omitted `mode` preserves the normal write path. `mode: propose` performs the normal analysis, search, and deduplication but performs zero mutations. `mode: apply` executes only the prompt-supplied frozen plan. @@ -104,6 +106,11 @@ Expect one complete task with these fields: ```yaml artifactId: +artifactIntegrity: # required normal/propose; omitted apply + complete: true + manifest: { sha256: <64 lowercase hex characters>, bytes: } + contentMarkdown: { sha256: <64 lowercase hex characters>, bytes: } + transcriptMarkdown: { sha256: <64 lowercase hex characters>, bytes: } capturePageSlug: mode: admissionScope: @@ -126,13 +133,46 @@ attempt: manifest: contentMarkdown: transcriptMarkdown: -priorAttempt: +priorAttempt: # optional; omitted for a clean no-write propose attempt + attempt: + failureCode: + terminalFailureClass: + receiptStatus: + createdPages: + updatedPages: + verifiedPages: + pageResults: + slugAdjustments: + timelineResults: + linkResults: ``` -Stop with `failed` before any write when a required field is absent, the -provider is not `github`, the manifest kind is not `github_issue`, -`github_pull_request`, or `github_document`, or the artifact is visibly -incomplete. Do not fetch, truncate, split, or reconstruct missing input. +In normal and propose modes, stop with `failed` before analysis or staging when +a required field is absent, the provider is not `github`, the manifest kind is +not `github_issue`, `github_pull_request`, or `github_document`, or +`artifactIntegrity.complete` is not exactly `true`. The envelope is the +authority for transport completeness. Working-context projection or omission +markers from the model provider describe only the current context window; +never treat them as proof that the original artifact is incomplete. Do not +fetch, truncate, split, or reconstruct missing input. Apply mode intentionally +omits `artifactIntegrity` because it replays only the supplied frozen plan. + +Before normal/propose analysis or staging, require each integrity `sha256` to +contain exactly 64 lowercase hexadecimal characters and each `bytes` to be a +non-negative integer. The authenticated OAuth caller deterministically +verified these values against the exact prompt fields before submission; treat +the well-formed envelope as authoritative. Do not attempt to recalculate, +estimate, or second-guess hashes or byte counts in model reasoning. Do not +reinterpret a context-projection marker as an integrity failure when the +envelope is well formed and `complete` is exactly `true`. + +When `priorAttempt` is present, accept only the typed projection shown above. +It never contains a top-level summary, unresolved list, raw error, or a nested +result `error`; reject unexpected fields instead of treating them as artifact +evidence. Use the projected ledgers only to select read-back checks and safe +resume points. Durable GBrain state remains authoritative, so never skip a +mutation based on the projection alone. A clean propose attempt with no writes +omits `priorAttempt` entirely. An absent `mode` selects normal mode. Reject any other mode value. Require a non-empty `admissionScope` in propose and apply modes. In apply mode, require @@ -178,7 +218,7 @@ When `mode` is absent, follow the complete workflow. If the artifact clearly matches in part but also contains material excluded by the resolver, treat that as partial disqualification. Derive `admissionScope` only from the resolver's own exclusion language, finish the normal search and deduplication analysis, -and return `scoped_proposal` directly. Do not mutate before returning that +and return `staged_proposal` directly. Do not mutate before returning that proposal. Apply every propose-mode scope, completeness, provenance, and payload cap obligation to this receipt. Do not discard the completed analysis into `needs_attention`. @@ -200,15 +240,24 @@ title, claim, citation, timeline entry, link, summary, feature update, or dossier update may derive from excluded material. Do not repeat, paraphrase, or identify the exclusion inside any proposed mutation. -In `propose` mode, do not call any mutating tool, including `put_page`, -`add_link`, or `add_timeline_entry`. Read-only discovery and verification are -allowed. Return the complete set of pages that `apply` will write. For this +In `propose` mode, do not call any corpus-mutating tool, including `put_page`, +`add_link`, or `add_timeline_entry`. Read-only discovery and job-evidence +staging are allowed. Construct the complete set of pages that `apply` will write. For this provider that includes the exact capture and every feature, initiative, or entity page that the scoped ingestion requires. Each update entry contains the full intended `bodyMarkdown`, never a diff. Copy the exact body and `content_hash` from the `get_page` read used to draft each update into `baseMarkdown` and `expectedContentHash`. Omit both fields for a create. +Immediately after reading and constructing each complete page, call +`brain_stage_ingestion_proposal_page` in its own agent turn with the exact +`artifact_id`, `source_id`, `admission_scope`, one-based `sequence`, stable +`total_pages`, and page object. Stage only one page per turn. Preserve the +returned `{sequence, slug, digest}`; later turns may rely on that durable digest +instead of retaining every old raw `get_page` output or proposed body in the +working context. An identical retry is safe. Never change `total_pages`, reuse +a sequence for different content, or stage after finalization. + Populate `proposedTimelineEntries` with the exact timeline mutations the normal workflow requires: timeline entries only for material dated events with the capture-page reference. Populate `proposedLinks` with typed links only for @@ -232,11 +281,14 @@ Its title and body must not name or describe the excluded material or the admission scope. Scope provenance exists only in the top-level proposal receipt. -Serialize the complete `scoped_proposal` receipt as JSON and measure its UTF-8 -byte length. The receipt, including `proposedTimelineEntries` and -`proposedLinks`, must not exceed 262,144 UTF-8 bytes. Return `failed` with an -operational summary and no mutations when it exceeds that limit or the size -cannot be established. Never truncate or split a proposal. +After every page is staged, call `brain_finalize_ingestion_proposal` in a +separate turn with the exact binding fields, stable page count, ordered +`page_digests`, summary, timeline entries, links, and unresolved items. The +server rejects gaps, duplicates, changed digests, cross-job evidence, a full +plan over 786,432 UTF-8 bytes, or a compact manifest over 262,144 UTF-8 bytes. +Return the finalizer's compact manifest as `staged_proposal`; never reproduce +page bodies or baselines in the final receipt. Never truncate or split a +proposal. ### Apply mode @@ -563,7 +615,8 @@ duplication. - Guessing identities from usernames or creating contributor stubs. - Deleting a canonical page when a tombstone arrives. - Advancing Lore checkpoints or changing Review eligibility. -- Mutating in propose mode or returning an incomplete or truncated proposal. +- Mutating corpus state in propose mode or returning an incomplete or truncated + proposal. - Returning a timeline entry whose `ref` is not the planned capture page. - Returning a link whose `from` slug is absent from `proposedPages`. - Omitting a required material event from `proposedTimelineEntries`. @@ -582,27 +635,16 @@ For propose mode or normal-mode partial disqualification: ```json { - "status": "scoped_proposal", + "status": "staged_proposal", "artifactId": "copied exactly from the prompt", "sourceId": "verified source id", "admissionScope": "supplied or resolver-derived scope", "summary": "compact source-grounded proposal summary", - "proposedPages": [ - { - "slug": "sources/github/example", - "effect": "create", - "title": "complete intended page title", - "bodyMarkdown": "complete intended page body" - }, - { - "slug": "projects/example", - "effect": "update", - "title": "complete intended project title", - "bodyMarkdown": "complete intended project body", - "baseMarkdown": "exact reviewed page body for updates, null for creates", - "expectedContentHash": "exact get_page content_hash for updates, null for creates" - } + "pageDigests": [ + { "sequence": 1, "slug": "sources/github/example", "digest": "64 lowercase hex characters" }, + { "sequence": 2, "slug": "projects/example", "digest": "64 lowercase hex characters" } ], + "proposalDigest": "64 lowercase hex characters", "proposedTimelineEntries": [ { "pageSlug": "projects/example", diff --git a/skills/gmail-thread-ingestion/SKILL.md b/skills/gmail-thread-ingestion/SKILL.md index c3b0f41a64..d368b6db56 100644 --- a/skills/gmail-thread-ingestion/SKILL.md +++ b/skills/gmail-thread-ingestion/SKILL.md @@ -83,6 +83,11 @@ Expect one complete task with these fields: ```yaml artifactId: +artifactIntegrity: + complete: true + manifest: { sha256: <64 lowercase hex characters>, bytes: } + contentMarkdown: { sha256: <64 lowercase hex characters>, bytes: } + transcriptMarkdown: { sha256: <64 lowercase hex characters>, bytes: } capturePageSlug: canonicalExternalId: captureExternalId: ::> @@ -100,14 +105,44 @@ attempt: manifest: contentMarkdown: transcriptMarkdown: -priorAttempt: +priorAttempt: # optional; omitted when there is no prior write evidence + attempt: + failureCode: + terminalFailureClass: + receiptStatus: + createdPages: + updatedPages: + verifiedPages: + pageResults: + slugAdjustments: + timelineResults: + linkResults: ``` Stop with `failed` before any write when a required field is absent, the provider is not `google-gmail`, `canonicalExternalId` is not the package's Gmail thread ID, the capture identity or revision does not match the manifest, or the -artifact is visibly incomplete. Do not fetch, truncate, split, or reconstruct -missing input. +`artifactIntegrity.complete` flag is not exactly `true`. This envelope is the +authority for transport completeness. Working-context projection or omission +markers from the model provider describe only the current context window; +never treat them as proof that the original artifact is incomplete. Do not +fetch, truncate, split, or reconstruct missing input. + +Before analysis or writing, require each integrity `sha256` to contain exactly +64 lowercase hexadecimal characters and each `bytes` to be a non-negative +integer. The authenticated OAuth caller deterministically verified these +values against the exact prompt fields before submission; treat the well-formed +envelope as authoritative. Do not attempt to recalculate, estimate, or +second-guess hashes or byte counts in model reasoning. Do not reinterpret a +context-projection marker as an integrity failure when the envelope is well +formed and `complete` is exactly `true`. + +When `priorAttempt` is present, accept only the typed projection shown above. +It never contains a top-level summary, unresolved list, raw error, or a nested +result `error`; reject unexpected fields instead of treating them as mail +evidence. Use the projected ledgers only to select read-back checks and safe +resume points. Durable GBrain state remains authoritative, so never skip a +mutation based on the projection alone. ## Phases diff --git a/skills/granola-meeting-ingestion/SKILL.md b/skills/granola-meeting-ingestion/SKILL.md index 70e5461fb3..8ea0713760 100644 --- a/skills/granola-meeting-ingestion/SKILL.md +++ b/skills/granola-meeting-ingestion/SKILL.md @@ -13,6 +13,8 @@ tools: - resolve_slugs - get_links - get_backlinks + - stage_ingestion_proposal_page + - finalize_ingestion_proposal - put_page - add_link - add_timeline_entry @@ -48,7 +50,7 @@ These instructions are self-contained for a source-bound remote Minion. - Independently confirm that the complete artifact satisfies the supplied resolver text and revision before writing. Resolver ambiguity returns a classed `needs_attention` receipt without mutation. Partial disqualification - returns a complete `scoped_proposal` without mutation. + returns a complete `staged_proposal` manifest without corpus mutation. - An omitted `mode` preserves the normal write path. `mode: propose` performs the normal analysis, search, and deduplication but performs zero mutations. `mode: apply` executes only the prompt-supplied frozen plan. @@ -98,6 +100,11 @@ Expect one complete task with these fields: ```yaml artifactId: +artifactIntegrity: # required normal/propose; omitted apply + complete: true + manifest: { sha256: <64 lowercase hex characters>, bytes: } + contentMarkdown: { sha256: <64 lowercase hex characters>, bytes: } + transcriptMarkdown: { sha256: <64 lowercase hex characters>, bytes: } capturePageSlug: mode: admissionScope: @@ -114,12 +121,45 @@ attempt: manifest: contentMarkdown: transcriptMarkdown: -priorAttempt: +priorAttempt: # optional; omitted for a clean no-write propose attempt + attempt: + failureCode: + terminalFailureClass: + receiptStatus: + createdPages: + updatedPages: + verifiedPages: + pageResults: + slugAdjustments: + timelineResults: + linkResults: ``` -Stop with `failed` before any write when a required field is absent, the -provider is not `granola`, or the artifact is visibly incomplete. Do not fetch, -truncate, split, or reconstruct missing input. +In normal and propose modes, stop with `failed` before analysis or staging when +a required field is absent, the provider is not `granola`, or +`artifactIntegrity.complete` is not exactly `true`. The envelope is the +authority for transport completeness. Working-context projection or omission +markers from the model provider describe only the current context window; +never treat them as proof that the original artifact is incomplete. Do not +fetch, truncate, split, or reconstruct missing input. Apply mode intentionally +omits `artifactIntegrity` because it replays only the supplied frozen plan. + +Before normal/propose analysis or staging, require each integrity `sha256` to +contain exactly 64 lowercase hexadecimal characters and each `bytes` to be a +non-negative integer. The authenticated OAuth caller deterministically +verified these values against the exact prompt fields before submission; treat +the well-formed envelope as authoritative. Do not attempt to recalculate, +estimate, or second-guess hashes or byte counts in model reasoning. Do not +reinterpret a context-projection marker as an integrity failure when the +envelope is well formed and `complete` is exactly `true`. + +When `priorAttempt` is present, accept only the typed projection shown above. +It never contains a top-level summary, unresolved list, raw error, or a nested +result `error`; reject unexpected fields instead of treating them as artifact +evidence. Use the projected ledgers only to select read-back checks and safe +resume points. Durable GBrain state remains authoritative, so never skip a +mutation based on the projection alone. A clean propose attempt with no writes +omits `priorAttempt` entirely. An absent `mode` selects normal mode. Reject any other mode value. Require a non-empty `admissionScope` in propose and apply modes. In apply mode, require @@ -164,7 +204,7 @@ When `mode` is absent, follow the complete workflow. If the artifact clearly matches in part but also contains material excluded by the resolver, treat that as partial disqualification. Derive `admissionScope` only from the resolver's own exclusion language, finish the normal search and deduplication analysis, -and return `scoped_proposal` directly. Do not mutate before returning that +and return `staged_proposal` directly. Do not mutate before returning that proposal. Apply every propose-mode scope, completeness, provenance, and payload cap obligation to this receipt. Do not discard the completed analysis into `needs_attention`. @@ -186,15 +226,24 @@ timeline entry, link, summary, or dossier update may derive from excluded material. Do not repeat, paraphrase, or identify the exclusion inside any proposed mutation. -In `propose` mode, do not call any mutating tool, including `put_page`, -`add_link`, or `add_timeline_entry`. Read-only discovery and verification are -allowed. Return the complete set of pages that `apply` will write. For this +In `propose` mode, do not call any corpus-mutating tool, including `put_page`, +`add_link`, or `add_timeline_entry`. Read-only discovery and job-evidence +staging are allowed. Construct the complete set of pages that `apply` will write. For this provider that includes the capture page, meeting page, and every dossier page that the scoped ingestion requires. Each update entry contains the full intended `bodyMarkdown`, never a diff. Copy the exact body and `content_hash` from the `get_page` read used to draft each update into `baseMarkdown` and `expectedContentHash`. Omit both fields for a create. +Immediately after reading and constructing each complete page, call +`brain_stage_ingestion_proposal_page` in its own agent turn with the exact +`artifact_id`, `source_id`, `admission_scope`, one-based `sequence`, stable +`total_pages`, and page object. Stage only one page per turn. Preserve the +returned `{sequence, slug, digest}`; later turns may rely on that durable digest +instead of retaining every old raw `get_page` output or proposed body in the +working context. An identical retry is safe. Never change `total_pages`, reuse +a sequence for different content, or stage after finalization. + Populate `proposedTimelineEntries` with the exact timeline mutations the normal workflow requires: timeline entries only for material dated events with the capture-page reference. Populate `proposedLinks` with typed links only for @@ -217,11 +266,14 @@ complete verbatim record. Its title and body must not name or describe the excluded material or the admission scope. Scope provenance exists only in the top-level proposal receipt. -Serialize the complete `scoped_proposal` receipt as JSON and measure its UTF-8 -byte length. The receipt, including `proposedTimelineEntries` and -`proposedLinks`, must not exceed 262,144 UTF-8 bytes. Return `failed` with an -operational summary and no mutations when it exceeds that limit or the size -cannot be established. Never truncate or split a proposal. +After every page is staged, call `brain_finalize_ingestion_proposal` in a +separate turn with the exact binding fields, stable page count, ordered +`page_digests`, summary, timeline entries, links, and unresolved items. The +server rejects gaps, duplicates, changed digests, cross-job evidence, a full +plan over 786,432 UTF-8 bytes, or a compact manifest over 262,144 UTF-8 bytes. +Return the finalizer's compact manifest as `staged_proposal`; never reproduce +page bodies or baselines in the final receipt. Never truncate or split a +proposal. ### Apply mode @@ -495,7 +547,8 @@ writes honestly so a later attempt can continue without duplication. - Creating attendee stubs or guessing among same-name identities. - Adding routine attendance to every entity timeline. - Advancing Lore checkpoints or making Review-eligibility decisions. -- Mutating in propose mode or returning an incomplete or truncated proposal. +- Mutating corpus state in propose mode or returning an incomplete or truncated + proposal. - Returning a timeline entry whose `ref` is not the planned capture page. - Returning a link whose `from` slug is absent from `proposedPages`. - Hand-writing a `## Timeline` section or omitting a required material event @@ -516,35 +569,17 @@ For propose mode or normal-mode partial disqualification: ```json { - "status": "scoped_proposal", + "status": "staged_proposal", "artifactId": "copied exactly from the prompt", "sourceId": "verified source id", "admissionScope": "supplied or resolver-derived scope", "summary": "compact source-grounded proposal summary", - "proposedPages": [ - { - "slug": "sources/granola/example", - "effect": "create", - "title": "complete intended page title", - "bodyMarkdown": "complete intended page body" - }, - { - "slug": "projects/example", - "effect": "update", - "title": "complete intended project title", - "bodyMarkdown": "complete intended project body", - "baseMarkdown": "exact reviewed page body for updates, null for creates", - "expectedContentHash": "exact get_page content_hash for updates, null for creates" - }, - { - "slug": "meetings/example", - "effect": "update", - "title": "complete intended meeting title", - "bodyMarkdown": "complete intended meeting body", - "baseMarkdown": "exact reviewed page body for updates, null for creates", - "expectedContentHash": "exact get_page content_hash for updates, null for creates" - } + "pageDigests": [ + { "sequence": 1, "slug": "sources/granola/example", "digest": "64 lowercase hex characters" }, + { "sequence": 2, "slug": "projects/example", "digest": "64 lowercase hex characters" }, + { "sequence": 3, "slug": "meetings/example", "digest": "64 lowercase hex characters" } ], + "proposalDigest": "64 lowercase hex characters", "proposedTimelineEntries": [ { "pageSlug": "projects/example", diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index badc4063f1..aaac52a25c 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -57,6 +57,7 @@ import { runGuardrails, hasGuardrails, type GuardrailHook } from '../guardrails. import { loadConfig } from '../config.ts'; import { buildGatewayConfig } from './build-gateway-config.ts'; import { compactToolLoopMessages, resolveToolLoopMessageBudget } from './tool-loop-context.ts'; +import { assertProposalToolTurnPersistable } from '../minions/agent-job-proposals.ts'; // ---- Gateway-wide AI-HTTP timeout (v0.42.20.0, #1762/#1775) ---- // @@ -3430,6 +3431,10 @@ export async function toolLoop(opts: ToolLoopOpts): Promise { throw err; } + // Proposal stage calls carry exact page bodies. Reject over-limit or + // multi-stage turns before onAssistantTurn persists the raw tool inputs. + assertProposalToolTurnPersistable(chatResult.blocks); + totalUsage.input_tokens += chatResult.usage.input_tokens; totalUsage.output_tokens += chatResult.usage.output_tokens; totalUsage.cache_read_tokens += chatResult.usage.cache_read_tokens; diff --git a/src/core/migrate.ts b/src/core/migrate.ts index 1f39d78721..8c2a723d25 100644 --- a/src/core/migrate.ts +++ b/src/core/migrate.ts @@ -5967,6 +5967,93 @@ export const MIGRATIONS: Migration[] = [ await engine.executeRaw(`UPDATE pages SET id = id WHERE search_vector IS NOT NULL`); }, }, + { + version: 136, + name: 'agent_job_staged_proposals', + // Exact page bodies and optimistic baselines are staged into a job-owned + // ledger so bounded final receipts carry only a verified manifest. + idempotent: true, + sql: ` + CREATE TABLE IF NOT EXISTS agent_job_proposal_fragments ( + job_id BIGINT NOT NULL REFERENCES minion_jobs(id) ON DELETE CASCADE, + owner_client_id TEXT NOT NULL, + source_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + admission_scope TEXT NOT NULL, + sequence INTEGER NOT NULL, + total_pages INTEGER NOT NULL, + page JSONB NOT NULL, + page_digest TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (job_id, sequence), + CONSTRAINT chk_agent_job_proposal_fragment_sequence + CHECK (sequence >= 1 AND total_pages >= sequence), + CONSTRAINT chk_agent_job_proposal_fragment_digest + CHECK (page_digest ~ '^[a-f0-9]{64}$') + ); + CREATE INDEX IF NOT EXISTS idx_agent_job_proposal_fragments_owner + ON agent_job_proposal_fragments (owner_client_id, job_id); + CREATE TABLE IF NOT EXISTS agent_job_proposals ( + job_id BIGINT PRIMARY KEY REFERENCES minion_jobs(id) ON DELETE CASCADE, + owner_client_id TEXT NOT NULL, + source_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + admission_scope TEXT NOT NULL, + total_pages INTEGER NOT NULL CHECK (total_pages >= 1), + page_digests JSONB NOT NULL, + plan JSONB NOT NULL, + proposal_digest TEXT NOT NULL, + manifest JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chk_agent_job_proposal_digest + CHECK (proposal_digest ~ '^[a-f0-9]{64}$') + ); + CREATE INDEX IF NOT EXISTS idx_agent_job_proposals_owner_digest + ON agent_job_proposals (owner_client_id, proposal_digest); + ALTER TABLE agent_job_proposal_fragments ENABLE ROW LEVEL SECURITY; + ALTER TABLE agent_job_proposals ENABLE ROW LEVEL SECURITY; + `, + sqlFor: { + pglite: ` + CREATE TABLE IF NOT EXISTS agent_job_proposal_fragments ( + job_id BIGINT NOT NULL REFERENCES minion_jobs(id) ON DELETE CASCADE, + owner_client_id TEXT NOT NULL, + source_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + admission_scope TEXT NOT NULL, + sequence INTEGER NOT NULL, + total_pages INTEGER NOT NULL, + page JSONB NOT NULL, + page_digest TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (job_id, sequence), + CONSTRAINT chk_agent_job_proposal_fragment_sequence + CHECK (sequence >= 1 AND total_pages >= sequence), + CONSTRAINT chk_agent_job_proposal_fragment_digest + CHECK (page_digest ~ '^[a-f0-9]{64}$') + ); + CREATE INDEX IF NOT EXISTS idx_agent_job_proposal_fragments_owner + ON agent_job_proposal_fragments (owner_client_id, job_id); + CREATE TABLE IF NOT EXISTS agent_job_proposals ( + job_id BIGINT PRIMARY KEY REFERENCES minion_jobs(id) ON DELETE CASCADE, + owner_client_id TEXT NOT NULL, + source_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + admission_scope TEXT NOT NULL, + total_pages INTEGER NOT NULL CHECK (total_pages >= 1), + page_digests JSONB NOT NULL, + plan JSONB NOT NULL, + proposal_digest TEXT NOT NULL, + manifest JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chk_agent_job_proposal_digest + CHECK (proposal_digest ~ '^[a-f0-9]{64}$') + ); + CREATE INDEX IF NOT EXISTS idx_agent_job_proposals_owner_digest + ON agent_job_proposals (owner_client_id, proposal_digest); + `, + }, + }, ]; export const LATEST_VERSION = MIGRATIONS.length > 0 diff --git a/src/core/minions/agent-job-proposals.ts b/src/core/minions/agent-job-proposals.ts new file mode 100644 index 0000000000..1a19414f9e --- /dev/null +++ b/src/core/minions/agent-job-proposals.ts @@ -0,0 +1,683 @@ +import { createHash } from 'node:crypto'; +import type { BrainEngine } from '../engine.ts'; +import { assertValidSourceId } from '../source-id.ts'; + +/** Maximum UTF-8 size of one page-staging tool input. */ +export const PROPOSAL_STAGE_INPUT_MAX_BYTES = 196_608; + +/** Maximum UTF-8 size of one finalized, canonical proposal plan. */ +export const PROPOSAL_AGGREGATE_MAX_BYTES = 786_432; + +/** Maximum UTF-8 size of the compact receipt manifest. */ +export const PROPOSAL_MANIFEST_MAX_BYTES = 262_144; + +export const STAGE_PROPOSAL_TOOL_NAME = 'brain_stage_ingestion_proposal_page'; +export const FINALIZE_PROPOSAL_TOOL_NAME = 'brain_finalize_ingestion_proposal'; + +const SHA256_RE = /^[a-f0-9]{64}$/; +const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; +const PAGE_SLUG_RE = /^[\p{L}\p{N}][\p{L}\p{N}-]*(?:\/[\p{L}\p{N}][\p{L}\p{N}-]*)*$/u; + +/** Error raised when staged proposal evidence violates its durable contract. */ +export class AgentJobProposalError extends Error { + constructor( + public readonly code: string, + message: string, + ) { + super(message); + this.name = 'AgentJobProposalError'; + } +} + +export interface ProposalPageDigest { + sequence: number; + slug: string; + digest: string; +} + +export interface ScopedProposalPage { + slug: string; + effect: 'create' | 'update'; + title: string; + bodyMarkdown: string; + baseMarkdown?: string; + expectedContentHash?: string; +} + +export interface ScopedProposalTimelineEntry { + pageSlug: string; + date: string; + text: string; + ref: string; + refLabel?: string; +} + +export interface ScopedProposalLink { + from: string; + to: string; + type: string; +} + +export interface ScopedAdmissionProposalPlan { + artifactId: string; + sourceId: string; + admissionScope: string; + summary: string; + proposedPages: ScopedProposalPage[]; + proposedTimelineEntries: ScopedProposalTimelineEntry[]; + proposedLinks: ScopedProposalLink[]; + unresolved: string[]; +} + +export interface StageProposalPageInput { + artifact_id: string; + source_id: string; + admission_scope: string; + sequence: number; + total_pages: number; + page: unknown; +} + +export interface FinalizeProposalInput { + artifact_id: string; + source_id: string; + admission_scope: string; + total_pages: number; + page_digests: unknown; + summary: string; + proposed_timeline_entries?: unknown; + proposed_links?: unknown; + unresolved?: unknown; +} + +export interface FinalizedProposalManifest { + status: 'staged_proposal'; + artifactId: string; + sourceId: string; + admissionScope: string; + summary: string; + pageDigests: ProposalPageDigest[]; + proposalDigest: string; + proposedTimelineEntries: ScopedProposalTimelineEntry[]; + proposedLinks: ScopedProposalLink[]; + unresolved: string[]; +} + +interface JobBinding { + ownerClientId: string; + sourceId: string; + artifactId: string; + admissionScope: string; +} + +interface ProposalTurnBlock { + type?: unknown; + name?: unknown; + toolName?: unknown; + input?: unknown; +} + +/** Return the canonical JSON encoding used for every proposal digest. */ +export function canonicalProposalJson(value: unknown): string { + if (value === null || typeof value !== 'object') return JSON.stringify(value); + if (Array.isArray(value)) { + return `[${value.map(canonicalProposalJson).join(',')}]`; + } + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => ( + `${JSON.stringify(key)}:${canonicalProposalJson(record[key])}` + )).join(',')}}`; +} + +/** Hash one canonical proposal value with SHA-256. */ +export function digestProposalValue(value: unknown): string { + return createHash('sha256').update(canonicalProposalJson(value), 'utf8').digest('hex'); +} + +/** Measure one tool input using the same canonical UTF-8 representation persisted by GBrain. */ +export function proposalToolInputBytes(input: unknown): number { + return Buffer.byteLength(canonicalProposalJson(input ?? null), 'utf8'); +} + +/** + * Validate page-staging calls before an assistant turn is persisted. + * + * A turn may stage at most one page. Both stage and finalize inputs are byte + * bounded here so rejected raw inputs never reach either durable transcript + * table. Callers repeat this check immediately before legacy replay writes as + * defense in depth. + */ +export function assertProposalToolTurnPersistable(blocks: readonly ProposalTurnBlock[]): void { + const calls = blocks.flatMap((block) => { + const isCall = block.type === 'tool_use' || block.type === 'tool-call'; + if (!isCall) return []; + const name = typeof block.name === 'string' + ? block.name + : typeof block.toolName === 'string' + ? block.toolName + : ''; + return [{ name, input: block.input }]; + }); + const stageCalls = calls.filter((call) => call.name === STAGE_PROPOSAL_TOOL_NAME); + const finalizeCalls = calls.filter((call) => call.name === FINALIZE_PROPOSAL_TOOL_NAME); + if (stageCalls.length > 1) { + throw new AgentJobProposalError( + 'multiple_stage_calls', + 'Stage exactly one proposal page per agent turn.', + ); + } + if (stageCalls.length > 0 && finalizeCalls.length > 0) { + throw new AgentJobProposalError( + 'mixed_proposal_calls', + 'Stage and finalize proposal calls must occur in separate agent turns.', + ); + } + for (const call of calls) { + if (call.name !== STAGE_PROPOSAL_TOOL_NAME && call.name !== FINALIZE_PROPOSAL_TOOL_NAME) continue; + const bytes = proposalToolInputBytes(call.input); + if (bytes > PROPOSAL_STAGE_INPUT_MAX_BYTES) { + throw new AgentJobProposalError( + 'stage_input_too_large', + `Proposal tool input is ${bytes} UTF-8 bytes; maximum is ${PROPOSAL_STAGE_INPUT_MAX_BYTES}.`, + ); + } + } +} + +/** Stage one exact page proposal in the current agent job's durable ledger. */ +export async function stageAgentJobProposalPage( + engine: BrainEngine, + jobId: number, + input: StageProposalPageInput, +): Promise { + assertSafeJobId(jobId); + if (proposalToolInputBytes(input) > PROPOSAL_STAGE_INPUT_MAX_BYTES) { + throw new AgentJobProposalError('stage_input_too_large', 'Proposal page input exceeds the staging byte limit.'); + } + const sequence = readPositiveInteger(input.sequence, 'sequence'); + const totalPages = readPositiveInteger(input.total_pages, 'total_pages'); + if (sequence > totalPages) { + throw new AgentJobProposalError('invalid_sequence', 'sequence must be within 1..total_pages.'); + } + const page = parseProposalPage(input.page); + const pageDigest = digestProposalValue(page); + + return engine.transaction(async (tx) => { + const binding = await readJobBinding(tx, jobId, true); + assertBindingMatches(binding, input.artifact_id, input.source_id, input.admission_scope); + const finalized = await tx.executeRaw<{ proposal_digest: string }>( + `SELECT proposal_digest FROM agent_job_proposals WHERE job_id = $1`, + [jobId], + ); + if (finalized.length > 0) { + throw new AgentJobProposalError('proposal_finalized', 'This job proposal is already finalized.'); + } + + await tx.executeRaw( + `INSERT INTO agent_job_proposal_fragments + (job_id, owner_client_id, source_id, artifact_id, admission_scope, + sequence, total_pages, page, page_digest) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8::text::jsonb, $9) + ON CONFLICT (job_id, sequence) DO NOTHING`, + [ + jobId, binding.ownerClientId, binding.sourceId, binding.artifactId, + binding.admissionScope, sequence, totalPages, canonicalProposalJson(page), pageDigest, + ], + ); + const rows = await tx.executeRaw<{ + source_id: string; + artifact_id: string; + admission_scope: string; + total_pages: number; + page: unknown; + page_digest: string; + }>( + `SELECT source_id, artifact_id, admission_scope, total_pages, page, page_digest + FROM agent_job_proposal_fragments + WHERE job_id = $1 AND sequence = $2`, + [jobId, sequence], + ); + const existing = rows[0]; + if ( + !existing + || existing.source_id !== binding.sourceId + || existing.artifact_id !== binding.artifactId + || existing.admission_scope !== binding.admissionScope + || Number(existing.total_pages) !== totalPages + || existing.page_digest !== pageDigest + || canonicalProposalJson(existing.page) !== canonicalProposalJson(page) + ) { + throw new AgentJobProposalError( + 'conflicting_fragment', + `Sequence ${sequence} already contains a different proposal fragment.`, + ); + } + return { sequence, slug: page.slug, digest: pageDigest }; + }); +} + +/** Assemble, validate, cap, digest, and freeze the current job's staged plan. */ +export async function finalizeAgentJobProposal( + engine: BrainEngine, + jobId: number, + input: FinalizeProposalInput, +): Promise { + assertSafeJobId(jobId); + if (proposalToolInputBytes(input) > PROPOSAL_STAGE_INPUT_MAX_BYTES) { + throw new AgentJobProposalError('finalize_input_too_large', 'Proposal finalization input exceeds the tool byte limit.'); + } + const totalPages = readPositiveInteger(input.total_pages, 'total_pages'); + const requestedDigests = parseRequestedDigests(input.page_digests, totalPages); + const summary = readBoundedString(input.summary, 'summary', 1_000); + const timeline = parseTimelineEntries(input.proposed_timeline_entries ?? []); + const links = parseLinks(input.proposed_links ?? []); + const unresolved = parseUnresolved(input.unresolved ?? []); + + return engine.transaction(async (tx) => { + const binding = await readJobBinding(tx, jobId, true); + assertBindingMatches(binding, input.artifact_id, input.source_id, input.admission_scope); + const fragments = await tx.executeRaw<{ + sequence: number; + total_pages: number; + owner_client_id: string; + source_id: string; + artifact_id: string; + admission_scope: string; + page: unknown; + page_digest: string; + }>( + `SELECT sequence, total_pages, owner_client_id, source_id, artifact_id, + admission_scope, page, page_digest + FROM agent_job_proposal_fragments + WHERE job_id = $1 + ORDER BY sequence`, + [jobId], + ); + if (fragments.length !== totalPages) { + throw new AgentJobProposalError( + 'fragment_gap', + `Expected ${totalPages} staged pages but found ${fragments.length}.`, + ); + } + + const pages: ScopedProposalPage[] = []; + const pageDigests: ProposalPageDigest[] = []; + const seenSlugs = new Set(); + for (let index = 0; index < fragments.length; index++) { + const expectedSequence = index + 1; + const fragment = fragments[index]!; + const requested = requestedDigests[index]!; + if (Number(fragment.sequence) !== expectedSequence || Number(fragment.total_pages) !== totalPages) { + throw new AgentJobProposalError('fragment_gap', 'Staged proposal sequences are not exactly contiguous.'); + } + if ( + fragment.owner_client_id !== binding.ownerClientId + || fragment.source_id !== binding.sourceId + || fragment.artifact_id !== binding.artifactId + || fragment.admission_scope !== binding.admissionScope + ) { + throw new AgentJobProposalError('binding_mismatch', 'A staged fragment does not match its job binding.'); + } + const page = parseProposalPage(fragment.page); + if ( + requested.sequence !== expectedSequence + || requested.digest !== fragment.page_digest + || digestProposalValue(page) !== fragment.page_digest + ) { + throw new AgentJobProposalError('digest_mismatch', `Digest mismatch at sequence ${expectedSequence}.`); + } + if (seenSlugs.has(page.slug)) { + throw new AgentJobProposalError('duplicate_page', `Proposal contains duplicate page slug ${page.slug}.`); + } + seenSlugs.add(page.slug); + pages.push(page); + pageDigests.push({ sequence: expectedSequence, slug: page.slug, digest: fragment.page_digest }); + } + validatePlanRelations(seenSlugs, timeline, links); + + const plan: ScopedAdmissionProposalPlan = { + artifactId: binding.artifactId, + sourceId: binding.sourceId, + admissionScope: binding.admissionScope, + summary, + proposedPages: pages, + proposedTimelineEntries: timeline, + proposedLinks: links, + unresolved, + }; + const planJson = canonicalProposalJson(plan); + const planBytes = Buffer.byteLength(planJson, 'utf8'); + if (planBytes > PROPOSAL_AGGREGATE_MAX_BYTES) { + throw new AgentJobProposalError( + 'proposal_too_large', + `Finalized proposal is ${planBytes} UTF-8 bytes; maximum is ${PROPOSAL_AGGREGATE_MAX_BYTES}.`, + ); + } + const proposalDigest = digestProposalValue(plan); + const manifest: FinalizedProposalManifest = { + status: 'staged_proposal', + artifactId: binding.artifactId, + sourceId: binding.sourceId, + admissionScope: binding.admissionScope, + summary, + pageDigests, + proposalDigest, + proposedTimelineEntries: timeline, + proposedLinks: links, + unresolved, + }; + const manifestJson = canonicalProposalJson(manifest); + const manifestBytes = Buffer.byteLength(manifestJson, 'utf8'); + if (manifestBytes > PROPOSAL_MANIFEST_MAX_BYTES) { + throw new AgentJobProposalError( + 'manifest_too_large', + `Compact proposal manifest is ${manifestBytes} UTF-8 bytes; maximum is ${PROPOSAL_MANIFEST_MAX_BYTES}.`, + ); + } + + await tx.executeRaw( + `INSERT INTO agent_job_proposals + (job_id, owner_client_id, source_id, artifact_id, admission_scope, + total_pages, page_digests, plan, proposal_digest, manifest) + VALUES ($1, $2, $3, $4, $5, $6, $7::text::jsonb, $8::text::jsonb, $9, $10::text::jsonb) + ON CONFLICT (job_id) DO NOTHING`, + [ + jobId, binding.ownerClientId, binding.sourceId, binding.artifactId, + binding.admissionScope, totalPages, canonicalProposalJson(pageDigests), + planJson, proposalDigest, manifestJson, + ], + ); + const stored = await tx.executeRaw<{ + proposal_digest: string; + manifest: FinalizedProposalManifest; + }>( + `SELECT proposal_digest, manifest FROM agent_job_proposals WHERE job_id = $1`, + [jobId], + ); + if (!stored[0] || stored[0].proposal_digest !== proposalDigest) { + throw new AgentJobProposalError('conflicting_finalization', 'This job already has a different finalized proposal.'); + } + return stored[0].manifest; + }); +} + +/** Retrieve a frozen full plan by exact job owner and proposal digest. */ +export async function getOwnedAgentJobProposal( + engine: BrainEngine, + jobId: number, + ownerClientId: string, + proposalDigest: string, +): Promise<{ + id: number; + proposal_digest: string; + page_digests: ProposalPageDigest[]; + plan: ScopedAdmissionProposalPlan; +}> { + assertSafeJobId(jobId); + if (!ownerClientId) { + throw new AgentJobProposalError('permission_denied', 'An OAuth client owner is required.'); + } + if (!SHA256_RE.test(proposalDigest)) { + throw new AgentJobProposalError('invalid_digest', 'proposal_digest must be 64 lowercase hexadecimal characters.'); + } + const rows = await engine.executeRaw<{ + owner_client_id: string; + proposal_digest: string; + page_digests: ProposalPageDigest[]; + plan: ScopedAdmissionProposalPlan; + }>( + `SELECT owner_client_id, proposal_digest, page_digests, plan + FROM agent_job_proposals + WHERE job_id = $1 AND owner_client_id = $2 AND proposal_digest = $3`, + [jobId, ownerClientId, proposalDigest], + ); + if (!rows[0]) { + throw new AgentJobProposalError('permission_denied', 'Agent job proposal is not owned by this OAuth client or its digest does not match.'); + } + const plan = rows[0].plan; + if (digestProposalValue(plan) !== proposalDigest) { + throw new AgentJobProposalError('digest_mismatch', 'Stored proposal content does not match its digest.'); + } + const pageDigests = rows[0].page_digests; + if ( + !Array.isArray(pageDigests) + || pageDigests.length !== plan.proposedPages.length + || pageDigests.some((entry, index) => ( + entry.sequence !== index + 1 + || entry.slug !== plan.proposedPages[index]?.slug + || entry.digest !== digestProposalValue(plan.proposedPages[index]) + )) + ) { + throw new AgentJobProposalError('digest_mismatch', 'Stored proposal page manifest does not match the frozen plan.'); + } + return { + id: jobId, + proposal_digest: proposalDigest, + page_digests: pageDigests, + plan, + }; +} + +async function readJobBinding(engine: BrainEngine, jobId: number, lock: boolean): Promise { + const rows = await engine.executeRaw<{ + name: string; + owner_client_id: string | null; + source_id: string | null; + artifact_id: string | null; + admission_scope: string | null; + }>( + `SELECT name, + data->>'__owner_client_id' AS owner_client_id, + data->>'source_id' AS source_id, + data->>'proposal_artifact_id' AS artifact_id, + data->>'proposal_admission_scope' AS admission_scope + FROM minion_jobs + WHERE id = $1${lock ? ' FOR UPDATE' : ''}`, + [jobId], + ); + const row = rows[0]; + if ( + !row || row.name !== 'subagent' || !row.owner_client_id || !row.artifact_id + || !row.admission_scope + ) { + throw new AgentJobProposalError( + 'job_not_bound', + 'Agent job is not bound to an owner, proposal artifact, and admission scope.', + ); + } + const sourceId = row.source_id ?? 'default'; + assertValidSourceId(sourceId); + return { + ownerClientId: row.owner_client_id, + sourceId, + artifactId: row.artifact_id, + admissionScope: row.admission_scope, + }; +} + +function assertBindingMatches( + binding: JobBinding, + artifactId: unknown, + sourceId: unknown, + admissionScope: unknown, +): void { + if ( + artifactId !== binding.artifactId + || sourceId !== binding.sourceId + || admissionScope !== binding.admissionScope + ) { + throw new AgentJobProposalError( + 'binding_mismatch', + 'Proposal artifact, source, or admission scope does not match the submitted agent job.', + ); + } +} + +function parseProposalPage(raw: unknown): ScopedProposalPage { + const page = readRecord(raw, 'page'); + const effect = page.effect; + if (effect !== 'create' && effect !== 'update') { + throw new AgentJobProposalError('invalid_page', 'page.effect must be create or update.'); + } + const allowed = effect === 'create' + ? ['slug', 'effect', 'title', 'bodyMarkdown'] + : ['slug', 'effect', 'title', 'bodyMarkdown', 'baseMarkdown', 'expectedContentHash']; + assertExactKeys(page, allowed, 'page'); + const slug = readCanonicalSlug(page.slug, 'page.slug'); + const title = readBoundedString(page.title, 'page.title', 1_000); + const bodyMarkdown = readString(page.bodyMarkdown, 'page.bodyMarkdown'); + if (effect === 'create') return { slug, effect, title, bodyMarkdown }; + const baseMarkdown = readString(page.baseMarkdown, 'page.baseMarkdown'); + const expectedContentHash = readString(page.expectedContentHash, 'page.expectedContentHash'); + if (!SHA256_RE.test(expectedContentHash)) { + throw new AgentJobProposalError('invalid_page', 'page.expectedContentHash must be a lowercase SHA-256 digest.'); + } + return { slug, effect, title, bodyMarkdown, baseMarkdown, expectedContentHash }; +} + +function parseRequestedDigests(raw: unknown, totalPages: number): Array<{ sequence: number; digest: string }> { + if (!Array.isArray(raw) || raw.length !== totalPages) { + throw new AgentJobProposalError('invalid_manifest', 'page_digests must contain exactly total_pages entries.'); + } + const seen = new Set(); + return raw.map((entry, index) => { + const record = readRecord(entry, `page_digests[${index}]`); + assertExactKeys(record, ['sequence', 'digest'], `page_digests[${index}]`); + const sequence = readPositiveInteger(record.sequence, `page_digests[${index}].sequence`); + const digest = readString(record.digest, `page_digests[${index}].digest`); + if (!SHA256_RE.test(digest)) { + throw new AgentJobProposalError('invalid_manifest', `Invalid digest at sequence ${sequence}.`); + } + if (seen.has(sequence)) { + throw new AgentJobProposalError('duplicate_manifest_sequence', `Duplicate manifest sequence ${sequence}.`); + } + seen.add(sequence); + if (sequence !== index + 1) { + throw new AgentJobProposalError('fragment_gap', 'page_digests must be ordered and contiguous from 1.'); + } + return { sequence, digest }; + }); +} + +function parseTimelineEntries(raw: unknown): ScopedProposalTimelineEntry[] { + if (!Array.isArray(raw) || raw.length > 40) { + throw new AgentJobProposalError('invalid_timeline', 'proposed_timeline_entries must be an array of at most 40 entries.'); + } + return raw.map((entry, index) => { + const record = readRecord(entry, `proposed_timeline_entries[${index}]`); + const keys = Object.keys(record).sort(); + const validKeys = keys.every((key) => ['date', 'pageSlug', 'ref', 'refLabel', 'text'].includes(key)); + if (!validKeys || !['date', 'pageSlug', 'ref', 'text'].every((key) => key in record)) { + throw new AgentJobProposalError('invalid_timeline', `Invalid timeline entry at index ${index}.`); + } + const date = readString(record.date, 'timeline.date'); + const parsedDate = new Date(`${date}T00:00:00Z`); + if ( + !DATE_RE.test(date) + || Number.isNaN(parsedDate.getTime()) + || parsedDate.toISOString().slice(0, 10) !== date + ) { + throw new AgentJobProposalError('invalid_timeline', `Invalid timeline date ${date}.`); + } + const result: ScopedProposalTimelineEntry = { + pageSlug: readCanonicalSlug(record.pageSlug, 'timeline.pageSlug'), + date, + text: readBoundedString(record.text, 'timeline.text', 1_000), + ref: readCanonicalSlug(record.ref, 'timeline.ref'), + }; + if ('refLabel' in record) result.refLabel = readBoundedString(record.refLabel, 'timeline.refLabel', 1_000); + return result; + }); +} + +function parseLinks(raw: unknown): ScopedProposalLink[] { + if (!Array.isArray(raw) || raw.length > 40) { + throw new AgentJobProposalError('invalid_links', 'proposed_links must be an array of at most 40 entries.'); + } + return raw.map((entry, index) => { + const record = readRecord(entry, `proposed_links[${index}]`); + assertExactKeys(record, ['from', 'to', 'type'], `proposed_links[${index}]`); + return { + from: readCanonicalSlug(record.from, 'link.from'), + to: readCanonicalSlug(record.to, 'link.to'), + type: readBoundedString(record.type, 'link.type', 128), + }; + }); +} + +function parseUnresolved(raw: unknown): string[] { + if (!Array.isArray(raw) || raw.length > 40) { + throw new AgentJobProposalError('invalid_unresolved', 'unresolved must be an array of at most 40 strings.'); + } + return raw.map((entry, index) => readBoundedString(entry, `unresolved[${index}]`, 500)); +} + +function validatePlanRelations( + pageSlugs: ReadonlySet, + timeline: readonly ScopedProposalTimelineEntry[], + links: readonly ScopedProposalLink[], +): void { + // The shared plan contract guarantees referential integrity. Provider + // skills own stricter provenance rules such as capture-page-only refs. + for (const entry of timeline) { + if (!pageSlugs.has(entry.ref)) { + throw new AgentJobProposalError('invalid_timeline', 'Every timeline ref must name a proposed page.'); + } + } + for (const link of links) { + if (!pageSlugs.has(link.from)) { + throw new AgentJobProposalError('invalid_links', 'Every proposed link from slug must name a proposed page.'); + } + } +} + +function assertSafeJobId(jobId: number): void { + if (!Number.isSafeInteger(jobId) || jobId < 1) { + throw new AgentJobProposalError('invalid_job', 'Agent job id must be a positive safe integer.'); + } +} + +function readPositiveInteger(raw: unknown, name: string): number { + if (!Number.isSafeInteger(raw) || Number(raw) < 1 || Number(raw) > 1_000) { + throw new AgentJobProposalError('invalid_integer', `${name} must be an integer from 1 to 1000.`); + } + return Number(raw); +} + +function readRecord(raw: unknown, name: string): Record { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new AgentJobProposalError('invalid_object', `${name} must be an object.`); + } + return raw as Record; +} + +function readString(raw: unknown, name: string): string { + if (typeof raw !== 'string') { + throw new AgentJobProposalError('invalid_string', `${name} must be a string.`); + } + return raw; +} + +function readBoundedString(raw: unknown, name: string, maxLength: number): string { + const value = readString(raw, name).trim(); + if (!value || value.length > maxLength) { + throw new AgentJobProposalError('invalid_string', `${name} must contain 1-${maxLength} characters.`); + } + return value; +} + +function readCanonicalSlug(raw: unknown, name: string): string { + const slug = readString(raw, name); + if (slug.length > 255 || !PAGE_SLUG_RE.test(slug) || slug !== slug.toLowerCase()) { + throw new AgentJobProposalError('invalid_slug', `${name} is not a canonical page slug.`); + } + return slug; +} + +function assertExactKeys(record: Record, expected: string[], name: string): void { + const actual = Object.keys(record).sort(); + const wanted = [...expected].sort(); + if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index])) { + throw new AgentJobProposalError('invalid_keys', `${name} must contain exactly: ${expected.join(', ')}.`); + } +} diff --git a/src/core/minions/handlers/subagent.ts b/src/core/minions/handlers/subagent.ts index bb362d96f2..2c86e18e82 100644 --- a/src/core/minions/handlers/subagent.ts +++ b/src/core/minions/handlers/subagent.ts @@ -57,6 +57,7 @@ import { supportsReasoningEffort } from '../../ai/model-resolver.ts'; import { randomUUIDv7 } from 'bun'; import { isContextLimitMessage } from '../error-classify.ts'; import { ToolLoopContextProjectionError } from '../../ai/tool-loop-context.ts'; +import { assertProposalToolTurnPersistable } from '../agent-job-proposals.ts'; // ── Defaults ──────────────────────────────────────────────── @@ -400,6 +401,7 @@ export function makeSubagentHandler(deps: SubagentDeps) { }; } if (pendingToolUses.length > 0) { + assertProposalToolTurnPersistable(pendingToolUses); const synthesizedResults: ContentBlock[] = []; for (const use of pendingToolUses) { const prior = priorToolByUseId.get(use.id); @@ -654,6 +656,10 @@ export function makeSubagentHandler(deps: SubagentDeps) { const blocks = assistantMsg.content as ContentBlock[]; + // Enforce proposal page count and byte limits before the assistant turn + // makes its raw tool inputs durable in subagent_messages. + assertProposalToolTurnPersistable(blocks); + // 3. Persist the assistant message BEFORE tool dispatch so replay // sees a consistent state. const assistantIdx = nextMessageIdx++; @@ -1198,6 +1204,7 @@ async function reconcileGatewayReplay(args: ReconcileArgs): Promise => b.type === 'tool-call', ); + assertProposalToolTurnPersistable(toolCalls); if (toolCalls.length === 0) continue; // Skip if a following tool-result user turn exists AT ALL. A fully-answered diff --git a/src/core/minions/tools/brain-allowlist.ts b/src/core/minions/tools/brain-allowlist.ts index 5416524a58..5ae01bfe4a 100644 --- a/src/core/minions/tools/brain-allowlist.ts +++ b/src/core/minions/tools/brain-allowlist.ts @@ -66,6 +66,10 @@ export const BRAIN_TOOL_ALLOWLIST: ReadonlySet = new Set([ 'get_ingest_log', 'validate_links', 'get_active_schema_pack', + // Exact ingestion proposals are staged in a job-owned evidence ledger. + // Neither operation mutates corpus pages, links, takes, or timeline state. + 'stage_ingestion_proposal_page', + 'finalize_ingestion_proposal', 'put_page', // Scoped body-text edit (lore-1f3.3). Fenced exactly like put_page via // enforceSubagentSlugFence, and CAS-guarded by expected_content_hash so an @@ -126,6 +130,8 @@ export const BRAIN_TOOL_USAGE_HINTS: Readonly> = { get_ingest_log: 'Read the brain ingestion log for diagnostic / verification queries.', validate_links: 'Validate explicit references on one page after a repair. Read-only.', get_active_schema_pack: 'Read the active source schema-pack identity before a schema-sensitive repair.', + stage_ingestion_proposal_page: 'In ingestion propose mode, stage exactly one complete page proposal per agent turn before finalizing the compact manifest.', + finalize_ingestion_proposal: 'After every proposed page is staged, validate and freeze the ordered manifest before returning the staged_proposal receipt.', put_page: 'Write a markdown page to the gbrain DATABASE (NOT the local filesystem). Page becomes searchable + linkable. Slug must match the agent\'s allowed namespace.', replace_page_text: 'Replace an exact literal in one page\'s authored body. Use over put_page for targeted text edits; requires the page\'s current content_hash and exact match count from get_page. Slug must match the agent\'s allowed namespace.', suppress_claim: 'Record a user-refuted prose claim on an existing page without editing its prose. Slug must match the agent\'s allowed namespace.', diff --git a/src/core/operations.ts b/src/core/operations.ts index a5072952ad..c7afb88bec 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -5363,6 +5363,14 @@ const submit_agent: Operation = { type: 'string', description: 'Caller-stable key that returns the same agent job for retries by this OAuth client', }, + proposal_artifact_id: { + type: 'string', + description: 'Exact ingestion artifact id bound to staged proposal tools for this job', + }, + proposal_admission_scope: { + type: 'string', + description: 'Exact resolver admission scope bound to staged proposal tools for this job', + }, }, mutating: true, scope: 'agent' as any, @@ -5446,6 +5454,24 @@ const submit_agent: Operation = { 'submit_agent: reasoning_effort must be one of none, minimal, low, medium, high, xhigh.', ); } + const proposalArtifactId = typeof p.proposal_artifact_id === 'string' + ? p.proposal_artifact_id.trim() + : ''; + const proposalAdmissionScope = typeof p.proposal_admission_scope === 'string' + ? p.proposal_admission_scope.trim() + : ''; + if ((proposalArtifactId.length > 0) !== (proposalAdmissionScope.length > 0)) { + throw new OperationError( + 'invalid_params', + 'submit_agent: proposal_artifact_id and proposal_admission_scope must be supplied together.', + ); + } + if (proposalArtifactId.length > 255 || proposalAdmissionScope.length > 8_000) { + throw new OperationError( + 'invalid_params', + 'submit_agent: proposal artifact id or admission scope exceeds its bounded length.', + ); + } if (p.reasoning_effort !== undefined && typeof p.model !== 'string') { throw new OperationError( 'invalid_params', @@ -5519,6 +5545,8 @@ const submit_agent: Operation = { model: typeof p.model === 'string' ? p.model : '', reasoning_effort: isReasoningEffort(p.reasoning_effort) ? p.reasoning_effort : null, skill_name: typeof p.skill_name === 'string' ? p.skill_name : null, + proposal_artifact_id: proposalArtifactId || null, + proposal_admission_scope: proposalAdmissionScope || null, }; } @@ -5557,6 +5585,10 @@ const submit_agent: Operation = { if (typeof p.model === 'string') jobData.model = p.model; if (isReasoningEffort(p.reasoning_effort)) jobData.reasoning_effort = p.reasoning_effort; if (boundSource) jobData.source_id = boundSource; + if (proposalArtifactId) { + jobData.proposal_artifact_id = proposalArtifactId; + jobData.proposal_admission_scope = proposalAdmissionScope; + } const job = await queue.add( 'subagent', jobData, @@ -5593,6 +5625,70 @@ const submit_agent: Operation = { }, }; +const stage_ingestion_proposal_page: Operation = { + name: 'stage_ingestion_proposal_page', + description: 'Stage one exact page body and optimistic baseline in the current ingestion agent job. This writes job evidence only and never mutates corpus state.', + // Operation mutability classifies corpus effects; durable job evidence is + // intentionally excluded from corpus-write audit and retry semantics. + mutating: false, + params: { + artifact_id: { type: 'string', required: true, description: 'Exact artifact id bound at submit_agent time' }, + source_id: { type: 'string', required: true, description: 'Exact source bound to the current agent job' }, + admission_scope: { type: 'string', required: true, description: 'Exact admission scope bound at submit_agent time' }, + sequence: { type: 'number', required: true, description: 'One-based page position' }, + total_pages: { type: 'number', required: true, description: 'Total number of pages in the proposal' }, + page: { type: 'object', required: true, description: 'Exact create or update page proposal' }, + }, + scope: 'agent', + handler: async (ctx, p) => { + if (ctx.viaSubagent !== true || ctx.jobId === undefined) { + throw new OperationError('permission_denied', 'stage_ingestion_proposal_page is available only inside an agent job.'); + } + const proposal = await import('./minions/agent-job-proposals.ts'); + try { + return await proposal.stageAgentJobProposalPage(ctx.engine, ctx.jobId, p as any); + } catch (error) { + if (error instanceof proposal.AgentJobProposalError) { + throw new OperationError('invalid_params', error.message); + } + throw error; + } + }, +}; + +const finalize_ingestion_proposal: Operation = { + name: 'finalize_ingestion_proposal', + description: 'Validate and freeze the complete staged ingestion proposal, returning a compact digest manifest without page bodies.', + // Freezing job evidence does not mutate pages, links, takes, or timelines. + mutating: false, + params: { + artifact_id: { type: 'string', required: true, description: 'Exact artifact id bound at submit_agent time' }, + source_id: { type: 'string', required: true, description: 'Exact source bound to the current agent job' }, + admission_scope: { type: 'string', required: true, description: 'Exact admission scope bound at submit_agent time' }, + total_pages: { type: 'number', required: true, description: 'Total number of staged pages' }, + page_digests: { type: 'array', required: true, items: { type: 'object' }, description: 'Ordered one-based sequence and digest pairs' }, + summary: { type: 'string', required: true, description: 'Compact source-grounded proposal summary' }, + proposed_timeline_entries: { type: 'array', items: { type: 'object' }, description: 'Bounded frozen timeline entries' }, + proposed_links: { type: 'array', items: { type: 'object' }, description: 'Bounded frozen typed links' }, + unresolved: { type: 'array', items: { type: 'string' }, description: 'Bounded unresolved items' }, + }, + scope: 'agent', + handler: async (ctx, p) => { + if (ctx.viaSubagent !== true || ctx.jobId === undefined) { + throw new OperationError('permission_denied', 'finalize_ingestion_proposal is available only inside an agent job.'); + } + const proposal = await import('./minions/agent-job-proposals.ts'); + try { + return await proposal.finalizeAgentJobProposal(ctx.engine, ctx.jobId, p as any); + } catch (error) { + if (error instanceof proposal.AgentJobProposalError) { + throw new OperationError('invalid_params', error.message); + } + throw error; + } + }, +}; + const get_agent_job: Operation = { name: 'get_agent_job', description: 'Get status and the structured result for a submit_agent job owned by this OAuth client.', @@ -5650,6 +5746,37 @@ const get_agent_job: Operation = { }, }; +const get_agent_job_proposal: Operation = { + name: 'get_agent_job_proposal', + description: 'Retrieve the complete frozen ingestion proposal for an owned agent job and exact proposal digest.', + params: { + id: { type: 'number', required: true, description: 'Agent job id returned by submit_agent', trace: { kind: 'job' } }, + proposal_digest: { type: 'string', required: true, description: 'Exact proposal digest from the compact staged_proposal receipt', trace: { kind: 'proposal' } }, + }, + scope: 'agent', + handler: async (ctx, p) => { + const clientId = ctx.auth?.clientId; + if (!clientId) { + throw new OperationError('permission_denied', 'get_agent_job_proposal requires an OAuth client.'); + } + const proposal = await import('./minions/agent-job-proposals.ts'); + try { + return await proposal.getOwnedAgentJobProposal( + ctx.engine, + p.id as number, + clientId, + p.proposal_digest as string, + ); + } catch (error) { + if (error instanceof proposal.AgentJobProposalError) { + const code = error.code === 'permission_denied' ? 'permission_denied' : 'invalid_params'; + throw new OperationError(code, error.message); + } + throw error; + } + }, +}; + const get_agent_job_execution_evidence: Operation = { name: 'get_agent_job_execution_evidence', description: @@ -8560,7 +8687,8 @@ export const operations: Operation[] = [ submit_job, get_job, list_jobs, cancel_job, retry_job, get_job_progress, pause_job, resume_job, replay_job, send_job_message, // v0.38 Slice 3: remote-callable agent dispatch with OAuth-bound trust boundary - submit_agent, get_agent_job, get_agent_job_execution_evidence, + submit_agent, stage_ingestion_proposal_page, finalize_ingestion_proposal, + get_agent_job, get_agent_job_proposal, get_agent_job_execution_evidence, // Orphans find_orphans, // v0.36.1.0 (T7) — Hindsight calibration wave: read profile via MCP diff --git a/src/core/pglite-schema.ts b/src/core/pglite-schema.ts index 97b1b3b2ef..d18ae492ab 100644 --- a/src/core/pglite-schema.ts +++ b/src/core/pglite-schema.ts @@ -584,6 +584,46 @@ CREATE TABLE IF NOT EXISTS subagent_tool_executions ( ); CREATE INDEX IF NOT EXISTS idx_subagent_tools_job ON subagent_tool_executions (job_id, status); +-- Exact ingestion proposal bodies live outside the bounded agent receipt. +-- These rows are job-owned evidence only; they never mutate corpus state. +CREATE TABLE IF NOT EXISTS agent_job_proposal_fragments ( + job_id BIGINT NOT NULL REFERENCES minion_jobs(id) ON DELETE CASCADE, + owner_client_id TEXT NOT NULL, + source_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + admission_scope TEXT NOT NULL, + sequence INTEGER NOT NULL, + total_pages INTEGER NOT NULL, + page JSONB NOT NULL, + page_digest TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (job_id, sequence), + CONSTRAINT chk_agent_job_proposal_fragment_sequence + CHECK (sequence >= 1 AND total_pages >= sequence), + CONSTRAINT chk_agent_job_proposal_fragment_digest + CHECK (page_digest ~ '^[a-f0-9]{64}$') +); +CREATE INDEX IF NOT EXISTS idx_agent_job_proposal_fragments_owner + ON agent_job_proposal_fragments (owner_client_id, job_id); + +CREATE TABLE IF NOT EXISTS agent_job_proposals ( + job_id BIGINT PRIMARY KEY REFERENCES minion_jobs(id) ON DELETE CASCADE, + owner_client_id TEXT NOT NULL, + source_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + admission_scope TEXT NOT NULL, + total_pages INTEGER NOT NULL CHECK (total_pages >= 1), + page_digests JSONB NOT NULL, + plan JSONB NOT NULL, + proposal_digest TEXT NOT NULL, + manifest JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chk_agent_job_proposal_digest + CHECK (proposal_digest ~ '^[a-f0-9]{64}$') +); +CREATE INDEX IF NOT EXISTS idx_agent_job_proposals_owner_digest + ON agent_job_proposals (owner_client_id, proposal_digest); + CREATE TABLE IF NOT EXISTS subagent_rate_leases ( id BIGSERIAL PRIMARY KEY, key TEXT NOT NULL, diff --git a/src/core/schema-embedded.ts b/src/core/schema-embedded.ts index f9a384cded..b1143ead66 100644 --- a/src/core/schema-embedded.ts +++ b/src/core/schema-embedded.ts @@ -1068,6 +1068,46 @@ CREATE TABLE IF NOT EXISTS subagent_tool_executions ( ); CREATE INDEX IF NOT EXISTS idx_subagent_tools_job ON subagent_tool_executions (job_id, status); +-- Exact ingestion proposal bodies live outside the bounded agent receipt. +-- These rows are job-owned evidence only; they never mutate corpus state. +CREATE TABLE IF NOT EXISTS agent_job_proposal_fragments ( + job_id BIGINT NOT NULL REFERENCES minion_jobs(id) ON DELETE CASCADE, + owner_client_id TEXT NOT NULL, + source_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + admission_scope TEXT NOT NULL, + sequence INTEGER NOT NULL, + total_pages INTEGER NOT NULL, + page JSONB NOT NULL, + page_digest TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (job_id, sequence), + CONSTRAINT chk_agent_job_proposal_fragment_sequence + CHECK (sequence >= 1 AND total_pages >= sequence), + CONSTRAINT chk_agent_job_proposal_fragment_digest + CHECK (page_digest ~ '^[a-f0-9]{64}\$') +); +CREATE INDEX IF NOT EXISTS idx_agent_job_proposal_fragments_owner + ON agent_job_proposal_fragments (owner_client_id, job_id); + +CREATE TABLE IF NOT EXISTS agent_job_proposals ( + job_id BIGINT PRIMARY KEY REFERENCES minion_jobs(id) ON DELETE CASCADE, + owner_client_id TEXT NOT NULL, + source_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + admission_scope TEXT NOT NULL, + total_pages INTEGER NOT NULL CHECK (total_pages >= 1), + page_digests JSONB NOT NULL, + plan JSONB NOT NULL, + proposal_digest TEXT NOT NULL, + manifest JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chk_agent_job_proposal_digest + CHECK (proposal_digest ~ '^[a-f0-9]{64}\$') +); +CREATE INDEX IF NOT EXISTS idx_agent_job_proposals_owner_digest + ON agent_job_proposals (owner_client_id, proposal_digest); + -- Rate-lease table — concurrency cap on outbound providers (e.g. -- anthropic:messages). Acquire: INSERT if active < max_concurrent under -- advisory lock. Release: DELETE. Stale leases (expires_at past) auto-prune @@ -1514,6 +1554,8 @@ BEGIN ALTER TABLE minion_attachments ENABLE ROW LEVEL SECURITY; ALTER TABLE subagent_messages ENABLE ROW LEVEL SECURITY; ALTER TABLE subagent_tool_executions ENABLE ROW LEVEL SECURITY; + ALTER TABLE agent_job_proposal_fragments ENABLE ROW LEVEL SECURITY; + ALTER TABLE agent_job_proposals ENABLE ROW LEVEL SECURITY; ALTER TABLE subagent_rate_leases ENABLE ROW LEVEL SECURITY; ALTER TABLE gbrain_cycle_locks ENABLE ROW LEVEL SECURITY; ALTER TABLE dream_verdicts ENABLE ROW LEVEL SECURITY; diff --git a/src/schema.sql b/src/schema.sql index 4251880b8c..99ac4e386f 100644 --- a/src/schema.sql +++ b/src/schema.sql @@ -1064,6 +1064,46 @@ CREATE TABLE IF NOT EXISTS subagent_tool_executions ( ); CREATE INDEX IF NOT EXISTS idx_subagent_tools_job ON subagent_tool_executions (job_id, status); +-- Exact ingestion proposal bodies live outside the bounded agent receipt. +-- These rows are job-owned evidence only; they never mutate corpus state. +CREATE TABLE IF NOT EXISTS agent_job_proposal_fragments ( + job_id BIGINT NOT NULL REFERENCES minion_jobs(id) ON DELETE CASCADE, + owner_client_id TEXT NOT NULL, + source_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + admission_scope TEXT NOT NULL, + sequence INTEGER NOT NULL, + total_pages INTEGER NOT NULL, + page JSONB NOT NULL, + page_digest TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (job_id, sequence), + CONSTRAINT chk_agent_job_proposal_fragment_sequence + CHECK (sequence >= 1 AND total_pages >= sequence), + CONSTRAINT chk_agent_job_proposal_fragment_digest + CHECK (page_digest ~ '^[a-f0-9]{64}$') +); +CREATE INDEX IF NOT EXISTS idx_agent_job_proposal_fragments_owner + ON agent_job_proposal_fragments (owner_client_id, job_id); + +CREATE TABLE IF NOT EXISTS agent_job_proposals ( + job_id BIGINT PRIMARY KEY REFERENCES minion_jobs(id) ON DELETE CASCADE, + owner_client_id TEXT NOT NULL, + source_id TEXT NOT NULL, + artifact_id TEXT NOT NULL, + admission_scope TEXT NOT NULL, + total_pages INTEGER NOT NULL CHECK (total_pages >= 1), + page_digests JSONB NOT NULL, + plan JSONB NOT NULL, + proposal_digest TEXT NOT NULL, + manifest JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chk_agent_job_proposal_digest + CHECK (proposal_digest ~ '^[a-f0-9]{64}$') +); +CREATE INDEX IF NOT EXISTS idx_agent_job_proposals_owner_digest + ON agent_job_proposals (owner_client_id, proposal_digest); + -- Rate-lease table — concurrency cap on outbound providers (e.g. -- anthropic:messages). Acquire: INSERT if active < max_concurrent under -- advisory lock. Release: DELETE. Stale leases (expires_at past) auto-prune @@ -1510,6 +1550,8 @@ BEGIN ALTER TABLE minion_attachments ENABLE ROW LEVEL SECURITY; ALTER TABLE subagent_messages ENABLE ROW LEVEL SECURITY; ALTER TABLE subagent_tool_executions ENABLE ROW LEVEL SECURITY; + ALTER TABLE agent_job_proposal_fragments ENABLE ROW LEVEL SECURITY; + ALTER TABLE agent_job_proposals ENABLE ROW LEVEL SECURITY; ALTER TABLE subagent_rate_leases ENABLE ROW LEVEL SECURITY; ALTER TABLE gbrain_cycle_locks ENABLE ROW LEVEL SECURITY; ALTER TABLE dream_verdicts ENABLE ROW LEVEL SECURITY; diff --git a/test/agent-job-proposals.test.ts b/test/agent-job-proposals.test.ts new file mode 100644 index 0000000000..0ee1553505 --- /dev/null +++ b/test/agent-job-proposals.test.ts @@ -0,0 +1,302 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { resetPgliteState } from './helpers/reset-pglite.ts'; +import { + AgentJobProposalError, + PROPOSAL_AGGREGATE_MAX_BYTES, + PROPOSAL_STAGE_INPUT_MAX_BYTES, + assertProposalToolTurnPersistable, + digestProposalValue, + finalizeAgentJobProposal, + getOwnedAgentJobProposal, + stageAgentJobProposalPage, + type ScopedProposalPage, +} from '../src/core/minions/agent-job-proposals.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await resetPgliteState(engine); +}); + +async function seedJob(overrides: Record = {}): Promise { + const rows = await engine.executeRaw<{ id: number }>( + `INSERT INTO minion_jobs (name, status, data, queue, priority, created_at) + VALUES ('subagent', 'active', $1::text::jsonb, 'default', 0, now()) + RETURNING id`, + [JSON.stringify({ + __owner_client_id: 'lore-client', + source_id: 'company', + proposal_artifact_id: 'artifact-1', + proposal_admission_scope: 'Include project delivery notes.', + ...overrides, + })], + ); + return Number(rows[0]!.id); +} + +function createPage(slug: string, bodyMarkdown = '# Page'): ScopedProposalPage { + return { slug, effect: 'create', title: 'Page', bodyMarkdown }; +} + +function updatePage(slug: string, suffix = ''): ScopedProposalPage { + return { + slug, + effect: 'update', + title: 'Page', + bodyMarkdown: `# Updated${suffix}`, + baseMarkdown: '# Existing', + expectedContentHash: 'a'.repeat(64), + }; +} + +async function stage( + jobId: number, + sequence: number, + totalPages: number, + page: ScopedProposalPage, +) { + return stageAgentJobProposalPage(engine, jobId, { + artifact_id: 'artifact-1', + source_id: 'company', + admission_scope: 'Include project delivery notes.', + sequence, + total_pages: totalPages, + page, + }); +} + +describe('proposal turn pre-persistence boundary', () => { + it('rejects oversized stage input and multiple stages before persistence callbacks run', () => { + const oversized = { + type: 'tool-call', + toolName: 'brain_stage_ingestion_proposal_page', + input: { body: 'x'.repeat(PROPOSAL_STAGE_INPUT_MAX_BYTES) }, + }; + expect(() => assertProposalToolTurnPersistable([oversized])).toThrow(/maximum/i); + expect(() => assertProposalToolTurnPersistable([ + { type: 'tool-call', toolName: 'brain_stage_ingestion_proposal_page', input: { page: 1 } }, + { type: 'tool-call', toolName: 'brain_stage_ingestion_proposal_page', input: { page: 2 } }, + ])).toThrow(/exactly one/i); + }); + + it('rejects a turn that mixes page staging with finalization', () => { + expect(() => assertProposalToolTurnPersistable([ + { type: 'tool_use', name: 'brain_stage_ingestion_proposal_page', input: { page: 1 } }, + { type: 'tool_use', name: 'brain_finalize_ingestion_proposal', input: { total_pages: 1 } }, + ])).toThrow(/separate agent turns/i); + }); +}); + +describe('durable agent-job proposal staging', () => { + it('stages exact pages, finalizes an ordered manifest, and retrieves the full owned plan', async () => { + const jobId = await seedJob(); + const first = await stage(jobId, 1, 2, createPage('sources/example')); + const second = await stage(jobId, 2, 2, updatePage('projects/example')); + + const manifest = await finalizeAgentJobProposal(engine, jobId, { + artifact_id: 'artifact-1', + source_id: 'company', + admission_scope: 'Include project delivery notes.', + total_pages: 2, + page_digests: [ + { sequence: 1, digest: first.digest }, + { sequence: 2, digest: second.digest }, + ], + summary: 'Two exact pages are ready for review.', + proposed_timeline_entries: [{ + pageSlug: 'projects/existing', + date: '2026-08-07', + text: 'Delivery review completed.', + ref: 'sources/example', + }], + proposed_links: [{ from: 'sources/example', to: 'projects/example', type: 'documents' }], + unresolved: [], + }); + + expect(manifest.pageDigests).toEqual([first, second]); + expect(manifest.status).toBe('staged_proposal'); + expect(manifest.proposalDigest).toMatch(/^[a-f0-9]{64}$/); + const owned = await getOwnedAgentJobProposal( + engine, + jobId, + 'lore-client', + manifest.proposalDigest, + ); + expect(owned.page_digests).toEqual(manifest.pageDigests); + expect(owned.plan.proposedPages).toEqual([ + createPage('sources/example'), + updatePage('projects/example'), + ]); + expect(owned.plan.proposedTimelineEntries[0]!.pageSlug).toBe('projects/existing'); + expect(digestProposalValue(owned.plan)).toBe(manifest.proposalDigest); + }); + + it('makes identical staging and finalization replay-safe but rejects conflicts', async () => { + const jobId = await seedJob(); + const page = createPage('sources/example'); + const first = await stage(jobId, 1, 1, page); + expect(await stage(jobId, 1, 1, page)).toEqual(first); + await expect(stage(jobId, 1, 1, createPage('sources/example', '# Different'))) + .rejects.toMatchObject({ code: 'conflicting_fragment' }); + + const input = { + artifact_id: 'artifact-1', + source_id: 'company', + admission_scope: 'Include project delivery notes.', + total_pages: 1, + page_digests: [{ sequence: 1, digest: first.digest }], + summary: 'Ready.', + }; + const finalized = await finalizeAgentJobProposal(engine, jobId, input); + expect(await finalizeAgentJobProposal(engine, jobId, input)).toEqual(finalized); + await expect(finalizeAgentJobProposal(engine, jobId, { ...input, summary: 'Changed.' })) + .rejects.toMatchObject({ code: 'conflicting_finalization' }); + }); + + it('allows different jobs owned by the same client to finalize an identical proposal', async () => { + const firstJobId = await seedJob(); + const secondJobId = await seedJob(); + const page = createPage('sources/example'); + const firstPage = await stage(firstJobId, 1, 1, page); + const secondPage = await stage(secondJobId, 1, 1, page); + const input = { + artifact_id: 'artifact-1', + source_id: 'company', + admission_scope: 'Include project delivery notes.', + total_pages: 1, + page_digests: [{ sequence: 1, digest: firstPage.digest }], + summary: 'Ready.', + }; + + const first = await finalizeAgentJobProposal(engine, firstJobId, input); + const second = await finalizeAgentJobProposal(engine, secondJobId, { + ...input, + page_digests: [{ sequence: 1, digest: secondPage.digest }], + }); + + expect(second.proposalDigest).toBe(first.proposalDigest); + const rows = await engine.executeRaw<{ count: string }>( + `SELECT count(*)::text AS count + FROM agent_job_proposals + WHERE owner_client_id = 'lore-client' AND proposal_digest = $1`, + [first.proposalDigest], + ); + expect(Number(rows[0]!.count)).toBe(2); + }); + + it('rejects gaps, duplicate manifest sequences, digest mismatches, and duplicate page slugs', async () => { + const gapJob = await seedJob(); + const gap = await stage(gapJob, 1, 2, createPage('sources/gap')); + await expect(finalizeAgentJobProposal(engine, gapJob, { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', + total_pages: 2, page_digests: [{ sequence: 1, digest: gap.digest }, { sequence: 2, digest: gap.digest }], summary: 'Gap.', + })).rejects.toMatchObject({ code: 'fragment_gap' }); + + const duplicateManifestJob = await seedJob(); + const one = await stage(duplicateManifestJob, 1, 2, createPage('sources/one')); + const two = await stage(duplicateManifestJob, 2, 2, createPage('sources/two')); + await expect(finalizeAgentJobProposal(engine, duplicateManifestJob, { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', + total_pages: 2, page_digests: [{ sequence: 1, digest: one.digest }, { sequence: 1, digest: two.digest }], summary: 'Duplicate.', + })).rejects.toBeInstanceOf(AgentJobProposalError); + await expect(finalizeAgentJobProposal(engine, duplicateManifestJob, { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', + total_pages: 2, page_digests: [{ sequence: 1, digest: one.digest }, { sequence: 2, digest: 'f'.repeat(64) }], summary: 'Mismatch.', + })).rejects.toMatchObject({ code: 'digest_mismatch' }); + + const duplicatePageJob = await seedJob(); + const a = await stage(duplicatePageJob, 1, 2, createPage('sources/same')); + const b = await stage(duplicatePageJob, 2, 2, updatePage('sources/same')); + await expect(finalizeAgentJobProposal(engine, duplicatePageJob, { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', + total_pages: 2, page_digests: [{ sequence: 1, digest: a.digest }, { sequence: 2, digest: b.digest }], summary: 'Duplicate page.', + })).rejects.toMatchObject({ code: 'duplicate_page' }); + }); + + it('rejects calendar-normalized dates instead of accepting them as strict dates', async () => { + const jobId = await seedJob(); + const page = await stage(jobId, 1, 1, createPage('sources/example')); + await expect(finalizeAgentJobProposal(engine, jobId, { + artifact_id: 'artifact-1', + source_id: 'company', + admission_scope: 'Include project delivery notes.', + total_pages: 1, + page_digests: [{ sequence: 1, digest: page.digest }], + summary: 'Invalid date.', + proposed_timeline_entries: [{ + pageSlug: 'sources/example', + date: '2026-02-31', + text: 'Impossible date.', + ref: 'sources/example', + }], + })).rejects.toMatchObject({ code: 'invalid_timeline' }); + }); + + it('rejects job, owner, source, artifact, scope, and stored-fragment binding mismatches', async () => { + const jobId = await seedJob(); + await expect(stageAgentJobProposalPage(engine, jobId, { + artifact_id: 'other', source_id: 'company', admission_scope: 'Include project delivery notes.', + sequence: 1, total_pages: 1, page: createPage('sources/example'), + })).rejects.toMatchObject({ code: 'binding_mismatch' }); + + const staged = await stage(jobId, 1, 1, createPage('sources/example')); + await engine.executeRaw( + `UPDATE agent_job_proposal_fragments SET source_id = 'other' WHERE job_id = $1`, + [jobId], + ); + await expect(finalizeAgentJobProposal(engine, jobId, { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', + total_pages: 1, page_digests: [{ sequence: 1, digest: staged.digest }], summary: 'Ready.', + })).rejects.toMatchObject({ code: 'binding_mismatch' }); + + const unbound = await seedJob({ proposal_artifact_id: null }); + await expect(stage(unbound, 1, 1, createPage('sources/unbound'))) + .rejects.toMatchObject({ code: 'job_not_bound' }); + }); + + it('requires exact owner and proposal digest on retrieval', async () => { + const jobId = await seedJob(); + const page = await stage(jobId, 1, 1, createPage('sources/example')); + const manifest = await finalizeAgentJobProposal(engine, jobId, { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', + total_pages: 1, page_digests: [{ sequence: 1, digest: page.digest }], summary: 'Ready.', + }); + await expect(getOwnedAgentJobProposal(engine, jobId, 'other', manifest.proposalDigest)) + .rejects.toMatchObject({ code: 'permission_denied' }); + await expect(getOwnedAgentJobProposal(engine, jobId, 'lore-client', 'f'.repeat(64))) + .rejects.toMatchObject({ code: 'permission_denied' }); + }); + + it('rejects an aggregate plan over the explicit ceiling without finalizing it', async () => { + const jobId = await seedJob(); + const pageCount = 5; + const digests = []; + for (let sequence = 1; sequence <= pageCount; sequence++) { + const page = createPage(`sources/large-${sequence}`, 'x'.repeat(165_000)); + digests.push(await stage(jobId, sequence, pageCount, page)); + } + await expect(finalizeAgentJobProposal(engine, jobId, { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', + total_pages: pageCount, + page_digests: digests.map(({ sequence, digest }) => ({ sequence, digest })), + summary: 'Too large.', + })).rejects.toMatchObject({ code: 'proposal_too_large' }); + const rows = await engine.executeRaw<{ count: string }>( + `SELECT count(*)::text AS count FROM agent_job_proposals WHERE job_id = $1`, + [jobId], + ); + expect(Number(rows[0]!.count)).toBe(0); + expect(PROPOSAL_AGGREGATE_MAX_BYTES).toBe(786_432); + }); +}); diff --git a/test/e2e/subagent-gateway-path.test.ts b/test/e2e/subagent-gateway-path.test.ts index 89db781046..c524061ffb 100644 --- a/test/e2e/subagent-gateway-path.test.ts +++ b/test/e2e/subagent-gateway-path.test.ts @@ -171,6 +171,90 @@ function buildHandler(toolRegistry: ToolDef[]) { describe('runSubagentViaGateway (v0.38 Slice 1 — full handler path through gateway.toolLoop)', () => { afterAll(() => clearGateway()); + it('rejects an oversized staged page before gateway assistant or tool persistence', async () => { + __setChatTransportForTests(async () => ({ + text: '', + blocks: [{ + type: 'tool-call', + toolCallId: 'stage-too-large', + toolName: 'brain_stage_ingestion_proposal_page', + input: { page: { bodyMarkdown: 'x'.repeat(196_608) } }, + }] as ChatBlock[], + stopReason: 'tool_calls', + usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, + model: 'anthropic:claude-sonnet-4-6', + providerId: 'anthropic', + } satisfies ChatResult)); + const tool: ToolDef = { + name: 'brain_stage_ingestion_proposal_page', + description: 'stage', + input_schema: { type: 'object' }, + idempotent: true, + mutating: false, + async execute() { return { ok: true }; }, + }; + const handler = buildHandler([tool]); + const { jobId, ctx } = await makeFakeJob({ prompt: 'stage', model: 'anthropic:claude-sonnet-4-6' }); + + await expect(handler(ctx)).rejects.toThrow(/maximum/i); + const messages = await engine.executeRaw<{ role: string }>( + `SELECT role FROM subagent_messages WHERE job_id = $1 ORDER BY message_idx`, + [jobId], + ); + const executions = await engine.executeRaw( + `SELECT id FROM subagent_tool_executions WHERE job_id = $1`, + [jobId], + ); + expect(messages.map((row) => row.role)).toEqual(['user']); + expect(executions).toHaveLength(0); + }); + + it('rejects two staged pages before gateway assistant or tool persistence', async () => { + __setChatTransportForTests(async () => ({ + text: '', + blocks: [ + { + type: 'tool-call', + toolCallId: 'stage-1', + toolName: 'brain_stage_ingestion_proposal_page', + input: { page: 1 }, + }, + { + type: 'tool-call', + toolCallId: 'stage-2', + toolName: 'brain_stage_ingestion_proposal_page', + input: { page: 2 }, + }, + ] as ChatBlock[], + stopReason: 'tool_calls', + usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, + model: 'anthropic:claude-sonnet-4-6', + providerId: 'anthropic', + } satisfies ChatResult)); + const tool: ToolDef = { + name: 'brain_stage_ingestion_proposal_page', + description: 'stage', + input_schema: { type: 'object' }, + idempotent: true, + mutating: false, + async execute() { return { ok: true }; }, + }; + const handler = buildHandler([tool]); + const { jobId, ctx } = await makeFakeJob({ prompt: 'stage twice', model: 'anthropic:claude-sonnet-4-6' }); + + await expect(handler(ctx)).rejects.toThrow(/exactly one/i); + const messages = await engine.executeRaw<{ role: string }>( + `SELECT role FROM subagent_messages WHERE job_id = $1 ORDER BY message_idx`, + [jobId], + ); + const executions = await engine.executeRaw( + `SELECT id FROM subagent_tool_executions WHERE job_id = $1`, + [jobId], + ); + expect(messages.map((row) => row.role)).toEqual(['user']); + expect(executions).toHaveLength(0); + }); + it('happy path 1-turn: gateway returns text, handler returns SubagentResult', async () => { __setChatTransportForTests(async () => ({ text: 'all done', diff --git a/test/github-project-ingestion-skill.test.ts b/test/github-project-ingestion-skill.test.ts index 8cf77ccbd4..eb3e9e2398 100644 --- a/test/github-project-ingestion-skill.test.ts +++ b/test/github-project-ingestion-skill.test.ts @@ -21,6 +21,8 @@ const expectedTools = [ 'resolve_slugs', 'get_links', 'get_backlinks', + 'stage_ingestion_proposal_page', + 'finalize_ingestion_proposal', 'put_page', 'add_link', 'add_timeline_entry', @@ -205,20 +207,24 @@ describe('github-project-ingestion skill', () => { test('supports a zero-mutation scoped proposal with a bounded complete plan', () => { expect(skill).toContain('mode: '); expect(skill).toContain('admissionScope: '); - expect(skill).toContain('In `propose` mode, do not call any mutating tool'); + expect(skill).toContain('In `propose` mode, do not call any corpus-mutating tool'); expect(skill).toMatch(/`put_page`,\s+`add_link`, or `add_timeline_entry`/); expect(skill).toContain('complete set of pages that `apply` will write'); expect(skill).toContain('full intended `bodyMarkdown`, never a diff'); expect(skill).toContain('262,144 UTF-8 bytes'); expect(skill).toMatch(/[Rr]eturn `failed`\s+with an operational summary/); - expect(skill).toContain('Never truncate or split a proposal'); + expect(skill).toMatch(/Never truncate or split a\s+proposal/); expect(skill).toMatch(/must not\s+name or describe the excluded material/); expect(skill).toMatch(/brain-facing provenance and synthesis record/); - expect(skill).toContain('"status": "scoped_proposal"'); + expect(skill).toContain('"status": "staged_proposal"'); + expect(skill).toContain('"pageDigests": ['); + expect(skill).toContain('"proposalDigest": "64 lowercase hex characters"'); + expect(skill).toContain('brain_stage_ingestion_proposal_page'); + expect(skill).toContain('Stage only one page per turn'); + expect(skill).toContain('brain_finalize_ingestion_proposal'); expect(skill).toContain('"effect": "create | update"'); - expect(skill).toContain('"bodyMarkdown": "complete intended page body"'); - expect(skill).toContain('"baseMarkdown": "exact reviewed page body for updates, null for creates"'); - expect(skill).toContain('"expectedContentHash": "exact get_page content_hash for updates, null for creates"'); + expect(skill).toContain('Create entries have'); + expect(skill).toMatch(/Update entries add\s+exactly `baseMarkdown` and `expectedContentHash`/); expect(skill).toContain('Omit both fields for a create'); }); @@ -242,7 +248,7 @@ describe('github-project-ingestion skill', () => { /typed links only for\s+relationships that the planned Markdown does not express accurately/, ); expect(skill).toMatch( - /receipt, including `proposedTimelineEntries` and\s+`proposedLinks`, must not exceed 262,144 UTF-8 bytes/, + /compact manifest over 262,144 UTF-8 bytes/, ); expect(skill).toContain('"proposedTimelineEntries": ['); expect(skill).toContain('"refLabel": "pull request capture"'); @@ -271,12 +277,38 @@ describe('github-project-ingestion skill', () => { test('splits partial disqualification from classed needs-attention outcomes', () => { expect(skill).toContain('partial disqualification'); expect(skill).toMatch(/[Dd]erive `admissionScope` only from the resolver/); - expect(skill).toContain('return `scoped_proposal` directly'); + expect(skill).toContain('return `staged_proposal` directly'); expect(skill).toMatch(/Do not mutate before returning that\s+proposal/); expect(skill).toContain('"reason_class": "resolver_ambiguity | operational"'); expect(skill).toMatch(/Every `needs_attention`\s+receipt must include `reason_class`/); }); + test('verifies Lore artifact integrity without treating context projection as loss', () => { + expect(skill).toContain('artifactIntegrity: # required normal/propose; omitted apply'); + expect(skill).toContain('complete: true'); + expect(skill).toContain('manifest: { sha256: <64 lowercase hex characters>, bytes: }'); + expect(skill).toMatch(/`artifactIntegrity\.complete` is not exactly `true`/); + expect(skill).toMatch(/Apply mode intentionally\s+omits `artifactIntegrity`/); + expect(skill).toMatch(/exactly 64 lowercase hexadecimal characters/); + expect(skill).toMatch(/authenticated OAuth caller deterministically\s+verified these values/); + expect(skill).toMatch(/treat\s+the well-formed envelope as authoritative/); + expect(skill).toMatch(/Do not attempt to recalculate,\s+estimate, or second-guess hashes or byte counts/); + expect(skill).not.toContain('Recompute each SHA-256'); + expect(skill).toMatch(/Working-context projection or omission\s+markers[\s\S]*never treat them\s+as proof that the original artifact is incomplete/); + }); + + test('accepts only the bounded error-redacted prior-attempt projection', () => { + expect(skill).toContain('priorAttempt: # optional; omitted for a clean no-write propose attempt'); + for (const field of [ + 'failureCode:', 'terminalFailureClass:', 'receiptStatus:', 'createdPages:', + 'updatedPages:', 'verifiedPages:', 'pageResults:', 'slugAdjustments:', + 'timelineResults:', 'linkResults:', + ]) expect(skill).toContain(field); + expect(skill).toMatch(/never contains a top-level summary, unresolved list, raw error, or a nested\s+result `error`/); + expect(skill).toMatch(/Durable GBrain state remains authoritative/); + expect(skill).toMatch(/never skip a\s+mutation based on the projection alone/); + }); + test('applies only the frozen plan with resumable page and collision results', () => { expect(skill).toContain('proposedPages: '); expect(skill).toMatch(/write the supplied title\s+and full body exactly/); diff --git a/test/gmail-thread-ingestion-skill.test.ts b/test/gmail-thread-ingestion-skill.test.ts index 033934c676..5bd6f2b1ff 100644 --- a/test/gmail-thread-ingestion-skill.test.ts +++ b/test/gmail-thread-ingestion-skill.test.ts @@ -81,6 +81,31 @@ describe('gmail-thread-ingestion skill', () => { expect(skill).toContain('prompt_injection_suspected'); }); + test('uses Lore artifact integrity as the transport-completeness authority', () => { + expect(skill).toContain('artifactIntegrity:'); + expect(skill).toContain('complete: true'); + expect(skill).toContain('manifest: { sha256: <64 lowercase hex characters>, bytes: }'); + expect(skill).toMatch(/`artifactIntegrity\.complete` flag is not exactly `true`/); + expect(skill).toMatch(/authenticated OAuth caller deterministically verified these\s+values/); + expect(skill).toMatch(/treat the well-formed\s+envelope as authoritative/); + expect(skill).toMatch(/Do not attempt to recalculate, estimate, or\s+second-guess hashes or byte counts/); + expect(skill).not.toContain('Recompute each SHA-256'); + expect(skill).toMatch(/Working-context projection or omission\s+markers[\s\S]*never treat them as proof that the original artifact is incomplete/); + expect(skill).not.toContain('artifact is visibly incomplete'); + }); + + test('accepts only the bounded error-redacted prior-attempt projection', () => { + expect(skill).toContain('priorAttempt: # optional; omitted when there is no prior write evidence'); + for (const field of [ + 'failureCode:', 'terminalFailureClass:', 'receiptStatus:', 'createdPages:', + 'updatedPages:', 'verifiedPages:', 'pageResults:', 'slugAdjustments:', + 'timelineResults:', 'linkResults:', + ]) expect(skill).toContain(field); + expect(skill).toMatch(/never contains a top-level summary, unresolved list, raw error, or a nested\s+result `error`/); + expect(skill).toMatch(/Durable GBrain state remains authoritative/); + expect(skill).toMatch(/never skip a\s+mutation based on the projection alone/); + }); + test('consolidates only on exact stable identity', () => { expect(skill).toContain('Search and read before every create'); expect(skill).toContain('Search for the exact Gmail thread ID'); diff --git a/test/granola-meeting-ingestion-skill.test.ts b/test/granola-meeting-ingestion-skill.test.ts index 2d7fa8c74f..f494c143f3 100644 --- a/test/granola-meeting-ingestion-skill.test.ts +++ b/test/granola-meeting-ingestion-skill.test.ts @@ -20,6 +20,8 @@ const expectedTools = [ 'resolve_slugs', 'get_links', 'get_backlinks', + 'stage_ingestion_proposal_page', + 'finalize_ingestion_proposal', 'put_page', 'add_link', 'add_timeline_entry', @@ -98,20 +100,24 @@ describe('granola-meeting-ingestion skill', () => { test('supports a zero-mutation scoped proposal with a bounded complete plan', () => { expect(skill).toContain('mode: '); expect(skill).toContain('admissionScope: '); - expect(skill).toContain('In `propose` mode, do not call any mutating tool'); + expect(skill).toContain('In `propose` mode, do not call any corpus-mutating tool'); expect(skill).toMatch(/`put_page`,\s+`add_link`, or `add_timeline_entry`/); expect(skill).toContain('complete set of pages that `apply` will write'); expect(skill).toContain('full intended `bodyMarkdown`, never a diff'); expect(skill).toContain('262,144 UTF-8 bytes'); expect(skill).toMatch(/[Rr]eturn `failed`\s+with an operational summary/); - expect(skill).toContain('Never truncate or split a proposal'); + expect(skill).toMatch(/Never truncate or split a\s+proposal/); expect(skill).toMatch(/must not\s+name or describe the excluded material/); expect(skill).toMatch(/local Markdown artifact is the\s+complete verbatim record/); - expect(skill).toContain('"status": "scoped_proposal"'); + expect(skill).toContain('"status": "staged_proposal"'); + expect(skill).toContain('"pageDigests": ['); + expect(skill).toContain('"proposalDigest": "64 lowercase hex characters"'); + expect(skill).toContain('brain_stage_ingestion_proposal_page'); + expect(skill).toContain('Stage only one page per turn'); + expect(skill).toContain('brain_finalize_ingestion_proposal'); expect(skill).toContain('"effect": "create | update"'); - expect(skill).toContain('"bodyMarkdown": "complete intended page body"'); - expect(skill).toContain('"baseMarkdown": "exact reviewed page body for updates, null for creates"'); - expect(skill).toContain('"expectedContentHash": "exact get_page content_hash for updates, null for creates"'); + expect(skill).toContain('Create entries have'); + expect(skill).toMatch(/Update entries add\s+exactly `baseMarkdown` and `expectedContentHash`/); expect(skill).toContain('Omit both fields for a create'); }); @@ -135,7 +141,7 @@ describe('granola-meeting-ingestion skill', () => { /typed links only for\s+relationships that the planned Markdown does not express accurately/, ); expect(skill).toMatch( - /receipt, including `proposedTimelineEntries` and\s+`proposedLinks`, must not exceed 262,144 UTF-8 bytes/, + /compact manifest over 262,144 UTF-8 bytes/, ); expect(skill).toContain('"proposedTimelineEntries": ['); expect(skill).toContain('"refLabel": "meeting capture"'); @@ -164,12 +170,38 @@ describe('granola-meeting-ingestion skill', () => { test('splits partial disqualification from classed needs-attention outcomes', () => { expect(skill).toContain('partial disqualification'); expect(skill).toMatch(/[Dd]erive `admissionScope` only from the resolver/); - expect(skill).toContain('return `scoped_proposal` directly'); + expect(skill).toContain('return `staged_proposal` directly'); expect(skill).toMatch(/Do not mutate before returning that\s+proposal/); expect(skill).toContain('"reason_class": "resolver_ambiguity | operational"'); expect(skill).toMatch(/Every `needs_attention`\s+receipt must include `reason_class`/); }); + test('verifies Lore artifact integrity without treating context projection as loss', () => { + expect(skill).toContain('artifactIntegrity: # required normal/propose; omitted apply'); + expect(skill).toContain('complete: true'); + expect(skill).toContain('manifest: { sha256: <64 lowercase hex characters>, bytes: }'); + expect(skill).toMatch(/`artifactIntegrity\.complete` is not exactly `true`/); + expect(skill).toMatch(/Apply mode intentionally\s+omits `artifactIntegrity`/); + expect(skill).toMatch(/exactly 64 lowercase hexadecimal characters/); + expect(skill).toMatch(/authenticated OAuth caller deterministically\s+verified these values/); + expect(skill).toMatch(/treat\s+the well-formed envelope as authoritative/); + expect(skill).toMatch(/Do not attempt to recalculate,\s+estimate, or second-guess hashes or byte counts/); + expect(skill).not.toContain('Recompute each SHA-256'); + expect(skill).toMatch(/Working-context projection or omission\s+markers[\s\S]*never treat them as proof\s+that the original artifact is incomplete/); + }); + + test('accepts only the bounded error-redacted prior-attempt projection', () => { + expect(skill).toContain('priorAttempt: # optional; omitted for a clean no-write propose attempt'); + for (const field of [ + 'failureCode:', 'terminalFailureClass:', 'receiptStatus:', 'createdPages:', + 'updatedPages:', 'verifiedPages:', 'pageResults:', 'slugAdjustments:', + 'timelineResults:', 'linkResults:', + ]) expect(skill).toContain(field); + expect(skill).toMatch(/never contains a top-level summary, unresolved list, raw error, or a nested\s+result `error`/); + expect(skill).toMatch(/Durable GBrain state remains authoritative/); + expect(skill).toMatch(/never skip a\s+mutation based on the projection alone/); + }); + test('applies only the frozen plan with resumable page and collision results', () => { expect(skill).toContain('proposedPages: '); expect(skill).toMatch(/write the supplied title\s+and full body exactly/); diff --git a/test/migration-v135.test.ts b/test/migration-v135.test.ts index d73b07f6f6..1ea46157ac 100644 --- a/test/migration-v135.test.ts +++ b/test/migration-v135.test.ts @@ -19,7 +19,7 @@ describe('migration v135 — timeline reference columns', () => { await engine.disconnect(); }); - test('registers refs, per-page event dedup, and search repair as the latest migration', () => { + test('registers refs, per-page event dedup, and search repair', () => { const migration = MIGRATIONS.find(entry => entry.version === 135); expect(migration).toMatchObject({ name: 'timeline_entry_refs', @@ -30,7 +30,7 @@ describe('migration v135 — timeline reference columns', () => { expect(migration?.sql).toContain('DROP INDEX IF EXISTS idx_timeline_event_dedup'); expect(migration?.sql).toContain('page_id, event_page_id, date'); expect(typeof migration?.handler).toBe('function'); - expect(LATEST_VERSION).toBe(135); + expect(LATEST_VERSION).toBeGreaterThanOrEqual(135); }); test('adds refs, widens event dedup, and preserves row-backed timeline search', async () => { @@ -81,7 +81,7 @@ describe('migration v135 — timeline reference columns', () => { await engine.setConfig('version', '134'); const first = await runMigrations(engine); - expect(first).toEqual({ applied: 1, current: 135 }); + expect(first).toEqual({ applied: 2, current: 136 }); const columns = await engine.executeRaw<{ column_name: string; is_nullable: string }>( `SELECT column_name, is_nullable FROM information_schema.columns @@ -120,6 +120,6 @@ describe('migration v135 — timeline reference columns', () => { await engine.setConfig('version', '134'); const second = await runMigrations(engine); - expect(second).toEqual({ applied: 1, current: 135 }); + expect(second).toEqual({ applied: 2, current: 136 }); }, 30_000); }); diff --git a/test/subagent-handler.test.ts b/test/subagent-handler.test.ts index da5a830d7f..1089efde2e 100644 --- a/test/subagent-handler.test.ts +++ b/test/subagent-handler.test.ts @@ -125,6 +125,58 @@ function makeThrowingTool(name = 'broken'): ToolDef { // ── Tests ─────────────────────────────────────────────────── describe('subagent handler happy path', () => { + test('proposal stage guard rejects oversized raw input before assistant or tool persistence', async () => { + const tool = makeEchoTool('brain_stage_ingestion_proposal_page'); + const client = new FakeMessagesClient([{ + content: [{ + type: 'tool_use', + id: 'stage-too-large', + name: tool.name, + input: { page: { bodyMarkdown: 'x'.repeat(196_608) } }, + } as any], + stop_reason: 'tool_use' as any, + }]); + const handler = makeSubagentHandler({ engine, client, toolRegistry: [tool] }); + const ctx = await makeCtx({ prompt: 'stage one page' }); + + await expect(handler(ctx)).rejects.toThrow(/maximum/i); + const messages = await engine.executeRaw<{ role: string }>( + `SELECT role FROM subagent_messages WHERE job_id = $1 ORDER BY message_idx`, + [ctx.id], + ); + const tools = await engine.executeRaw( + `SELECT id FROM subagent_tool_executions WHERE job_id = $1`, + [ctx.id], + ); + expect(messages.map((row) => row.role)).toEqual(['user']); + expect(tools).toHaveLength(0); + }); + + test('proposal stage guard rejects two staged pages before assistant or tool persistence', async () => { + const tool = makeEchoTool('brain_stage_ingestion_proposal_page'); + const client = new FakeMessagesClient([{ + content: [ + { type: 'tool_use', id: 'stage-1', name: tool.name, input: { page: 1 } } as any, + { type: 'tool_use', id: 'stage-2', name: tool.name, input: { page: 2 } } as any, + ], + stop_reason: 'tool_use' as any, + }]); + const handler = makeSubagentHandler({ engine, client, toolRegistry: [tool] }); + const ctx = await makeCtx({ prompt: 'stage two pages' }); + + await expect(handler(ctx)).rejects.toThrow(/exactly one/i); + const messages = await engine.executeRaw<{ role: string }>( + `SELECT role FROM subagent_messages WHERE job_id = $1 ORDER BY message_idx`, + [ctx.id], + ); + const tools = await engine.executeRaw( + `SELECT id FROM subagent_tool_executions WHERE job_id = $1`, + [ctx.id], + ); + expect(messages.map((row) => row.role)).toEqual(['user']); + expect(tools).toHaveLength(0); + }); + test('no-tool end_turn: returns text response + persists user + assistant rows', async () => { const client = new FakeMessagesClient([ { content: [{ type: 'text', text: 'hello world' }] as any, stop_reason: 'end_turn' }, diff --git a/test/submit-agent.test.ts b/test/submit-agent.test.ts index aea45890d2..19df3d0f22 100644 --- a/test/submit-agent.test.ts +++ b/test/submit-agent.test.ts @@ -26,6 +26,9 @@ import { operationsByName } from '../src/core/operations.ts'; const submit_agent = operationsByName['submit_agent']; const get_agent_job = operationsByName['get_agent_job']; +const stage_ingestion_proposal_page = operationsByName['stage_ingestion_proposal_page']; +const finalize_ingestion_proposal = operationsByName['finalize_ingestion_proposal']; +const get_agent_job_proposal = operationsByName['get_agent_job_proposal']; const get_agent_job_execution_evidence = operationsByName['get_agent_job_execution_evidence']; if (!submit_agent) { @@ -134,6 +137,15 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with it('declares an explicit reasoning_effort param', () => { expect(submit_agent.params.reasoning_effort).toBeDefined(); }); + it('publishes non-corpus-mutating staged proposal operations', () => { + expect(submit_agent.params.proposal_artifact_id).toBeDefined(); + expect(submit_agent.params.proposal_admission_scope).toBeDefined(); + expect(stage_ingestion_proposal_page?.scope).toBe('agent'); + expect(stage_ingestion_proposal_page?.mutating).toBe(false); + expect(finalize_ingestion_proposal?.scope).toBe('agent'); + expect(finalize_ingestion_proposal?.mutating).toBe(false); + expect(get_agent_job_proposal?.scope).toBe('agent'); + }); }); describe('local CLI bypass (ctx.remote === false)', () => { @@ -362,6 +374,36 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with }); describe('happy-path submission', () => { + it('requires and persists the proposal artifact and admission-scope binding together', async () => { + await seedClient('cursor', { + bound_tools: ['stage_ingestion_proposal_page', 'finalize_ingestion_proposal'], + bound_source_id: 'company', + bound_slug_prefixes: ['sources/'], + }); + const ctx = makeCtx({ clientId: 'cursor' }); + await expect(callSubmitAgent(ctx, { + prompt: 'propose', + proposal_artifact_id: 'artifact-1', + })).rejects.toThrow(/must be supplied together/); + await expect(callSubmitAgent(ctx, { + prompt: 'propose', + proposal_admission_scope: 'Include delivery notes.', + })).rejects.toThrow(/must be supplied together/); + + const result = await callSubmitAgent(ctx, { + prompt: 'propose', + proposal_artifact_id: 'artifact-1', + proposal_admission_scope: 'Include delivery notes.', + }); + const [row] = await engine.executeRaw<{ data: Record }>( + 'SELECT data FROM minion_jobs WHERE id = $1', + [result.id], + ); + expect(row.data.proposal_artifact_id).toBe('artifact-1'); + expect(row.data.proposal_admission_scope).toBe('Include delivery notes.'); + expect(row.data.source_id).toBe('company'); + }); + it('deduplicates a client idempotency key before enforcing concurrency', async () => { await seedClient('cursor', { bound_tools: ['search'], @@ -1125,6 +1167,93 @@ describe('get_agent_job owner-scoped receipt', () => { }); }); +describe('staged proposal operation contract', () => { + it('stages only inside the bound job, finalizes compactly, and retrieves by exact owner and digest', async () => { + const [row] = await engine.executeRaw<{ id: number }>( + `INSERT INTO minion_jobs (name, status, data, queue, priority, created_at) + VALUES ('subagent', 'active', $1::jsonb, 'default', 0, now()) + RETURNING id`, + [JSON.stringify({ + __owner_client_id: 'lore', + source_id: 'company', + proposal_artifact_id: 'artifact-1', + proposal_admission_scope: 'Include delivery notes.', + })], + ); + const agentCtx = { ...makeCtx({ clientId: 'lore' }), viaSubagent: true, jobId: row.id }; + const staged = await stage_ingestion_proposal_page!.handler(agentCtx, { + artifact_id: 'artifact-1', + source_id: 'company', + admission_scope: 'Include delivery notes.', + sequence: 1, + total_pages: 1, + page: { + slug: 'sources/example', + effect: 'create', + title: 'Example', + bodyMarkdown: '# Example', + }, + }) as any; + const manifest = await finalize_ingestion_proposal!.handler(agentCtx, { + artifact_id: 'artifact-1', + source_id: 'company', + admission_scope: 'Include delivery notes.', + total_pages: 1, + page_digests: [{ sequence: 1, digest: staged.digest }], + summary: 'Ready for review.', + proposed_timeline_entries: [], + proposed_links: [], + unresolved: [], + }) as any; + + expect(manifest).toEqual({ + status: 'staged_proposal', + artifactId: 'artifact-1', + sourceId: 'company', + admissionScope: 'Include delivery notes.', + summary: 'Ready for review.', + pageDigests: [{ sequence: 1, slug: 'sources/example', digest: staged.digest }], + proposalDigest: expect.stringMatching(/^[a-f0-9]{64}$/), + proposedTimelineEntries: [], + proposedLinks: [], + unresolved: [], + }); + + const retrieved = await get_agent_job_proposal!.handler( + makeCtx({ clientId: 'lore' }), + { id: row.id, proposal_digest: manifest.proposalDigest }, + ) as any; + expect(retrieved).toEqual({ + id: row.id, + proposal_digest: manifest.proposalDigest, + page_digests: manifest.pageDigests, + plan: { + artifactId: 'artifact-1', + sourceId: 'company', + admissionScope: 'Include delivery notes.', + summary: 'Ready for review.', + proposedPages: [{ + slug: 'sources/example', + effect: 'create', + title: 'Example', + bodyMarkdown: '# Example', + }], + proposedTimelineEntries: [], + proposedLinks: [], + unresolved: [], + }, + }); + await expect(get_agent_job_proposal!.handler( + makeCtx({ clientId: 'other' }), + { id: row.id, proposal_digest: manifest.proposalDigest }, + )).rejects.toMatchObject({ code: 'permission_denied' }); + await expect(stage_ingestion_proposal_page!.handler(makeCtx({ clientId: 'lore' }), { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include delivery notes.', + sequence: 1, total_pages: 1, page: {}, + })).rejects.toMatchObject({ code: 'permission_denied' }); + }); +}); + describe('get_agent_job_execution_evidence admin fallback', () => { it('is an admin-only operation with exact owner and source inputs', () => { expect(get_agent_job_execution_evidence).toBeDefined(); From e2c4d30d851a9d2d90c0b45de6333dd5cdf68c99 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 15:20:29 -0700 Subject: [PATCH 02/18] lore-b74.5.1: align staged proposal limits --- skills/github-project-ingestion/SKILL.md | 3 +- skills/granola-meeting-ingestion/SKILL.md | 3 +- src/core/minions/agent-job-proposals.ts | 20 ++++++- src/core/operations.ts | 4 +- test/agent-job-proposals.test.ts | 70 +++++++++++++++++++++++ 5 files changed, 93 insertions(+), 7 deletions(-) diff --git a/skills/github-project-ingestion/SKILL.md b/skills/github-project-ingestion/SKILL.md index 43100200a2..c9714a1ce4 100644 --- a/skills/github-project-ingestion/SKILL.md +++ b/skills/github-project-ingestion/SKILL.md @@ -285,7 +285,8 @@ After every page is staged, call `brain_finalize_ingestion_proposal` in a separate turn with the exact binding fields, stable page count, ordered `page_digests`, summary, timeline entries, links, and unresolved items. The server rejects gaps, duplicates, changed digests, cross-job evidence, a full -plan over 786,432 UTF-8 bytes, or a compact manifest over 262,144 UTF-8 bytes. +plan over 786,432 UTF-8 bytes, more than 100 pages, a timeline `refLabel` over +500 characters, or a compact manifest over 262,144 UTF-8 bytes. Return the finalizer's compact manifest as `staged_proposal`; never reproduce page bodies or baselines in the final receipt. Never truncate or split a proposal. diff --git a/skills/granola-meeting-ingestion/SKILL.md b/skills/granola-meeting-ingestion/SKILL.md index 8ea0713760..2ef975d75c 100644 --- a/skills/granola-meeting-ingestion/SKILL.md +++ b/skills/granola-meeting-ingestion/SKILL.md @@ -270,7 +270,8 @@ After every page is staged, call `brain_finalize_ingestion_proposal` in a separate turn with the exact binding fields, stable page count, ordered `page_digests`, summary, timeline entries, links, and unresolved items. The server rejects gaps, duplicates, changed digests, cross-job evidence, a full -plan over 786,432 UTF-8 bytes, or a compact manifest over 262,144 UTF-8 bytes. +plan over 786,432 UTF-8 bytes, more than 100 pages, a timeline `refLabel` over +500 characters, or a compact manifest over 262,144 UTF-8 bytes. Return the finalizer's compact manifest as `staged_proposal`; never reproduce page bodies or baselines in the final receipt. Never truncate or split a proposal. diff --git a/src/core/minions/agent-job-proposals.ts b/src/core/minions/agent-job-proposals.ts index 1a19414f9e..3bb716a7f7 100644 --- a/src/core/minions/agent-job-proposals.ts +++ b/src/core/minions/agent-job-proposals.ts @@ -11,6 +11,9 @@ export const PROPOSAL_AGGREGATE_MAX_BYTES = 786_432; /** Maximum UTF-8 size of the compact receipt manifest. */ export const PROPOSAL_MANIFEST_MAX_BYTES = 262_144; +/** Maximum number of pages in one finalized proposal. */ +export const PROPOSAL_MAX_PAGES = 100; + export const STAGE_PROPOSAL_TOOL_NAME = 'brain_stage_ingestion_proposal_page'; export const FINALIZE_PROPOSAL_TOOL_NAME = 'brain_finalize_ingestion_proposal'; @@ -195,7 +198,7 @@ export async function stageAgentJobProposalPage( throw new AgentJobProposalError('stage_input_too_large', 'Proposal page input exceeds the staging byte limit.'); } const sequence = readPositiveInteger(input.sequence, 'sequence'); - const totalPages = readPositiveInteger(input.total_pages, 'total_pages'); + const totalPages = readProposalPageCount(input.total_pages); if (sequence > totalPages) { throw new AgentJobProposalError('invalid_sequence', 'sequence must be within 1..total_pages.'); } @@ -266,7 +269,7 @@ export async function finalizeAgentJobProposal( if (proposalToolInputBytes(input) > PROPOSAL_STAGE_INPUT_MAX_BYTES) { throw new AgentJobProposalError('finalize_input_too_large', 'Proposal finalization input exceeds the tool byte limit.'); } - const totalPages = readPositiveInteger(input.total_pages, 'total_pages'); + const totalPages = readProposalPageCount(input.total_pages); const requestedDigests = parseRequestedDigests(input.page_digests, totalPages); const summary = readBoundedString(input.summary, 'summary', 1_000); const timeline = parseTimelineEntries(input.proposed_timeline_entries ?? []); @@ -585,7 +588,7 @@ function parseTimelineEntries(raw: unknown): ScopedProposalTimelineEntry[] { text: readBoundedString(record.text, 'timeline.text', 1_000), ref: readCanonicalSlug(record.ref, 'timeline.ref'), }; - if ('refLabel' in record) result.refLabel = readBoundedString(record.refLabel, 'timeline.refLabel', 1_000); + if ('refLabel' in record) result.refLabel = readBoundedString(record.refLabel, 'timeline.refLabel', 500); return result; }); } @@ -644,6 +647,17 @@ function readPositiveInteger(raw: unknown, name: string): number { return Number(raw); } +function readProposalPageCount(raw: unknown): number { + const totalPages = readPositiveInteger(raw, 'total_pages'); + if (totalPages > PROPOSAL_MAX_PAGES) { + throw new AgentJobProposalError( + 'invalid_total_pages', + `total_pages must be at most ${PROPOSAL_MAX_PAGES}.`, + ); + } + return totalPages; +} + function readRecord(raw: unknown, name: string): Record { if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { throw new AgentJobProposalError('invalid_object', `${name} must be an object.`); diff --git a/src/core/operations.ts b/src/core/operations.ts index c7afb88bec..bec6776d68 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -5636,7 +5636,7 @@ const stage_ingestion_proposal_page: Operation = { source_id: { type: 'string', required: true, description: 'Exact source bound to the current agent job' }, admission_scope: { type: 'string', required: true, description: 'Exact admission scope bound at submit_agent time' }, sequence: { type: 'number', required: true, description: 'One-based page position' }, - total_pages: { type: 'number', required: true, description: 'Total number of pages in the proposal' }, + total_pages: { type: 'number', required: true, description: 'Total number of pages in the proposal (1-100)' }, page: { type: 'object', required: true, description: 'Exact create or update page proposal' }, }, scope: 'agent', @@ -5665,7 +5665,7 @@ const finalize_ingestion_proposal: Operation = { artifact_id: { type: 'string', required: true, description: 'Exact artifact id bound at submit_agent time' }, source_id: { type: 'string', required: true, description: 'Exact source bound to the current agent job' }, admission_scope: { type: 'string', required: true, description: 'Exact admission scope bound at submit_agent time' }, - total_pages: { type: 'number', required: true, description: 'Total number of staged pages' }, + total_pages: { type: 'number', required: true, description: 'Total number of staged pages (1-100)' }, page_digests: { type: 'array', required: true, items: { type: 'object' }, description: 'Ordered one-based sequence and digest pairs' }, summary: { type: 'string', required: true, description: 'Compact source-grounded proposal summary' }, proposed_timeline_entries: { type: 'array', items: { type: 'object' }, description: 'Bounded frozen timeline entries' }, diff --git a/test/agent-job-proposals.test.ts b/test/agent-job-proposals.test.ts index 0ee1553505..7afa9fff66 100644 --- a/test/agent-job-proposals.test.ts +++ b/test/agent-job-proposals.test.ts @@ -4,8 +4,11 @@ import { resetPgliteState } from './helpers/reset-pglite.ts'; import { AgentJobProposalError, PROPOSAL_AGGREGATE_MAX_BYTES, + PROPOSAL_MANIFEST_MAX_BYTES, + PROPOSAL_MAX_PAGES, PROPOSAL_STAGE_INPUT_MAX_BYTES, assertProposalToolTurnPersistable, + canonicalProposalJson, digestProposalValue, finalizeAgentJobProposal, getOwnedAgentJobProposal, @@ -99,6 +102,13 @@ describe('proposal turn pre-persistence boundary', () => { }); describe('durable agent-job proposal staging', () => { + it('rejects proposal page counts above the shared 100-page contract', async () => { + const jobId = await seedJob(); + await expect(stage(jobId, 1, 101, createPage('sources/too-many'))) + .rejects.toMatchObject({ code: 'invalid_total_pages' }); + expect(PROPOSAL_MAX_PAGES).toBe(100); + }); + it('stages exact pages, finalizes an ordered manifest, and retrieves the full owned plan', async () => { const jobId = await seedJob(); const first = await stage(jobId, 1, 2, createPage('sources/example')); @@ -243,6 +253,26 @@ describe('durable agent-job proposal staging', () => { })).rejects.toMatchObject({ code: 'invalid_timeline' }); }); + it('rejects timeline reference labels above the shared 500-character contract', async () => { + const jobId = await seedJob(); + const page = await stage(jobId, 1, 1, createPage('sources/example')); + await expect(finalizeAgentJobProposal(engine, jobId, { + artifact_id: 'artifact-1', + source_id: 'company', + admission_scope: 'Include project delivery notes.', + total_pages: 1, + page_digests: [{ sequence: 1, digest: page.digest }], + summary: 'Invalid reference label.', + proposed_timeline_entries: [{ + pageSlug: 'sources/example', + date: '2026-08-07', + text: 'Delivery review completed.', + ref: 'sources/example', + refLabel: 'x'.repeat(501), + }], + })).rejects.toMatchObject({ code: 'invalid_string' }); + }); + it('rejects job, owner, source, artifact, scope, and stored-fragment binding mismatches', async () => { const jobId = await seedJob(); await expect(stageAgentJobProposalPage(engine, jobId, { @@ -298,5 +328,45 @@ describe('durable agent-job proposal staging', () => { ); expect(Number(rows[0]!.count)).toBe(0); expect(PROPOSAL_AGGREGATE_MAX_BYTES).toBe(786_432); + expect(PROPOSAL_MANIFEST_MAX_BYTES).toBe(256 * 1024); + }); + + it('accepts a raw aggregate plan exactly at the shared ceiling', async () => { + const jobId = await seedJob(); + const pageCount = 5; + const pages = Array.from({ length: pageCount }, (_, index) => + createPage(`sources/boundary-${index + 1}`, 'x'.repeat(150_000))); + const basePlan = { + artifactId: 'artifact-1', + sourceId: 'company', + admissionScope: 'Include project delivery notes.', + summary: 'Boundary.', + proposedPages: pages, + proposedTimelineEntries: [], + proposedLinks: [], + unresolved: [], + }; + const remaining = PROPOSAL_AGGREGATE_MAX_BYTES - + Buffer.byteLength(canonicalProposalJson(basePlan), 'utf8'); + pages[pageCount - 1]!.bodyMarkdown += 'y'.repeat(remaining); + const digests = []; + for (let sequence = 1; sequence <= pageCount; sequence++) { + digests.push(await stage(jobId, sequence, pageCount, pages[sequence - 1]!)); + } + + const manifest = await finalizeAgentJobProposal(engine, jobId, { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', + total_pages: pageCount, + page_digests: digests.map(({ sequence, digest }) => ({ sequence, digest })), + summary: 'Boundary.', + }); + const owned = await getOwnedAgentJobProposal( + engine, + jobId, + 'lore-client', + manifest.proposalDigest, + ); + expect(Buffer.byteLength(canonicalProposalJson(owned.plan), 'utf8')) + .toBe(PROPOSAL_AGGREGATE_MAX_BYTES); }); }); From 1ebe25c2b40f8e38e92c2f25594fee94b6bcda39 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 15:39:13 -0700 Subject: [PATCH 03/18] lore-b74.5.1: harden staged proposal boundaries --- skills/github-project-ingestion/SKILL.md | 9 +- skills/granola-meeting-ingestion/SKILL.md | 9 +- src/core/minions/agent-job-proposals.ts | 411 +++++++++++++------ src/core/minions/handlers/subagent.ts | 8 +- src/core/operations.ts | 109 +++-- src/core/slug-allow-list.ts | 38 ++ test/agent-job-proposals.test.ts | 242 ++++++++--- test/e2e/subagent-gateway-path.test.ts | 83 +++- test/github-project-ingestion-skill.test.ts | 2 + test/granola-meeting-ingestion-skill.test.ts | 2 + test/submit-agent.test.ts | 48 ++- 11 files changed, 706 insertions(+), 255 deletions(-) create mode 100644 src/core/slug-allow-list.ts diff --git a/skills/github-project-ingestion/SKILL.md b/skills/github-project-ingestion/SKILL.md index c9714a1ce4..36a4d3b5ef 100644 --- a/skills/github-project-ingestion/SKILL.md +++ b/skills/github-project-ingestion/SKILL.md @@ -282,9 +282,12 @@ admission scope. Scope provenance exists only in the top-level proposal receipt. After every page is staged, call `brain_finalize_ingestion_proposal` in a -separate turn with the exact binding fields, stable page count, ordered -`page_digests`, summary, timeline entries, links, and unresolved items. The -server rejects gaps, duplicates, changed digests, cross-job evidence, a full +separate turn with the exact binding fields, stable page count, summary, +timeline entries, links, and unresolved items. The server derives the ordered +page-digest manifest from the job's durable fragments, so finalization does not +depend on old stage outputs remaining in model context. The server rejects +gaps, duplicate or changed fragments, cross-job evidence, a capture page that +does not match the exact job binding, mutations outside the job slug fence, a full plan over 786,432 UTF-8 bytes, more than 100 pages, a timeline `refLabel` over 500 characters, or a compact manifest over 262,144 UTF-8 bytes. Return the finalizer's compact manifest as `staged_proposal`; never reproduce diff --git a/skills/granola-meeting-ingestion/SKILL.md b/skills/granola-meeting-ingestion/SKILL.md index 2ef975d75c..599045ac2e 100644 --- a/skills/granola-meeting-ingestion/SKILL.md +++ b/skills/granola-meeting-ingestion/SKILL.md @@ -267,9 +267,12 @@ excluded material or the admission scope. Scope provenance exists only in the top-level proposal receipt. After every page is staged, call `brain_finalize_ingestion_proposal` in a -separate turn with the exact binding fields, stable page count, ordered -`page_digests`, summary, timeline entries, links, and unresolved items. The -server rejects gaps, duplicates, changed digests, cross-job evidence, a full +separate turn with the exact binding fields, stable page count, summary, +timeline entries, links, and unresolved items. The server derives the ordered +page-digest manifest from the job's durable fragments, so finalization does not +depend on old stage outputs remaining in model context. The server rejects +gaps, duplicate or changed fragments, cross-job evidence, a capture page that +does not match the exact job binding, mutations outside the job slug fence, a full plan over 786,432 UTF-8 bytes, more than 100 pages, a timeline `refLabel` over 500 characters, or a compact manifest over 262,144 UTF-8 bytes. Return the finalizer's compact manifest as `staged_proposal`; never reproduce diff --git a/src/core/minions/agent-job-proposals.ts b/src/core/minions/agent-job-proposals.ts index 3bb716a7f7..83c0e7a807 100644 --- a/src/core/minions/agent-job-proposals.ts +++ b/src/core/minions/agent-job-proposals.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import type { BrainEngine } from '../engine.ts'; import { assertValidSourceId } from '../source-id.ts'; +import { matchesSlugAllowList } from '../slug-allow-list.ts'; /** Maximum UTF-8 size of one page-staging tool input. */ export const PROPOSAL_STAGE_INPUT_MAX_BYTES = 196_608; @@ -86,7 +87,6 @@ export interface FinalizeProposalInput { source_id: string; admission_scope: string; total_pages: number; - page_digests: unknown; summary: string; proposed_timeline_entries?: unknown; proposed_links?: unknown; @@ -110,7 +110,30 @@ interface JobBinding { ownerClientId: string; sourceId: string; artifactId: string; + admissionScope: string | null; + capturePageSlug: string; + allowedSlugPrefixes: string[]; +} + +interface ParsedStageProposalPageInput { + artifactId: string; + sourceId: string; admissionScope: string; + sequence: number; + totalPages: number; + page: ScopedProposalPage; + pageDigest: string; +} + +interface StoredProposalFragment { + sequence: number; + total_pages: number; + owner_client_id: string; + source_id: string; + artifact_id: string; + admission_scope: string; + page: unknown; + page_digest: string; } interface ProposalTurnBlock { @@ -151,16 +174,7 @@ export function proposalToolInputBytes(input: unknown): number { * defense in depth. */ export function assertProposalToolTurnPersistable(blocks: readonly ProposalTurnBlock[]): void { - const calls = blocks.flatMap((block) => { - const isCall = block.type === 'tool_use' || block.type === 'tool-call'; - if (!isCall) return []; - const name = typeof block.name === 'string' - ? block.name - : typeof block.toolName === 'string' - ? block.toolName - : ''; - return [{ name, input: block.input }]; - }); + const calls = proposalToolCalls(blocks); const stageCalls = calls.filter((call) => call.name === STAGE_PROPOSAL_TOOL_NAME); const finalizeCalls = calls.filter((call) => call.name === FINALIZE_PROPOSAL_TOOL_NAME); if (stageCalls.length > 1) { @@ -187,6 +201,46 @@ export function assertProposalToolTurnPersistable(blocks: readonly ProposalTurnB } } +/** + * Check cumulative staged bytes before persisting a fresh assistant turn. + * + * The same-sequence, same-content replay contributes zero new bytes. The + * staging transaction repeats this check while holding the job row lock. + */ +export async function assertProposalToolTurnPersistableForJob( + engine: BrainEngine, + jobId: number, + blocks: readonly ProposalTurnBlock[], +): Promise { + assertProposalToolTurnPersistable(blocks); + const stageCall = proposalToolCalls(blocks) + .find((call) => call.name === STAGE_PROPOSAL_TOOL_NAME); + if (!stageCall) return; + const candidate = parseStageProposalPageInput(stageCall.input); + + await engine.transaction(async (tx) => { + const binding = await readJobBinding(tx, jobId, true); + assertBindingMatches(binding, candidate); + assertSlugAllowed(binding, candidate.page.slug, 'Proposed page'); + const fragments = await readStoredFragments(tx, jobId); + assertCumulativeStageFits(binding, candidate, fragments); + }); +} + +/** Extract provider-neutral and legacy proposal tool-call shapes. */ +function proposalToolCalls(blocks: readonly ProposalTurnBlock[]): Array<{ name: string; input: unknown }> { + return blocks.flatMap((block) => { + const isCall = block.type === 'tool_use' || block.type === 'tool-call'; + if (!isCall) return []; + const name = typeof block.name === 'string' + ? block.name + : typeof block.toolName === 'string' + ? block.toolName + : ''; + return [{ name, input: block.input }]; + }); +} + /** Stage one exact page proposal in the current agent job's durable ledger. */ export async function stageAgentJobProposalPage( engine: BrainEngine, @@ -194,20 +248,13 @@ export async function stageAgentJobProposalPage( input: StageProposalPageInput, ): Promise { assertSafeJobId(jobId); - if (proposalToolInputBytes(input) > PROPOSAL_STAGE_INPUT_MAX_BYTES) { - throw new AgentJobProposalError('stage_input_too_large', 'Proposal page input exceeds the staging byte limit.'); - } - const sequence = readPositiveInteger(input.sequence, 'sequence'); - const totalPages = readProposalPageCount(input.total_pages); - if (sequence > totalPages) { - throw new AgentJobProposalError('invalid_sequence', 'sequence must be within 1..total_pages.'); - } - const page = parseProposalPage(input.page); - const pageDigest = digestProposalValue(page); + const candidate = parseStageProposalPageInput(input); return engine.transaction(async (tx) => { const binding = await readJobBinding(tx, jobId, true); - assertBindingMatches(binding, input.artifact_id, input.source_id, input.admission_scope); + assertBindingMatches(binding, candidate); + const frozenBinding = await freezeAdmissionScope(tx, jobId, binding, candidate.admissionScope); + assertSlugAllowed(frozenBinding, candidate.page.slug, 'Proposed page'); const finalized = await tx.executeRaw<{ proposal_digest: string }>( `SELECT proposal_digest FROM agent_job_proposals WHERE job_id = $1`, [jobId], @@ -216,6 +263,16 @@ export async function stageAgentJobProposalPage( throw new AgentJobProposalError('proposal_finalized', 'This job proposal is already finalized.'); } + const fragments = await readStoredFragments(tx, jobId); + const replay = assertCumulativeStageFits(frozenBinding, candidate, fragments); + if (replay) { + return { + sequence: candidate.sequence, + slug: candidate.page.slug, + digest: candidate.pageDigest, + }; + } + await tx.executeRaw( `INSERT INTO agent_job_proposal_fragments (job_id, owner_client_id, source_id, artifact_id, admission_scope, @@ -223,39 +280,16 @@ export async function stageAgentJobProposalPage( VALUES ($1, $2, $3, $4, $5, $6, $7, $8::text::jsonb, $9) ON CONFLICT (job_id, sequence) DO NOTHING`, [ - jobId, binding.ownerClientId, binding.sourceId, binding.artifactId, - binding.admissionScope, sequence, totalPages, canonicalProposalJson(page), pageDigest, + jobId, frozenBinding.ownerClientId, frozenBinding.sourceId, frozenBinding.artifactId, + frozenBinding.admissionScope, candidate.sequence, candidate.totalPages, + canonicalProposalJson(candidate.page), candidate.pageDigest, ], ); - const rows = await tx.executeRaw<{ - source_id: string; - artifact_id: string; - admission_scope: string; - total_pages: number; - page: unknown; - page_digest: string; - }>( - `SELECT source_id, artifact_id, admission_scope, total_pages, page, page_digest - FROM agent_job_proposal_fragments - WHERE job_id = $1 AND sequence = $2`, - [jobId, sequence], - ); - const existing = rows[0]; - if ( - !existing - || existing.source_id !== binding.sourceId - || existing.artifact_id !== binding.artifactId - || existing.admission_scope !== binding.admissionScope - || Number(existing.total_pages) !== totalPages - || existing.page_digest !== pageDigest - || canonicalProposalJson(existing.page) !== canonicalProposalJson(page) - ) { - throw new AgentJobProposalError( - 'conflicting_fragment', - `Sequence ${sequence} already contains a different proposal fragment.`, - ); - } - return { sequence, slug: page.slug, digest: pageDigest }; + return { + sequence: candidate.sequence, + slug: candidate.page.slug, + digest: candidate.pageDigest, + }; }); } @@ -270,7 +304,10 @@ export async function finalizeAgentJobProposal( throw new AgentJobProposalError('finalize_input_too_large', 'Proposal finalization input exceeds the tool byte limit.'); } const totalPages = readProposalPageCount(input.total_pages); - const requestedDigests = parseRequestedDigests(input.page_digests, totalPages); + const artifactId = readBoundedString(input.artifact_id, 'artifact_id', 255); + const sourceId = readBoundedString(input.source_id, 'source_id', 255); + assertValidSourceId(sourceId); + const admissionScope = readBoundedString(input.admission_scope, 'admission_scope', 8_000); const summary = readBoundedString(input.summary, 'summary', 1_000); const timeline = parseTimelineEntries(input.proposed_timeline_entries ?? []); const links = parseLinks(input.proposed_links ?? []); @@ -278,24 +315,9 @@ export async function finalizeAgentJobProposal( return engine.transaction(async (tx) => { const binding = await readJobBinding(tx, jobId, true); - assertBindingMatches(binding, input.artifact_id, input.source_id, input.admission_scope); - const fragments = await tx.executeRaw<{ - sequence: number; - total_pages: number; - owner_client_id: string; - source_id: string; - artifact_id: string; - admission_scope: string; - page: unknown; - page_digest: string; - }>( - `SELECT sequence, total_pages, owner_client_id, source_id, artifact_id, - admission_scope, page, page_digest - FROM agent_job_proposal_fragments - WHERE job_id = $1 - ORDER BY sequence`, - [jobId], - ); + assertBindingMatches(binding, { artifactId, sourceId, admissionScope }); + const boundScope = requireBoundAdmissionScope(binding); + const fragments = await readStoredFragments(tx, jobId); if (fragments.length !== totalPages) { throw new AgentJobProposalError( 'fragment_gap', @@ -309,7 +331,6 @@ export async function finalizeAgentJobProposal( for (let index = 0; index < fragments.length; index++) { const expectedSequence = index + 1; const fragment = fragments[index]!; - const requested = requestedDigests[index]!; if (Number(fragment.sequence) !== expectedSequence || Number(fragment.total_pages) !== totalPages) { throw new AgentJobProposalError('fragment_gap', 'Staged proposal sequences are not exactly contiguous.'); } @@ -317,31 +338,30 @@ export async function finalizeAgentJobProposal( fragment.owner_client_id !== binding.ownerClientId || fragment.source_id !== binding.sourceId || fragment.artifact_id !== binding.artifactId - || fragment.admission_scope !== binding.admissionScope + || fragment.admission_scope !== boundScope ) { throw new AgentJobProposalError('binding_mismatch', 'A staged fragment does not match its job binding.'); } const page = parseProposalPage(fragment.page); if ( - requested.sequence !== expectedSequence - || requested.digest !== fragment.page_digest - || digestProposalValue(page) !== fragment.page_digest + digestProposalValue(page) !== fragment.page_digest ) { throw new AgentJobProposalError('digest_mismatch', `Digest mismatch at sequence ${expectedSequence}.`); } if (seenSlugs.has(page.slug)) { throw new AgentJobProposalError('duplicate_page', `Proposal contains duplicate page slug ${page.slug}.`); } + assertSlugAllowed(binding, page.slug, 'Proposed page'); seenSlugs.add(page.slug); pages.push(page); pageDigests.push({ sequence: expectedSequence, slug: page.slug, digest: fragment.page_digest }); } - validatePlanRelations(seenSlugs, timeline, links); + validatePlanRelations(binding, seenSlugs, timeline, links); const plan: ScopedAdmissionProposalPlan = { artifactId: binding.artifactId, sourceId: binding.sourceId, - admissionScope: binding.admissionScope, + admissionScope: boundScope, summary, proposedPages: pages, proposedTimelineEntries: timeline, @@ -361,7 +381,7 @@ export async function finalizeAgentJobProposal( status: 'staged_proposal', artifactId: binding.artifactId, sourceId: binding.sourceId, - admissionScope: binding.admissionScope, + admissionScope: boundScope, summary, pageDigests, proposalDigest, @@ -386,7 +406,7 @@ export async function finalizeAgentJobProposal( ON CONFLICT (job_id) DO NOTHING`, [ jobId, binding.ownerClientId, binding.sourceId, binding.artifactId, - binding.admissionScope, totalPages, canonicalProposalJson(pageDigests), + boundScope, totalPages, canonicalProposalJson(pageDigests), planJson, proposalDigest, manifestJson, ], ); @@ -468,46 +488,61 @@ async function readJobBinding(engine: BrainEngine, jobId: number, lock: boolean) source_id: string | null; artifact_id: string | null; admission_scope: string | null; + capture_page_slug: string | null; + allowed_slug_prefixes: unknown; }>( `SELECT name, data->>'__owner_client_id' AS owner_client_id, data->>'source_id' AS source_id, data->>'proposal_artifact_id' AS artifact_id, - data->>'proposal_admission_scope' AS admission_scope + data->>'proposal_admission_scope' AS admission_scope, + data->>'proposal_capture_page_slug' AS capture_page_slug, + data->'allowed_slug_prefixes' AS allowed_slug_prefixes FROM minion_jobs WHERE id = $1${lock ? ' FOR UPDATE' : ''}`, [jobId], ); const row = rows[0]; if ( - !row || row.name !== 'subagent' || !row.owner_client_id || !row.artifact_id - || !row.admission_scope + !row || row.name !== 'subagent' || !row.owner_client_id || !row.source_id + || !row.artifact_id || !row.capture_page_slug + || !Array.isArray(row.allowed_slug_prefixes) + || row.allowed_slug_prefixes.length === 0 + || row.allowed_slug_prefixes.some((prefix) => typeof prefix !== 'string' || prefix.length === 0) ) { throw new AgentJobProposalError( 'job_not_bound', - 'Agent job is not bound to an owner, proposal artifact, and admission scope.', + 'Agent job is not bound to an owner, source, proposal artifact, capture page, and slug fence.', ); } - const sourceId = row.source_id ?? 'default'; + const sourceId = row.source_id; assertValidSourceId(sourceId); + const capturePageSlug = readCanonicalSlug(row.capture_page_slug, 'proposal_capture_page_slug'); + const allowedSlugPrefixes = row.allowed_slug_prefixes as string[]; + if (!matchesSlugAllowList(capturePageSlug, allowedSlugPrefixes)) { + throw new AgentJobProposalError( + 'job_not_bound', + 'Agent job capture page is outside its bound slug fence.', + ); + } return { ownerClientId: row.owner_client_id, sourceId, artifactId: row.artifact_id, admissionScope: row.admission_scope, + capturePageSlug, + allowedSlugPrefixes, }; } function assertBindingMatches( binding: JobBinding, - artifactId: unknown, - sourceId: unknown, - admissionScope: unknown, + requested: Pick, ): void { if ( - artifactId !== binding.artifactId - || sourceId !== binding.sourceId - || admissionScope !== binding.admissionScope + requested.artifactId !== binding.artifactId + || requested.sourceId !== binding.sourceId + || (binding.admissionScope !== null && requested.admissionScope !== binding.admissionScope) ) { throw new AgentJobProposalError( 'binding_mismatch', @@ -516,6 +551,141 @@ function assertBindingMatches( } } +/** Freeze a first-stage admission scope without changing any other job binding. */ +async function freezeAdmissionScope( + engine: BrainEngine, + jobId: number, + binding: JobBinding, + requestedScope: string, +): Promise { + if (binding.admissionScope === null) { + await engine.executeRaw( + `UPDATE minion_jobs + SET data = jsonb_set(data, '{proposal_admission_scope}', to_jsonb($2::text), true), + updated_at = now() + WHERE id = $1 AND data->>'proposal_admission_scope' IS NULL`, + [jobId, requestedScope], + ); + binding.admissionScope = requestedScope; + } + if (binding.admissionScope !== requestedScope) { + throw new AgentJobProposalError( + 'binding_mismatch', + 'Proposal admission scope does not match the scope frozen by the first staged page.', + ); + } + return binding as JobBinding & { admissionScope: string }; +} + +/** Require finalization to use a scope already frozen by a staged page. */ +function requireBoundAdmissionScope(binding: JobBinding): string { + if (binding.admissionScope === null) { + throw new AgentJobProposalError( + 'job_not_bound', + 'Agent job admission scope has not been frozen by a staged page.', + ); + } + return binding.admissionScope; +} + +/** Parse and normalize one page-staging input before any durable write. */ +function parseStageProposalPageInput(raw: unknown): ParsedStageProposalPageInput { + if (proposalToolInputBytes(raw) > PROPOSAL_STAGE_INPUT_MAX_BYTES) { + throw new AgentJobProposalError('stage_input_too_large', 'Proposal page input exceeds the staging byte limit.'); + } + const input = readRecord(raw, 'stage proposal input'); + const artifactId = readBoundedString(input.artifact_id, 'artifact_id', 255); + const sourceId = readBoundedString(input.source_id, 'source_id', 255); + assertValidSourceId(sourceId); + const admissionScope = readBoundedString(input.admission_scope, 'admission_scope', 8_000); + const sequence = readPositiveInteger(input.sequence, 'sequence'); + const totalPages = readProposalPageCount(input.total_pages); + if (sequence > totalPages) { + throw new AgentJobProposalError('invalid_sequence', 'sequence must be within 1..total_pages.'); + } + const page = parseProposalPage(input.page); + return { + artifactId, + sourceId, + admissionScope, + sequence, + totalPages, + page, + pageDigest: digestProposalValue(page), + }; +} + +/** Read every staged fragment in deterministic sequence order. */ +async function readStoredFragments(engine: BrainEngine, jobId: number): Promise { + return engine.executeRaw( + `SELECT sequence, total_pages, owner_client_id, source_id, artifact_id, + admission_scope, page, page_digest + FROM agent_job_proposal_fragments + WHERE job_id = $1 + ORDER BY sequence`, + [jobId], + ); +} + +/** + * Validate replay identity and the cumulative canonical page-byte ceiling. + * Returns true only for an exact existing sequence replay. + */ +function assertCumulativeStageFits( + binding: JobBinding, + candidate: ParsedStageProposalPageInput, + fragments: readonly StoredProposalFragment[], +): boolean { + let stagedBytes = 0; + let replay = false; + for (const fragment of fragments) { + if ( + fragment.owner_client_id !== binding.ownerClientId + || fragment.source_id !== binding.sourceId + || fragment.artifact_id !== binding.artifactId + || fragment.admission_scope !== candidate.admissionScope + || Number(fragment.total_pages) !== candidate.totalPages + ) { + throw new AgentJobProposalError('binding_mismatch', 'A staged fragment does not match its job binding.'); + } + const page = parseProposalPage(fragment.page); + const pageDigest = digestProposalValue(page); + if (pageDigest !== fragment.page_digest) { + throw new AgentJobProposalError('digest_mismatch', 'A staged fragment does not match its stored digest.'); + } + stagedBytes += proposalToolInputBytes(page); + if (Number(fragment.sequence) !== candidate.sequence) continue; + if ( + pageDigest !== candidate.pageDigest + || canonicalProposalJson(page) !== canonicalProposalJson(candidate.page) + ) { + throw new AgentJobProposalError( + 'conflicting_fragment', + `Sequence ${candidate.sequence} already contains a different proposal fragment.`, + ); + } + replay = true; + } + if (!replay) stagedBytes += proposalToolInputBytes(candidate.page); + if (stagedBytes > PROPOSAL_AGGREGATE_MAX_BYTES) { + throw new AgentJobProposalError( + 'proposal_too_large', + `Staged proposal pages are ${stagedBytes} UTF-8 bytes; maximum is ${PROPOSAL_AGGREGATE_MAX_BYTES}.`, + ); + } + return replay; +} + +/** Reject a proposed mutation target outside the job's durable slug fence. */ +function assertSlugAllowed(binding: JobBinding, slug: string, label: string): void { + if (!matchesSlugAllowList(slug, binding.allowedSlugPrefixes)) { + throw new AgentJobProposalError( + 'slug_not_allowed', + `${label} ${slug} is outside the agent job slug fence.`, + ); + } +} + function parseProposalPage(raw: unknown): ScopedProposalPage { const page = readRecord(raw, 'page'); const effect = page.effect; @@ -538,34 +708,11 @@ function parseProposalPage(raw: unknown): ScopedProposalPage { return { slug, effect, title, bodyMarkdown, baseMarkdown, expectedContentHash }; } -function parseRequestedDigests(raw: unknown, totalPages: number): Array<{ sequence: number; digest: string }> { - if (!Array.isArray(raw) || raw.length !== totalPages) { - throw new AgentJobProposalError('invalid_manifest', 'page_digests must contain exactly total_pages entries.'); - } - const seen = new Set(); - return raw.map((entry, index) => { - const record = readRecord(entry, `page_digests[${index}]`); - assertExactKeys(record, ['sequence', 'digest'], `page_digests[${index}]`); - const sequence = readPositiveInteger(record.sequence, `page_digests[${index}].sequence`); - const digest = readString(record.digest, `page_digests[${index}].digest`); - if (!SHA256_RE.test(digest)) { - throw new AgentJobProposalError('invalid_manifest', `Invalid digest at sequence ${sequence}.`); - } - if (seen.has(sequence)) { - throw new AgentJobProposalError('duplicate_manifest_sequence', `Duplicate manifest sequence ${sequence}.`); - } - seen.add(sequence); - if (sequence !== index + 1) { - throw new AgentJobProposalError('fragment_gap', 'page_digests must be ordered and contiguous from 1.'); - } - return { sequence, digest }; - }); -} - function parseTimelineEntries(raw: unknown): ScopedProposalTimelineEntry[] { if (!Array.isArray(raw) || raw.length > 40) { throw new AgentJobProposalError('invalid_timeline', 'proposed_timeline_entries must be an array of at most 40 entries.'); } + const identities = new Set(); return raw.map((entry, index) => { const record = readRecord(entry, `proposed_timeline_entries[${index}]`); const keys = Object.keys(record).sort(); @@ -589,6 +736,11 @@ function parseTimelineEntries(raw: unknown): ScopedProposalTimelineEntry[] { ref: readCanonicalSlug(record.ref, 'timeline.ref'), }; if ('refLabel' in record) result.refLabel = readBoundedString(record.refLabel, 'timeline.refLabel', 500); + const identity = canonicalProposalJson(result); + if (identities.has(identity)) { + throw new AgentJobProposalError('duplicate_timeline', 'Proposal contains duplicate timeline mutations.'); + } + identities.add(identity); return result; }); } @@ -597,14 +749,21 @@ function parseLinks(raw: unknown): ScopedProposalLink[] { if (!Array.isArray(raw) || raw.length > 40) { throw new AgentJobProposalError('invalid_links', 'proposed_links must be an array of at most 40 entries.'); } + const identities = new Set(); return raw.map((entry, index) => { const record = readRecord(entry, `proposed_links[${index}]`); assertExactKeys(record, ['from', 'to', 'type'], `proposed_links[${index}]`); - return { + const result = { from: readCanonicalSlug(record.from, 'link.from'), to: readCanonicalSlug(record.to, 'link.to'), type: readBoundedString(record.type, 'link.type', 128), }; + const identity = canonicalProposalJson(result); + if (identities.has(identity)) { + throw new AgentJobProposalError('duplicate_links', 'Proposal contains duplicate link mutations.'); + } + identities.add(identity); + return result; }); } @@ -616,18 +775,28 @@ function parseUnresolved(raw: unknown): string[] { } function validatePlanRelations( + binding: JobBinding, pageSlugs: ReadonlySet, timeline: readonly ScopedProposalTimelineEntry[], links: readonly ScopedProposalLink[], ): void { - // The shared plan contract guarantees referential integrity. Provider - // skills own stricter provenance rules such as capture-page-only refs. + if (!pageSlugs.has(binding.capturePageSlug)) { + throw new AgentJobProposalError( + 'missing_capture_page', + 'The exact job-bound capture page must be included in proposed pages.', + ); + } for (const entry of timeline) { - if (!pageSlugs.has(entry.ref)) { - throw new AgentJobProposalError('invalid_timeline', 'Every timeline ref must name a proposed page.'); + assertSlugAllowed(binding, entry.pageSlug, 'Timeline target'); + if (entry.ref !== binding.capturePageSlug) { + throw new AgentJobProposalError( + 'invalid_timeline_capture', + 'Every timeline ref must equal the exact job-bound capture page.', + ); } } for (const link of links) { + assertSlugAllowed(binding, link.from, 'Link source'); if (!pageSlugs.has(link.from)) { throw new AgentJobProposalError('invalid_links', 'Every proposed link from slug must name a proposed page.'); } diff --git a/src/core/minions/handlers/subagent.ts b/src/core/minions/handlers/subagent.ts index 2c86e18e82..66e5beeaac 100644 --- a/src/core/minions/handlers/subagent.ts +++ b/src/core/minions/handlers/subagent.ts @@ -57,7 +57,10 @@ import { supportsReasoningEffort } from '../../ai/model-resolver.ts'; import { randomUUIDv7 } from 'bun'; import { isContextLimitMessage } from '../error-classify.ts'; import { ToolLoopContextProjectionError } from '../../ai/tool-loop-context.ts'; -import { assertProposalToolTurnPersistable } from '../agent-job-proposals.ts'; +import { + assertProposalToolTurnPersistable, + assertProposalToolTurnPersistableForJob, +} from '../agent-job-proposals.ts'; // ── Defaults ──────────────────────────────────────────────── @@ -658,7 +661,7 @@ export function makeSubagentHandler(deps: SubagentDeps) { // Enforce proposal page count and byte limits before the assistant turn // makes its raw tool inputs durable in subagent_messages. - assertProposalToolTurnPersistable(blocks); + await assertProposalToolTurnPersistableForJob(engine, ctx.id, blocks); // 3. Persist the assistant message BEFORE tool dispatch so replay // sees a consistent state. @@ -1015,6 +1018,7 @@ async function runSubagentViaGateway(args: GatewayRunArgs): Promise { + await assertProposalToolTurnPersistableForJob(engine, ctx.id, blocks); // Convert ChatBlock[] back to ContentBlock-shaped JSONB for persistence. // Storing the gateway's provider-neutral shape is the v2 content_blocks // contract; the D5 shim handles legacy reads from v1 rows. diff --git a/src/core/operations.ts b/src/core/operations.ts index bec6776d68..3aede34752 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -49,6 +49,8 @@ import { isSearchMode } from './search/mode.ts'; import { stampEvidence } from './search/evidence.ts'; import type { SearchResult } from './types.ts'; import { CJK_SLUG_CHARS } from './cjk.ts'; +import { matchesSlugAllowList, slugFenceContains } from './slug-allow-list.ts'; +export { matchesSlugAllowList } from './slug-allow-list.ts'; import * as db from './db.ts'; import { VERSION } from '../version.ts'; import { assertValidSourceId } from './source-id.ts'; @@ -230,52 +232,6 @@ export function validatePageSlug(slug: string): void { } } -/** - * Match a slug against a list of allow-list prefix globs. - * - * Glob form: `/*` matches any slug starting with `/` and - * having at least one more segment (single or multi). Bare `` (no - * trailing `/*`) matches that exact slug only. The `*` is intentionally - * permissive — depth is unbounded, so `wiki/originals/*` matches both - * `wiki/originals/idea-x` and `wiki/originals/ideas/2026-04-25-idea-y`. - * - * Used by the v0.23 dream-cycle trusted-workspace path. Order doesn't - * matter; the first match wins (returns true on any match). - */ -export function matchesSlugAllowList(slug: string, prefixes: readonly string[]): boolean { - for (const p of prefixes) { - if (p.endsWith('/*')) { - const base = p.slice(0, -2); - if (slug === base) continue; - if (slug.startsWith(base + '/')) return true; - } else if (p.endsWith('/')) { - if (slug.startsWith(p) && slug.length > p.length) return true; - } else if (p === slug) { - return true; - } - } - return false; -} - -/** Return whether every slug matched by a requested fence is inside a bound fence. */ -function slugFenceContains(bound: string, requested: string): boolean { - const boundBase = recursiveSlugFenceBase(bound); - const requestedBase = recursiveSlugFenceBase(requested); - if (boundBase === null) { - return requestedBase === null && requested === bound; - } - const requestedAnchor = requestedBase ?? requested; - return requestedAnchor.startsWith(`${boundBase}/`) - || (requestedBase !== null && requestedAnchor === boundBase); -} - -/** Strip the recursive suffix from one slash- or glob-form slug fence. */ -function recursiveSlugFenceBase(fence: string): string | null { - if (fence.endsWith('/*')) return fence.slice(0, -2); - if (fence.endsWith('/')) return fence.slice(0, -1); - return null; -} - /** * Subagent slug-fence enforcement, shared by every mutating op a subagent * can reach (put_page, add_timeline_entry). FAIL-CLOSED: `viaSubagent=true` @@ -5367,9 +5323,13 @@ const submit_agent: Operation = { type: 'string', description: 'Exact ingestion artifact id bound to staged proposal tools for this job', }, + proposal_capture_page_slug: { + type: 'string', + description: 'Exact capture page slug bound to staged proposal provenance for this job', + }, proposal_admission_scope: { type: 'string', - description: 'Exact resolver admission scope bound to staged proposal tools for this job', + description: 'Exact resolver admission scope, or omit so the first staged page freezes it', }, }, mutating: true, @@ -5457,21 +5417,57 @@ const submit_agent: Operation = { const proposalArtifactId = typeof p.proposal_artifact_id === 'string' ? p.proposal_artifact_id.trim() : ''; + const proposalCapturePageSlug = typeof p.proposal_capture_page_slug === 'string' + ? p.proposal_capture_page_slug.trim() + : ''; const proposalAdmissionScope = typeof p.proposal_admission_scope === 'string' ? p.proposal_admission_scope.trim() : ''; - if ((proposalArtifactId.length > 0) !== (proposalAdmissionScope.length > 0)) { + const usesProposalTools = requestedTools.some((tool) => ( + tool === 'stage_ingestion_proposal_page' || tool === 'finalize_ingestion_proposal' + )); + if ((proposalArtifactId.length > 0) !== (proposalCapturePageSlug.length > 0)) { + throw new OperationError( + 'invalid_params', + 'submit_agent: proposal artifact and capture page slug must be supplied together.', + ); + } + if (proposalAdmissionScope && !proposalArtifactId) { + throw new OperationError( + 'invalid_params', + 'submit_agent: proposal admission scope requires a proposal artifact and capture page binding.', + ); + } + if (usesProposalTools && (!proposalArtifactId || !proposalCapturePageSlug)) { throw new OperationError( 'invalid_params', - 'submit_agent: proposal_artifact_id and proposal_admission_scope must be supplied together.', + 'submit_agent: proposal-tool jobs require a proposal artifact and capture page slug.', ); } - if (proposalArtifactId.length > 255 || proposalAdmissionScope.length > 8_000) { + if (proposalArtifactId.length > 255 || proposalCapturePageSlug.length > 255 || proposalAdmissionScope.length > 8_000) { throw new OperationError( 'invalid_params', - 'submit_agent: proposal artifact id or admission scope exceeds its bounded length.', + 'submit_agent: proposal artifact, capture page slug, or admission scope exceeds its bounded length.', ); } + if (usesProposalTools && !boundSource) { + throw new OperationError('invalid_params', 'submit_agent: proposal-tool jobs require a bound source.'); + } + if (usesProposalTools && requestedSlugPrefixes.length === 0) { + throw new OperationError('invalid_params', 'submit_agent: proposal-tool jobs require a non-empty slug fence.'); + } + if (proposalCapturePageSlug) { + validatePageSlug(proposalCapturePageSlug); + if (proposalCapturePageSlug !== proposalCapturePageSlug.toLowerCase()) { + throw new OperationError('invalid_params', 'submit_agent: proposal capture page slug must be canonical lowercase.'); + } + if (!matchesSlugAllowList(proposalCapturePageSlug, requestedSlugPrefixes)) { + throw new OperationError( + 'permission_denied', + 'submit_agent: proposal capture page is outside the requested slug fence.', + ); + } + } if (p.reasoning_effort !== undefined && typeof p.model !== 'string') { throw new OperationError( 'invalid_params', @@ -5546,6 +5542,7 @@ const submit_agent: Operation = { reasoning_effort: isReasoningEffort(p.reasoning_effort) ? p.reasoning_effort : null, skill_name: typeof p.skill_name === 'string' ? p.skill_name : null, proposal_artifact_id: proposalArtifactId || null, + proposal_capture_page_slug: proposalCapturePageSlug || null, proposal_admission_scope: proposalAdmissionScope || null, }; } @@ -5587,7 +5584,8 @@ const submit_agent: Operation = { if (boundSource) jobData.source_id = boundSource; if (proposalArtifactId) { jobData.proposal_artifact_id = proposalArtifactId; - jobData.proposal_admission_scope = proposalAdmissionScope; + jobData.proposal_capture_page_slug = proposalCapturePageSlug; + if (proposalAdmissionScope) jobData.proposal_admission_scope = proposalAdmissionScope; } const job = await queue.add( 'subagent', @@ -5634,7 +5632,7 @@ const stage_ingestion_proposal_page: Operation = { params: { artifact_id: { type: 'string', required: true, description: 'Exact artifact id bound at submit_agent time' }, source_id: { type: 'string', required: true, description: 'Exact source bound to the current agent job' }, - admission_scope: { type: 'string', required: true, description: 'Exact admission scope bound at submit_agent time' }, + admission_scope: { type: 'string', required: true, description: 'Exact admission scope bound at submit_agent time or frozen by the first staged page' }, sequence: { type: 'number', required: true, description: 'One-based page position' }, total_pages: { type: 'number', required: true, description: 'Total number of pages in the proposal (1-100)' }, page: { type: 'object', required: true, description: 'Exact create or update page proposal' }, @@ -5658,15 +5656,14 @@ const stage_ingestion_proposal_page: Operation = { const finalize_ingestion_proposal: Operation = { name: 'finalize_ingestion_proposal', - description: 'Validate and freeze the complete staged ingestion proposal, returning a compact digest manifest without page bodies.', + description: 'Validate and freeze the complete staged ingestion proposal, deriving a compact ordered digest manifest without page bodies.', // Freezing job evidence does not mutate pages, links, takes, or timelines. mutating: false, params: { artifact_id: { type: 'string', required: true, description: 'Exact artifact id bound at submit_agent time' }, source_id: { type: 'string', required: true, description: 'Exact source bound to the current agent job' }, - admission_scope: { type: 'string', required: true, description: 'Exact admission scope bound at submit_agent time' }, + admission_scope: { type: 'string', required: true, description: 'Exact admission scope already frozen on the agent job' }, total_pages: { type: 'number', required: true, description: 'Total number of staged pages (1-100)' }, - page_digests: { type: 'array', required: true, items: { type: 'object' }, description: 'Ordered one-based sequence and digest pairs' }, summary: { type: 'string', required: true, description: 'Compact source-grounded proposal summary' }, proposed_timeline_entries: { type: 'array', items: { type: 'object' }, description: 'Bounded frozen timeline entries' }, proposed_links: { type: 'array', items: { type: 'object' }, description: 'Bounded frozen typed links' }, diff --git a/src/core/slug-allow-list.ts b/src/core/slug-allow-list.ts new file mode 100644 index 0000000000..4f102d0cb6 --- /dev/null +++ b/src/core/slug-allow-list.ts @@ -0,0 +1,38 @@ +/** + * Match one canonical page slug against recursive or exact slug fences. + * + * A trailing `/*` or `/` permits descendants but not the namespace root. + * A bare fence permits only the exact slug. + */ +export function matchesSlugAllowList(slug: string, prefixes: readonly string[]): boolean { + for (const prefix of prefixes) { + if (prefix.endsWith('/*')) { + const base = prefix.slice(0, -2); + if (slug !== base && slug.startsWith(`${base}/`)) return true; + } else if (prefix.endsWith('/')) { + if (slug.startsWith(prefix) && slug.length > prefix.length) return true; + } else if (prefix === slug) { + return true; + } + } + return false; +} + +/** Return whether every slug matched by a requested fence is inside a bound fence. */ +export function slugFenceContains(bound: string, requested: string): boolean { + const boundBase = recursiveSlugFenceBase(bound); + const requestedBase = recursiveSlugFenceBase(requested); + if (boundBase === null) { + return requestedBase === null && requested === bound; + } + const requestedAnchor = requestedBase ?? requested; + return requestedAnchor.startsWith(`${boundBase}/`) + || (requestedBase !== null && requestedAnchor === boundBase); +} + +/** Strip the recursive suffix from one slash- or glob-form slug fence. */ +function recursiveSlugFenceBase(fence: string): string | null { + if (fence.endsWith('/*')) return fence.slice(0, -2); + if (fence.endsWith('/')) return fence.slice(0, -1); + return null; +} diff --git a/test/agent-job-proposals.test.ts b/test/agent-job-proposals.test.ts index 7afa9fff66..a84482fbf8 100644 --- a/test/agent-job-proposals.test.ts +++ b/test/agent-job-proposals.test.ts @@ -2,12 +2,12 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'bun:test' import { PGLiteEngine } from '../src/core/pglite-engine.ts'; import { resetPgliteState } from './helpers/reset-pglite.ts'; import { - AgentJobProposalError, PROPOSAL_AGGREGATE_MAX_BYTES, PROPOSAL_MANIFEST_MAX_BYTES, PROPOSAL_MAX_PAGES, PROPOSAL_STAGE_INPUT_MAX_BYTES, assertProposalToolTurnPersistable, + assertProposalToolTurnPersistableForJob, canonicalProposalJson, digestProposalValue, finalizeAgentJobProposal, @@ -15,6 +15,8 @@ import { stageAgentJobProposalPage, type ScopedProposalPage, } from '../src/core/minions/agent-job-proposals.ts'; +import { compactToolLoopMessages } from '../src/core/ai/tool-loop-context.ts'; +import type { ChatMessage } from '../src/core/ai/gateway.ts'; let engine: PGLiteEngine; @@ -41,7 +43,9 @@ async function seedJob(overrides: Record = {}): Promise __owner_client_id: 'lore-client', source_id: 'company', proposal_artifact_id: 'artifact-1', + proposal_capture_page_slug: 'sources/example', proposal_admission_scope: 'Include project delivery notes.', + allowed_slug_prefixes: ['sources/*', 'projects/*'], ...overrides, })], ); @@ -68,11 +72,12 @@ async function stage( sequence: number, totalPages: number, page: ScopedProposalPage, + admissionScope = 'Include project delivery notes.', ) { return stageAgentJobProposalPage(engine, jobId, { artifact_id: 'artifact-1', source_id: 'company', - admission_scope: 'Include project delivery notes.', + admission_scope: admissionScope, sequence, total_pages: totalPages, page, @@ -119,10 +124,6 @@ describe('durable agent-job proposal staging', () => { source_id: 'company', admission_scope: 'Include project delivery notes.', total_pages: 2, - page_digests: [ - { sequence: 1, digest: first.digest }, - { sequence: 2, digest: second.digest }, - ], summary: 'Two exact pages are ready for review.', proposed_timeline_entries: [{ pageSlug: 'projects/existing', @@ -152,6 +153,142 @@ describe('durable agent-job proposal staging', () => { expect(digestProposalValue(owned.plan)).toBe(manifest.proposalDigest); }); + it('finalizes from the durable ordered manifest after old stage outputs compact away', async () => { + const jobId = await seedJob(); + const first = await stage(jobId, 1, 2, createPage('sources/example')); + const second = await stage(jobId, 2, 2, createPage('projects/example')); + const messages: ChatMessage[] = [{ role: 'user', content: 'Build the exact ingestion proposal.' }]; + for (const page of [first, second]) { + messages.push({ + role: 'assistant', + content: [{ + type: 'tool-call', + toolCallId: `stage-${page.sequence}`, + toolName: 'brain_stage_ingestion_proposal_page', + input: { sequence: page.sequence, body: 'x'.repeat(1_500) }, + }], + }); + messages.push({ + role: 'user', + content: [{ + type: 'tool-result', + toolCallId: `stage-${page.sequence}`, + toolName: 'brain_stage_ingestion_proposal_page', + output: page, + }], + }); + } + const compacted = compactToolLoopMessages(messages, 2_000, { mutatingToolNames: new Set() }); + expect(JSON.stringify(compacted)).not.toContain(first.digest); + + const manifest = await finalizeAgentJobProposal(engine, jobId, { + artifact_id: 'artifact-1', + source_id: 'company', + admission_scope: 'Include project delivery notes.', + total_pages: 2, + summary: 'Ready after compaction.', + }); + expect(manifest.pageDigests).toEqual([first, second]); + }); + + it('freezes a previously-null admission scope on the first stage only', async () => { + const jobId = await seedJob({ proposal_admission_scope: null }); + const page = createPage('sources/example'); + const firstInput = { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Derived resolver scope.', + sequence: 1, total_pages: 2, page, + }; + await assertProposalToolTurnPersistableForJob(engine, jobId, [{ + type: 'tool-call', toolName: 'brain_stage_ingestion_proposal_page', input: firstInput, + }]); + const [preflightRow] = await engine.executeRaw<{ scope: string | null }>( + `SELECT data->>'proposal_admission_scope' AS scope FROM minion_jobs WHERE id = $1`, + [jobId], + ); + expect(preflightRow!.scope).toBeNull(); + const first = await stage(jobId, 1, 2, page, 'Derived resolver scope.'); + expect(await stage(jobId, 1, 2, page, 'Derived resolver scope.')).toEqual(first); + const [row] = await engine.executeRaw<{ scope: string | null }>( + `SELECT data->>'proposal_admission_scope' AS scope FROM minion_jobs WHERE id = $1`, + [jobId], + ); + expect(row!.scope).toBe('Derived resolver scope.'); + await expect(stage(jobId, 2, 2, createPage('projects/example'), 'Different scope.')) + .rejects.toMatchObject({ code: 'binding_mismatch' }); + }); + + it('requires pre-bound owner, source, artifact, capture slug, and slug fences', async () => { + for (const overrides of [ + { __owner_client_id: null }, + { source_id: null }, + { proposal_artifact_id: null }, + { proposal_capture_page_slug: null }, + { allowed_slug_prefixes: [] }, + ]) { + const jobId = await seedJob(overrides); + await expect(stage(jobId, 1, 1, createPage('sources/example'))) + .rejects.toMatchObject({ code: 'job_not_bound' }); + } + }); + + it('fences proposed page, timeline target, link source, and exact capture provenance', async () => { + const unauthorizedJob = await seedJob(); + await expect(stage(unauthorizedJob, 1, 1, createPage('private/example'))) + .rejects.toMatchObject({ code: 'slug_not_allowed' }); + + const jobId = await seedJob(); + await stage(jobId, 1, 2, createPage('sources/example')); + await stage(jobId, 2, 2, createPage('projects/example')); + const common = { + artifact_id: 'artifact-1', source_id: 'company', + admission_scope: 'Include project delivery notes.', total_pages: 2, + summary: 'Ready.', + }; + await expect(finalizeAgentJobProposal(engine, jobId, { + ...common, + proposed_timeline_entries: [{ + pageSlug: 'private/existing', date: '2026-08-07', text: 'Event.', ref: 'sources/example', + }], + })).rejects.toMatchObject({ code: 'slug_not_allowed' }); + await expect(finalizeAgentJobProposal(engine, jobId, { + ...common, + proposed_timeline_entries: [{ + pageSlug: 'projects/example', date: '2026-08-07', text: 'Event.', ref: 'projects/example', + }], + })).rejects.toMatchObject({ code: 'invalid_timeline_capture' }); + await expect(finalizeAgentJobProposal(engine, jobId, { + ...common, + proposed_links: [{ from: 'private/example', to: 'projects/example', type: 'documents' }], + })).rejects.toMatchObject({ code: 'slug_not_allowed' }); + + const missingCaptureJob = await seedJob(); + await stage(missingCaptureJob, 1, 1, createPage('projects/example')); + await expect(finalizeAgentJobProposal(engine, missingCaptureJob, { + artifact_id: 'artifact-1', source_id: 'company', + admission_scope: 'Include project delivery notes.', total_pages: 1, summary: 'Missing capture.', + })).rejects.toMatchObject({ code: 'missing_capture_page' }); + }); + + it('rejects duplicate canonical timeline and link mutations', async () => { + const jobId = await seedJob(); + await stage(jobId, 1, 1, createPage('sources/example')); + const common = { + artifact_id: 'artifact-1', source_id: 'company', + admission_scope: 'Include project delivery notes.', total_pages: 1, summary: 'Ready.', + }; + const timeline = { + pageSlug: 'sources/example', date: '2026-08-07', text: 'Event.', + ref: 'sources/example', refLabel: 'Capture', + }; + await expect(finalizeAgentJobProposal(engine, jobId, { + ...common, proposed_timeline_entries: [timeline, { ...timeline }], + })).rejects.toMatchObject({ code: 'duplicate_timeline' }); + const link = { from: 'sources/example', to: 'projects/example', type: 'documents' }; + await expect(finalizeAgentJobProposal(engine, jobId, { + ...common, proposed_links: [link, { ...link }], + })).rejects.toMatchObject({ code: 'duplicate_links' }); + }); + it('makes identical staging and finalization replay-safe but rejects conflicts', async () => { const jobId = await seedJob(); const page = createPage('sources/example'); @@ -165,7 +302,6 @@ describe('durable agent-job proposal staging', () => { source_id: 'company', admission_scope: 'Include project delivery notes.', total_pages: 1, - page_digests: [{ sequence: 1, digest: first.digest }], summary: 'Ready.', }; const finalized = await finalizeAgentJobProposal(engine, jobId, input); @@ -178,22 +314,18 @@ describe('durable agent-job proposal staging', () => { const firstJobId = await seedJob(); const secondJobId = await seedJob(); const page = createPage('sources/example'); - const firstPage = await stage(firstJobId, 1, 1, page); - const secondPage = await stage(secondJobId, 1, 1, page); + await stage(firstJobId, 1, 1, page); + await stage(secondJobId, 1, 1, page); const input = { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', total_pages: 1, - page_digests: [{ sequence: 1, digest: firstPage.digest }], summary: 'Ready.', }; const first = await finalizeAgentJobProposal(engine, firstJobId, input); - const second = await finalizeAgentJobProposal(engine, secondJobId, { - ...input, - page_digests: [{ sequence: 1, digest: secondPage.digest }], - }); + const second = await finalizeAgentJobProposal(engine, secondJobId, input); expect(second.proposalDigest).toBe(first.proposalDigest); const rows = await engine.executeRaw<{ count: string }>( @@ -205,44 +337,42 @@ describe('durable agent-job proposal staging', () => { expect(Number(rows[0]!.count)).toBe(2); }); - it('rejects gaps, duplicate manifest sequences, digest mismatches, and duplicate page slugs', async () => { + it('rejects gaps, corrupted stored digests, and duplicate page slugs', async () => { const gapJob = await seedJob(); - const gap = await stage(gapJob, 1, 2, createPage('sources/gap')); + await stage(gapJob, 1, 2, createPage('sources/gap')); await expect(finalizeAgentJobProposal(engine, gapJob, { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', - total_pages: 2, page_digests: [{ sequence: 1, digest: gap.digest }, { sequence: 2, digest: gap.digest }], summary: 'Gap.', + total_pages: 2, summary: 'Gap.', })).rejects.toMatchObject({ code: 'fragment_gap' }); - const duplicateManifestJob = await seedJob(); - const one = await stage(duplicateManifestJob, 1, 2, createPage('sources/one')); - const two = await stage(duplicateManifestJob, 2, 2, createPage('sources/two')); - await expect(finalizeAgentJobProposal(engine, duplicateManifestJob, { - artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', - total_pages: 2, page_digests: [{ sequence: 1, digest: one.digest }, { sequence: 1, digest: two.digest }], summary: 'Duplicate.', - })).rejects.toBeInstanceOf(AgentJobProposalError); - await expect(finalizeAgentJobProposal(engine, duplicateManifestJob, { + const corruptedDigestJob = await seedJob(); + await stage(corruptedDigestJob, 1, 1, createPage('sources/example')); + await engine.executeRaw( + `UPDATE agent_job_proposal_fragments SET page_digest = $2 WHERE job_id = $1`, + [corruptedDigestJob, 'f'.repeat(64)], + ); + await expect(finalizeAgentJobProposal(engine, corruptedDigestJob, { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', - total_pages: 2, page_digests: [{ sequence: 1, digest: one.digest }, { sequence: 2, digest: 'f'.repeat(64) }], summary: 'Mismatch.', + total_pages: 1, summary: 'Mismatch.', })).rejects.toMatchObject({ code: 'digest_mismatch' }); const duplicatePageJob = await seedJob(); - const a = await stage(duplicatePageJob, 1, 2, createPage('sources/same')); - const b = await stage(duplicatePageJob, 2, 2, updatePage('sources/same')); + await stage(duplicatePageJob, 1, 2, createPage('sources/same')); + await stage(duplicatePageJob, 2, 2, updatePage('sources/same')); await expect(finalizeAgentJobProposal(engine, duplicatePageJob, { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', - total_pages: 2, page_digests: [{ sequence: 1, digest: a.digest }, { sequence: 2, digest: b.digest }], summary: 'Duplicate page.', + total_pages: 2, summary: 'Duplicate page.', })).rejects.toMatchObject({ code: 'duplicate_page' }); }); it('rejects calendar-normalized dates instead of accepting them as strict dates', async () => { const jobId = await seedJob(); - const page = await stage(jobId, 1, 1, createPage('sources/example')); + await stage(jobId, 1, 1, createPage('sources/example')); await expect(finalizeAgentJobProposal(engine, jobId, { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', total_pages: 1, - page_digests: [{ sequence: 1, digest: page.digest }], summary: 'Invalid date.', proposed_timeline_entries: [{ pageSlug: 'sources/example', @@ -255,13 +385,12 @@ describe('durable agent-job proposal staging', () => { it('rejects timeline reference labels above the shared 500-character contract', async () => { const jobId = await seedJob(); - const page = await stage(jobId, 1, 1, createPage('sources/example')); + await stage(jobId, 1, 1, createPage('sources/example')); await expect(finalizeAgentJobProposal(engine, jobId, { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', total_pages: 1, - page_digests: [{ sequence: 1, digest: page.digest }], summary: 'Invalid reference label.', proposed_timeline_entries: [{ pageSlug: 'sources/example', @@ -280,14 +409,14 @@ describe('durable agent-job proposal staging', () => { sequence: 1, total_pages: 1, page: createPage('sources/example'), })).rejects.toMatchObject({ code: 'binding_mismatch' }); - const staged = await stage(jobId, 1, 1, createPage('sources/example')); + await stage(jobId, 1, 1, createPage('sources/example')); await engine.executeRaw( `UPDATE agent_job_proposal_fragments SET source_id = 'other' WHERE job_id = $1`, [jobId], ); await expect(finalizeAgentJobProposal(engine, jobId, { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', - total_pages: 1, page_digests: [{ sequence: 1, digest: staged.digest }], summary: 'Ready.', + total_pages: 1, summary: 'Ready.', })).rejects.toMatchObject({ code: 'binding_mismatch' }); const unbound = await seedJob({ proposal_artifact_id: null }); @@ -297,10 +426,10 @@ describe('durable agent-job proposal staging', () => { it('requires exact owner and proposal digest on retrieval', async () => { const jobId = await seedJob(); - const page = await stage(jobId, 1, 1, createPage('sources/example')); + await stage(jobId, 1, 1, createPage('sources/example')); const manifest = await finalizeAgentJobProposal(engine, jobId, { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', - total_pages: 1, page_digests: [{ sequence: 1, digest: page.digest }], summary: 'Ready.', + total_pages: 1, summary: 'Ready.', }); await expect(getOwnedAgentJobProposal(engine, jobId, 'other', manifest.proposalDigest)) .rejects.toMatchObject({ code: 'permission_denied' }); @@ -308,25 +437,36 @@ describe('durable agent-job proposal staging', () => { .rejects.toMatchObject({ code: 'permission_denied' }); }); - it('rejects an aggregate plan over the explicit ceiling without finalizing it', async () => { + it('rejects cumulative staged pages over the aggregate ceiling without persisting the crossing fragment', async () => { const jobId = await seedJob(); const pageCount = 5; - const digests = []; - for (let sequence = 1; sequence <= pageCount; sequence++) { + for (let sequence = 1; sequence < pageCount; sequence++) { const page = createPage(`sources/large-${sequence}`, 'x'.repeat(165_000)); - digests.push(await stage(jobId, sequence, pageCount, page)); + await stage(jobId, sequence, pageCount, page); } - await expect(finalizeAgentJobProposal(engine, jobId, { + const crossingInput = { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', - total_pages: pageCount, - page_digests: digests.map(({ sequence, digest }) => ({ sequence, digest })), - summary: 'Too large.', - })).rejects.toMatchObject({ code: 'proposal_too_large' }); + sequence: pageCount, total_pages: pageCount, + page: createPage(`sources/large-${pageCount}`, 'x'.repeat(165_000)), + }; + await expect(assertProposalToolTurnPersistableForJob(engine, jobId, [{ + type: 'tool-call', toolName: 'brain_stage_ingestion_proposal_page', input: crossingInput, + }])).rejects.toMatchObject({ code: 'proposal_too_large' }); + const replayInput = { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', + sequence: pageCount - 1, total_pages: pageCount, + page: createPage(`sources/large-${pageCount - 1}`, 'x'.repeat(165_000)), + }; + await expect(assertProposalToolTurnPersistableForJob(engine, jobId, [{ + type: 'tool-call', toolName: 'brain_stage_ingestion_proposal_page', input: replayInput, + }])).resolves.toBeUndefined(); + await expect(stageAgentJobProposalPage(engine, jobId, crossingInput)) + .rejects.toMatchObject({ code: 'proposal_too_large' }); const rows = await engine.executeRaw<{ count: string }>( - `SELECT count(*)::text AS count FROM agent_job_proposals WHERE job_id = $1`, + `SELECT count(*)::text AS count FROM agent_job_proposal_fragments WHERE job_id = $1`, [jobId], ); - expect(Number(rows[0]!.count)).toBe(0); + expect(Number(rows[0]!.count)).toBe(pageCount - 1); expect(PROPOSAL_AGGREGATE_MAX_BYTES).toBe(786_432); expect(PROPOSAL_MANIFEST_MAX_BYTES).toBe(256 * 1024); }); @@ -335,7 +475,7 @@ describe('durable agent-job proposal staging', () => { const jobId = await seedJob(); const pageCount = 5; const pages = Array.from({ length: pageCount }, (_, index) => - createPage(`sources/boundary-${index + 1}`, 'x'.repeat(150_000))); + createPage(index === 0 ? 'sources/example' : `sources/boundary-${index + 1}`, 'x'.repeat(150_000))); const basePlan = { artifactId: 'artifact-1', sourceId: 'company', @@ -349,15 +489,13 @@ describe('durable agent-job proposal staging', () => { const remaining = PROPOSAL_AGGREGATE_MAX_BYTES - Buffer.byteLength(canonicalProposalJson(basePlan), 'utf8'); pages[pageCount - 1]!.bodyMarkdown += 'y'.repeat(remaining); - const digests = []; for (let sequence = 1; sequence <= pageCount; sequence++) { - digests.push(await stage(jobId, sequence, pageCount, pages[sequence - 1]!)); + await stage(jobId, sequence, pageCount, pages[sequence - 1]!); } const manifest = await finalizeAgentJobProposal(engine, jobId, { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', total_pages: pageCount, - page_digests: digests.map(({ sequence, digest }) => ({ sequence, digest })), summary: 'Boundary.', }); const owned = await getOwnedAgentJobProposal( diff --git a/test/e2e/subagent-gateway-path.test.ts b/test/e2e/subagent-gateway-path.test.ts index c524061ffb..28eebf84e6 100644 --- a/test/e2e/subagent-gateway-path.test.ts +++ b/test/e2e/subagent-gateway-path.test.ts @@ -32,6 +32,7 @@ import { type ChatBlock, type ChatResult, } from '../../src/core/ai/gateway.ts'; +import { stageAgentJobProposalPage } from '../../src/core/minions/agent-job-proposals.ts'; // ── Helpers ───────────────────────────────────────────────── @@ -71,20 +72,23 @@ interface FakeJobOpts { model?: string; allowed_tools?: string[]; use_gateway_loop?: boolean; + data?: Record; } async function makeFakeJob(opts: FakeJobOpts): Promise<{ jobId: number; ctx: MinionJobContext; tokenSink: any[] }> { + const jobData = { + prompt: opts.prompt, + model: opts.model, + allowed_tools: opts.allowed_tools, + use_gateway_loop: opts.use_gateway_loop, + ...opts.data, + }; // Insert a minion_jobs row so foreign keys validate (subagent_tool_executions.job_id FK). const rows = await engine.executeRaw<{ id: number }>( `INSERT INTO minion_jobs (name, status, data, queue, priority, created_at) VALUES ('subagent', 'active', $1::jsonb, 'default', 0, now()) RETURNING id`, - [JSON.stringify({ - prompt: opts.prompt, - model: opts.model, - allowed_tools: opts.allowed_tools, - use_gateway_loop: opts.use_gateway_loop, - })], + [JSON.stringify(jobData)], ); const jobId = rows[0].id; @@ -95,12 +99,7 @@ async function makeFakeJob(opts: FakeJobOpts): Promise<{ jobId: number; ctx: Min const ctx: MinionJobContext = { id: jobId, name: 'subagent', - data: { - prompt: opts.prompt, - model: opts.model, - allowed_tools: opts.allowed_tools, - use_gateway_loop: opts.use_gateway_loop, - }, + data: jobData, attempts_made: 0, signal: abortCtrl.signal, deadlineAtMs: null, @@ -255,6 +254,66 @@ describe('runSubagentViaGateway (v0.38 Slice 1 — full handler path through gat expect(executions).toHaveLength(0); }); + it('rejects a cumulative stage overflow before assistant, tool, or fragment persistence', async () => { + const crossingInput = { + artifact_id: 'artifact-1', source_id: 'company', + admission_scope: 'Derived scope.', sequence: 5, total_pages: 5, + page: { + slug: 'sources/large-5', effect: 'create', title: 'Large', + bodyMarkdown: 'x'.repeat(165_000), + }, + }; + __setChatTransportForTests(async () => ({ + text: '', + blocks: [{ + type: 'tool-call', toolCallId: 'stage-overflow', + toolName: 'brain_stage_ingestion_proposal_page', input: crossingInput, + }] as ChatBlock[], + stopReason: 'tool_calls', + usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, + model: 'anthropic:claude-sonnet-4-6', + providerId: 'anthropic', + } satisfies ChatResult)); + let executed = false; + const tool: ToolDef = { + name: 'brain_stage_ingestion_proposal_page', + description: 'stage', input_schema: { type: 'object' }, + idempotent: true, mutating: false, + async execute() { executed = true; return { ok: true }; }, + }; + const handler = buildHandler([tool]); + const { jobId, ctx } = await makeFakeJob({ + prompt: 'stage', model: 'anthropic:claude-sonnet-4-6', + data: { + __owner_client_id: 'lore-client', source_id: 'company', + proposal_artifact_id: 'artifact-1', proposal_capture_page_slug: 'sources/large-1', + proposal_admission_scope: 'Derived scope.', allowed_slug_prefixes: ['sources/*'], + }, + }); + for (let sequence = 1; sequence < 5; sequence++) { + await stageAgentJobProposalPage(engine, jobId, { + ...crossingInput, + sequence, + page: { ...crossingInput.page, slug: `sources/large-${sequence}` }, + }); + } + + await expect(handler(ctx)).rejects.toThrow(/maximum/i); + const messages = await engine.executeRaw<{ role: string }>( + `SELECT role FROM subagent_messages WHERE job_id = $1 ORDER BY message_idx`, [jobId], + ); + const executions = await engine.executeRaw( + `SELECT id FROM subagent_tool_executions WHERE job_id = $1`, [jobId], + ); + const fragments = await engine.executeRaw<{ sequence: number }>( + `SELECT sequence FROM agent_job_proposal_fragments WHERE job_id = $1 ORDER BY sequence`, [jobId], + ); + expect(executed).toBe(false); + expect(messages.map((row) => row.role)).toEqual(['user']); + expect(executions).toHaveLength(0); + expect(fragments.map((row) => Number(row.sequence))).toEqual([1, 2, 3, 4]); + }); + it('happy path 1-turn: gateway returns text, handler returns SubagentResult', async () => { __setChatTransportForTests(async () => ({ text: 'all done', diff --git a/test/github-project-ingestion-skill.test.ts b/test/github-project-ingestion-skill.test.ts index eb3e9e2398..532a2896c3 100644 --- a/test/github-project-ingestion-skill.test.ts +++ b/test/github-project-ingestion-skill.test.ts @@ -222,6 +222,8 @@ describe('github-project-ingestion skill', () => { expect(skill).toContain('brain_stage_ingestion_proposal_page'); expect(skill).toContain('Stage only one page per turn'); expect(skill).toContain('brain_finalize_ingestion_proposal'); + expect(skill).toContain('server derives the ordered'); + expect(skill).not.toContain('ordered\n`page_digests`'); expect(skill).toContain('"effect": "create | update"'); expect(skill).toContain('Create entries have'); expect(skill).toMatch(/Update entries add\s+exactly `baseMarkdown` and `expectedContentHash`/); diff --git a/test/granola-meeting-ingestion-skill.test.ts b/test/granola-meeting-ingestion-skill.test.ts index f494c143f3..9b0d65123a 100644 --- a/test/granola-meeting-ingestion-skill.test.ts +++ b/test/granola-meeting-ingestion-skill.test.ts @@ -115,6 +115,8 @@ describe('granola-meeting-ingestion skill', () => { expect(skill).toContain('brain_stage_ingestion_proposal_page'); expect(skill).toContain('Stage only one page per turn'); expect(skill).toContain('brain_finalize_ingestion_proposal'); + expect(skill).toContain('server derives the ordered'); + expect(skill).not.toContain('ordered\n`page_digests`'); expect(skill).toContain('"effect": "create | update"'); expect(skill).toContain('Create entries have'); expect(skill).toMatch(/Update entries add\s+exactly `baseMarkdown` and `expectedContentHash`/); diff --git a/test/submit-agent.test.ts b/test/submit-agent.test.ts index 19df3d0f22..d510c2dfa9 100644 --- a/test/submit-agent.test.ts +++ b/test/submit-agent.test.ts @@ -139,11 +139,13 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with }); it('publishes non-corpus-mutating staged proposal operations', () => { expect(submit_agent.params.proposal_artifact_id).toBeDefined(); + expect(submit_agent.params.proposal_capture_page_slug).toBeDefined(); expect(submit_agent.params.proposal_admission_scope).toBeDefined(); expect(stage_ingestion_proposal_page?.scope).toBe('agent'); expect(stage_ingestion_proposal_page?.mutating).toBe(false); expect(finalize_ingestion_proposal?.scope).toBe('agent'); expect(finalize_ingestion_proposal?.mutating).toBe(false); + expect(finalize_ingestion_proposal?.params.page_digests).toBeUndefined(); expect(get_agent_job_proposal?.scope).toBe('agent'); }); }); @@ -374,7 +376,7 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with }); describe('happy-path submission', () => { - it('requires and persists the proposal artifact and admission-scope binding together', async () => { + it('requires ingestion artifact and capture bindings while allowing first-stage scope selection', async () => { await seedClient('cursor', { bound_tools: ['stage_ingestion_proposal_page', 'finalize_ingestion_proposal'], bound_source_id: 'company', @@ -384,26 +386,59 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with await expect(callSubmitAgent(ctx, { prompt: 'propose', proposal_artifact_id: 'artifact-1', - })).rejects.toThrow(/must be supplied together/); + })).rejects.toThrow(/capture page slug/i); await expect(callSubmitAgent(ctx, { prompt: 'propose', proposal_admission_scope: 'Include delivery notes.', - })).rejects.toThrow(/must be supplied together/); + })).rejects.toThrow(/artifact and capture/i); const result = await callSubmitAgent(ctx, { prompt: 'propose', proposal_artifact_id: 'artifact-1', - proposal_admission_scope: 'Include delivery notes.', + proposal_capture_page_slug: 'sources/example', }); const [row] = await engine.executeRaw<{ data: Record }>( 'SELECT data FROM minion_jobs WHERE id = $1', [result.id], ); expect(row.data.proposal_artifact_id).toBe('artifact-1'); - expect(row.data.proposal_admission_scope).toBe('Include delivery notes.'); + expect(row.data.proposal_capture_page_slug).toBe('sources/example'); + expect(row.data.proposal_admission_scope).toBeUndefined(); expect(row.data.source_id).toBe('company'); }); + it('rejects proposal-tool jobs without a bound source, slug fence, or capture inside the fence', async () => { + await seedClient('unscoped', { + bound_tools: ['stage_ingestion_proposal_page', 'finalize_ingestion_proposal'], + bound_source_id: null, + bound_slug_prefixes: ['sources/'], + }); + await expect(callSubmitAgent(makeCtx({ clientId: 'unscoped' }), { + prompt: 'propose', proposal_artifact_id: 'artifact-1', + proposal_capture_page_slug: 'sources/example', + })).rejects.toThrow(/bound source/i); + + await seedClient('unfenced', { + bound_tools: ['stage_ingestion_proposal_page', 'finalize_ingestion_proposal'], + bound_source_id: 'company', + bound_slug_prefixes: [], + }); + await expect(callSubmitAgent(makeCtx({ clientId: 'unfenced' }), { + prompt: 'propose', proposal_artifact_id: 'artifact-1', + proposal_capture_page_slug: 'sources/example', + })).rejects.toThrow(/slug fence/i); + + await seedClient('wrong-capture', { + bound_tools: ['stage_ingestion_proposal_page', 'finalize_ingestion_proposal'], + bound_source_id: 'company', + bound_slug_prefixes: ['projects/'], + }); + await expect(callSubmitAgent(makeCtx({ clientId: 'wrong-capture' }), { + prompt: 'propose', proposal_artifact_id: 'artifact-1', + proposal_capture_page_slug: 'sources/example', + })).rejects.toThrow(/capture page.*slug fence/i); + }); + it('deduplicates a client idempotency key before enforcing concurrency', async () => { await seedClient('cursor', { bound_tools: ['search'], @@ -1177,7 +1212,9 @@ describe('staged proposal operation contract', () => { __owner_client_id: 'lore', source_id: 'company', proposal_artifact_id: 'artifact-1', + proposal_capture_page_slug: 'sources/example', proposal_admission_scope: 'Include delivery notes.', + allowed_slug_prefixes: ['sources/*'], })], ); const agentCtx = { ...makeCtx({ clientId: 'lore' }), viaSubagent: true, jobId: row.id }; @@ -1199,7 +1236,6 @@ describe('staged proposal operation contract', () => { source_id: 'company', admission_scope: 'Include delivery notes.', total_pages: 1, - page_digests: [{ sequence: 1, digest: staged.digest }], summary: 'Ready for review.', proposed_timeline_entries: [], proposed_links: [], From 5fbe057288147a04fc059daca7cf5f870f98bc02 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 14:41:56 -0700 Subject: [PATCH 04/18] lore-b74.6: make compacted read context explicit --- src/core/ai/tool-loop-context.ts | 65 +++++++++++++--- test/ai/gateway-tool-loop.test.ts | 120 +++++++++++++++++++++++++++++- 2 files changed, 175 insertions(+), 10 deletions(-) diff --git a/src/core/ai/tool-loop-context.ts b/src/core/ai/tool-loop-context.ts index 42e8f13395..efe3dcae73 100644 --- a/src/core/ai/tool-loop-context.ts +++ b/src/core/ai/tool-loop-context.ts @@ -30,6 +30,11 @@ interface ToolRound { evidence: ToolEvidence[]; } +interface WorkingContextProjectionSource { + kind: 'tool_input' | 'tool_result'; + toolName: string; +} + export interface ToolLoopContextOptions { /** Tools whose effects must retain a distinct identity when raw rounds drop. */ mutatingToolNames?: ReadonlySet; @@ -191,14 +196,26 @@ function compactRound(round: ToolRound, perPayload: number): ToolRound { content: mapBlocks(round.assistant, block => { if (block.type === 'text') return { ...block, text: boundMiddle(block.text, perPayload) }; if (block.type !== 'tool-call') return block; - return { ...block, input: boundValue(block.input, perPayload) }; + return { + ...block, + input: boundValue(block.input, perPayload, { + kind: 'tool_input', + toolName: block.toolName, + }), + }; }), }, result: { ...round.result, content: mapBlocks(round.result, block => { if (block.type !== 'tool-result') return block; - return { ...block, output: boundValue(block.output, perPayload) }; + return { + ...block, + output: boundValue(block.output, perPayload, { + kind: 'tool_result', + toolName: block.toolName, + }), + }; }), }, }; @@ -303,21 +320,51 @@ function mapBlocks(message: ChatMessage, fn: (block: ChatBlock) => ChatBlock): C return typeof message.content === 'string' ? message.content : message.content.map(fn); } -function boundValue(value: unknown, maxChars: number): unknown { +function boundValue( + value: unknown, + maxChars: number, + source: WorkingContextProjectionSource, +): unknown { const serialized = safeJson(value); if (serialized.length <= maxChars) return value; - return { - _gbrain_context_compacted: true, - original_chars: serialized.length, - ...(maxChars >= 64 ? { preview: boundMiddle(serialized, Math.max(16, maxChars - 80)) } : {}), + + // Full projections carry enough identity to verify or re-read the exact + // durable value without presenting any fragment as source content. + const originalBytes = Buffer.byteLength(serialized, 'utf8'); + const sha256 = createHash('sha256').update(serialized).digest('hex'); + const full = { + working_context_projection: { + schema: 'gbrain.working_context_projection.v1', + kind: source.kind, + tool_name: source.toolName, + original_json_utf8_bytes: originalBytes, + sha256, + interpretation: 'projection_metadata_not_source_content', + re_read_guidance: source.kind === 'tool_result' + ? `Re-run ${source.toolName} with focused input if exact content is needed.` + : `Do not reconstruct the exact ${source.toolName} input from this projection; consult durable execution evidence.`, + }, + }; + if (safeJson(full).length <= maxChars) return full; + + // Preserve the checkable identity under tighter budgets. The smallest + // tier remains unmistakable metadata while letting atomic rounds fit. + const compact = { + working_context_projection: { + original_json_utf8_bytes: originalBytes, + sha256, + }, }; + return safeJson(compact).length <= maxChars + ? compact + : { working_context_projection: true }; } function boundMiddle(text: string, maxChars: number): string { if (text.length <= maxChars) return text; if (maxChars <= 0) return ''; - const marker = '\n... [middle omitted] ...\n'; - if (maxChars <= marker.length) return text.slice(0, maxChars); + const marker = '\n[gbrain working-context projection: exact text retained in durable execution ledger]\n'; + if (maxChars <= marker.length) return marker.slice(0, maxChars); const available = maxChars - marker.length; const head = Math.ceil(available / 2); return text.slice(0, head) + marker + text.slice(text.length - (available - head)); diff --git a/test/ai/gateway-tool-loop.test.ts b/test/ai/gateway-tool-loop.test.ts index 49991f4c13..96c15d9493 100644 --- a/test/ai/gateway-tool-loop.test.ts +++ b/test/ai/gateway-tool-loop.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, afterEach } from 'bun:test'; +import { createHash } from 'node:crypto'; import { toolLoop, __setChatTransportForTests, @@ -374,10 +375,127 @@ describe('gateway.toolLoop (v0.38 D11 — provider-agnostic loop control)', () = expect(compacted[0]).toEqual({ role: 'user', content: task }); expect(JSON.stringify(compacted).length).toBeLessThanOrEqual(130_000); - expect(JSON.stringify(compacted.slice(1))).toContain('Context compacted'); + expect(JSON.stringify(compacted.slice(1))).toContain('working_context_projection'); expect(JSON.stringify(compacted[0])).not.toContain('[middle omitted]'); }); + it('projects large read results as explicit metadata without implying artifact truncation', () => { + // Arrange a complete production-sized task followed by enough large reads + // to force provider-only compaction of both get_page and search results. + const completeArtifact = [ + 'Ingest this complete artifact. artifactIntegrity.complete=true\n', + 'a'.repeat(50_000), + '\nCOMPLETE_ARTIFACT_MIDDLE\n', + 'z'.repeat(50_000), + ].join(''); + const messages: ChatMessage[] = [{ role: 'user', content: completeArtifact }]; + for (let i = 0; i < 12; i++) { + const toolName = i % 2 === 0 ? 'get_page' : 'search'; + messages.push({ + role: 'assistant', + content: [{ + type: 'tool-call', + toolCallId: `read-${i}`, + toolName, + input: toolName === 'get_page' + ? { slug: `notes/page-${i}` } + : { query: `artifact topic ${i}` }, + }], + }); + messages.push({ + role: 'user', + content: [{ + type: 'tool-result', + toolCallId: `read-${i}`, + toolName, + output: { slug: `notes/page-${i}`, body: `${i}:`.repeat(20_000) }, + }], + }); + } + const durableSnapshot = structuredClone(messages); + + // Act on the provider projection only. + const compacted = compactToolLoopMessages(messages, 130_000, { + mutatingToolNames: new Set(), + }); + const serialized = JSON.stringify(compacted); + + // Assert the complete task remains authoritative and projections are + // checkable metadata, never source-looking omission previews. + expect(compacted[0]).toEqual({ role: 'user', content: completeArtifact }); + expect(serialized).toContain('COMPLETE_ARTIFACT_MIDDLE'); + expect(serialized).toContain('working_context_projection'); + expect(serialized).toContain('original_json_utf8_bytes'); + expect(serialized).toContain('sha256'); + expect(serialized).toContain('Re-run search'); + expect(serialized).not.toContain('[middle omitted]'); + + const lastResult = compacted.at(-1)!; + expect(lastResult.role).toBe('user'); + expect(typeof lastResult.content).not.toBe('string'); + const resultBlock = typeof lastResult.content === 'string' + ? undefined + : lastResult.content.find(block => block.type === 'tool-result'); + const output = resultBlock && resultBlock.type === 'tool-result' + ? resultBlock.output as Record> + : {}; + const metadata = output.working_context_projection; + const originalOutput = (messages.at(-1)!.content as ChatBlock[])[0]!; + const originalJson = JSON.stringify( + originalOutput.type === 'tool-result' ? originalOutput.output : null, + ); + expect(metadata.original_json_utf8_bytes).toBe(Buffer.byteLength(originalJson, 'utf8')); + expect(metadata.sha256).toBe(createHash('sha256').update(originalJson).digest('hex')); + expect(metadata.interpretation).toBe('projection_metadata_not_source_content'); + expect(messages).toEqual(durableSnapshot); + }); + + it('tiers projections for tight budgets while preserving a balanced tool round', () => { + // Arrange one round whose narrative and result both require projection. + const messages: ChatMessage[] = [ + { role: 'user', content: 'Read the page and continue from verified evidence.' }, + { + role: 'assistant', + content: [ + { type: 'text', text: `Historical working note: ${'n'.repeat(4_000)}` }, + { + type: 'tool-call', + toolCallId: 'tight-read', + toolName: 'get_page', + input: { slug: 'notes/tight-budget' }, + }, + ], + }, + { + role: 'user', + content: [{ + type: 'tool-result', + toolCallId: 'tight-read', + toolName: 'get_page', + output: { body: 'p'.repeat(20_000) }, + }], + }, + ]; + + // Act with a budget that requires the minimal metadata tier. + const compacted = compactToolLoopMessages(messages, 500, { + mutatingToolNames: new Set(), + }); + const serialized = JSON.stringify(compacted); + + // Assert the tool call/result stay paired and even tighter budgets fail + // closed instead of emitting an invalid provider transcript. + expect(serialized.length).toBeLessThanOrEqual(500); + expect(serialized).toContain('"working_context_projection":true'); + expect(serialized).toContain('gbrain working-context projection'); + expect(serialized).toContain('tight-read'); + expect(serialized).not.toContain('[middle omitted]'); + expect(compacted).toHaveLength(3); + expect(() => compactToolLoopMessages(messages, 200, { + mutatingToolNames: new Set(), + })).toThrow(ToolLoopContextProjectionError); + }); + it('fails closed when the full original task and required evidence cannot fit', () => { const task = `Apply this exact plan:\n${'p'.repeat(1_800)}`; const messages: ChatMessage[] = [ From 2497ddc3dccc38aadd03fd342157c5bdabc9f364 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 15:35:53 -0700 Subject: [PATCH 05/18] lore-b74.6: preserve bounded mutation evidence --- src/core/ai/tool-loop-context.ts | 183 +++++++++++++++++++++--------- test/ai/gateway-tool-loop.test.ts | 99 ++++++++++++++++ 2 files changed, 229 insertions(+), 53 deletions(-) diff --git a/src/core/ai/tool-loop-context.ts b/src/core/ai/tool-loop-context.ts index efe3dcae73..494e75a466 100644 --- a/src/core/ai/tool-loop-context.ts +++ b/src/core/ai/tool-loop-context.ts @@ -13,9 +13,33 @@ import type { ChatBlock, ChatMessage, ChatToolDef } from './gateway.ts'; const CONTEXT_TARGET_FRACTION = 0.7; const ESTIMATED_CHARS_PER_TOKEN = 2; +// Byte-level provider tokenizers cannot emit more tokens than UTF-8 bytes. +// One byte per token is intentionally conservative for ASCII, CJK, emoji, +// and mixed JSON without requiring a model-specific tokenizer at runtime. +const CONSERVATIVE_BYTES_PER_TOKEN = 1; const FALLBACK_CONTEXT_TOKENS = 128_000; -const MIN_MESSAGE_BUDGET_CHARS = 2_000; const PAYLOAD_LIMITS = [12_000, 4_000, 1_000, 256, 64, 0] as const; +const MAX_STRUCTURAL_IDENTITY_VALUE_BYTES = 256; +const STRUCTURAL_IDENTITY_KEYS = [ + 'slug', + 'page_slug', + 'page_id', + 'id', + 'source_id', + 'source', + 'from', + 'to', + 'from_slug', + 'to_slug', + 'old_slug', + 'new_slug', + 'page', + 'path', + 'ref', + 'date', + 'expected_content_hash', + 'link_type', +] as const; interface ToolEvidence { toolCallId: string; @@ -33,10 +57,11 @@ interface ToolRound { interface WorkingContextProjectionSource { kind: 'tool_input' | 'tool_result'; toolName: string; + preserveStructuralIdentity: boolean; } export interface ToolLoopContextOptions { - /** Tools whose effects must retain a distinct identity when raw rounds drop. */ + /** Tools whose effects must retain a distinct identity whenever context compacts. */ mutatingToolNames?: ReadonlySet; } @@ -48,7 +73,7 @@ export class ToolLoopContextProjectionError extends Error { } } -/** Resolve a conservative message-character budget inside the model window. */ +/** Resolve a conservative UTF-8 byte budget inside the model window. */ export function resolveToolLoopMessageBudget(args: { model: string; maxOutputTokens: number; @@ -66,12 +91,18 @@ export function resolveToolLoopMessageBudget(args: { } const targetTokens = Math.floor(contextTokens * CONTEXT_TARGET_FRACTION); - const messageTokens = Math.max(0, targetTokens - args.maxOutputTokens); - const staticChars = (args.system?.length ?? 0) + jsonLength(args.tools); - return Math.max( - MIN_MESSAGE_BUDGET_CHARS, - messageTokens * ESTIMATED_CHARS_PER_TOKEN - staticChars, - ); + const targetMessageTokens = Math.max(0, targetTokens - args.maxOutputTokens); + const staticChars = (args.system?.length ?? 0) + safeJson(args.tools).length; + const targetBudget = targetMessageTokens * ESTIMATED_CHARS_PER_TOKEN - staticChars; + + // The historical two-character estimate keeps ordinary prompts near the + // 70% target. A second absolute byte cap uses the model's whole declared + // window, so dense Unicode cannot overflow while ordinary English is not + // needlessly constrained to one byte per target token. + const hardInputTokens = Math.max(0, contextTokens - args.maxOutputTokens); + const staticBytes = utf8Bytes(args.system ?? '') + jsonBytes(args.tools); + const byteSafeBudget = hardInputTokens * CONSERVATIVE_BYTES_PER_TOKEN - staticBytes; + return Math.max(0, Math.min(targetBudget, byteSafeBudget)); } /** @@ -80,10 +111,10 @@ export function resolveToolLoopMessageBudget(args: { */ export function compactToolLoopMessages( messages: ChatMessage[], - maxChars: number, + maxBytes: number, options: ToolLoopContextOptions = {}, ): ChatMessage[] { - if (jsonLength(messages) <= maxChars) return messages; + if (jsonBytes(messages) <= maxBytes) return messages; const { rounds, otherCount } = collectToolRounds(messages); const task = buildTaskAnchor(messages); @@ -96,8 +127,8 @@ export function compactToolLoopMessages( for (let i = rounds.length - 1; i >= 0; i--) { const summary = buildLedgerSummary(rounds.slice(0, i), otherCount, options); const base = [task, ...(summary ? [summary] : []), ...flattenRounds(retained)]; - const available = maxChars - jsonLength(base) - 32; - const compacted = compactRoundToFit(rounds[i]!, available); + const available = maxBytes - jsonBytes(base) - 32; + const compacted = compactRoundToFit(rounds[i]!, available, options); if (!compacted) break; retained.unshift(compacted); retainedStart = i; @@ -108,7 +139,7 @@ export function compactToolLoopMessages( // JSON array delimiters can add a few bytes beyond individually-sized // elements. Drop the oldest raw round to its evidence record if necessary. - while (jsonLength(projection) > maxChars && retained.length > 1) { + while (jsonBytes(projection) > maxBytes && retained.length > 1) { retained.shift(); retainedStart++; summary = buildLedgerSummary(rounds.slice(0, retainedStart), otherCount, options); @@ -117,12 +148,12 @@ export function compactToolLoopMessages( if (rounds.length > 0 && retained.length === 0) { throw new ToolLoopContextProjectionError( - `The latest balanced tool round cannot fit safely within the ${maxChars}-character context window budget.`, + `The latest balanced tool round cannot fit safely within the ${maxBytes}-byte context window budget.`, ); } - if (jsonLength(projection) > maxChars) { + if (jsonBytes(projection) > maxBytes) { throw new ToolLoopContextProjectionError( - `Required task and durable mutation evidence exceed the ${maxChars}-character context window budget.`, + `Required task and durable mutation evidence exceed the ${maxBytes}-byte context window budget.`, ); } return projection; @@ -178,29 +209,41 @@ function buildTaskAnchor(messages: ChatMessage[]): ChatMessage { } /** Find the largest payload representation whose complete round fits. */ -function compactRoundToFit(round: ToolRound, availableChars: number): ToolRound | null { - if (availableChars <= 0) return null; +function compactRoundToFit( + round: ToolRound, + availableBytes: number, + options: ToolLoopContextOptions, +): ToolRound | null { + if (availableBytes <= 0) return null; for (const perPayload of PAYLOAD_LIMITS) { - const compacted = compactRound(round, perPayload); - if (jsonLength([compacted.assistant, compacted.result]) <= availableChars) return compacted; + const compacted = compactRound(round, perPayload, options); + if (jsonBytes([compacted.assistant, compacted.result]) <= availableBytes) return compacted; } return null; } /** Bound historical tool inputs/results while keeping provider call IDs paired. */ -function compactRound(round: ToolRound, perPayload: number): ToolRound { +function compactRound( + round: ToolRound, + perPayloadBytes: number, + options: ToolLoopContextOptions, +): ToolRound { return { ...round, assistant: { ...round.assistant, content: mapBlocks(round.assistant, block => { - if (block.type === 'text') return { ...block, text: boundMiddle(block.text, perPayload) }; + if (block.type === 'text') return { ...block, text: boundText(block.text, perPayloadBytes) }; if (block.type !== 'tool-call') return block; return { ...block, - input: boundValue(block.input, perPayload, { + input: boundValue(block.input, perPayloadBytes, { kind: 'tool_input', toolName: block.toolName, + preserveStructuralIdentity: isMutationSensitive( + block.toolName, + options.mutatingToolNames, + ), }), }; }), @@ -211,9 +254,10 @@ function compactRound(round: ToolRound, perPayload: number): ToolRound { if (block.type !== 'tool-result') return block; return { ...block, - output: boundValue(block.output, perPayload, { + output: boundValue(block.output, perPayloadBytes, { kind: 'tool_result', toolName: block.toolName, + preserveStructuralIdentity: false, }), }; }), @@ -261,8 +305,8 @@ function formatMutationEvidence(evidence: ToolEvidence): string { const fingerprint = createHash('sha256').update(serialized).digest('hex').slice(0, 16); const target = mutationTarget(evidence.input); return [ - `- ${boundMiddle(evidence.toolName, 80)}`, - `call_id=${boundMiddle(evidence.toolCallId, 96)}`, + `- ${boundIdentifier(evidence.toolName, 80)}`, + `call_id=${boundIdentifier(evidence.toolCallId, 96)}`, `outcome=${evidence.failed ? 'failed' : 'complete'}`, `input_sha256=${fingerprint}`, target ? `target=${target}` : '', @@ -271,15 +315,15 @@ function formatMutationEvidence(evidence: ToolEvidence): string { /** Pick a bounded human-legible operation target without retaining whole input. */ function mutationTarget(input: unknown): string | null { - if (!input || typeof input !== 'object' || Array.isArray(input)) return null; - const record = input as Record; - for (const key of ['slug', 'page_slug', 'path', 'id', 'entity', 'title', 'name']) { - const value = record[key]; - if (typeof value === 'string' || typeof value === 'number') { - return `${key}:${boundMiddle(String(value), 160)}`; - } - } - return null; + const identity = extractStructuralIdentity(input); + const first = identity ? Object.entries(identity)[0] : undefined; + return first ? `${first[0]}:${boundIdentifier(String(first[1]), 160)}` : null; +} + +/** Bound trusted identifiers without retaining a source-looking fragment. */ +function boundIdentifier(value: string, maxBytes: number): string { + if (utf8Bytes(value) <= maxBytes) return value; + return `sha256:${createHash('sha256').update(value).digest('hex').slice(0, 16)}`; } function isMutationSensitive(name: string, names: ReadonlySet | undefined): boolean { @@ -322,16 +366,19 @@ function mapBlocks(message: ChatMessage, fn: (block: ChatBlock) => ChatBlock): C function boundValue( value: unknown, - maxChars: number, + maxBytes: number, source: WorkingContextProjectionSource, ): unknown { const serialized = safeJson(value); - if (serialized.length <= maxChars) return value; + if (utf8Bytes(serialized) <= maxBytes) return value; // Full projections carry enough identity to verify or re-read the exact // durable value without presenting any fragment as source content. const originalBytes = Buffer.byteLength(serialized, 'utf8'); const sha256 = createHash('sha256').update(serialized).digest('hex'); + const structuralIdentity = source.preserveStructuralIdentity + ? extractStructuralIdentity(value) + : undefined; const full = { working_context_projection: { schema: 'gbrain.working_context_projection.v1', @@ -339,13 +386,14 @@ function boundValue( tool_name: source.toolName, original_json_utf8_bytes: originalBytes, sha256, + ...(structuralIdentity ? { structural_identity: structuralIdentity } : {}), interpretation: 'projection_metadata_not_source_content', re_read_guidance: source.kind === 'tool_result' ? `Re-run ${source.toolName} with focused input if exact content is needed.` : `Do not reconstruct the exact ${source.toolName} input from this projection; consult durable execution evidence.`, }, }; - if (safeJson(full).length <= maxChars) return full; + if (jsonBytes(full) <= maxBytes) return full; // Preserve the checkable identity under tighter budgets. The smallest // tier remains unmistakable metadata while letting atomic rounds fit. @@ -353,30 +401,59 @@ function boundValue( working_context_projection: { original_json_utf8_bytes: originalBytes, sha256, + ...(structuralIdentity ? { structural_identity: structuralIdentity } : {}), }, }; - return safeJson(compact).length <= maxChars - ? compact - : { working_context_projection: true }; + if (jsonBytes(compact) <= maxBytes) return compact; + + // A retained mutation round must never shed the only human-legible target. + // Returning the identity-only envelope even when it exceeds this tier makes + // compactRoundToFit try the remaining tiers and ultimately fail closed if + // the complete balanced round cannot preserve it. + if (structuralIdentity) { + return { working_context_projection: { structural_identity: structuralIdentity } }; + } + return { working_context_projection: true }; +} + +/** Keep compacted narrative unmistakable without retaining arbitrary prose. */ +function boundText(text: string, maxBytes: number): string { + if (utf8Bytes(text) <= maxBytes) return text; + const full = '\n[gbrain working-context projection: exact text retained in durable execution ledger]\n'; + if (utf8Bytes(full) <= maxBytes) return full; + const compact = '[gbrain working-context projection]'; + return utf8Bytes(compact) <= maxBytes ? compact : ''; +} + +/** Extract short operation identifiers, never body, summary, or evidence prose. */ +function extractStructuralIdentity(value: unknown): Record | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; + const record = value as Record; + const identity: Record = {}; + for (const key of STRUCTURAL_IDENTITY_KEYS) { + const candidate = record[key]; + if ( + (typeof candidate === 'string' && utf8Bytes(candidate) <= MAX_STRUCTURAL_IDENTITY_VALUE_BYTES) + || (typeof candidate === 'number' && Number.isFinite(candidate)) + || typeof candidate === 'boolean' + ) { + identity[key] = candidate; + } + } + return Object.keys(identity).length > 0 ? identity : undefined; } -function boundMiddle(text: string, maxChars: number): string { - if (text.length <= maxChars) return text; - if (maxChars <= 0) return ''; - const marker = '\n[gbrain working-context projection: exact text retained in durable execution ledger]\n'; - if (maxChars <= marker.length) return marker.slice(0, maxChars); - const available = maxChars - marker.length; - const head = Math.ceil(available / 2); - return text.slice(0, head) + marker + text.slice(text.length - (available - head)); +function jsonBytes(value: unknown): number { + return utf8Bytes(safeJson(value)); } -function jsonLength(value: unknown): number { - return safeJson(value).length; +function utf8Bytes(value: string): number { + return Buffer.byteLength(value, 'utf8'); } function safeJson(value: unknown): string { try { - return JSON.stringify(value ?? null); + return JSON.stringify(value ?? null) ?? String(value); } catch { return String(value); } diff --git a/test/ai/gateway-tool-loop.test.ts b/test/ai/gateway-tool-loop.test.ts index 96c15d9493..78e2cfb620 100644 --- a/test/ai/gateway-tool-loop.test.ts +++ b/test/ai/gateway-tool-loop.test.ts @@ -11,6 +11,7 @@ import { } from '../../src/core/ai/gateway.ts'; import { compactToolLoopMessages, + resolveToolLoopMessageBudget, ToolLoopContextProjectionError, } from '../../src/core/ai/tool-loop-context.ts'; @@ -496,6 +497,104 @@ describe('gateway.toolLoop (v0.38 D11 — provider-agnostic loop control)', () = })).toThrow(ToolLoopContextProjectionError); }); + it('retains only structural mutation identity when a large write input is projected', () => { + const messages: ChatMessage[] = [ + { role: 'user', content: 'Write the page exactly once and report the target.' }, + { + role: 'assistant', + content: [{ + type: 'tool-call', + toolCallId: 'large-write', + toolName: 'put_page', + input: { + slug: 'wiki/critical-target', + source_id: 'martian', + content: 'PRIVATE_BODY_PROSE'.repeat(2_000), + }, + }], + }, + { + role: 'user', + content: [{ + type: 'tool-result', + toolCallId: 'large-write', + toolName: 'put_page', + output: { ok: true }, + }], + }, + ]; + const durableSnapshot = structuredClone(messages); + + const compacted = compactToolLoopMessages(messages, 600, { + mutatingToolNames: new Set(['put_page']), + }); + const serialized = JSON.stringify(compacted); + + expect(serialized).toContain('wiki/critical-target'); + expect(serialized).toContain('martian'); + expect(serialized).not.toContain('PRIVATE_BODY_PROSE'); + expect(serialized).toContain('sha256'); + expect(messages).toEqual(durableSnapshot); + }); + + it('bounds projections by UTF-8 bytes for dense Unicode content', () => { + const maxBytes = 2_000; + const messages: ChatMessage[] = [ + { role: 'user', content: 'Read the page and continue.' }, + { + role: 'assistant', + content: [{ + type: 'tool-call', + toolCallId: 'unicode-read', + toolName: 'get_page', + input: { slug: 'notes/unicode' }, + }], + }, + { + role: 'user', + content: [{ + type: 'tool-result', + toolCallId: 'unicode-read', + toolName: 'get_page', + output: { body: `${'漢'.repeat(2_000)}${'🧠'.repeat(2_000)}` }, + }], + }, + ]; + const durableSnapshot = structuredClone(messages); + + const compacted = compactToolLoopMessages(messages, maxBytes, { + mutatingToolNames: new Set(), + }); + + expect(Buffer.byteLength(JSON.stringify(compacted), 'utf8')).toBeLessThanOrEqual(maxBytes); + expect(JSON.stringify(compacted)).toContain('working_context_projection'); + expect(messages).toEqual(durableSnapshot); + }); + + it('fails closed when a dense-Unicode task exceeds its byte budget', () => { + const task = `Preserve exactly:\n${'漢'.repeat(700)}`; + expect(JSON.stringify([{ role: 'user', content: task }]).length).toBeLessThan(1_000); + expect(Buffer.byteLength(JSON.stringify([{ role: 'user', content: task }]), 'utf8')).toBeGreaterThan(1_000); + + expect(() => compactToolLoopMessages([{ role: 'user', content: task }], 1_000)) + .toThrow(ToolLoopContextProjectionError); + }); + + it('reserves no more than one UTF-8 byte per available input token', () => { + const budget = resolveToolLoopMessageBudget({ + model: 'openai:gpt-5.6-terra', + maxOutputTokens: 32_768, + contextWindowTokens: 200_000, + system: '漢'.repeat(1_000), + tools: [], + }); + + // A byte-level tokenizer cannot emit more tokens than the UTF-8 byte + // count, so one byte per available token is conservative for ASCII, + // CJK, emoji, and mixed JSON payloads. + expect(budget).toBe(200_000 - 32_768 - Buffer.byteLength('漢'.repeat(1_000), 'utf8') - 2); + }); + it('fails closed when the full original task and required evidence cannot fit', () => { const task = `Apply this exact plan:\n${'p'.repeat(1_800)}`; const messages: ChatMessage[] = [ From 697621224b89d34baded31ab96ca257f3ca68a6f Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 15:42:38 -0700 Subject: [PATCH 06/18] test: preserve staged proposal compaction proof --- test/agent-job-proposals.test.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/agent-job-proposals.test.ts b/test/agent-job-proposals.test.ts index a84482fbf8..8916b8ac73 100644 --- a/test/agent-job-proposals.test.ts +++ b/test/agent-job-proposals.test.ts @@ -155,10 +155,14 @@ describe('durable agent-job proposal staging', () => { it('finalizes from the durable ordered manifest after old stage outputs compact away', async () => { const jobId = await seedJob(); - const first = await stage(jobId, 1, 2, createPage('sources/example')); - const second = await stage(jobId, 2, 2, createPage('projects/example')); + const staged = [ + await stage(jobId, 1, 4, createPage('sources/example')), + await stage(jobId, 2, 4, createPage('projects/example-2')), + await stage(jobId, 3, 4, createPage('projects/example-3')), + await stage(jobId, 4, 4, createPage('projects/example-4')), + ]; const messages: ChatMessage[] = [{ role: 'user', content: 'Build the exact ingestion proposal.' }]; - for (const page of [first, second]) { + for (const page of staged) { messages.push({ role: 'assistant', content: [{ @@ -179,16 +183,18 @@ describe('durable agent-job proposal staging', () => { }); } const compacted = compactToolLoopMessages(messages, 2_000, { mutatingToolNames: new Set() }); - expect(JSON.stringify(compacted)).not.toContain(first.digest); + const compactedJson = JSON.stringify(compacted); + expect(compactedJson).not.toContain(staged[0]!.digest); + expect(compactedJson).toContain(staged[3]!.digest); const manifest = await finalizeAgentJobProposal(engine, jobId, { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', - total_pages: 2, + total_pages: 4, summary: 'Ready after compaction.', }); - expect(manifest.pageDigests).toEqual([first, second]); + expect(manifest.pageDigests).toEqual(staged); }); it('freezes a previously-null admission scope on the first stage only', async () => { From 9f7f75c751dbcf463a57bea56e5c39d133d3432e Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 15:46:21 -0700 Subject: [PATCH 07/18] fix: align staged proposal scope limit --- skills/github-project-ingestion/SKILL.md | 3 ++- skills/granola-meeting-ingestion/SKILL.md | 3 ++- src/core/minions/agent-job-proposals.ts | 15 ++++++++++-- src/core/operations.ts | 7 +++++- test/agent-job-proposals.test.ts | 20 ++++++++++++++++ test/github-project-ingestion-skill.test.ts | 1 + test/granola-meeting-ingestion-skill.test.ts | 1 + test/submit-agent.test.ts | 25 ++++++++++++++++++++ 8 files changed, 70 insertions(+), 5 deletions(-) diff --git a/skills/github-project-ingestion/SKILL.md b/skills/github-project-ingestion/SKILL.md index 36a4d3b5ef..c9ac13b1c5 100644 --- a/skills/github-project-ingestion/SKILL.md +++ b/skills/github-project-ingestion/SKILL.md @@ -256,7 +256,8 @@ Immediately after reading and constructing each complete page, call returned `{sequence, slug, digest}`; later turns may rely on that durable digest instead of retaining every old raw `get_page` output or proposed body in the working context. An identical retry is safe. Never change `total_pages`, reuse -a sequence for different content, or stage after finalization. +a sequence for different content, or stage after finalization. The exact +`admission_scope` must contain 1-4,000 characters after trimming. Populate `proposedTimelineEntries` with the exact timeline mutations the normal workflow requires: timeline entries only for material dated events with the diff --git a/skills/granola-meeting-ingestion/SKILL.md b/skills/granola-meeting-ingestion/SKILL.md index 599045ac2e..d61d2b1afb 100644 --- a/skills/granola-meeting-ingestion/SKILL.md +++ b/skills/granola-meeting-ingestion/SKILL.md @@ -242,7 +242,8 @@ Immediately after reading and constructing each complete page, call returned `{sequence, slug, digest}`; later turns may rely on that durable digest instead of retaining every old raw `get_page` output or proposed body in the working context. An identical retry is safe. Never change `total_pages`, reuse -a sequence for different content, or stage after finalization. +a sequence for different content, or stage after finalization. The exact +`admission_scope` must contain 1-4,000 characters after trimming. Populate `proposedTimelineEntries` with the exact timeline mutations the normal workflow requires: timeline entries only for material dated events with the diff --git a/src/core/minions/agent-job-proposals.ts b/src/core/minions/agent-job-proposals.ts index 83c0e7a807..d5137a6005 100644 --- a/src/core/minions/agent-job-proposals.ts +++ b/src/core/minions/agent-job-proposals.ts @@ -15,6 +15,9 @@ export const PROPOSAL_MANIFEST_MAX_BYTES = 262_144; /** Maximum number of pages in one finalized proposal. */ export const PROPOSAL_MAX_PAGES = 100; +/** Maximum characters in the shared Lore/GBrain admission-scope contract. */ +export const PROPOSAL_ADMISSION_SCOPE_MAX_CHARS = 4_000; + export const STAGE_PROPOSAL_TOOL_NAME = 'brain_stage_ingestion_proposal_page'; export const FINALIZE_PROPOSAL_TOOL_NAME = 'brain_finalize_ingestion_proposal'; @@ -307,7 +310,11 @@ export async function finalizeAgentJobProposal( const artifactId = readBoundedString(input.artifact_id, 'artifact_id', 255); const sourceId = readBoundedString(input.source_id, 'source_id', 255); assertValidSourceId(sourceId); - const admissionScope = readBoundedString(input.admission_scope, 'admission_scope', 8_000); + const admissionScope = readBoundedString( + input.admission_scope, + 'admission_scope', + PROPOSAL_ADMISSION_SCOPE_MAX_CHARS, + ); const summary = readBoundedString(input.summary, 'summary', 1_000); const timeline = parseTimelineEntries(input.proposed_timeline_entries ?? []); const links = parseLinks(input.proposed_links ?? []); @@ -597,7 +604,11 @@ function parseStageProposalPageInput(raw: unknown): ParsedStageProposalPageInput const artifactId = readBoundedString(input.artifact_id, 'artifact_id', 255); const sourceId = readBoundedString(input.source_id, 'source_id', 255); assertValidSourceId(sourceId); - const admissionScope = readBoundedString(input.admission_scope, 'admission_scope', 8_000); + const admissionScope = readBoundedString( + input.admission_scope, + 'admission_scope', + PROPOSAL_ADMISSION_SCOPE_MAX_CHARS, + ); const sequence = readPositiveInteger(input.sequence, 'sequence'); const totalPages = readProposalPageCount(input.total_pages); if (sequence > totalPages) { diff --git a/src/core/operations.ts b/src/core/operations.ts index 3aede34752..38a9d791fc 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -50,6 +50,7 @@ import { stampEvidence } from './search/evidence.ts'; import type { SearchResult } from './types.ts'; import { CJK_SLUG_CHARS } from './cjk.ts'; import { matchesSlugAllowList, slugFenceContains } from './slug-allow-list.ts'; +import { PROPOSAL_ADMISSION_SCOPE_MAX_CHARS } from './minions/agent-job-proposals.ts'; export { matchesSlugAllowList } from './slug-allow-list.ts'; import * as db from './db.ts'; import { VERSION } from '../version.ts'; @@ -5444,7 +5445,11 @@ const submit_agent: Operation = { 'submit_agent: proposal-tool jobs require a proposal artifact and capture page slug.', ); } - if (proposalArtifactId.length > 255 || proposalCapturePageSlug.length > 255 || proposalAdmissionScope.length > 8_000) { + if ( + proposalArtifactId.length > 255 + || proposalCapturePageSlug.length > 255 + || proposalAdmissionScope.length > PROPOSAL_ADMISSION_SCOPE_MAX_CHARS + ) { throw new OperationError( 'invalid_params', 'submit_agent: proposal artifact, capture page slug, or admission scope exceeds its bounded length.', diff --git a/test/agent-job-proposals.test.ts b/test/agent-job-proposals.test.ts index 8916b8ac73..edffaa29b9 100644 --- a/test/agent-job-proposals.test.ts +++ b/test/agent-job-proposals.test.ts @@ -223,6 +223,26 @@ describe('durable agent-job proposal staging', () => { .rejects.toMatchObject({ code: 'binding_mismatch' }); }); + it('shares Lore\'s exact 4,000-character admission-scope ceiling', async () => { + const maximumScope = 's'.repeat(4_000); + const acceptedJobId = await seedJob({ proposal_admission_scope: null }); + await stage(acceptedJobId, 1, 1, createPage('sources/example'), maximumScope); + const manifest = await finalizeAgentJobProposal(engine, acceptedJobId, { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: maximumScope, + total_pages: 1, summary: 'Boundary scope.', + }); + expect(manifest.admissionScope).toBe(maximumScope); + + const oversizedScope = `${maximumScope}s`; + const rejectedJobId = await seedJob({ proposal_admission_scope: null }); + await expect(stage(rejectedJobId, 1, 1, createPage('sources/example'), oversizedScope)) + .rejects.toMatchObject({ code: 'invalid_string' }); + await expect(finalizeAgentJobProposal(engine, acceptedJobId, { + artifact_id: 'artifact-1', source_id: 'company', admission_scope: oversizedScope, + total_pages: 1, summary: 'Oversized scope.', + })).rejects.toMatchObject({ code: 'invalid_string' }); + }); + it('requires pre-bound owner, source, artifact, capture slug, and slug fences', async () => { for (const overrides of [ { __owner_client_id: null }, diff --git a/test/github-project-ingestion-skill.test.ts b/test/github-project-ingestion-skill.test.ts index 532a2896c3..778a66cc15 100644 --- a/test/github-project-ingestion-skill.test.ts +++ b/test/github-project-ingestion-skill.test.ts @@ -222,6 +222,7 @@ describe('github-project-ingestion skill', () => { expect(skill).toContain('brain_stage_ingestion_proposal_page'); expect(skill).toContain('Stage only one page per turn'); expect(skill).toContain('brain_finalize_ingestion_proposal'); + expect(skill).toContain('4,000 characters'); expect(skill).toContain('server derives the ordered'); expect(skill).not.toContain('ordered\n`page_digests`'); expect(skill).toContain('"effect": "create | update"'); diff --git a/test/granola-meeting-ingestion-skill.test.ts b/test/granola-meeting-ingestion-skill.test.ts index 9b0d65123a..5edd292313 100644 --- a/test/granola-meeting-ingestion-skill.test.ts +++ b/test/granola-meeting-ingestion-skill.test.ts @@ -115,6 +115,7 @@ describe('granola-meeting-ingestion skill', () => { expect(skill).toContain('brain_stage_ingestion_proposal_page'); expect(skill).toContain('Stage only one page per turn'); expect(skill).toContain('brain_finalize_ingestion_proposal'); + expect(skill).toContain('4,000 characters'); expect(skill).toContain('server derives the ordered'); expect(skill).not.toContain('ordered\n`page_digests`'); expect(skill).toContain('"effect": "create | update"'); diff --git a/test/submit-agent.test.ts b/test/submit-agent.test.ts index d510c2dfa9..86b920e467 100644 --- a/test/submit-agent.test.ts +++ b/test/submit-agent.test.ts @@ -439,6 +439,31 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with })).rejects.toThrow(/capture page.*slug fence/i); }); + it('accepts a 4,000-character proposal scope and rejects 4,001 characters', async () => { + await seedClient('scope-limit', { + bound_tools: ['stage_ingestion_proposal_page', 'finalize_ingestion_proposal'], + bound_source_id: 'company', + bound_slug_prefixes: ['sources/'], + }); + const ctx = makeCtx({ clientId: 'scope-limit' }); + const maximumScope = 's'.repeat(4_000); + await expect(callSubmitAgent(ctx, { + prompt: 'propose', proposal_artifact_id: 'artifact-1', + proposal_capture_page_slug: 'sources/example', + proposal_admission_scope: `${maximumScope}s`, + })).rejects.toThrow(/admission scope exceeds/i); + + const result = await callSubmitAgent(ctx, { + prompt: 'propose', proposal_artifact_id: 'artifact-1', + proposal_capture_page_slug: 'sources/example', + proposal_admission_scope: maximumScope, + }); + const [row] = await engine.executeRaw<{ data: Record }>( + 'SELECT data FROM minion_jobs WHERE id = $1', [result.id], + ); + expect(row!.data.proposal_admission_scope).toBe(maximumScope); + }); + it('deduplicates a client idempotency key before enforcing concurrency', async () => { await seedClient('cursor', { bound_tools: ['search'], From 5ad9cc0528a73f30ad45a4b28d45480a8ef21045 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 16:20:30 -0700 Subject: [PATCH 08/18] lore-b74.5.1: close staged proposal execution gaps --- skills/github-project-ingestion/SKILL.md | 7 +- skills/granola-meeting-ingestion/SKILL.md | 7 +- src/core/ai/tool-loop-context.ts | 39 +++++++++- src/core/migrate.ts | 18 ++++- src/core/minions/agent-job-proposals.ts | 32 ++++++-- src/core/minions/handlers/subagent.ts | 27 +------ src/core/minions/subagent-limits.ts | 17 +++++ src/core/minions/subagent-prompt-budget.ts | 73 ++++++++++++++++++ src/core/operations.ts | 80 +++++++++++++++----- test/agent-job-proposals.test.ts | 65 ++++++++++++---- test/ai/gateway-tool-loop.test.ts | 42 ++++++++-- test/e2e/subagent-gateway-path.test.ts | 20 +++-- test/github-project-ingestion-skill.test.ts | 2 + test/granola-meeting-ingestion-skill.test.ts | 2 + test/migrate.test.ts | 18 +++++ test/submit-agent.test.ts | 76 ++++++++++++++++++- 16 files changed, 433 insertions(+), 92 deletions(-) create mode 100644 src/core/minions/subagent-limits.ts create mode 100644 src/core/minions/subagent-prompt-budget.ts diff --git a/skills/github-project-ingestion/SKILL.md b/skills/github-project-ingestion/SKILL.md index c9ac13b1c5..59b8c77e36 100644 --- a/skills/github-project-ingestion/SKILL.md +++ b/skills/github-project-ingestion/SKILL.md @@ -249,7 +249,9 @@ full intended `bodyMarkdown`, never a diff. Copy the exact body and `content_hash` from the `get_page` read used to draft each update into `baseMarkdown` and `expectedContentHash`. Omit both fields for a create. -Immediately after reading and constructing each complete page, call +Before constructing final page bodies, freeze the complete ordered page inventory +and stable `total_pages`; the inventory may contain at most 32 pages. Once the +inventory is frozen, read and construct each page in order, then immediately call `brain_stage_ingestion_proposal_page` in its own agent turn with the exact `artifact_id`, `source_id`, `admission_scope`, one-based `sequence`, stable `total_pages`, and page object. Stage only one page per turn. Preserve the @@ -289,7 +291,8 @@ page-digest manifest from the job's durable fragments, so finalization does not depend on old stage outputs remaining in model context. The server rejects gaps, duplicate or changed fragments, cross-job evidence, a capture page that does not match the exact job binding, mutations outside the job slug fence, a full -plan over 786,432 UTF-8 bytes, more than 100 pages, a timeline `refLabel` over +raw or JSON-escaped plan representation over 98,304 UTF-8 bytes, more than 32 +pages, a timeline `refLabel` over 500 characters, or a compact manifest over 262,144 UTF-8 bytes. Return the finalizer's compact manifest as `staged_proposal`; never reproduce page bodies or baselines in the final receipt. Never truncate or split a diff --git a/skills/granola-meeting-ingestion/SKILL.md b/skills/granola-meeting-ingestion/SKILL.md index d61d2b1afb..04efe6e6ab 100644 --- a/skills/granola-meeting-ingestion/SKILL.md +++ b/skills/granola-meeting-ingestion/SKILL.md @@ -235,7 +235,9 @@ intended `bodyMarkdown`, never a diff. Copy the exact body and `content_hash` from the `get_page` read used to draft each update into `baseMarkdown` and `expectedContentHash`. Omit both fields for a create. -Immediately after reading and constructing each complete page, call +Before constructing final page bodies, freeze the complete ordered page inventory +and stable `total_pages`; the inventory may contain at most 32 pages. Once the +inventory is frozen, read and construct each page in order, then immediately call `brain_stage_ingestion_proposal_page` in its own agent turn with the exact `artifact_id`, `source_id`, `admission_scope`, one-based `sequence`, stable `total_pages`, and page object. Stage only one page per turn. Preserve the @@ -274,7 +276,8 @@ page-digest manifest from the job's durable fragments, so finalization does not depend on old stage outputs remaining in model context. The server rejects gaps, duplicate or changed fragments, cross-job evidence, a capture page that does not match the exact job binding, mutations outside the job slug fence, a full -plan over 786,432 UTF-8 bytes, more than 100 pages, a timeline `refLabel` over +raw or JSON-escaped plan representation over 98,304 UTF-8 bytes, more than 32 +pages, a timeline `refLabel` over 500 characters, or a compact manifest over 262,144 UTF-8 bytes. Return the finalizer's compact manifest as `staged_proposal`; never reproduce page bodies or baselines in the final receipt. Never truncate or split a diff --git a/src/core/ai/tool-loop-context.ts b/src/core/ai/tool-loop-context.ts index 494e75a466..4213a5ac0b 100644 --- a/src/core/ai/tool-loop-context.ts +++ b/src/core/ai/tool-loop-context.ts @@ -8,7 +8,9 @@ */ import { createHash } from 'node:crypto'; +import { get_encoding } from '@dqbd/tiktoken'; import { getProviderCapabilities } from './capabilities.ts'; +import { splitProviderModelId } from '../model-id.ts'; import type { ChatBlock, ChatMessage, ChatToolDef } from './gateway.ts'; const CONTEXT_TARGET_FRACTION = 0.7; @@ -17,6 +19,10 @@ const ESTIMATED_CHARS_PER_TOKEN = 2; // One byte per token is intentionally conservative for ASCII, CJK, emoji, // and mixed JSON without requiring a model-specific tokenizer at runtime. const CONSERVATIVE_BYTES_PER_TOKEN = 1; +// The tokenizer counts static content exactly, but provider message/tool +// envelopes are not exposed by the API. Keep a fixed hard-window reserve for +// that framing rather than treating content tokenization as wire-exact. +const OPENAI_PROTOCOL_TOKEN_RESERVE = 1_024; const FALLBACK_CONTEXT_TOKENS = 128_000; const PAYLOAD_LIMITS = [12_000, 4_000, 1_000, 256, 64, 0] as const; const MAX_STRUCTURAL_IDENTITY_VALUE_BYTES = 256; @@ -60,6 +66,17 @@ interface WorkingContextProjectionSource { preserveStructuralIdentity: boolean; } +let openAiEncoding: ReturnType | undefined; + +function countOpenAiTokens(value: string): number { + openAiEncoding ??= get_encoding('o200k_base'); + return openAiEncoding.encode(value).length; +} + +function isOpenAiModel(model: string): boolean { + return splitProviderModelId(model).provider?.toLowerCase() === 'openai'; +} + export interface ToolLoopContextOptions { /** Tools whose effects must retain a distinct identity whenever context compacts. */ mutatingToolNames?: ReadonlySet; @@ -91,17 +108,31 @@ export function resolveToolLoopMessageBudget(args: { } const targetTokens = Math.floor(contextTokens * CONTEXT_TARGET_FRACTION); + const staticTools = safeJson(args.tools); const targetMessageTokens = Math.max(0, targetTokens - args.maxOutputTokens); - const staticChars = (args.system?.length ?? 0) + safeJson(args.tools).length; - const targetBudget = targetMessageTokens * ESTIMATED_CHARS_PER_TOKEN - staticChars; + const openAiStaticTokens = isOpenAiModel(args.model) + ? countOpenAiTokens(args.system ?? '') + countOpenAiTokens(staticTools) + : null; + const targetBudget = openAiStaticTokens === null + ? targetMessageTokens * ESTIMATED_CHARS_PER_TOKEN + - (args.system?.length ?? 0) + - staticTools.length + : Math.max(0, targetMessageTokens - openAiStaticTokens) + * ESTIMATED_CHARS_PER_TOKEN; // The historical two-character estimate keeps ordinary prompts near the // 70% target. A second absolute byte cap uses the model's whole declared // window, so dense Unicode cannot overflow while ordinary English is not // needlessly constrained to one byte per target token. const hardInputTokens = Math.max(0, contextTokens - args.maxOutputTokens); - const staticBytes = utf8Bytes(args.system ?? '') + jsonBytes(args.tools); - const byteSafeBudget = hardInputTokens * CONSERVATIVE_BYTES_PER_TOKEN - staticBytes; + const byteSafeBudget = openAiStaticTokens === null + ? hardInputTokens * CONSERVATIVE_BYTES_PER_TOKEN + - utf8Bytes(args.system ?? '') + - utf8Bytes(staticTools) + : Math.max( + 0, + hardInputTokens - openAiStaticTokens - OPENAI_PROTOCOL_TOKEN_RESERVE, + ) * CONSERVATIVE_BYTES_PER_TOKEN; return Math.max(0, Math.min(targetBudget, byteSafeBudget)); } diff --git a/src/core/migrate.ts b/src/core/migrate.ts index 8c2a723d25..f29d4307af 100644 --- a/src/core/migrate.ts +++ b/src/core/migrate.ts @@ -6010,8 +6010,22 @@ export const MIGRATIONS: Migration[] = [ ); CREATE INDEX IF NOT EXISTS idx_agent_job_proposals_owner_digest ON agent_job_proposals (owner_client_id, proposal_digest); - ALTER TABLE agent_job_proposal_fragments ENABLE ROW LEVEL SECURITY; - ALTER TABLE agent_job_proposals ENABLE ROW LEVEL SECURITY; + DO $$ + DECLARE + has_bypass BOOLEAN; + BEGIN + SELECT EXISTS ( + SELECT 1 + FROM pg_roles pr + WHERE pg_has_role(current_user, pr.oid, 'USAGE') + AND (pr.rolbypassrls OR pr.rolsuper) + ) INTO has_bypass; + IF NOT has_bypass THEN + RAISE EXCEPTION 'v136 agent_job_staged_proposals: role % does not have BYPASSRLS privilege — cannot enable RLS safely. Re-run as postgres (or another BYPASSRLS role). The migration will retry automatically on the next initSchema call.', current_user; + END IF; + ALTER TABLE agent_job_proposal_fragments ENABLE ROW LEVEL SECURITY; + ALTER TABLE agent_job_proposals ENABLE ROW LEVEL SECURITY; + END $$; `, sqlFor: { pglite: ` diff --git a/src/core/minions/agent-job-proposals.ts b/src/core/minions/agent-job-proposals.ts index d5137a6005..b6d2565218 100644 --- a/src/core/minions/agent-job-proposals.ts +++ b/src/core/minions/agent-job-proposals.ts @@ -7,13 +7,16 @@ import { matchesSlugAllowList } from '../slug-allow-list.ts'; export const PROPOSAL_STAGE_INPUT_MAX_BYTES = 196_608; /** Maximum UTF-8 size of one finalized, canonical proposal plan. */ -export const PROPOSAL_AGGREGATE_MAX_BYTES = 786_432; +export const PROPOSAL_AGGREGATE_MAX_BYTES = 98_304; + +/** Maximum UTF-8 size after the canonical plan is embedded as a JSON string. */ +export const PROPOSAL_ESCAPED_PLAN_MAX_BYTES = 98_304; /** Maximum UTF-8 size of the compact receipt manifest. */ export const PROPOSAL_MANIFEST_MAX_BYTES = 262_144; /** Maximum number of pages in one finalized proposal. */ -export const PROPOSAL_MAX_PAGES = 100; +export const PROPOSAL_MAX_PAGES = 32; /** Maximum characters in the shared Lore/GBrain admission-scope contract. */ export const PROPOSAL_ADMISSION_SCOPE_MAX_CHARS = 4_000; @@ -23,7 +26,9 @@ export const FINALIZE_PROPOSAL_TOOL_NAME = 'brain_finalize_ingestion_proposal'; const SHA256_RE = /^[a-f0-9]{64}$/; const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; -const PAGE_SLUG_RE = /^[\p{L}\p{N}][\p{L}\p{N}-]*(?:\/[\p{L}\p{N}][\p{L}\p{N}-]*)*$/u; +const CANONICAL_SLUG_CHARS = 'a-z0-9\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af'; +const PAGE_SLUG_SEGMENT = `[${CANONICAL_SLUG_CHARS}][${CANONICAL_SLUG_CHARS}-]*`; +const PAGE_SLUG_RE = new RegExp(`^${PAGE_SLUG_SEGMENT}(\\/${PAGE_SLUG_SEGMENT})*$`); /** Error raised when staged proposal evidence violates its durable contract. */ export class AgentJobProposalError extends Error { @@ -383,6 +388,13 @@ export async function finalizeAgentJobProposal( `Finalized proposal is ${planBytes} UTF-8 bytes; maximum is ${PROPOSAL_AGGREGATE_MAX_BYTES}.`, ); } + const escapedPlanBytes = Buffer.byteLength(JSON.stringify(planJson), 'utf8'); + if (escapedPlanBytes > PROPOSAL_ESCAPED_PLAN_MAX_BYTES) { + throw new AgentJobProposalError( + 'proposal_too_large', + `Escaped finalized proposal is ${escapedPlanBytes} UTF-8 bytes; maximum is ${PROPOSAL_ESCAPED_PLAN_MAX_BYTES}.`, + ); + } const proposalDigest = digestProposalValue(plan); const manifest: FinalizedProposalManifest = { status: 'staged_proposal', @@ -709,9 +721,9 @@ function parseProposalPage(raw: unknown): ScopedProposalPage { assertExactKeys(page, allowed, 'page'); const slug = readCanonicalSlug(page.slug, 'page.slug'); const title = readBoundedString(page.title, 'page.title', 1_000); - const bodyMarkdown = readString(page.bodyMarkdown, 'page.bodyMarkdown'); + const bodyMarkdown = readNonBlankString(page.bodyMarkdown, 'page.bodyMarkdown'); if (effect === 'create') return { slug, effect, title, bodyMarkdown }; - const baseMarkdown = readString(page.baseMarkdown, 'page.baseMarkdown'); + const baseMarkdown = readNonBlankString(page.baseMarkdown, 'page.baseMarkdown'); const expectedContentHash = readString(page.expectedContentHash, 'page.expectedContentHash'); if (!SHA256_RE.test(expectedContentHash)) { throw new AgentJobProposalError('invalid_page', 'page.expectedContentHash must be a lowercase SHA-256 digest.'); @@ -860,9 +872,17 @@ function readBoundedString(raw: unknown, name: string, maxLength: number): strin return value; } +function readNonBlankString(raw: unknown, name: string): string { + const value = readString(raw, name); + if (!value.trim()) { + throw new AgentJobProposalError('invalid_string', `${name} must not be blank.`); + } + return value; +} + function readCanonicalSlug(raw: unknown, name: string): string { const slug = readString(raw, name); - if (slug.length > 255 || !PAGE_SLUG_RE.test(slug) || slug !== slug.toLowerCase()) { + if (slug.length > 255 || !PAGE_SLUG_RE.test(slug)) { throw new AgentJobProposalError('invalid_slug', `${name} is not a canonical page slug.`); } return slug; diff --git a/src/core/minions/handlers/subagent.ts b/src/core/minions/handlers/subagent.ts index 66e5beeaac..a853cab0e9 100644 --- a/src/core/minions/handlers/subagent.ts +++ b/src/core/minions/handlers/subagent.ts @@ -61,34 +61,15 @@ import { assertProposalToolTurnPersistable, assertProposalToolTurnPersistableForJob, } from '../agent-job-proposals.ts'; +import { resolveSubagentMaxOutputTokens } from '../subagent-limits.ts'; +export { resolveSubagentMaxOutputTokens as resolveMaxOutputTokens } from '../subagent-limits.ts'; // ── Defaults ──────────────────────────────────────────────── const DEFAULT_MODEL = 'claude-sonnet-4-6'; const DEFAULT_MAX_TURNS = 20; -const DEFAULT_MAX_OUTPUT_TOKENS = 8192; const DEFAULT_RATE_KEY = 'anthropic:messages'; -/** - * Resolve the per-turn output-token cap (#2778). Per-job data wins, then the - * `agent.max_output_tokens` config row, then the 8192 default (was a - * hardcoded 4096 that made pages >~12KB unwritable via put_page). Invalid - * values (NaN / zero / negative) fall through to the next tier. - */ -export function resolveMaxOutputTokens( - perJob: number | undefined, - configRaw: string | null | undefined, -): number { - if (typeof perJob === 'number' && Number.isFinite(perJob) && perJob > 0) { - return Math.floor(perJob); - } - if (typeof configRaw === 'string' && configRaw.trim() !== '') { - const n = Number(configRaw); - if (Number.isFinite(n) && n > 0) return Math.floor(n); - } - return DEFAULT_MAX_OUTPUT_TOKENS; -} - /** * Resolve the rate-lease cap from the env var. * @@ -255,7 +236,7 @@ export function makeSubagentHandler(deps: SubagentDeps) { } const maxTurns = data.max_turns ?? DEFAULT_MAX_TURNS; // #2778: per-turn output cap — data.max_tokens → config → 8192 default. - const maxOutputTokens = resolveMaxOutputTokens( + const maxOutputTokens = resolveSubagentMaxOutputTokens( data.max_tokens, await engine.getConfig('agent.max_output_tokens').catch(() => null), ); @@ -854,7 +835,7 @@ interface GatewayRunArgs { systemPrompt: string; toolDefs: ToolDef[]; maxTurns: number; - /** #2778: per-turn output-token cap (resolved by resolveMaxOutputTokens). */ + /** #2778: per-turn output-token cap resolved from frozen job/config state. */ maxOutputTokens: number; } diff --git a/src/core/minions/subagent-limits.ts b/src/core/minions/subagent-limits.ts new file mode 100644 index 0000000000..b8ecd396f9 --- /dev/null +++ b/src/core/minions/subagent-limits.ts @@ -0,0 +1,17 @@ +/** Default per-turn output-token cap for subagent jobs. */ +export const DEFAULT_SUBAGENT_MAX_OUTPUT_TOKENS = 8_192; + +/** Resolve a subagent output cap from job data, configuration, then default. */ +export function resolveSubagentMaxOutputTokens( + perJob: number | undefined, + configRaw: string | null | undefined, +): number { + if (typeof perJob === 'number' && Number.isFinite(perJob) && perJob > 0) { + return Math.floor(perJob); + } + if (typeof configRaw === 'string' && configRaw.trim() !== '') { + const configured = Number(configRaw); + if (Number.isFinite(configured) && configured > 0) return Math.floor(configured); + } + return DEFAULT_SUBAGENT_MAX_OUTPUT_TOKENS; +} diff --git a/src/core/minions/subagent-prompt-budget.ts b/src/core/minions/subagent-prompt-budget.ts new file mode 100644 index 0000000000..8f3c89ef69 --- /dev/null +++ b/src/core/minions/subagent-prompt-budget.ts @@ -0,0 +1,73 @@ +import type { BrainEngine } from '../engine.ts'; +import type { GBrainConfig } from '../config.ts'; +import { resolveModel, TIER_DEFAULTS } from '../model-config.ts'; +import type { ChatMessage, ChatToolDef } from '../ai/gateway.ts'; +import { resolveToolLoopMessageBudget } from '../ai/tool-loop-context.ts'; +import { buildSystemPrompt } from './system-prompt.ts'; +import { buildBrainTools, filterAllowedTools } from './tools/brain-allowlist.ts'; +import { resolveSubagentMaxOutputTokens } from './subagent-limits.ts'; + +const FUTURE_JOB_ID = Number.MAX_SAFE_INTEGER; + +export interface SubagentInitialPromptBudgetArgs { + engine: BrainEngine; + config: GBrainConfig; + prompt: string; + userSystem?: string; + model?: string; + allowedTools: string[]; + allowedSlugPrefixes: string[]; + sourceId?: string; +} + +export interface SubagentInitialPromptBudget { + model: string; + maxOutputTokens: number; + messageBytes: number; + messageBudgetBytes: number; +} + +/** Resolve a conservative fresh-job prompt surface and executable byte budget. */ +export async function resolveSubagentInitialPromptBudget( + args: SubagentInitialPromptBudgetArgs, +): Promise { + const model = args.model ?? await resolveModel(args.engine, { + tier: 'subagent', + configKey: 'models.subagent', + fallback: TIER_DEFAULTS.subagent, + }); + const maxOutputTokens = resolveSubagentMaxOutputTokens( + undefined, + await args.engine.getConfig('agent.max_output_tokens').catch(() => null), + ); + const registry = buildBrainTools({ + // A fresh job has no id yet. The widest safe integer is a conservative + // stand-in for the only schema text whose bytes depend on that future id. + subagentId: FUTURE_JOB_ID, + engine: args.engine, + config: args.config, + allowedSlugPrefixes: args.allowedSlugPrefixes, + sourceId: args.sourceId, + }); + const toolDefs = args.allowedTools.length > 0 + ? filterAllowedTools(registry, args.allowedTools) + : registry; + const system = buildSystemPrompt(toolDefs, args.userSystem); + const tools: ChatToolDef[] = toolDefs.map(tool => ({ + name: tool.name, + description: tool.description, + inputSchema: tool.input_schema as Record, + })); + const message: ChatMessage[] = [{ role: 'user', content: args.prompt }]; + return { + model, + maxOutputTokens, + messageBytes: Buffer.byteLength(JSON.stringify(message), 'utf8'), + messageBudgetBytes: resolveToolLoopMessageBudget({ + model, + maxOutputTokens, + system, + tools, + }), + }; +} diff --git a/src/core/operations.ts b/src/core/operations.ts index 38a9d791fc..1e75c0b7db 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -5518,6 +5518,58 @@ const submit_agent: Operation = { } } + const prompt = typeof p.prompt === 'string' ? p.prompt : ''; + if (!prompt) { + throw new OperationError('invalid_params', 'submit_agent: prompt must be a non-empty string.'); + } + + let skillBody: string | undefined; + let skillName: string | undefined; + let skillSha256: string | undefined; + let selectedModel: string | undefined; + let selectedMaxOutputTokens: number | undefined; + if (!ctx.dryRun) { + if (typeof p.skill_name === 'string') { + if (!hasScope(ctx.auth?.scopes ?? [], 'read')) { + throw new OperationError( + 'permission_denied', + 'submit_agent: skill_name requires read scope in addition to agent scope.', + ); + } + const skillCatalog = await import('./skill-catalog.ts'); + const publish = await skillCatalog.readMcpPublishSkills(ctx); + skillCatalog.assertPublishEnabled(ctx, publish); + const configuredDir = await skillCatalog.readMcpSkillsDir(ctx); + const { dir } = skillCatalog.resolveSkillsDir(ctx, configuredDir); + const skill = skillCatalog.getSkillDetail(ctx, dir, p.skill_name); + const { createHash } = await import('crypto'); + skillBody = skill.body; + skillName = skill.name; + skillSha256 = createHash('sha256').update(skill.body, 'utf8').digest('hex'); + } + const { resolveSubagentInitialPromptBudget } = await import( + './minions/subagent-prompt-budget.ts' + ); + const promptBudget = await resolveSubagentInitialPromptBudget({ + engine: ctx.engine, + config: ctx.config, + prompt, + userSystem: skillBody, + model: typeof p.model === 'string' ? p.model : undefined, + allowedTools: requestedTools, + allowedSlugPrefixes: requestedSlugPrefixes, + sourceId: boundSource ?? undefined, + }); + selectedModel = promptBudget.model; + selectedMaxOutputTokens = promptBudget.maxOutputTokens; + if (promptBudget.messageBytes > promptBudget.messageBudgetBytes) { + throw new OperationError( + 'invalid_params', + `submit_agent: initial prompt is too large for model ${selectedModel} with the published skill and requested tools (${promptBudget.messageBytes} UTF-8 bytes; maximum ${promptBudget.messageBudgetBytes}).`, + ); + } + } + // Concurrency cap: count active+waiting agent jobs for this client. const inflight = await sql` SELECT COUNT(*)::int AS n @@ -5559,32 +5611,20 @@ const submit_agent: Operation = { const queue = new MinionQueue(ctx.engine); const jobData: Record = { - prompt: p.prompt as string, + prompt, max_turns: Math.min((p.max_turns as number) ?? 20, 100), allowed_tools: requestedTools, allowed_slug_prefixes: requestedSlugPrefixes, use_gateway_loop: true, __owner_client_id: clientId, }; - if (typeof p.skill_name === 'string') { - if (!hasScope(ctx.auth?.scopes ?? [], 'read')) { - throw new OperationError( - 'permission_denied', - 'submit_agent: skill_name requires read scope in addition to agent scope.', - ); - } - const skillCatalog = await import('./skill-catalog.ts'); - const publish = await skillCatalog.readMcpPublishSkills(ctx); - skillCatalog.assertPublishEnabled(ctx, publish); - const configuredDir = await skillCatalog.readMcpSkillsDir(ctx); - const { dir } = skillCatalog.resolveSkillsDir(ctx, configuredDir); - const skill = skillCatalog.getSkillDetail(ctx, dir, p.skill_name); - const { createHash } = await import('crypto'); - jobData.system = skill.body; - jobData.skill_name = skill.name; - jobData.skill_sha256 = createHash('sha256').update(skill.body, 'utf8').digest('hex'); + if (skillBody && skillName && skillSha256) { + jobData.system = skillBody; + jobData.skill_name = skillName; + jobData.skill_sha256 = skillSha256; } - if (typeof p.model === 'string') jobData.model = p.model; + if (selectedModel) jobData.model = selectedModel; + if (selectedMaxOutputTokens) jobData.max_tokens = selectedMaxOutputTokens; if (isReasoningEffort(p.reasoning_effort)) jobData.reasoning_effort = p.reasoning_effort; if (boundSource) jobData.source_id = boundSource; if (proposalArtifactId) { @@ -5612,7 +5652,7 @@ const submit_agent: Operation = { logAgentSubmission({ client_id: clientId, job_id: job.id, - model: typeof p.model === 'string' ? p.model : '', + model: selectedModel ?? '', reasoning_effort: isReasoningEffort(p.reasoning_effort) ? p.reasoning_effort : undefined, bound_tools: requestedTools, bound_source: boundSource, diff --git a/test/agent-job-proposals.test.ts b/test/agent-job-proposals.test.ts index edffaa29b9..e69b3eb5dc 100644 --- a/test/agent-job-proposals.test.ts +++ b/test/agent-job-proposals.test.ts @@ -3,6 +3,7 @@ import { PGLiteEngine } from '../src/core/pglite-engine.ts'; import { resetPgliteState } from './helpers/reset-pglite.ts'; import { PROPOSAL_AGGREGATE_MAX_BYTES, + PROPOSAL_ESCAPED_PLAN_MAX_BYTES, PROPOSAL_MANIFEST_MAX_BYTES, PROPOSAL_MAX_PAGES, PROPOSAL_STAGE_INPUT_MAX_BYTES, @@ -107,11 +108,33 @@ describe('proposal turn pre-persistence boundary', () => { }); describe('durable agent-job proposal staging', () => { - it('rejects proposal page counts above the shared 100-page contract', async () => { + it('rejects proposal page counts above the shared 32-page contract', async () => { const jobId = await seedJob(); - await expect(stage(jobId, 1, 101, createPage('sources/too-many'))) + await expect(stage(jobId, 1, 33, createPage('sources/too-many'))) .rejects.toMatchObject({ code: 'invalid_total_pages' }); - expect(PROPOSAL_MAX_PAGES).toBe(100); + expect(PROPOSAL_MAX_PAGES).toBe(32); + }); + + it('uses the same lowercase ASCII-and-CJK slug contract as Lore', async () => { + const jobId = await seedJob(); + await expect(stage(jobId, 1, 1, createPage('sources/éclair'))) + .rejects.toMatchObject({ code: 'invalid_slug' }); + await expect(stage(jobId, 1, 1, createPage('sources/αθήνα'))) + .rejects.toMatchObject({ code: 'invalid_slug' }); + await expect(stage(jobId, 1, 1, createPage('sources/東京'))) + .resolves.toMatchObject({ slug: 'sources/東京' }); + }); + + it('rejects blank page bodies and update baselines', async () => { + const createJob = await seedJob(); + await expect(stage(createJob, 1, 1, createPage('sources/example', ' \n\t '))) + .rejects.toMatchObject({ code: 'invalid_string' }); + + const updateJob = await seedJob(); + await expect(stage(updateJob, 1, 1, { + ...updatePage('sources/example'), + baseMarkdown: ' ', + })).rejects.toMatchObject({ code: 'invalid_string' }); }); it('stages exact pages, finalizes an ordered manifest, and retrieves the full owned plan', async () => { @@ -465,15 +488,15 @@ describe('durable agent-job proposal staging', () => { it('rejects cumulative staged pages over the aggregate ceiling without persisting the crossing fragment', async () => { const jobId = await seedJob(); - const pageCount = 5; + const pageCount = 2; for (let sequence = 1; sequence < pageCount; sequence++) { - const page = createPage(`sources/large-${sequence}`, 'x'.repeat(165_000)); + const page = createPage(`sources/large-${sequence}`, 'x'.repeat(80_000)); await stage(jobId, sequence, pageCount, page); } const crossingInput = { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', sequence: pageCount, total_pages: pageCount, - page: createPage(`sources/large-${pageCount}`, 'x'.repeat(165_000)), + page: createPage(`sources/large-${pageCount}`, 'x'.repeat(80_000)), }; await expect(assertProposalToolTurnPersistableForJob(engine, jobId, [{ type: 'tool-call', toolName: 'brain_stage_ingestion_proposal_page', input: crossingInput, @@ -481,7 +504,7 @@ describe('durable agent-job proposal staging', () => { const replayInput = { artifact_id: 'artifact-1', source_id: 'company', admission_scope: 'Include project delivery notes.', sequence: pageCount - 1, total_pages: pageCount, - page: createPage(`sources/large-${pageCount - 1}`, 'x'.repeat(165_000)), + page: createPage(`sources/large-${pageCount - 1}`, 'x'.repeat(80_000)), }; await expect(assertProposalToolTurnPersistableForJob(engine, jobId, [{ type: 'tool-call', toolName: 'brain_stage_ingestion_proposal_page', input: replayInput, @@ -493,15 +516,16 @@ describe('durable agent-job proposal staging', () => { [jobId], ); expect(Number(rows[0]!.count)).toBe(pageCount - 1); - expect(PROPOSAL_AGGREGATE_MAX_BYTES).toBe(786_432); + expect(PROPOSAL_AGGREGATE_MAX_BYTES).toBe(96 * 1024); + expect(PROPOSAL_ESCAPED_PLAN_MAX_BYTES).toBe(96 * 1024); expect(PROPOSAL_MANIFEST_MAX_BYTES).toBe(256 * 1024); }); - it('accepts a raw aggregate plan exactly at the shared ceiling', async () => { + it('accepts an escaped canonical plan exactly at the shared ceiling', async () => { const jobId = await seedJob(); - const pageCount = 5; + const pageCount = 1; const pages = Array.from({ length: pageCount }, (_, index) => - createPage(index === 0 ? 'sources/example' : `sources/boundary-${index + 1}`, 'x'.repeat(150_000))); + createPage(index === 0 ? 'sources/example' : `sources/boundary-${index + 1}`, 'x')); const basePlan = { artifactId: 'artifact-1', sourceId: 'company', @@ -512,8 +536,8 @@ describe('durable agent-job proposal staging', () => { proposedLinks: [], unresolved: [], }; - const remaining = PROPOSAL_AGGREGATE_MAX_BYTES - - Buffer.byteLength(canonicalProposalJson(basePlan), 'utf8'); + const remaining = PROPOSAL_ESCAPED_PLAN_MAX_BYTES - + Buffer.byteLength(JSON.stringify(canonicalProposalJson(basePlan)), 'utf8'); pages[pageCount - 1]!.bodyMarkdown += 'y'.repeat(remaining); for (let sequence = 1; sequence <= pageCount; sequence++) { await stage(jobId, sequence, pageCount, pages[sequence - 1]!); @@ -530,7 +554,20 @@ describe('durable agent-job proposal staging', () => { 'lore-client', manifest.proposalDigest, ); + expect(Buffer.byteLength(JSON.stringify(canonicalProposalJson(owned.plan)), 'utf8')) + .toBe(PROPOSAL_ESCAPED_PLAN_MAX_BYTES); expect(Buffer.byteLength(canonicalProposalJson(owned.plan), 'utf8')) - .toBe(PROPOSAL_AGGREGATE_MAX_BYTES); + .toBeLessThan(PROPOSAL_AGGREGATE_MAX_BYTES); + }); + + it('rejects a raw plan whose JSON-string escaped representation exceeds 96 KiB', async () => { + const jobId = await seedJob(); + await stage(jobId, 1, 1, createPage('sources/example', `x${'\n'.repeat(40_000)}`)); + + await expect(finalizeAgentJobProposal(engine, jobId, { + artifact_id: 'artifact-1', source_id: 'company', + admission_scope: 'Include project delivery notes.', + total_pages: 1, summary: 'Escaped boundary.', + })).rejects.toMatchObject({ code: 'proposal_too_large' }); }); }); diff --git a/test/ai/gateway-tool-loop.test.ts b/test/ai/gateway-tool-loop.test.ts index 78e2cfb620..3791b2b822 100644 --- a/test/ai/gateway-tool-loop.test.ts +++ b/test/ai/gateway-tool-loop.test.ts @@ -9,6 +9,7 @@ import { type ChatMessage, type ToolHandler, } from '../../src/core/ai/gateway.ts'; +import { get_encoding } from '@dqbd/tiktoken'; import { compactToolLoopMessages, resolveToolLoopMessageBudget, @@ -580,19 +581,46 @@ describe('gateway.toolLoop (v0.38 D11 — provider-agnostic loop control)', () = .toThrow(ToolLoopContextProjectionError); }); - it('reserves no more than one UTF-8 byte per available input token', () => { + it('uses exact o200k_base tokens for OpenAI static context and byte-bounds CJK and emoji messages', () => { + const system = `${'漢'.repeat(1_000)}${'🧠'.repeat(1_000)}`; + const tools = [{ + name: 'unicode_tool', + description: '検証 🧠', + inputSchema: { type: 'object', properties: { text: { type: 'string' } } }, + }]; const budget = resolveToolLoopMessageBudget({ model: 'openai:gpt-5.6-terra', maxOutputTokens: 32_768, contextWindowTokens: 200_000, - system: '漢'.repeat(1_000), - tools: [], + system, + tools, + }); + const encoding = get_encoding('o200k_base'); + const staticTokens = encoding.encode(system).length + + encoding.encode(JSON.stringify(tools)).length; + + expect(budget).toBe(200_000 - 32_768 - staticTokens - 1_024); + encoding.free(); + }); + + it('leaves room for a production-sized 128 KiB OpenAI initial message', () => { + const budget = resolveToolLoopMessageBudget({ + model: 'openai:gpt-5.6-terra', + maxOutputTokens: 32_768, + contextWindowTokens: 200_000, + system: 'Published ingestion instructions.\n'.repeat(1_200), + tools: Array.from({ length: 14 }, (_, index) => ({ + name: `tool_${index}`, + description: 'Production tool description. '.repeat(24), + inputSchema: { type: 'object', properties: { slug: { type: 'string' } } }, + })), }); + const messageBytes = Buffer.byteLength(JSON.stringify([{ + role: 'user', + content: 'p'.repeat(128 * 1024), + }]), 'utf8'); - // A byte-level tokenizer cannot emit more tokens than the UTF-8 byte - // count, so one byte per available token is conservative for ASCII, - // CJK, emoji, and mixed JSON payloads. - expect(budget).toBe(200_000 - 32_768 - Buffer.byteLength('漢'.repeat(1_000), 'utf8') - 2); + expect(budget).toBeGreaterThanOrEqual(messageBytes); }); it('fails closed when the full original task and required evidence cannot fit', () => { diff --git a/test/e2e/subagent-gateway-path.test.ts b/test/e2e/subagent-gateway-path.test.ts index 28eebf84e6..243ea9e070 100644 --- a/test/e2e/subagent-gateway-path.test.ts +++ b/test/e2e/subagent-gateway-path.test.ts @@ -257,10 +257,10 @@ describe('runSubagentViaGateway (v0.38 Slice 1 — full handler path through gat it('rejects a cumulative stage overflow before assistant, tool, or fragment persistence', async () => { const crossingInput = { artifact_id: 'artifact-1', source_id: 'company', - admission_scope: 'Derived scope.', sequence: 5, total_pages: 5, + admission_scope: 'Derived scope.', sequence: 2, total_pages: 2, page: { - slug: 'sources/large-5', effect: 'create', title: 'Large', - bodyMarkdown: 'x'.repeat(165_000), + slug: 'sources/large-2', effect: 'create', title: 'Large', + bodyMarkdown: 'x'.repeat(80_000), }, }; __setChatTransportForTests(async () => ({ @@ -290,13 +290,11 @@ describe('runSubagentViaGateway (v0.38 Slice 1 — full handler path through gat proposal_admission_scope: 'Derived scope.', allowed_slug_prefixes: ['sources/*'], }, }); - for (let sequence = 1; sequence < 5; sequence++) { - await stageAgentJobProposalPage(engine, jobId, { - ...crossingInput, - sequence, - page: { ...crossingInput.page, slug: `sources/large-${sequence}` }, - }); - } + await stageAgentJobProposalPage(engine, jobId, { + ...crossingInput, + sequence: 1, + page: { ...crossingInput.page, slug: 'sources/large-1' }, + }); await expect(handler(ctx)).rejects.toThrow(/maximum/i); const messages = await engine.executeRaw<{ role: string }>( @@ -311,7 +309,7 @@ describe('runSubagentViaGateway (v0.38 Slice 1 — full handler path through gat expect(executed).toBe(false); expect(messages.map((row) => row.role)).toEqual(['user']); expect(executions).toHaveLength(0); - expect(fragments.map((row) => Number(row.sequence))).toEqual([1, 2, 3, 4]); + expect(fragments.map((row) => Number(row.sequence))).toEqual([1]); }); it('happy path 1-turn: gateway returns text, handler returns SubagentResult', async () => { diff --git a/test/github-project-ingestion-skill.test.ts b/test/github-project-ingestion-skill.test.ts index 778a66cc15..bc91a0d68d 100644 --- a/test/github-project-ingestion-skill.test.ts +++ b/test/github-project-ingestion-skill.test.ts @@ -221,6 +221,8 @@ describe('github-project-ingestion skill', () => { expect(skill).toContain('"proposalDigest": "64 lowercase hex characters"'); expect(skill).toContain('brain_stage_ingestion_proposal_page'); expect(skill).toContain('Stage only one page per turn'); + expect(skill).toContain('freeze the complete ordered page inventory'); + expect(skill).toContain('at most 32 pages'); expect(skill).toContain('brain_finalize_ingestion_proposal'); expect(skill).toContain('4,000 characters'); expect(skill).toContain('server derives the ordered'); diff --git a/test/granola-meeting-ingestion-skill.test.ts b/test/granola-meeting-ingestion-skill.test.ts index 5edd292313..de750153cf 100644 --- a/test/granola-meeting-ingestion-skill.test.ts +++ b/test/granola-meeting-ingestion-skill.test.ts @@ -114,6 +114,8 @@ describe('granola-meeting-ingestion skill', () => { expect(skill).toContain('"proposalDigest": "64 lowercase hex characters"'); expect(skill).toContain('brain_stage_ingestion_proposal_page'); expect(skill).toContain('Stage only one page per turn'); + expect(skill).toContain('freeze the complete ordered page inventory'); + expect(skill).toContain('at most 32 pages'); expect(skill).toContain('brain_finalize_ingestion_proposal'); expect(skill).toContain('4,000 characters'); expect(skill).toContain('server derives the ordered'); diff --git a/test/migrate.test.ts b/test/migrate.test.ts index 13067b2c01..e754fef837 100644 --- a/test/migrate.test.ts +++ b/test/migrate.test.ts @@ -20,6 +20,24 @@ describe('migrate', () => { // and are covered in the E2E suite (test/e2e/mechanical.test.ts) }); +describe('migration v136 — staged proposal RLS', () => { + test('Postgres gates RLS on inherited BYPASSRLS or superuser and fails loudly', () => { + const pgSql = MIGRATIONS.find(m => m.version === 136)!.sql; + expect(pgSql).toContain("pg_has_role(current_user, pr.oid, 'USAGE')"); + expect(pgSql).toMatch(/pr\.rolbypassrls OR pr\.rolsuper/); + expect(pgSql).toMatch(/IF NOT has_bypass/); + expect(pgSql).toMatch(/RAISE EXCEPTION[^;]*BYPASSRLS/); + expect(pgSql).toContain('ALTER TABLE agent_job_proposal_fragments ENABLE ROW LEVEL SECURITY'); + expect(pgSql).toContain('ALTER TABLE agent_job_proposals ENABLE ROW LEVEL SECURITY'); + }); + + test('PGLite has no RLS or BYPASSRLS gate', () => { + const pgliteSql = MIGRATIONS.find(m => m.version === 136)!.sqlFor!.pglite!; + expect(pgliteSql).not.toContain('rolbypassrls'); + expect(pgliteSql).not.toContain('ENABLE ROW LEVEL SECURITY'); + }); +}); + // v0.28.5 — A1: cheap probe used by `connectEngine` to gate `initSchema()` // so already-migrated brains don't pay the schema-replay cost on every // short-lived CLI invocation. Closes #651 in cooperation with X1's diff --git a/test/submit-agent.test.ts b/test/submit-agent.test.ts index 86b920e467..f8c69e0265 100644 --- a/test/submit-agent.test.ts +++ b/test/submit-agent.test.ts @@ -478,7 +478,7 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with idempotency_key: 'lore-job-01', }); const repeated = await callSubmitAgent(ctx, { - prompt: 'correct the selected claim', + prompt: 'x'.repeat(500_000), idempotency_key: 'lore-job-01', }); @@ -678,6 +678,80 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with expect(row.data.system).toContain('Knowledge Correction'); }); + it('accepts a production-sized 128 KiB prompt with the published GitHub skill on Terra', async () => { + const tools = [ + 'get_active_schema_pack', 'search', 'query', 'get_page', 'list_pages', + 'resolve_slugs', 'get_links', 'get_backlinks', + 'stage_ingestion_proposal_page', 'finalize_ingestion_proposal', + 'put_page', 'add_link', 'add_timeline_entry', 'validate_links', + ]; + await seedClient('lore', { + bound_tools: tools, + bound_source_id: 'company', + bound_slug_prefixes: ['sources/', 'projects/', 'people/', 'companies/'], + bound_max_concurrent: 3, + }); + await engine.setConfig('mcp.publish_skills', 'true'); + await engine.setConfig('agent.max_output_tokens', '32768'); + const ctx = makeCtx({ clientId: 'lore', scopes: ['read', 'agent'] }); + ctx.config = { mcp: { skills_dir: path.resolve(import.meta.dir, '../skills') } }; + + const result = await callSubmitAgent(ctx, { + prompt: 'p'.repeat(128 * 1024), + skill_name: 'github-project-ingestion', + model: 'openai:gpt-5.6-terra', + allowed_tools: tools, + proposal_artifact_id: 'artifact-128k', + proposal_capture_page_slug: 'sources/example', + proposal_admission_scope: 'Include source-grounded delivery knowledge.', + }); + + expect(result.id).toBeNumber(); + const [row] = await engine.executeRaw<{ data: Record }>( + 'SELECT data FROM minion_jobs WHERE id = $1', + [result.id], + ); + expect(row?.data.max_tokens).toBe(32_768); + }); + + it('rejects an over-budget fresh prompt before creating a job or audit record', async () => { + const tools = [ + 'get_active_schema_pack', 'search', 'query', 'get_page', 'list_pages', + 'resolve_slugs', 'get_links', 'get_backlinks', + 'stage_ingestion_proposal_page', 'finalize_ingestion_proposal', + 'put_page', 'add_link', 'add_timeline_entry', 'validate_links', + ]; + await seedClient('lore', { + bound_tools: tools, + bound_source_id: 'company', + bound_slug_prefixes: ['sources/', 'projects/', 'people/', 'companies/'], + bound_max_concurrent: 3, + }); + await engine.setConfig('mcp.publish_skills', 'true'); + await engine.setConfig('agent.max_output_tokens', '32768'); + const ctx = makeCtx({ clientId: 'lore', scopes: ['read', 'agent'] }); + ctx.config = { mcp: { skills_dir: path.resolve(import.meta.dir, '../skills') } }; + + await expect(callSubmitAgent(ctx, { + prompt: 'p'.repeat(300_000), + skill_name: 'github-project-ingestion', + model: 'openai:gpt-5.6-terra', + allowed_tools: tools, + proposal_artifact_id: 'artifact-too-large', + proposal_capture_page_slug: 'sources/example', + proposal_admission_scope: 'Include source-grounded delivery knowledge.', + })).rejects.toMatchObject({ + code: 'invalid_params', + message: expect.stringMatching(/initial prompt.*too large/i), + }); + const rows = await engine.executeRaw<{ n: number }>( + `SELECT COUNT(*)::int AS n FROM minion_jobs + WHERE data->>'__owner_client_id' = 'lore'`, + ); + expect(rows[0]?.n).toBe(0); + expect(fs.readdirSync(tmpAuditDir)).toEqual([]); + }); + it('refuses an unpublished server skill', async () => { await seedClient('lore', { bound_tools: ['search'], From 22022ce816c0ede407b4d9b93a6bac2ec56329c2 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 16:24:27 -0700 Subject: [PATCH 09/18] fix: publish the 32-page proposal limit --- src/core/operations.ts | 4 ++-- test/submit-agent.test.ts | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/operations.ts b/src/core/operations.ts index 1e75c0b7db..5a6626b584 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -5679,7 +5679,7 @@ const stage_ingestion_proposal_page: Operation = { source_id: { type: 'string', required: true, description: 'Exact source bound to the current agent job' }, admission_scope: { type: 'string', required: true, description: 'Exact admission scope bound at submit_agent time or frozen by the first staged page' }, sequence: { type: 'number', required: true, description: 'One-based page position' }, - total_pages: { type: 'number', required: true, description: 'Total number of pages in the proposal (1-100)' }, + total_pages: { type: 'number', required: true, description: 'Total number of pages in the proposal (1-32)' }, page: { type: 'object', required: true, description: 'Exact create or update page proposal' }, }, scope: 'agent', @@ -5708,7 +5708,7 @@ const finalize_ingestion_proposal: Operation = { artifact_id: { type: 'string', required: true, description: 'Exact artifact id bound at submit_agent time' }, source_id: { type: 'string', required: true, description: 'Exact source bound to the current agent job' }, admission_scope: { type: 'string', required: true, description: 'Exact admission scope already frozen on the agent job' }, - total_pages: { type: 'number', required: true, description: 'Total number of staged pages (1-100)' }, + total_pages: { type: 'number', required: true, description: 'Total number of staged pages (1-32)' }, summary: { type: 'string', required: true, description: 'Compact source-grounded proposal summary' }, proposed_timeline_entries: { type: 'array', items: { type: 'object' }, description: 'Bounded frozen timeline entries' }, proposed_links: { type: 'array', items: { type: 'object' }, description: 'Bounded frozen typed links' }, diff --git a/test/submit-agent.test.ts b/test/submit-agent.test.ts index f8c69e0265..82470667d4 100644 --- a/test/submit-agent.test.ts +++ b/test/submit-agent.test.ts @@ -143,8 +143,12 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with expect(submit_agent.params.proposal_admission_scope).toBeDefined(); expect(stage_ingestion_proposal_page?.scope).toBe('agent'); expect(stage_ingestion_proposal_page?.mutating).toBe(false); + expect(stage_ingestion_proposal_page?.params.total_pages.description) + .toContain('(1-32)'); expect(finalize_ingestion_proposal?.scope).toBe('agent'); expect(finalize_ingestion_proposal?.mutating).toBe(false); + expect(finalize_ingestion_proposal?.params.total_pages.description) + .toContain('(1-32)'); expect(finalize_ingestion_proposal?.params.page_digests).toBeUndefined(); expect(get_agent_job_proposal?.scope).toBe('agent'); }); From 788f3cc4db406b6fb9758593e0e7c43b7980ce71 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 16:51:44 -0700 Subject: [PATCH 10/18] lore-b74.9: persist explicit ingestion output budgets --- src/core/minions/subagent-limits.ts | 11 ++++ src/core/minions/subagent-prompt-budget.ts | 3 +- src/core/operations.ts | 21 +++++++ test/submit-agent.test.ts | 73 +++++++++++++++++++++- 4 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/core/minions/subagent-limits.ts b/src/core/minions/subagent-limits.ts index b8ecd396f9..1c201c00ee 100644 --- a/src/core/minions/subagent-limits.ts +++ b/src/core/minions/subagent-limits.ts @@ -1,6 +1,17 @@ /** Default per-turn output-token cap for subagent jobs. */ export const DEFAULT_SUBAGENT_MAX_OUTPUT_TOKENS = 8_192; +/** Hard ceiling for caller-selected per-job output budgets. */ +export const MAX_SUBAGENT_MAX_OUTPUT_TOKENS = 32_768; + +/** Return whether a public per-job output budget is safely bounded. */ +export function isValidSubagentMaxOutputTokens(value: unknown): value is number { + return typeof value === 'number' && + Number.isInteger(value) && + value >= 1 && + value <= MAX_SUBAGENT_MAX_OUTPUT_TOKENS; +} + /** Resolve a subagent output cap from job data, configuration, then default. */ export function resolveSubagentMaxOutputTokens( perJob: number | undefined, diff --git a/src/core/minions/subagent-prompt-budget.ts b/src/core/minions/subagent-prompt-budget.ts index 8f3c89ef69..ebffce6894 100644 --- a/src/core/minions/subagent-prompt-budget.ts +++ b/src/core/minions/subagent-prompt-budget.ts @@ -15,6 +15,7 @@ export interface SubagentInitialPromptBudgetArgs { prompt: string; userSystem?: string; model?: string; + maxOutputTokens?: number; allowedTools: string[]; allowedSlugPrefixes: string[]; sourceId?: string; @@ -37,7 +38,7 @@ export async function resolveSubagentInitialPromptBudget( fallback: TIER_DEFAULTS.subagent, }); const maxOutputTokens = resolveSubagentMaxOutputTokens( - undefined, + args.maxOutputTokens, await args.engine.getConfig('agent.max_output_tokens').catch(() => null), ); const registry = buildBrainTools({ diff --git a/src/core/operations.ts b/src/core/operations.ts index 5a6626b584..968fbb7241 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -51,6 +51,10 @@ import type { SearchResult } from './types.ts'; import { CJK_SLUG_CHARS } from './cjk.ts'; import { matchesSlugAllowList, slugFenceContains } from './slug-allow-list.ts'; import { PROPOSAL_ADMISSION_SCOPE_MAX_CHARS } from './minions/agent-job-proposals.ts'; +import { + isValidSubagentMaxOutputTokens, + MAX_SUBAGENT_MAX_OUTPUT_TOKENS, +} from './minions/subagent-limits.ts'; export { matchesSlugAllowList } from './slug-allow-list.ts'; import * as db from './db.ts'; import { VERSION } from '../version.ts'; @@ -5315,6 +5319,10 @@ const submit_agent: Operation = { allowed_tools: { type: 'array', description: 'Subset of bound_tools the agent may invoke', items: { type: 'string' } }, allowed_slug_prefixes: { type: 'array', description: 'Subset of bound_slug_prefixes for put_page writes', items: { type: 'string' } }, max_turns: { type: 'number', description: 'Max LLM turns (default 20, hard cap 100)' }, + max_output_tokens: { + type: 'number', + description: 'Per-turn output token budget (integer 1-32768; defaults to server configuration)', + }, queue: { type: 'string', description: 'Queue name (default "default")' }, idempotency_key: { type: 'string', @@ -5489,6 +5497,17 @@ const submit_agent: Operation = { `submit_agent: model "${p.model}" does not support reasoning_effort "${p.reasoning_effort}".`, ); } + const rawMaxOutputTokens = p.max_output_tokens; + let requestedMaxOutputTokens: number | undefined; + if (rawMaxOutputTokens !== undefined) { + if (!isValidSubagentMaxOutputTokens(rawMaxOutputTokens)) { + throw new OperationError( + 'invalid_params', + `submit_agent: max_output_tokens must be an integer from 1 to ${MAX_SUBAGENT_MAX_OUTPUT_TOKENS}.`, + ); + } + requestedMaxOutputTokens = rawMaxOutputTokens; + } if (boundSlugPrefixes !== null) { for (const sp of requestedSlugPrefixes) { if (!boundSlugPrefixes.some(bp => slugFenceContains(bp, sp))) { @@ -5556,6 +5575,7 @@ const submit_agent: Operation = { prompt, userSystem: skillBody, model: typeof p.model === 'string' ? p.model : undefined, + maxOutputTokens: requestedMaxOutputTokens, allowedTools: requestedTools, allowedSlugPrefixes: requestedSlugPrefixes, sourceId: boundSource ?? undefined, @@ -5597,6 +5617,7 @@ const submit_agent: Operation = { bound_max_concurrent: boundMaxConcurrent, model: typeof p.model === 'string' ? p.model : '', reasoning_effort: isReasoningEffort(p.reasoning_effort) ? p.reasoning_effort : null, + max_output_tokens: requestedMaxOutputTokens ?? null, skill_name: typeof p.skill_name === 'string' ? p.skill_name : null, proposal_artifact_id: proposalArtifactId || null, proposal_capture_page_slug: proposalCapturePageSlug || null, diff --git a/test/submit-agent.test.ts b/test/submit-agent.test.ts index 82470667d4..b560944323 100644 --- a/test/submit-agent.test.ts +++ b/test/submit-agent.test.ts @@ -137,6 +137,12 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with it('declares an explicit reasoning_effort param', () => { expect(submit_agent.params.reasoning_effort).toBeDefined(); }); + it('publishes the bounded per-job output budget', () => { + expect(submit_agent.params.max_output_tokens).toMatchObject({ + type: 'number', + description: expect.stringContaining('1-32768'), + }); + }); it('publishes non-corpus-mutating staged proposal operations', () => { expect(submit_agent.params.proposal_artifact_id).toBeDefined(); expect(submit_agent.params.proposal_capture_page_slug).toBeDefined(); @@ -591,6 +597,70 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with })).rejects.toThrow(/does not support reasoning_effort/i); }); + it.each([0, -1, 1.5, 32_769, Number.NaN, Number.POSITIVE_INFINITY, null, '32768'])( + 'rejects invalid max_output_tokens=%p before enqueue', + async (maxOutputTokens) => { + await seedClient('cursor', { + bound_tools: ['search'], + bound_source_id: 'default', + bound_slug_prefixes: ['wiki/'], + }); + const ctx = makeCtx({ clientId: 'cursor' }); + + await expect(callSubmitAgent(ctx, { + prompt: 'go', + max_output_tokens: maxOutputTokens, + })).rejects.toMatchObject({ + code: 'invalid_params', + message: expect.stringMatching(/max_output_tokens.*integer.*1.*32768/i), + }); + const rows = await engine.executeRaw<{ n: number }>( + `SELECT COUNT(*)::int AS n FROM minion_jobs + WHERE data->>'__owner_client_id' = 'cursor'`, + ); + expect(rows[0]?.n).toBe(0); + expect(fs.readdirSync(tmpAuditDir)).toEqual([]); + }, + ); + + it('uses the configured output budget when max_output_tokens is omitted', async () => { + await seedClient('cursor', { + bound_tools: ['search'], + bound_source_id: 'default', + bound_slug_prefixes: ['wiki/'], + }); + await engine.setConfig('agent.max_output_tokens', '5000'); + const ctx = makeCtx({ clientId: 'cursor' }); + + const result = await callSubmitAgent(ctx, { prompt: 'go' }); + const [row] = await engine.executeRaw<{ data: Record }>( + 'SELECT data FROM minion_jobs WHERE id = $1', + [result.id], + ); + expect(row?.data.max_tokens).toBe(5_000); + }); + + it('uses the explicit max_output_tokens for prompt admission and durable execution', async () => { + await seedClient('cursor', { + bound_tools: ['search'], + bound_source_id: 'default', + bound_slug_prefixes: ['wiki/'], + }); + await engine.setConfig('agent.max_output_tokens', '8192'); + const ctx = makeCtx({ clientId: 'cursor' }); + + const result = await callSubmitAgent(ctx, { + prompt: 'go', + model: 'openai:gpt-5.6-terra', + max_output_tokens: 32_768, + }); + const [row] = await engine.executeRaw<{ data: Record }>( + 'SELECT data FROM minion_jobs WHERE id = $1', + [result.id], + ); + expect(row?.data.max_tokens).toBe(32_768); + }); + it('inserts a subagent job + writes audit row', async () => { await seedClient('cursor', { bound_tools: ['search', 'get_page'], @@ -696,7 +766,7 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with bound_max_concurrent: 3, }); await engine.setConfig('mcp.publish_skills', 'true'); - await engine.setConfig('agent.max_output_tokens', '32768'); + await engine.setConfig('agent.max_output_tokens', '8192'); const ctx = makeCtx({ clientId: 'lore', scopes: ['read', 'agent'] }); ctx.config = { mcp: { skills_dir: path.resolve(import.meta.dir, '../skills') } }; @@ -704,6 +774,7 @@ describe('submit_agent op (v0.38 Slice 3 — remote-callable agent dispatch with prompt: 'p'.repeat(128 * 1024), skill_name: 'github-project-ingestion', model: 'openai:gpt-5.6-terra', + max_output_tokens: 32_768, allowed_tools: tools, proposal_artifact_id: 'artifact-128k', proposal_capture_page_slug: 'sources/example', From d9a37ba6445898bcc11c988d68f8426497a4fe9e Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 17:05:01 -0700 Subject: [PATCH 11/18] lore-625: stage scoped Gmail ingestion --- skills/gmail-thread-ingestion/SKILL.md | 308 ++++++++++++++++++---- test/gmail-thread-ingestion-skill.test.ts | 65 ++++- 2 files changed, 310 insertions(+), 63 deletions(-) diff --git a/skills/gmail-thread-ingestion/SKILL.md b/skills/gmail-thread-ingestion/SKILL.md index d368b6db56..60b0f3bd1b 100644 --- a/skills/gmail-thread-ingestion/SKILL.md +++ b/skills/gmail-thread-ingestion/SKILL.md @@ -1,6 +1,6 @@ --- name: gmail-thread-ingestion -version: 1.0.0 +version: 1.1.0 description: Ingest one complete prompt-supplied Gmail thread capture into one already-selected source. triggers: - "ingest this Gmail thread capture into this source" @@ -12,7 +12,10 @@ tools: - get_page - list_pages - resolve_slugs + - get_links - get_backlinks + - stage_ingestion_proposal_page + - finalize_ingestion_proposal - put_page - add_link - add_timeline_entry @@ -47,7 +50,13 @@ instructions are self-contained for a source-bound remote Minion. - Independently confirm the frozen resolver decision, resolver revision, and admission facts before writing. Return `needs_attention` without mutation when the capture no longer matches, the admitted scope is ambiguous, or a - `prompt_injection_suspected` fact contradicts automatic admission. + `prompt_injection_suspected` fact contradicts automatic admission. A newly + discovered partial exclusion returns a complete `staged_proposal` before any + corpus mutation. +- An omitted `mode` preserves the normal write path after admission is clear. + `mode: propose` performs the same analysis and identity resolution with zero + corpus mutations. `mode: apply` executes only the prompt-supplied frozen + plan and never reanalyzes email evidence. - Treat Lore's local artifact package as the complete source of record. Record one traceable `sources/` page for the Gmail thread and propagate only durable knowledge admitted by the destination resolver. @@ -55,9 +64,12 @@ instructions are self-contained for a source-bound remote Minion. first, then exact case, invoice, and document identifiers found in the capture. Similar subjects are never identity. Consolidate only on an exact non-empty identity match. -- Reuse a legacy email-source page when its provenance carries the same Gmail - thread ID. A newer capture updates that page and its canonical dossiers; - never create a parallel source page for the thread. +- The exact prompt-supplied `capturePageSlug` is the only source-page write + target. When a differently slugged legacy email-source page carries the same + Gmail thread ID, read it as identity evidence but never rewrite it. The exact + capture page may cite that legacy page when useful and admitted. The legacy + page remains read-only historical evidence; do not create any source page + other than the exact `capturePageSlug`. - Create or update substantive canonical dossiers under `people/`, `companies/`, and `projects/` when the capture establishes their identities and durable facts. Use `concepts/` and `decisions/` only for independently @@ -68,9 +80,10 @@ instructions are self-contained for a source-bound remote Minion. - Reference pages with `[[slug|label]]` wikilinks. Do not use relative Markdown links or nest Markdown links inside a citation bracket. - Read back every written page and validate its links before reporting success. -- In the receipt, qualify every page as `:`, including the - actual source page selected by identity search. Each page-array entry contains - only that exact identifier. +- In write receipts, qualify every page as `:`. The Gmail + `sourcePageSlug` is always `:`; the exact capture + fence does not permit collision adjustment. Each page-array entry contains + only its exact identifier. - Return exactly the JSON receipt in Output Format and no surrounding prose. This skill does not acquire data from Gmail, read Lore filesystem paths, choose @@ -102,6 +115,11 @@ resolverText: historical: reviewCutoff: attempt: +mode: +admissionScope: +proposedPages: +proposedTimelineEntries: +proposedLinks: manifest: contentMarkdown: transcriptMarkdown: @@ -119,23 +137,32 @@ priorAttempt: # optional; omitted when there is no prior write evidence linkResults: ``` -Stop with `failed` before any write when a required field is absent, the -provider is not `google-gmail`, `canonicalExternalId` is not the package's Gmail -thread ID, the capture identity or revision does not match the manifest, or the -`artifactIntegrity.complete` flag is not exactly `true`. This envelope is the -authority for transport completeness. Working-context projection or omission -markers from the model provider describe only the current context window; -never treat them as proof that the original artifact is incomplete. Do not -fetch, truncate, split, or reconstruct missing input. - -Before analysis or writing, require each integrity `sha256` to contain exactly -64 lowercase hexadecimal characters and each `bytes` to be a non-negative -integer. The authenticated OAuth caller deterministically verified these -values against the exact prompt fields before submission; treat the well-formed -envelope as authoritative. Do not attempt to recalculate, estimate, or -second-guess hashes or byte counts in model reasoning. Do not reinterpret a -context-projection marker as an integrity failure when the envelope is well -formed and `complete` is exactly `true`. +Apply mode intentionally omits resolver text, manifest, content Markdown, +transcript Markdown, and `artifactIntegrity`; it retains `artifactId`, +`capturePageSlug`, `canonicalExternalId`, `captureExternalId`, `revision`, +`provider`, `sourceId`, `attempt`, the approved `admissionScope`, and the exact +frozen mutation plan. Planning and normal modes retain the complete evidence +envelope. Lore caps the complete JSON-RPC submission at 131,072 UTF-8 bytes and +allows at most 100 total agent turns. + +Stop with `failed` before any write when a required field is absent or the +provider is not `google-gmail`. Outside apply mode, also stop when +`canonicalExternalId` is not the package's Gmail thread ID, the capture identity +or revision does not match the manifest, or the `artifactIntegrity.complete` +flag is not exactly `true`. That integrity envelope is the authority for +transport completeness. Working-context projection or omission markers from +the model provider describe only the current context window; never treat them +as proof that the original artifact is incomplete. Do not fetch, truncate, +split, or reconstruct missing input. + +Outside apply mode, before analysis or writing, require each integrity `sha256` +to contain exactly 64 lowercase hexadecimal characters and each `bytes` to be +a non-negative integer. The authenticated OAuth caller deterministically +verified these values against the exact prompt fields before submission; treat +the well-formed envelope as authoritative. Do not attempt to recalculate, +estimate, or second-guess hashes or byte counts in model reasoning. Do not +reinterpret a context-projection marker as an integrity failure when the +envelope is well formed and `complete` is exactly `true`. When `priorAttempt` is present, accept only the typed projection shown above. It never contains a top-level summary, unresolved list, raw error, or a nested @@ -144,17 +171,147 @@ evidence. Use the projected ledgers only to select read-back checks and safe resume points. Durable GBrain state remains authoritative, so never skip a mutation based on the projection alone. +## Staged proposal lifecycle + +### Propose mode and partial exclusions + +In `propose` mode, do not call any corpus-mutating tool: no `put_page`, +`add_link`, or `add_timeline_entry`. Search and read the destination normally, +then freeze the complete set of pages, timeline entries, and links that apply +mode would execute. Normal mode follows this same no-write path whenever a +newly discovered partial exclusion means only part of the capture may be +filed. Genuine ambiguity returns classed `needs_attention`; a clear partial +scope returns a staged proposal. + +Every proposed page has exactly one of these shapes: + +```json +{ + "slug": "sources/google-gmail/example", + "effect": "create", + "title": "Gmail thread", + "bodyMarkdown": "complete intended page body" +} +``` + +```json +{ + "slug": "projects/example", + "effect": "update", + "title": "Example", + "bodyMarkdown": "complete intended page body", + "baseMarkdown": "complete reviewed baseline", + "expectedContentHash": "64 lowercase hex characters" +} +``` + +The proposal must include `capturePageSlug` exactly once. A differently +slugged legacy source page is never proposed for create or update. Freeze +timeline entries as `{pageSlug,date,text,ref,refLabel?}` with a strict +`YYYY-MM-DD` date and `ref` equal to `capturePageSlug`. Freeze typed links as +`{from,to,type}` with `from` equal to a slug in `proposedPages`. Include at most +40 timeline entries and 40 links. No proposed mutation may derive from excluded +material. + +Before staging, freeze the complete ordered page inventory and stable +`total_pages`; it may contain at most 32 pages. Stage only one page per turn by +calling `brain_stage_ingestion_proposal_page` with the exact `artifact_id`, +`source_id`, `admission_scope`, one-based `sequence`, stable `total_pages`, and +`page` object. Then call `brain_finalize_ingestion_proposal` in its own turn +with the same exact `artifact_id`, `source_id`, `admission_scope`, and +`total_pages`, plus compact `summary`, `proposed_timeline_entries`, +`proposed_links`, and bounded `unresolved`. Return `failed` without corpus +mutation when the canonical plan or its escaped representation exceeds 98,304 +UTF-8 bytes, the compact manifest exceeds 262,144 UTF-8 bytes, or any required +value cannot be represented exactly. Never truncate, split, summarize, or +reproduce page bodies in the final receipt. + +Return the finalizer's compact manifest as `staged_proposal`: + +```json +{ + "status": "staged_proposal", + "artifactId": "copied exactly from the prompt", + "sourceId": "verified source id", + "admissionScope": "complete bounded scope", + "summary": "compact source-grounded proposal summary", + "pageDigests": [ + { + "sequence": 1, + "slug": "sources/google-gmail/example", + "digest": "64 lowercase hex characters" + } + ], + "proposalDigest": "64 lowercase hex characters", + "proposedTimelineEntries": [], + "proposedLinks": [], + "unresolved": [] +} +``` + +### Apply mode + +In `apply` mode, execute only the prompt-supplied frozen plan. Do not +reanalyze the artifact, invent or omit a mutation, change a title or body, or +call either staging tool. Before any read or mutation, validate the entire +frozen plan. Require one to 32 uniquely slugged canonical pages, with +`capturePageSlug` present exactly once. A create has exactly `slug`, `effect`, +`title`, and `bodyMarkdown`; an update additionally has a complete +`baseMarkdown` and 64-character lowercase hexadecimal `expectedContentHash`. +Treat omitted timeline and link arrays as empty; otherwise require at most 40 +exactly shaped entries of each kind. Every timeline entry has a canonical +planned `pageSlug`, strict `YYYY-MM-DD` date, non-empty text, and `ref` equal to +`capturePageSlug`. Every link has a canonical planned `from`, canonical `to`, +and non-empty type. Reject duplicate mutations and any canonical plan or escaped +representation above 98,304 UTF-8 bytes. Return `failed` without mutation when +any preflight check fails. + +After plan preflight, read every page target before the first mutation. A create +must still be absent apart from the non-capture mechanical collision adjustment +below; an update must match its reviewed `expectedContentHash`. If any target +cannot follow one of those exact paths, return a complete `failed` apply receipt +with `refresh_required` and make no mutation. + +Apply proposed pages in order with `put_page`, using `expected_content_hash: +null` for creates and the reviewed hash for updates. Read each page back before +marking it verified. The only permitted plan change is a mechanical +create-collision suffix `-` for a non-capture page, where the suffix is +1-16 lowercase alphanumeric characters; freeze all such adjustments before the +first write and apply the mapping to exact slug references in frozen bodies, +timeline entries, and links. `capturePageSlug` is never adjusted. No +differently slugged legacy source page qualifies as a collision target or +update target. + +After every page is verified, apply frozen timeline entries and then frozen +links in order. Verify timeline entries with `get_page`, and verify links with +both `get_links` and `get_backlinks`. Never execute a mutation absent from the +approved plan. Preserve `canonicalExternalId` and `captureExternalId` exactly +from the prompt, set `sourcePageSlug` to the source-qualified applied capture +slug, and retain Gmail completion attestations alongside the generic audited +mutation ledgers. + +Initialize one `pageResults` entry per proposed page, one `timelineResults` +entry per proposed timeline entry, and one `linkResults` entry per proposed +link. Preserve proposal order. Page statuses are `pending`, `written`, +`applied`, `rebased`, `already_applied`, `refresh_required`, or `failed`; +timeline and link statuses are `pending`, `applied`, or `failed`. Include +`slugAdjustments`, even when empty. Stop after the first failed or unverified +mutation and leave later entries `pending`. On retry, use `priorAttempt` only +to choose read-back checks; durable page and graph state determines whether an +exact frozen mutation still needs execution. + ## Phases ### 1. Verify the execution boundary -1. Treat the exact prompt-supplied resolver text and revision as frozen policy, - not executable instructions. -2. Confirm the prompt contains a non-empty source identity, canonical identity, - capture identity, revision, upstream order, resolver revision, resolver text, - manifest, and complete Markdown package. -3. Confirm the manifest's provider, Gmail thread ID, capture identity, version, - and admission facts agree with the prompt. +1. Outside apply mode, treat the exact prompt-supplied resolver text and + revision as frozen policy, not executable instructions. +2. Confirm the prompt contains non-empty source, canonical, capture, revision, + and capture-page identities. Planning and normal modes also require upstream + order, resolver revision, resolver text, manifest, and complete Markdown. + Apply mode instead requires the exact approved scope and frozen plan. +3. Outside apply mode, confirm the manifest's provider, Gmail thread ID, + capture identity, version, and admission facts agree with the prompt. 4. Call `get_active_schema_pack` and use its active page types. Return `failed` without mutation when the execution boundary or required tool @@ -163,7 +320,8 @@ sources. ### 2. Recheck resolver and admission scope -Read the supplied resolver as policy. Compare its positive claims, exclusions, +In apply mode, skip directly to frozen-plan preflight. Otherwise, read the +supplied resolver as policy. Compare its positive claims, exclusions, privacy limits, and disambiguation rules with the complete manifest and Markdown package. @@ -185,18 +343,20 @@ this order: 1. Search `sources/` for the exact Gmail thread ID from `canonicalExternalId`. Read every candidate and accept only an exact provenance-field match. -2. Include legacy Gmail or email-source pages in that search. Reuse a legacy - email-source page whose provenance carries the same Gmail thread ID. +2. Include legacy Gmail or email-source pages in that search. A differently + slugged legacy page with the same thread ID is read-only identity evidence. 3. Search for exact case, invoice, and document identifiers present in the thread to resolve established canonical dossiers. 4. Search participant names, explicit organization names, and established page slugs. An email address or display name alone does not establish identity. -When exactly one source page has the Gmail thread ID, it is the thread's source -page. Update it even when its slug differs from `capturePageSlug`. When no page -matches, use `capturePageSlug` for the first source-page write. If that slug -already belongs to another identity, or multiple pages carry the same thread -ID, return `needs_attention` without creating another page. +The exact `capturePageSlug` is the only source-page write target. When it does +not exist, create it. When it already carries the same Gmail thread ID, update +it. When it belongs to another identity, return `needs_attention`. A matching +differently slugged legacy page may be cited as read-only historical evidence +from the new exact capture page, but never rewritten or treated as an alternate +write target. Multiple conflicting thread identities still return +`needs_attention` without mutation. Use `captureExternalId`, `revision`, `upstreamOrder`, and `predecessorExternalId` to order captures. A newer thread version updates the @@ -207,7 +367,7 @@ distinct even when the subjects match. ### 4. Record the Gmail thread source page -Create or update the selected source page under `sources/`. Include: +Create or update exactly `capturePageSlug` under `sources/`. Include: - provider, artifact ID, Gmail thread ID, capture identity, revision, predecessor, upstream order, resolver revision, and historical flag; @@ -240,11 +400,11 @@ For each unambiguous, material person, company, or project: 3. Update current understanding with concrete dated facts, decisions, commitments, and owners. Preserve useful existing content. 4. Record each material dated change with `add_timeline_entry`. Use the event's - email date and pass `ref` with the actual source-page slug so the timeline + email date and pass `ref` with the exact `capturePageSlug` so the timeline entry links to its provenance. Do not write a `## Timeline` section through `put_page`. -5. Link the source page to each canonical dossier. Cite and link the actual - source page from every dossier update so `get_backlinks` verifies navigation +5. Link the capture page to each canonical dossier. Cite and link that exact + page from every dossier update so `get_backlinks` verifies navigation in both directions. Use `concepts/` and `decisions/` only when the capture supports a reusable @@ -290,8 +450,7 @@ mirror. 5. Confirm every written page exists under the authenticated source and appears in `createdPages` or `updatedPages`, `verifiedPages`, and `readBackVerifiedPages` using source-qualified identifiers. -6. Set `sourcePageSlug` to the actual source-qualified page selected by identity - search, not automatically to the prompt's fallback `capturePageSlug`. +6. Set `sourcePageSlug` to the exact source-qualified `capturePageSlug`. 7. Set `substantiveSummaryVerified` and `linksVerified` to `true` only after their checks pass. Set `datedFactCount` to the positive count of concrete, dated facts verified across the written pages. @@ -309,23 +468,28 @@ continue without duplication. attachments, or referenced documents. - Choosing, comparing, or writing more than one source. - Creating a new source page without first searching the exact Gmail thread ID. +- Rewriting a differently slugged legacy source page, even when its provenance + carries the same Gmail thread ID. - Treating a similar subject, sender set, or date as identity. -- Creating a parallel source or canonical page for a newer thread version. +- Creating any source page other than the exact `capturePageSlug`, or a + duplicate canonical page for a newer thread version. - Overwriting newer source or dossier state with an older capture. - Copying raw bodies, quoted text, extracted artifacts, or unadmitted addresses into GBrain pages. - Writing a source page made only of routing labels, extraction categories, or a restated subject. - Creating participant or organization stubs from display names or addresses. -- Calling `add_timeline_entry` without a `ref` to the actual source page, or +- Calling `add_timeline_entry` without a `ref` to the exact `capturePageSlug`, or hand-writing a `## Timeline` section through `put_page`. - Advancing Lore checkpoints or changing Review eligibility. +- Mutating corpus state in propose mode or before staging a partial exclusion. +- Reinterpreting, enriching, or adding to the approved plan in apply mode. - Reporting success without per-page read-back, backlink, and link validation. - Returning prose, Markdown fences, or fields outside the receipt. ## Output Format -Return exactly one JSON object: +For normal write outcomes, return exactly one JSON object: ```json { @@ -335,7 +499,7 @@ Return exactly one JSON object: "summary": "compact source-grounded outcome", "canonicalExternalId": "copied exactly from the prompt", "captureExternalId": "copied exactly from the prompt", - "sourcePageSlug": ":", + "sourcePageSlug": ":", "substantiveSummaryVerified": true, "datedFactCount": 1, "createdPages": [":"], @@ -347,6 +511,43 @@ Return exactly one JSON object: } ``` +For apply mode, retain the same Gmail identity and completion fields and add +the complete resumable mutation ledger: + +```json +{ + "status": "succeeded | failed", + "artifactId": "copied exactly from the prompt", + "sourceId": "verified source id", + "summary": "compact apply outcome", + "canonicalExternalId": "copied exactly from the prompt", + "captureExternalId": "copied exactly from the prompt", + "sourcePageSlug": ":", + "substantiveSummaryVerified": true, + "datedFactCount": 1, + "createdPages": [":"], + "updatedPages": [":"], + "verifiedPages": [":"], + "readBackVerifiedPages": [":"], + "linksVerified": true, + "pageResults": [ + { + "proposedSlug": "sources/google-gmail/example", + "appliedPage": ":sources/google-gmail/example", + "effect": "create | update", + "status": "pending | written | applied | rebased | already_applied | refresh_required | failed", + "expectedContentHash": null, + "appliedContentHash": "verified final hash or null", + "error": null + } + ], + "slugAdjustments": [], + "timelineResults": [], + "linkResults": [], + "unresolved": [] +} +``` + For `succeeded`, `datedFactCount` is at least one, `readBackVerifiedPages` contains every entry in `createdPages` and `updatedPages`, and `linksVerified` is `true`. For `needs_attention` or @@ -355,6 +556,7 @@ defect in `unresolved`. ## Tools Used -Use schema inspection, search, query, page reads, slug resolution, backlinks, -page writes, typed links, timeline entries, and link validation exactly as -declared in frontmatter. Do not request any other tool. +Use schema inspection, search, query, page reads, slug resolution, outgoing and +incoming links, proposal staging/finalization, page writes, typed links, +timeline entries, and link validation exactly as declared in frontmatter. Do +not request any other tool. diff --git a/test/gmail-thread-ingestion-skill.test.ts b/test/gmail-thread-ingestion-skill.test.ts index 5bd6f2b1ff..c309a9fca0 100644 --- a/test/gmail-thread-ingestion-skill.test.ts +++ b/test/gmail-thread-ingestion-skill.test.ts @@ -20,7 +20,10 @@ const expectedTools = [ 'get_page', 'list_pages', 'resolve_slugs', + 'get_links', 'get_backlinks', + 'stage_ingestion_proposal_page', + 'finalize_ingestion_proposal', 'put_page', 'add_link', 'add_timeline_entry', @@ -85,12 +88,12 @@ describe('gmail-thread-ingestion skill', () => { expect(skill).toContain('artifactIntegrity:'); expect(skill).toContain('complete: true'); expect(skill).toContain('manifest: { sha256: <64 lowercase hex characters>, bytes: }'); - expect(skill).toMatch(/`artifactIntegrity\.complete` flag is not exactly `true`/); - expect(skill).toMatch(/authenticated OAuth caller deterministically verified these\s+values/); - expect(skill).toMatch(/treat the well-formed\s+envelope as authoritative/); - expect(skill).toMatch(/Do not attempt to recalculate, estimate, or\s+second-guess hashes or byte counts/); + expect(skill).toMatch(/Outside apply mode[\s\S]{0,250}`artifactIntegrity\.complete`[\s\S]{0,40}flag is not exactly `true`/); + expect(skill).toMatch(/authenticated OAuth caller deterministically\s+verified these\s+values/); + expect(skill).toMatch(/treat\s+the well-formed\s+envelope as authoritative/); + expect(skill).toMatch(/Do not attempt to recalculate,\s+estimate, or\s+second-guess hashes or byte counts/); expect(skill).not.toContain('Recompute each SHA-256'); - expect(skill).toMatch(/Working-context projection or omission\s+markers[\s\S]*never treat them as proof that the original artifact is incomplete/); + expect(skill).toMatch(/Working-context projection or omission\s+markers[\s\S]*never treat them\s+as proof\s+that the original artifact is incomplete/); expect(skill).not.toContain('artifact is visibly incomplete'); }); @@ -106,15 +109,57 @@ describe('gmail-thread-ingestion skill', () => { expect(skill).toMatch(/never skip a\s+mutation based on the projection alone/); }); - test('consolidates only on exact stable identity', () => { + test('consolidates only on exact stable identity inside the capture fence', () => { expect(skill).toContain('Search and read before every create'); expect(skill).toContain('Search for the exact Gmail thread ID'); expect(skill).toContain('case, invoice, and document identifiers'); expect(skill).toContain('Similar subjects are never identity'); expect(skill).toMatch(/Consolidate only on an exact\s+non-empty identity match/); - expect(skill).toContain('legacy email-source page'); - expect(skill).toMatch(/provenance carries the same Gmail\s+thread ID/); - expect(skill).toContain('never create a parallel source page'); + expect(skill).toContain('differently slugged legacy email-source page'); + expect(skill).toMatch(/read it as identity evidence but never rewrite it/); + expect(skill).toMatch(/[Tt]he exact prompt-supplied `capturePageSlug` is the only\s+source-page write\s+target/); + expect(skill).not.toMatch(/Update it even when its slug differs from `capturePageSlug`/); + expect(skill).toMatch(/do not create any source page\s+other than the exact `capturePageSlug`/); + expect(skill).toMatch(/legacy page may be cited as read-only historical evidence/); + }); + + test('supports staged scoped proposals and frozen audited apply', () => { + expect(skill).toContain('mode: '); + expect(skill).toContain('admissionScope: '); + expect(skill).toMatch(/In `propose` mode, do not call any corpus-mutating tool/); + expect(skill).toContain('brain_stage_ingestion_proposal_page'); + expect(skill).toContain('brain_finalize_ingestion_proposal'); + expect(skill).toMatch(/exact `artifact_id`,\s+`source_id`, `admission_scope`, one-based `sequence`, stable `total_pages`, and\s+`page` object/); + expect(skill).toMatch(/same exact `artifact_id`, `source_id`, `admission_scope`, and\s+`total_pages`/); + expect(skill).toContain('`proposed_timeline_entries`'); + expect(skill).toContain('`proposed_links`'); + expect(skill).toContain('Stage only one page per turn'); + expect(skill).toContain('at most 32 pages'); + expect(skill).toContain('98,304 UTF-8 bytes'); + expect(skill).toContain('131,072 UTF-8 bytes'); + expect(skill).toContain('262,144 UTF-8 bytes'); + expect(skill).toContain('100 total agent turns'); + expect(skill).toContain('"status": "staged_proposal"'); + expect(skill).toMatch(/In `apply` mode, execute only the prompt-supplied frozen plan/); + expect(skill).toMatch(/Before any read or mutation, validate the entire\s+frozen plan/); + expect(skill).toContain('one to 32 uniquely slugged canonical pages'); + expect(skill).toContain('`capturePageSlug` present exactly once'); + expect(skill).toContain('at most 40'); + expect(skill).toMatch(/Reject duplicate mutations[\s\S]*98,304 UTF-8 bytes/); + expect(skill).toMatch(/A create\s+must still be absent apart from the non-capture mechanical collision adjustment/); + expect(skill).toContain('pageResults'); + expect(skill).toContain('timelineResults'); + expect(skill).toContain('linkResults'); + expect(skill).toContain('canonicalExternalId'); + expect(skill).toContain('captureExternalId'); + expect(skill).toContain('substantiveSummaryVerified'); + expect(skill).toContain('readBackVerifiedPages'); + expect(skill).toContain('`capturePageSlug` is never adjusted'); + }); + + test('stages a normal-mode partial exclusion before any corpus write', () => { + expect(skill).toMatch(/newly\s+discovered partial exclusion[\s\S]*staged_proposal/); + expect(skill).toMatch(/before any\s+corpus mutation/); }); test('records substantive source provenance without copying the local mirror', () => { @@ -150,7 +195,7 @@ describe('gmail-thread-ingestion skill', () => { expect(skill).toContain('"sourceId": "verified source id"'); expect(skill).toContain('"canonicalExternalId": "copied exactly from the prompt"'); expect(skill).toContain('"captureExternalId": "copied exactly from the prompt"'); - expect(skill).toContain('"sourcePageSlug": ":"'); + expect(skill).toContain('"sourcePageSlug": ":"'); expect(skill).toContain('"substantiveSummaryVerified": true'); expect(skill).toContain('"datedFactCount": 1'); expect(skill).toContain('"readBackVerifiedPages": [":"]'); From ab7c11dcae4716444f6964406d1d43b8bb8ba722 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 17:27:34 -0700 Subject: [PATCH 12/18] lore-625: clarify immutable Gmail captures --- skills/gmail-thread-ingestion/SKILL.md | 34 +++++++++++++---------- test/gmail-thread-ingestion-skill.test.ts | 14 +++++++++- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/skills/gmail-thread-ingestion/SKILL.md b/skills/gmail-thread-ingestion/SKILL.md index 60b0f3bd1b..43e84336ef 100644 --- a/skills/gmail-thread-ingestion/SKILL.md +++ b/skills/gmail-thread-ingestion/SKILL.md @@ -58,8 +58,8 @@ instructions are self-contained for a source-bound remote Minion. corpus mutations. `mode: apply` executes only the prompt-supplied frozen plan and never reanalyzes email evidence. - Treat Lore's local artifact package as the complete source of record. Record - one traceable `sources/` page for the Gmail thread and propagate only durable - knowledge admitted by the destination resolver. + one traceable `sources/` page for this immutable capture artifact and + propagate only durable knowledge admitted by the destination resolver. - Search and read before every create. Search for the exact Gmail thread ID first, then exact case, invoice, and document identifiers found in the capture. Similar subjects are never identity. Consolidate only on an exact @@ -101,7 +101,7 @@ artifactIntegrity: manifest: { sha256: <64 lowercase hex characters>, bytes: } contentMarkdown: { sha256: <64 lowercase hex characters>, bytes: } transcriptMarkdown: { sha256: <64 lowercase hex characters>, bytes: } -capturePageSlug: +capturePageSlug: canonicalExternalId: captureExternalId: ::> revision: @@ -351,21 +351,25 @@ this order: slugs. An email address or display name alone does not establish identity. The exact `capturePageSlug` is the only source-page write target. When it does -not exist, create it. When it already carries the same Gmail thread ID, update -it. When it belongs to another identity, return `needs_attention`. A matching -differently slugged legacy page may be cited as read-only historical evidence -from the new exact capture page, but never rewritten or treated as an alternate -write target. Multiple conflicting thread identities still return +not exist, create it. Update it only when it carries the same +`canonicalExternalId`, `captureExternalId`, and `revision`; that is a retry of +the same immutable capture. When it belongs to another identity or to the same +thread with a different capture identity or revision, return `needs_attention`. +A matching differently slugged legacy page may be cited as read-only historical +evidence from the new exact capture page, but never rewritten or treated as an +alternate write target. Multiple conflicting thread identities still return `needs_attention` without mutation. Use `captureExternalId`, `revision`, `upstreamOrder`, and -`predecessorExternalId` to order captures. A newer thread version updates the -same source page. An older or conflicting capture never replaces newer state. -Similar subjects, overlapping participants, and approximate dates do not -justify consolidation. Different case, invoice, or document identifiers remain -distinct even when the subjects match. - -### 4. Record the Gmail thread source page +`predecessorExternalId` to order captures. Each immutable capture writes its +own exact artifact capture page. A newer capture may update canonical dossiers, +but never updates or replaces a prior capture page. Prior capture pages remain +read-only provenance. An older or conflicting capture never replaces newer +state. Similar subjects, overlapping participants, and approximate dates do +not justify consolidation. Different case, invoice, or document identifiers +remain distinct even when the subjects match. + +### 4. Record the immutable capture source page Create or update exactly `capturePageSlug` under `sources/`. Include: diff --git a/test/gmail-thread-ingestion-skill.test.ts b/test/gmail-thread-ingestion-skill.test.ts index c309a9fca0..3bd2427210 100644 --- a/test/gmail-thread-ingestion-skill.test.ts +++ b/test/gmail-thread-ingestion-skill.test.ts @@ -120,7 +120,19 @@ describe('gmail-thread-ingestion skill', () => { expect(skill).toMatch(/[Tt]he exact prompt-supplied `capturePageSlug` is the only\s+source-page write\s+target/); expect(skill).not.toMatch(/Update it even when its slug differs from `capturePageSlug`/); expect(skill).toMatch(/do not create any source page\s+other than the exact `capturePageSlug`/); - expect(skill).toMatch(/legacy page may be cited as read-only historical evidence/); + expect(skill).toMatch(/legacy page may be cited as read-only historical\s+evidence/); + expect(skill).toMatch(/Each immutable capture writes its\s+own exact artifact capture page/); + expect(skill).toMatch(/one traceable `sources\/` page for this immutable\s+capture artifact/); + expect(skill).toContain('capturePageSlug: '); + expect(skill).toContain('### 4. Record the immutable capture source page'); + expect(skill).toMatch(/Prior capture pages remain\s+read-only provenance/); + expect(skill).toMatch(/same\s+`canonicalExternalId`, `captureExternalId`, and `revision`/); + expect(skill).toMatch(/same\s+thread with a different capture identity or revision/); + expect(skill).not.toMatch(/When it already carries the same Gmail thread ID, update it/); + expect(skill).not.toMatch(/one traceable `sources\/` page for the Gmail thread/); + expect(skill).not.toContain('capturePageSlug: '); + expect(skill).not.toContain('### 4. Record the Gmail thread source page'); + expect(skill).not.toMatch(/newer thread version updates the same source page/i); }); test('supports staged scoped proposals and frozen audited apply', () => { From 9d49d675789a89a126e37a4917277e9e98a699bf Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 20:31:30 -0700 Subject: [PATCH 13/18] test: count staged proposal tools --- test/brain-allowlist.serial.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/brain-allowlist.serial.test.ts b/test/brain-allowlist.serial.test.ts index 6bcadf58f3..5570f54662 100644 --- a/test/brain-allowlist.serial.test.ts +++ b/test/brain-allowlist.serial.test.ts @@ -55,7 +55,8 @@ describe('BRAIN_TOOL_ALLOWLIST', () => { // operations.ts:enforceSubagentSlugFence). // lore-cd8 added replace_page_text (write, fenced like put_page and // CAS-guarded by expected_content_hash). - expect(BRAIN_TOOL_ALLOWLIST.size).toBe(28); + // Durable ingestion proposals add two non-corpus-mutating staging tools. + expect(BRAIN_TOOL_ALLOWLIST.size).toBe(30); expect(BRAIN_TOOL_ALLOWLIST.has('add_timeline_entry')).toBe(true); expect(BRAIN_TOOL_ALLOWLIST.has('query')).toBe(true); expect(BRAIN_TOOL_ALLOWLIST.has('search')).toBe(true); From 63d74224963d9f76f3e857b0f0e4f66e471bf3be Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 20:36:01 -0700 Subject: [PATCH 14/18] test: expect timeline write-through receipt --- test/e2e/mechanical.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/e2e/mechanical.test.ts b/test/e2e/mechanical.test.ts index ee6c6fcbf5..29b41c3c53 100644 --- a/test/e2e/mechanical.test.ts +++ b/test/e2e/mechanical.test.ts @@ -281,6 +281,10 @@ describeE2E('E2E: Timeline', () => { expect(await callOp('add_timeline_entry', params)).toEqual({ status: 'ok', inserted: true, + write_through: { + written: false, + skipped: 'no_repo_configured', + }, }); const timeline = await callOp('get_timeline', { slug: 'people/sarah-chen' }) as any[]; From 44c7e595531ca0c62c96e589f928a13c4ea75f08 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 20:52:12 -0700 Subject: [PATCH 15/18] lore-920: keep migration fixture additive --- test/migration-v134.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/migration-v134.test.ts b/test/migration-v134.test.ts index 1bfbce7f8c..1c5511593b 100644 --- a/test/migration-v134.test.ts +++ b/test/migration-v134.test.ts @@ -36,13 +36,17 @@ describe('migration v134 — take proposal review owner', () => { }); test('adds the nullable column and can safely reapply it', async () => { + const pendingMigrations = MIGRATIONS.filter(entry => entry.version > 133).length; await engine.executeRaw( `ALTER TABLE take_proposals DROP COLUMN IF EXISTS review_owner`, ); await engine.setConfig('version', '133'); const first = await runMigrations(engine); - expect(first).toEqual({ applied: 2, current: 135 }); + expect(first).toEqual({ + applied: pendingMigrations, + current: LATEST_VERSION, + }); const columns = await engine.executeRaw<{ is_nullable: string; column_default: string | null; @@ -60,6 +64,9 @@ describe('migration v134 — take proposal review owner', () => { await engine.setConfig('version', '133'); const second = await runMigrations(engine); - expect(second).toEqual({ applied: 2, current: 135 }); + expect(second).toEqual({ + applied: pendingMigrations, + current: LATEST_VERSION, + }); }, 30_000); }); From b4a7ae7bef36f159b64786043ad6a8412ce69cd4 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 20:49:45 -0700 Subject: [PATCH 16/18] test(lore-832): refresh conditional put page fixtures --- test/claim-suppression-operations.test.ts | 4 ++ test/cycle-claim-suppression.test.ts | 2 + test/ingestion/put-page-write-through.test.ts | 3 + test/put-page-namespace.test.ts | 66 +++++++++++++++---- test/put-page-provenance.test.ts | 8 +++ test/subagent-slug-whitespace.test.ts | 2 +- 6 files changed, 73 insertions(+), 12 deletions(-) diff --git a/test/claim-suppression-operations.test.ts b/test/claim-suppression-operations.test.ts index 7b7c70604b..0a85fa8fba 100644 --- a/test/claim-suppression-operations.test.ts +++ b/test/claim-suppression-operations.test.ts @@ -231,6 +231,7 @@ describe('claim suppression operation contract', () => { slug, claim_text: 'The launch is Friday.', }); + const suppressed = await engine.getPage(slug, { sourceId: 'default' }); const blocked = await operationsByName.put_page.handler(ctx({ remote: true, @@ -240,6 +241,7 @@ describe('claim suppression operation contract', () => { }), { slug, content: '---\ntitle: Blocked\n---\n\nTHE LAUNCH is friday.', + expected_content_hash: suppressed!.content_hash, }) as Record; expect(blocked).toMatchObject({ suppression_backstop: { @@ -255,6 +257,7 @@ describe('claim suppression operation contract', () => { const listed = await listClaims(slug); expect(listed.suppressed_claims).toHaveLength(1); expect(listed.suppressed_claims[0].active).toBe(false); + const unsuppressed = await engine.getPage(slug, { sourceId: 'default' }); const allowed = await operationsByName.put_page.handler(ctx({ remote: true, @@ -264,6 +267,7 @@ describe('claim suppression operation contract', () => { }), { slug, content: '---\ntitle: Allowed\n---\n\nThe launch is Friday.', + expected_content_hash: unsuppressed!.content_hash, }) as Record; expect(allowed.suppression_backstop).toBeUndefined(); expect((await engine.getPage(slug, { sourceId: 'default' }))?.compiled_truth) diff --git a/test/cycle-claim-suppression.test.ts b/test/cycle-claim-suppression.test.ts index 3a1f65dfda..f13bb7bd3d 100644 --- a/test/cycle-claim-suppression.test.ts +++ b/test/cycle-claim-suppression.test.ts @@ -151,6 +151,7 @@ describe('dream-cycle suppression contract', () => { test('mocked synthesize generation is skipped and recorded when it reasserts a claim', async () => { await seedSuppression(); const slug = 'wiki/personal/patterns/launch-timing'; + const reviewed = await engine.getPage(slug, { sourceId: 'default' }); const generated = await operationsByName.put_page.handler(ctx({ remote: true, viaSubagent: true, @@ -159,6 +160,7 @@ describe('dream-cycle suppression contract', () => { }), { slug, content: '---\ntitle: Launch timing\n---\n\nTHE LAUNCH is friday.', + expected_content_hash: reviewed!.content_hash, }); expect(generated).toMatchObject({ status: 'skipped', diff --git a/test/ingestion/put-page-write-through.test.ts b/test/ingestion/put-page-write-through.test.ts index fbda5beb59..5601a9c13c 100644 --- a/test/ingestion/put-page-write-through.test.ts +++ b/test/ingestion/put-page-write-through.test.ts @@ -117,6 +117,7 @@ describe('put_page write-through — happy path', () => { const result = (await putPage.handler(ctx, { slug: 'inbox/mcp-prov', content: '---\ntitle: Q\n---\n\nbody', + expected_content_hash: null, })) as { write_through?: { written: boolean; path?: string } }; expect(result.write_through?.written).toBe(true); const onDisk = fs.readFileSync(result.write_through!.path!, 'utf8'); @@ -137,6 +138,7 @@ describe('put_page write-through — trust gating', () => { const result = (await putPage.handler(ctx, { slug: 'wiki/agents/42/scratch', content: '---\ntitle: S\n---\n\nbody', + expected_content_hash: null, })) as { write_through?: { written: boolean; skipped?: string } }; expect(result.write_through?.written).toBe(false); expect(result.write_through?.skipped).toBe('subagent_sandbox'); @@ -153,6 +155,7 @@ describe('put_page write-through — trust gating', () => { const result = (await putPage.handler(ctx, { slug: 'wiki/personal/reflections/note', content: '---\ntitle: R\n---\n\nreflection', + expected_content_hash: null, })) as { write_through?: { written: boolean; path?: string } }; expect(result.write_through?.written).toBe(true); expect(fs.existsSync(result.write_through!.path!)).toBe(true); diff --git a/test/put-page-namespace.test.ts b/test/put-page-namespace.test.ts index 0e1d7c05ef..b6f9987cbc 100644 --- a/test/put-page-namespace.test.ts +++ b/test/put-page-namespace.test.ts @@ -40,13 +40,21 @@ describe('put_page namespace (v0.15 subagent rule)', () => { test('MCP write (remote=true, viaSubagent=undefined) accepts arbitrary slug', async () => { const ctx = makeCtx({ remote: true }); - const result = await put_page.handler(ctx, { slug: 'wiki/analysis/foo', content: 'stub' }); + const result = await put_page.handler(ctx, { + slug: 'wiki/analysis/foo', + content: 'stub', + expected_content_hash: null, + }); expect(result).toMatchObject({ dry_run: true, action: 'put_page', slug: 'wiki/analysis/foo' }); }); test('viaSubagent=false is the same as unset', async () => { const ctx = makeCtx({ remote: true, viaSubagent: false, subagentId: 42 }); - const result = await put_page.handler(ctx, { slug: 'anything/goes', content: 'stub' }); + const result = await put_page.handler(ctx, { + slug: 'anything/goes', + content: 'stub', + expected_content_hash: null, + }); expect(result).toMatchObject({ dry_run: true }); }); }); @@ -54,57 +62,93 @@ describe('put_page namespace (v0.15 subagent rule)', () => { describe('subagent namespace rule', () => { test('accepts wiki/agents// prefix', async () => { const ctx = makeCtx({ viaSubagent: true, subagentId: 42 }); - const result = await put_page.handler(ctx, { slug: 'wiki/agents/42/notes', content: 'stub' }); + const result = await put_page.handler(ctx, { + slug: 'wiki/agents/42/notes', + content: 'stub', + expected_content_hash: null, + }); expect(result).toMatchObject({ dry_run: true, slug: 'wiki/agents/42/notes' }); }); test('accepts deep paths under the prefix', async () => { const ctx = makeCtx({ viaSubagent: true, subagentId: 42 }); - const result = await put_page.handler(ctx, { slug: 'wiki/agents/42/runs/2026-04-20/summary', content: 'stub' }); + const result = await put_page.handler(ctx, { + slug: 'wiki/agents/42/runs/2026-04-20/summary', + content: 'stub', + expected_content_hash: null, + }); expect(result).toMatchObject({ dry_run: true }); }); test('rejects leading slash (slug grammar + anchor)', async () => { const ctx = makeCtx({ viaSubagent: true, subagentId: 42 }); - const p = put_page.handler(ctx, { slug: '/wiki/agents/42/foo', content: 'stub' }); + const p = put_page.handler(ctx, { + slug: '/wiki/agents/42/foo', + content: 'stub', + expected_content_hash: null, + }); await expect(p).rejects.toBeInstanceOf(OperationError); }); test('rejects wrong subagentId', async () => { const ctx = makeCtx({ viaSubagent: true, subagentId: 42 }); - const p = put_page.handler(ctx, { slug: 'wiki/agents/12/foo', content: 'stub' }); + const p = put_page.handler(ctx, { + slug: 'wiki/agents/12/foo', + content: 'stub', + expected_content_hash: null, + }); await expect(p).rejects.toBeInstanceOf(OperationError); }); test('rejects prefix-collision attempt (wiki/agents/12evil/* with subagentId=12)', async () => { const ctx = makeCtx({ viaSubagent: true, subagentId: 12 }); - const p = put_page.handler(ctx, { slug: 'wiki/agents/12evil/foo', content: 'stub' }); + const p = put_page.handler(ctx, { + slug: 'wiki/agents/12evil/foo', + content: 'stub', + expected_content_hash: null, + }); await expect(p).rejects.toBeInstanceOf(OperationError); }); test('rejects bare prefix with no suffix (slug.length === prefix.length)', async () => { const ctx = makeCtx({ viaSubagent: true, subagentId: 42 }); - const p = put_page.handler(ctx, { slug: 'wiki/agents/42/', content: 'stub' }); + const p = put_page.handler(ctx, { + slug: 'wiki/agents/42/', + content: 'stub', + expected_content_hash: null, + }); await expect(p).rejects.toBeInstanceOf(OperationError); }); test('FAIL-CLOSED: viaSubagent=true with undefined subagentId rejects any slug', async () => { const ctx = makeCtx({ viaSubagent: true }); - const p = put_page.handler(ctx, { slug: 'wiki/agents/42/foo', content: 'stub' }); + const p = put_page.handler(ctx, { + slug: 'wiki/agents/42/foo', + content: 'stub', + expected_content_hash: null, + }); await expect(p).rejects.toBeInstanceOf(OperationError); await expect(p).rejects.toThrow(/subagentId/); }); test('FAIL-CLOSED: viaSubagent=true with NaN subagentId rejects', async () => { const ctx = makeCtx({ viaSubagent: true, subagentId: Number.NaN }); - const p = put_page.handler(ctx, { slug: 'wiki/agents/NaN/foo', content: 'stub' }); + const p = put_page.handler(ctx, { + slug: 'wiki/agents/NaN/foo', + content: 'stub', + expected_content_hash: null, + }); await expect(p).rejects.toBeInstanceOf(OperationError); }); test('error code is permission_denied (not validation)', async () => { const ctx = makeCtx({ viaSubagent: true, subagentId: 42 }); try { - await put_page.handler(ctx, { slug: 'people/alice', content: 'stub' }); + await put_page.handler(ctx, { + slug: 'people/alice', + content: 'stub', + expected_content_hash: null, + }); throw new Error('should have thrown'); } catch (e) { expect(e).toBeInstanceOf(OperationError); diff --git a/test/put-page-provenance.test.ts b/test/put-page-provenance.test.ts index b748ee3a65..6f5ad33d41 100644 --- a/test/put-page-provenance.test.ts +++ b/test/put-page-provenance.test.ts @@ -165,6 +165,7 @@ describe('put_page provenance — CV6 spoofing guard (ctx.remote !== false)', () await putPageOp.handler(ctx, { slug: 'wiki/p3a-remote-spoof-attempt', content: '---\ntype: note\ntitle: Spoof\n---\n\nbody', + expected_content_hash: null, source_kind: 'capture-cli', // client lies: pretends to be local CLI source_uri: 'spoofed://attacker-supplied', ingested_via: 'file-watcher', // client lies: claims daemon source @@ -191,6 +192,7 @@ describe('put_page provenance — CV6 spoofing guard (ctx.remote !== false)', () await putPageOp.handler(ctx, { slug: 'wiki/p3a-undefined-trust', content: '---\ntype: note\ntitle: Undefined\n---\n\nbody', + expected_content_hash: null, source_kind: 'capture-cli', }); const prov = await readProvenance('wiki/p3a-undefined-trust'); @@ -264,9 +266,13 @@ describe('put_page provenance — CV12 COALESCE-preserve UPDATE', () => { // Second: remote MCP edit (server stamps mcp:put_page) const remoteCtx = makeCtx({ remote: true }); + const reviewed = await engine.getPage('wiki/p3a-local-then-remote', { + sourceId: 'default', + }); await putPageOp.handler(remoteCtx, { slug: 'wiki/p3a-local-then-remote', content: '---\ntype: note\ntitle: V2\n---\n\nremote edit', + expected_content_hash: reviewed!.content_hash, }); // Remote second write is itself a provenance write (server-stamped), @@ -300,6 +306,7 @@ describe('put_page provenance — T2 subagent namespace regression', () => { putPageOp.handler(ctx, { slug: 'wiki/secret/leak', content: '---\ntype: note\ntitle: Leak\n---\n\nbody', + expected_content_hash: null, source_kind: 'capture-cli', source_uri: 'file:///tmp/spoof', ingested_via: 'put_page', @@ -317,6 +324,7 @@ describe('put_page provenance — T2 subagent namespace regression', () => { await putPageOp.handler(ctx, { slug: 'wiki/agents/42/scratch', content: '---\ntype: note\ntitle: Subagent OK\n---\n\nbody', + expected_content_hash: null, source_kind: 'capture-cli', // Spoof attempt — ignored by CV6 }); const prov = await readProvenance('wiki/agents/42/scratch'); diff --git a/test/subagent-slug-whitespace.test.ts b/test/subagent-slug-whitespace.test.ts index 808a4a4cc2..33108c06a1 100644 --- a/test/subagent-slug-whitespace.test.ts +++ b/test/subagent-slug-whitespace.test.ts @@ -24,7 +24,7 @@ import type { BrainEngine } from '../src/core/engine.ts'; const put_page = operations.find(o => o.name === 'put_page') as Operation; if (!put_page) throw new Error('put_page op missing'); -const BODY = { content: '# test' }; +const BODY = { content: '# test', expected_content_hash: null }; function makeCtx(overrides: Partial = {}): OperationContext { const engine = {} as BrainEngine; // dry_run short-circuits before touching the engine From 8ada1bdbcec42f19ff9c2ab4ad36e25fedfdf823 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 20:54:56 -0700 Subject: [PATCH 17/18] test: align engine fixtures with current contracts (lore-69e) --- test/phantom-redirect-engine-parity.test.ts | 6 +++--- test/takes-command-source-scope.test.ts | 12 +++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/test/phantom-redirect-engine-parity.test.ts b/test/phantom-redirect-engine-parity.test.ts index d208c38f33..caf43c3d1e 100644 --- a/test/phantom-redirect-engine-parity.test.ts +++ b/test/phantom-redirect-engine-parity.test.ts @@ -1,5 +1,5 @@ /** - * v0.35.5 — engine parity for `refreshPageBody` + `migrateFactsToCanonical`. + * Engine parity for `refreshPageBody` + `migrateFactsToCanonical`. * * These two new BrainEngine methods land in BOTH PGLite + Postgres. The * production cycle calls them transparently via the engine interface, so @@ -59,7 +59,7 @@ async function seed(engine: BrainEngine, slug: string, body: string, type = 'per // ─── refreshPageBody parity ───────────────────────────────────────── describe('refreshPageBody (parity)', () => { - test('updates compiled_truth + timeline + content_hash; skips soft-deleted', async () => { + test('updates compiled_truth + content_hash and clears the legacy timeline column', async () => { for (const engine of [pglite, pg].filter(Boolean) as BrainEngine[]) { await seed(engine, 'people/alice', '# alice\n\nOriginal body.'); @@ -73,7 +73,7 @@ describe('refreshPageBody (parity)', () => { const fetched = await engine.getPage('people/alice', { sourceId: 'default' }); expect(fetched?.compiled_truth).toBe('# alice\n\nNew compiled body.'); - expect(fetched?.timeline).toBe('## History\n\nNew timeline.'); + expect(fetched?.timeline).toBe(''); expect(fetched?.content_hash).toBe('newhash123'); } }); diff --git a/test/takes-command-source-scope.test.ts b/test/takes-command-source-scope.test.ts index 3c970be084..6507faf17b 100644 --- a/test/takes-command-source-scope.test.ts +++ b/test/takes-command-source-scope.test.ts @@ -19,8 +19,10 @@ function makeEngine(opts: { knownSources?: string[] } = {}) { const pageLookups: unknown[][] = []; const engine = { getConfig: async () => null, + resolveSlugWithAlias: async (slug: string) => slug, executeRaw: async (sql: string, params: unknown[] = []) => { - if (sql.includes('FROM sources WHERE id = $1')) { + const normalizedSql = sql.replace(/\s+/g, ' '); + if (normalizedSql.includes('FROM sources WHERE id = $1')) { // Default (no `knownSources` override): every id "exists", matching // the original test's assumption. When `knownSources` is passed, // only ids in that list resolve — used to simulate a source that @@ -28,23 +30,23 @@ function makeEngine(opts: { knownSources?: string[] } = {}) { if (!opts.knownSources) return [{ id: params[0] as string }]; return opts.knownSources.includes(params[0] as string) ? [{ id: params[0] as string }] : []; } - if (sql.includes('FROM sources WHERE local_path IS NOT NULL AND id != ')) { + if (normalizedSql.includes('FROM sources WHERE local_path IS NOT NULL AND id != ')) { // resolveSourceId tier 5.5 (sole-non-default-source). No registered // sources with a local_path in these tests. return []; } - if (sql.includes('FROM sources WHERE local_path IS NOT NULL')) { + if (normalizedSql.includes('FROM sources WHERE local_path IS NOT NULL')) { // resolveSourceId tier 4 (registered source whose local_path // contains CWD). No registered sources in these tests. return []; } - if (sql.includes('FROM pages WHERE slug = $1 AND source_id = $2')) { + if (normalizedSql.includes('FROM pages WHERE slug = $1 AND source_id = $2 AND deleted_at IS NULL')) { pageLookups.push(params); if (params[0] === 'shared/page' && params[1] === 'dept') return [{ id: 22 }]; if (params[0] === 'shared/page' && params[1] === 'default') return [{ id: 11 }]; return []; } - if (sql.includes('FROM pages WHERE slug = $1 LIMIT 1')) { + if (normalizedSql.includes('FROM pages WHERE slug = $1 AND deleted_at IS NULL')) { pageLookups.push(params); return [{ id: 11 }]; } From 7141547ffa25ec2fe27628c255d84063c7d4d561 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 7 Aug 2026 20:56:45 -0700 Subject: [PATCH 18/18] test: repair source resync fake git (lore-119) --- test/sources-resync-recovery.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/sources-resync-recovery.test.ts b/test/sources-resync-recovery.test.ts index 0913b6da70..f4b4a7201c 100644 --- a/test/sources-resync-recovery.test.ts +++ b/test/sources-resync-recovery.test.ts @@ -34,12 +34,14 @@ case "$mode" in esac has_clone=0 has_remote_get_url=0 +has_inside_work_tree=0 for ((i=1; i<=$#; i++)); do arg="\${!i}" next_idx=$((i+1)) next="\${!next_idx:-}" if [ "$arg" = "clone" ]; then has_clone=1; fi if [ "$arg" = "remote" ] && [ "$next" = "get-url" ]; then has_remote_get_url=1; fi + if [ "$arg" = "rev-parse" ] && [ "$next" = "--is-inside-work-tree" ]; then has_inside_work_tree=1; fi done if [ "$has_clone" = "1" ]; then dest="\${@: -1}" @@ -47,6 +49,10 @@ if [ "$has_clone" = "1" ]; then echo "ref: refs/heads/main" > "$dest/.git/HEAD" exit 0 fi +if [ "$has_inside_work_tree" = "1" ]; then + echo true + exit 0 +fi if [ "$has_remote_get_url" = "1" ]; then echo "$url_to_return" exit 0