From f83660041f6fce601ae10b0440fe415262e7ac82 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Mon, 6 Jul 2026 16:09:57 +0800 Subject: [PATCH] Add agentic Gemini image inspection --- src/agent-harness/ledger.test.ts | 46 +++ src/agent-harness/ledger.ts | 239 +++++++++++- src/agent-harness/runtime.test.ts | 334 ++++++++++++++++- src/agent-harness/runtime.ts | 281 ++++++++++++++- src/agent-harness/types.ts | 39 +- src/agent-harness/validator.test.ts | 143 ++++++++ src/agent-harness/validator.ts | 81 +++++ src/domains/chat/contracts.ts | 4 +- src/domains/chat/index.test.ts | 291 +++++++++++++++ src/domains/chat/index.ts | 5 +- src/domains/chat/prompt.ts | 3 + src/domains/chat/route-answer.ts | 480 ++++++++++++++++++++++++- src/domains/chat/route-service.test.ts | 448 +++++++++++++++++++++++ src/domains/chat/service.ts | 2 + 14 files changed, 2369 insertions(+), 27 deletions(-) diff --git a/src/agent-harness/ledger.test.ts b/src/agent-harness/ledger.test.ts index 5685bc3..22e0f6a 100644 --- a/src/agent-harness/ledger.test.ts +++ b/src/agent-harness/ledger.test.ts @@ -43,6 +43,26 @@ describe("createEvidenceLedger", () => { contentSlice: "", }) }) + + it("creates page image assets from live page citation asset URLs without metadata", () => { + const ledger = createEvidenceLedger() + + const snapshot = ledger.addRetrievalResponse( + makePageAssetUrlRetrievalResponse(), + ) + + expect(snapshot.assets).toContainEqual( + expect.objectContaining({ + ref: "asset:r1:referenced:1", + chunkRef: "r1:referenced:1", + type: "image", + assetUrl: + "https://knowhere-storage.example/results/job_1/page_citation_assets/page-8.png?AWSAccessKeyId=test", + sourcePath: "page_citation_assets/page-8.png", + revisionKey: "job_1", + }), + ) + }) }) function makeRetrievalResponse(): RetrievalQueryResponse { @@ -89,3 +109,29 @@ function makeRetrievalResponse(): RetrievalQueryResponse { ], } } + +function makePageAssetUrlRetrievalResponse(): RetrievalQueryResponse { + return { + namespace: "notebook", + query: "承包人 进度计划 修改 违约金", + routerUsed: "workflow_single_step", + answerText: null, + evidenceText: + "Root / （6）现场工期进度管理方面的违约责任 [Page PDF (page 8)]", + stopReason: "answer_done", + failureReason: null, + results: [], + referencedChunks: [ + { + chunkId: "node_3a513cf7-77d7-5c62-a9bd-6a1109123e2c", + documentId: "doc_contract", + chunkType: "page", + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + filePath: null, + jobId: "job_1", + assetUrl: + "https://knowhere-storage.example/results/job_1/page_citation_assets/page-8.png?AWSAccessKeyId=test", + }, + ], + } +} diff --git a/src/agent-harness/ledger.ts b/src/agent-harness/ledger.ts index 103f9c0..f9d859d 100644 --- a/src/agent-harness/ledger.ts +++ b/src/agent-harness/ledger.ts @@ -22,6 +22,20 @@ type MutableLedger = { decisionTraces: unknown[] } +type EvidenceAssetCandidate = { + readonly type: EvidenceAsset["type"] + readonly assetUrl?: string + readonly sourcePath?: string + readonly label: string +} + +type PageCitationAssetCandidate = { + readonly pageNum: number + readonly artifactRef?: string + readonly assetUrl?: string + readonly contentType?: string +} + export type EvidenceLedger = ReturnType export function createEvidenceLedger() { @@ -81,6 +95,7 @@ export function createEvidenceLedger() { sourceFileName: null, sectionPath: chunk.sectionPath, }, + ...(chunk.jobId ? { revisionKey: chunk.jobId } : {}), ...(chunk.assetUrl ? { assetUrl: chunk.assetUrl } : {}), }, }) @@ -168,13 +183,12 @@ function addChunk(input: { readonly ledger: MutableLedger readonly chunk: Omit }): void { - const assetUrl = input.chunk.assetUrl?.trim() - if (!assetUrl || !isRenderableAsset(input.chunk.chunkType, assetUrl)) { + const asset = getEvidenceAssetCandidate(input.chunk) + if (!asset) { input.ledger.chunks.push(input.chunk) return } - const type = getAssetType(input.chunk.chunkType, assetUrl) const assetRef = `asset:${input.chunk.ref}` const chunk: EvidenceChunk = { ...input.chunk, @@ -184,10 +198,12 @@ function addChunk(input: { input.ledger.assets.push({ ref: assetRef, chunkRef: chunk.ref, - type, - assetUrl, + type: asset.type, + ...(asset.assetUrl ? { assetUrl: asset.assetUrl } : {}), + ...(asset.sourcePath ? { sourcePath: asset.sourcePath } : {}), + ...(chunk.revisionKey ? { revisionKey: chunk.revisionKey } : {}), source: chunk.source, - label: formatAssetLabel(chunk), + label: asset.label, }) } @@ -206,6 +222,56 @@ function isRenderableAsset(chunkType: string, assetUrl: string): boolean { ) } +function getEvidenceAssetCandidate( + chunk: Omit, +): EvidenceAssetCandidate | null { + const pageAsset = getPageCitationAssetCandidate(chunk) + if (pageAsset) { + return pageAsset + } + + const assetUrl = getTrimmedString(chunk.assetUrl) + if (!assetUrl || !isRenderableAsset(chunk.chunkType, assetUrl)) return null + + const sourcePath = getAssetSourcePath(chunk, assetUrl) + return { + type: getAssetType(chunk.chunkType, assetUrl), + assetUrl, + ...(sourcePath ? { sourcePath } : {}), + label: formatAssetLabel(chunk, sourcePath), + } +} + +function getPageCitationAssetCandidate( + chunk: Omit, +): EvidenceAssetCandidate | null { + if (chunk.chunkType.toLowerCase() !== "page") return null + + const candidates = parsePageCitationAssetCandidates(chunk.metadata?.pageAssets) + .filter(isSupportedPageCitationAsset) + if (candidates.length === 0) return null + + const pageNumbers = getPageNumbers(chunk.metadata) + const candidate = + pageNumbers.length > 0 + ? candidates.find((item) => pageNumbers.includes(item.pageNum)) ?? + candidates[0] + : candidates[0] + if (!candidate) return null + + const sourcePath = getTrimmedString(candidate.artifactRef) + const assetUrl = + getTrimmedString(candidate.assetUrl) ?? getTrimmedString(chunk.assetUrl) + if (!sourcePath && !assetUrl) return null + + return { + type: "image", + ...(assetUrl ? { assetUrl } : {}), + ...(sourcePath ? { sourcePath } : {}), + label: formatAssetLabel(chunk, sourcePath), + } +} + function getAssetType(chunkType: string, assetUrl: string): "image" | "table" { return chunkType.toLowerCase() === "table" && !isImageAssetUrl(assetUrl) ? "table" @@ -217,6 +283,37 @@ function isImageAssetUrl(assetUrl: string): boolean { return imageExtensions.some((extension) => pathname.endsWith(extension)) } +function getAssetSourcePath( + chunk: Omit, + assetUrl: string, +): string | null { + const candidates = [ + chunk.filePath, + chunk.sourceChunkPath, + chunk.source.sectionPath, + getUrlPathname(assetUrl), + ] + + for (const candidate of candidates) { + const sourcePath = getSupportedAssetPath(candidate) + if (sourcePath) return sourcePath + } + + return null +} + +function getSupportedAssetPath(value: string | null | undefined): string | null { + const normalizedText = normalizeSourcePathCandidate(value) + if (!normalizedText) return null + + const match = + /(?:^|\/)((?:images|tables|pages|page_citation_assets)\/[^?#]+)(?:[?#]|$)?/i.exec( + normalizedText, + ) + const matchedPath = match?.[1] + return matchedPath ? matchedPath.trim() : null +} + function getUrlPathname(assetUrl: string): string { try { return new URL(assetUrl).pathname @@ -225,14 +322,136 @@ function getUrlPathname(assetUrl: string): string { } } -function formatAssetLabel(chunk: EvidenceChunk): string { - return [ +function parsePageCitationAssetCandidates( + value: unknown, +): readonly PageCitationAssetCandidate[] { + if (!Array.isArray(value)) return [] + + return value.flatMap((item): PageCitationAssetCandidate[] => { + if (!isRecord(item)) return [] + const pageNum = getPositiveInteger(item.pageNum) + if (!pageNum) return [] + + return [ + { + pageNum, + ...(getTrimmedString(item.artifactRef) + ? { artifactRef: getTrimmedString(item.artifactRef) ?? undefined } + : {}), + ...(getTrimmedString(item.assetUrl) + ? { assetUrl: getTrimmedString(item.assetUrl) ?? undefined } + : {}), + ...(getTrimmedString(item.contentType) + ? { contentType: getTrimmedString(item.contentType) ?? undefined } + : {}), + }, + ] + }) +} + +function isSupportedPageCitationAsset( + candidate: PageCitationAssetCandidate, +): boolean { + const contentType = candidate.contentType?.toLowerCase() + return ( + contentType?.startsWith("image/") === true || + hasImageExtension(candidate.artifactRef) || + hasImageExtension(candidate.assetUrl) + ) +} + +function hasImageExtension(value: string | null | undefined): boolean { + const normalized = normalizeSourcePathCandidate(value)?.toLowerCase() + return ( + normalized !== undefined && + imageExtensions.some((extension) => normalized.endsWith(extension)) + ) +} + +function getPageNumbers( + metadata: Readonly> | undefined, +): readonly number[] { + if (!metadata) return [] + + const values = [metadata.pageNums, metadata.page_nums, metadata.pageNum] + const pageNumbers = new Set() + + for (const value of values) { + if (Array.isArray(value)) { + for (const item of value) { + const pageNum = getPositiveInteger(item) + if (pageNum) pageNumbers.add(pageNum) + } + continue + } + + const pageNum = getPositiveInteger(value) + if (pageNum) pageNumbers.add(pageNum) + } + + return [...pageNumbers].sort((left, right) => left - right) +} + +function getTrimmedString(value: unknown): string | null { + if (typeof value !== "string") return null + const trimmed = value.trim() + return trimmed.length > 0 ? trimmed : null +} + +function getPositiveInteger(value: unknown): number | null { + return typeof value === "number" && + Number.isSafeInteger(value) && + value > 0 + ? value + : null +} + +function normalizeSourcePathCandidate( + value: string | null | undefined, +): string | null { + const trimmedValue = getTrimmedString(value) + if (!trimmedValue) return null + + const normalized = decodeUrlText(trimmedValue) + .replaceAll("\\", "/") + .replace(/\s*\/\s*/g, "/") + .replace(/\s+/g, " ") + .trim() + + return normalized.length > 0 ? normalized : null +} + +function decodeUrlText(value: string): string { + try { + return decodeURIComponent(value) + } catch { + return value + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null +} + +function formatAssetLabel( + chunk: EvidenceChunk, + sourcePath?: string | null, +): string { + const labels = [ chunk.source.sourceFileName, chunk.source.sectionPath, + sourcePath, chunk.chunkType, ] - .filter((value): value is string => Boolean(value?.trim())) - .join(" / ") + const uniqueLabels: string[] = [] + + for (const label of labels) { + const normalized = label?.trim() + if (!normalized || uniqueLabels.includes(normalized)) continue + uniqueLabels.push(normalized) + } + + return uniqueLabels.join(" / ") } function snapshot(ledger: MutableLedger): EvidenceLedgerSnapshot { diff --git a/src/agent-harness/runtime.test.ts b/src/agent-harness/runtime.test.ts index f6abf7d..3e3c5fa 100644 --- a/src/agent-harness/runtime.test.ts +++ b/src/agent-harness/runtime.test.ts @@ -166,6 +166,198 @@ describe("agent harness runtime", () => { ]) }) + it("rejects image inspection before retrieval has returned image assets", async () => { + const inspectImages = vi.fn() + const tools = createHarnessTools({ + state: {}, + ledger: createEvidenceLedger(), + retrieval: { query: vi.fn() }, + inspectImages, + recentTurns: [], + }) + + const result = await executeTool(tools.inspectImage, { + refs: ["asset:r1:result:1"], + question: "What text is visible?", + }) + + expect(result).toEqual({ + ok: false, + message: "retrieve must be called before inspectImage.", + inspected: [], + skipped: [ + { + ref: "asset:r1:result:1", + reason: "No retrieval evidence is available yet.", + }, + ], + }) + expect(inspectImages).not.toHaveBeenCalled() + }) + + it("rejects image inspection refs that are unknown or not image assets", async () => { + const ledger = createEvidenceLedger() + ledger.addRetrievalResponse(makeTableRetrievalResponse()) + const inspectImages = vi.fn() + const tools = createHarnessTools({ + state: {}, + ledger, + retrieval: { query: vi.fn() }, + inspectImages, + recentTurns: [], + }) + + const result = await executeTool(tools.inspectImage, { + refs: ["asset:r1:result:1", "missing"], + question: "What is in these images?", + }) + + expect(result).toEqual({ + ok: false, + message: "No inspectable image asset refs were provided.", + inspected: [], + skipped: [ + { + ref: "asset:r1:result:1", + reason: "Ref is not an image asset.", + }, + { + ref: "missing", + reason: "Ref was not returned by retrieve as an asset.", + }, + ], + }) + expect(inspectImages).not.toHaveBeenCalled() + }) + + it("enforces the inspectImage six-ref limit", async () => { + const ledger = createEvidenceLedger() + ledger.addRetrievalResponse(makeImageRetrievalResponse(7)) + const tools = createHarnessTools({ + state: {}, + ledger, + retrieval: { query: vi.fn() }, + inspectImages: vi.fn(), + recentTurns: [], + }) + + const result = await executeTool(tools.inspectImage, { + refs: Array.from({ length: 7 }, (_, index) => `asset:r1:result:${index + 1}`), + question: "Compare these images.", + }) + + expect(result).toMatchObject({ + ok: false, + message: "inspectImage accepts at most 6 refs per call.", + }) + }) + + it("calls the visual inspection capability with retrieved image ledger assets", async () => { + const ledger = createEvidenceLedger() + ledger.addRetrievalResponse(makeRetrievalResponse()) + const inspectImages = vi.fn().mockResolvedValue({ + analysis: "The image shows a Q4 revenue chart with a rising line.", + inspected: [ + { + ref: "asset:r1:result:1", + label: "report.pdf / images/chart.png / image", + }, + ], + skipped: [], + }) + const tools = createHarnessTools({ + state: {}, + ledger, + retrieval: { query: vi.fn() }, + inspectImages, + recentTurns: [], + }) + + const result = await executeTool(tools.inspectImage, { + refs: ["asset:r1:result:1"], + question: "What does the chart show?", + }) + + expect(inspectImages).toHaveBeenCalledWith({ + question: "What does the chart show?", + assets: [ + { + ref: "asset:r1:result:1", + label: "report.pdf / images/chart.png / image", + assetUrl: "https://assets.example/chart.png", + sourcePath: "images/chart.png", + source: { + documentId: "doc_1", + sourceFileName: "report.pdf", + sectionPath: "images/chart.png", + }, + }, + ], + }) + expect(result).toEqual({ + ok: true, + analysis: "The image shows a Q4 revenue chart with a rising line.", + inspected: [ + { + ref: "asset:r1:result:1", + label: "report.pdf / images/chart.png / image", + }, + ], + skipped: [], + }) + }) + + it("calls the visual inspection capability with retrieved page citation assets", async () => { + const ledger = createEvidenceLedger() + ledger.addRetrievalResponse(makePageCitationRetrievalResponse()) + const inspectImages = vi.fn().mockResolvedValue({ + analysis: "The clause says the contractor pays 5000 yuan per occurrence.", + inspected: [ + { + ref: "asset:r1:referenced:1", + label: + "Root / （6）现场工期进度管理方面的违约责任 / page_citation_assets/page-8.png / page", + }, + ], + skipped: [], + }) + const tools = createHarnessTools({ + state: {}, + ledger, + retrieval: { query: vi.fn() }, + inspectImages, + recentTurns: [], + }) + + const result = await executeTool(tools.inspectImage, { + refs: ["asset:r1:referenced:1"], + question: "What liquidated damages amount is visible in this clause?", + }) + + expect(inspectImages).toHaveBeenCalledWith({ + question: "What liquidated damages amount is visible in this clause?", + assets: [ + { + ref: "asset:r1:referenced:1", + label: + "Root / （6）现场工期进度管理方面的违约责任 / page_citation_assets/page-8.png / page", + assetUrl: "https://assets.example/page-8.png", + sourcePath: "page_citation_assets/page-8.png", + revisionKey: "job_contract", + source: { + documentId: "doc_contract", + sourceFileName: null, + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + }, + }, + ], + }) + expect(result).toMatchObject({ + ok: true, + analysis: "The clause says the contractor pays 5000 yuan per occurrence.", + }) + }) + it("blocks finalize until intent and context policy are declared", async () => { const state: { intent?: IntentFrame @@ -416,9 +608,64 @@ describe("agent harness runtime", () => { }) }) - it("forces finalize at step 12 using existing tool results", () => { + it("forces image inspection before forced finalization when image assets are available", () => { const result = prepareHarnessStep({ stepNumber: 12, + hasUninspectedImageAssets: true, + messages: [ + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call_1", + toolName: "retrieve", + output: { + type: "json", + value: { + ok: true, + assets: [{ ref: "asset:r1:referenced:1", type: "image" }], + }, + }, + }, + ], + }, + ], + }) + + expect(result.activeTools).toEqual(["inspectImage"]) + expect(result.toolChoice).toEqual({ + type: "tool", + toolName: "inspectImage", + }) + expect(result.messages).toEqual([ + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call_1", + toolName: "retrieve", + output: { + type: "json", + value: { + ok: true, + assets: [{ ref: "asset:r1:referenced:1", type: "image" }], + }, + }, + }, + ], + }, + { + role: "user", + content: expect.stringContaining("call inspectImage now"), + }, + ]) + }) + + it("forces finalize at step 13 using existing tool results", () => { + const result = prepareHarnessStep({ + stepNumber: 13, messages: [ { role: "tool", @@ -521,3 +768,88 @@ function makeRetrievalResponse(): RetrievalQueryResponse { referencedChunks: [], } } + +function makeTableRetrievalResponse(): RetrievalQueryResponse { + return { + namespace: "notebook", + query: "q4 table", + routerUsed: "workflow_single_step", + answerText: null, + evidenceText: "Table evidence", + stopReason: "answer_done", + failureReason: null, + results: [ + { + content: "", + chunkType: "table", + score: 0.8, + assetUrl: "https://assets.example/tables/revenue.html", + source: { + documentId: "doc_1", + sourceFileName: "report.pdf", + sectionPath: "tables/revenue.html", + }, + }, + ], + referencedChunks: [], + } +} + +function makeImageRetrievalResponse(count: number): RetrievalQueryResponse { + return { + namespace: "notebook", + query: "q4 images", + routerUsed: "workflow_single_step", + answerText: null, + evidenceText: "Image evidence", + stopReason: "answer_done", + failureReason: null, + results: Array.from({ length: count }, (_, index) => ({ + content: "", + chunkType: "image", + score: 0.8, + assetUrl: `https://assets.example/images/chart-${index + 1}.png`, + source: { + documentId: "doc_1", + sourceFileName: "report.pdf", + sectionPath: `images/chart-${index + 1}.png`, + }, + })), + referencedChunks: [], + } +} + +function makePageCitationRetrievalResponse(): RetrievalQueryResponse { + return { + namespace: "notebook", + query: "进度计划", + routerUsed: "workflow_single_step", + answerText: null, + evidenceText: "Root / （6）现场工期进度管理方面的违约责任", + stopReason: "answer_done", + failureReason: null, + results: [], + referencedChunks: [ + { + chunkId: "chunk_page_8", + documentId: "doc_contract", + chunkType: "page", + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + filePath: null, + jobId: "job_contract", + assetUrl: "https://assets.example/page-8.png", + metadata: { + pageNums: [8], + pageAssets: [ + { + pageNum: 8, + artifactRef: "page_citation_assets/page-8.png", + assetUrl: "https://assets.example/page-8.png", + contentType: "image/png", + }, + ], + }, + }, + ], + } +} diff --git a/src/agent-harness/runtime.ts b/src/agent-harness/runtime.ts index b565918..d38028e 100644 --- a/src/agent-harness/runtime.ts +++ b/src/agent-harness/runtime.ts @@ -16,6 +16,9 @@ import type { HarnessRunResult, HarnessToolCallTrace, HarnessTrace, + ImageInspectionAsset, + ImageInspectionResponse, + InspectImages, IntentFrame, OutputManifest, RetrievalCapability, @@ -23,9 +26,11 @@ import type { } from "./types" import { validateOutputManifest } from "./validator" -const defaultMaxSteps = 13 +const defaultMaxSteps = 14 const defaultMaxRevisions = 1 -const forcedFinalizationStepNumber = 12 +const imageInspectionReminderStepNumber = 12 +const forcedFinalizationStepNumber = 13 +const imageInspectionRefLimit = 6 type ToolLoopAgentSettings = ConstructorParameters[0] @@ -35,6 +40,7 @@ export type RunAgentHarnessInput = { readonly model: AgentHarnessModel readonly turn: AgentTurnInput readonly retrieval: RetrievalCapability + readonly inspectImages?: InspectImages readonly maxSteps?: number /** * How many times the agent may revise after a failed validation pass before @@ -49,6 +55,7 @@ type HarnessToolState = { finalizedManifest?: OutputManifest finalized?: boolean priorTurnReads?: string[] + inspectedImageRefs?: string[] toolCalls?: HarnessToolCallTrace[] } @@ -160,6 +167,7 @@ export async function runAgentHarness( state, ledger, retrieval: input.retrieval, + inspectImages: input.inspectImages, recentTurns: input.turn.recentTurns, }) const agent = new ToolLoopAgent({ @@ -170,6 +178,9 @@ export async function runAgentHarness( prepareHarnessStep({ messages: stepMessages, stepNumber, + hasUninspectedImageAssets: + input.inspectImages !== undefined && + hasUninspectedImageAssets({ state, ledger }), }), stopWhen: [ hasToolCall("finalize"), @@ -194,6 +205,7 @@ export async function runAgentHarness( contextPolicy: state.contextPolicy, finalized: state.finalized === true, ledger: ledger.snapshot(), + toolCalls: state.toolCalls, surface: input.turn.surface, }) validationErrors = validation.errors @@ -235,9 +247,30 @@ export async function runAgentHarness( export function prepareHarnessStep(input: { readonly stepNumber: number readonly messages: readonly ModelMessage[] + readonly hasUninspectedImageAssets?: boolean }): HarnessStepPreparation { const messages = sanitizeHarnessModelMessagesForStep(input.messages) + if ( + input.stepNumber === imageInspectionReminderStepNumber && + input.hasUninspectedImageAssets === true + ) { + return { + messages: [ + ...messages, + { + role: "user", + content: buildImageInspectionReminderFeedback(), + }, + ], + activeTools: ["inspectImage"], + toolChoice: { + type: "tool", + toolName: "inspectImage", + }, + } + } + if (input.stepNumber < forcedFinalizationStepNumber) { return { messages } } @@ -339,10 +372,32 @@ function buildForcedFinalizationFeedback(): string { ].join("\n") } +function buildImageInspectionReminderFeedback(): string { + return [ + "The retrieval step budget is nearly reached and retrieved image assets are available.", + "If the exact answer depends on OCR, page-image text, visual details, or image verification, call inspectImage now with the most relevant retrieved image asset refs.", + "If image inspection is not needed for this answer, call finalize using the evidence already available.", + "Do not search again.", + ].join("\n") +} + +function hasUninspectedImageAssets(input: { + readonly state: HarnessToolState + readonly ledger: ReturnType +}): boolean { + const inspectedRefs = new Set(input.state.inspectedImageRefs ?? []) + return input.ledger + .snapshot() + .assets.some( + (asset) => asset.type === "image" && !inspectedRefs.has(asset.ref), + ) +} + export function createHarnessTools(input: { readonly state: HarnessToolState readonly ledger: ReturnType readonly retrieval: RetrievalCapability + readonly inspectImages?: InspectImages readonly recentTurns: readonly AgentTurn[] }) { return { @@ -451,6 +506,29 @@ export function createHarnessTools(input: { }), }), + inspectImage: tool({ + description: + "Inspect retrieved image asset refs visually for OCR, visual details, comparisons, or verification. Use only after retrieve has returned image assets.", + inputSchema: z.object({ + refs: z.array(z.string().min(1)).min(1).max(imageInspectionRefLimit), + question: z.string().min(1), + }), + execute: async (request) => + traceToolCall(input.state, { + toolName: "inspectImage", + inputSummary: summarizeInspectImageRequest(request), + execute: async () => + inspectRetrievedImages({ + state: input.state, + ledger: input.ledger, + inspectImages: input.inspectImages, + refs: request.refs, + question: request.question, + }), + summarizeOutput: summarizeInspectImageOutput, + }), + }), + readEvidence: tool({ description: "Read more text from an evidence chunk already returned by KNOWHERE.", @@ -569,6 +647,173 @@ export function createHarnessTools(input: { } as const } +async function inspectRetrievedImages(input: { + readonly state: HarnessToolState + readonly ledger: ReturnType + readonly inspectImages?: InspectImages + readonly refs: readonly string[] + readonly question: string +}): Promise< + | ({ readonly ok: true } & ImageInspectionResponse) + | { + readonly ok: false + readonly message: string + readonly inspected: readonly [] + readonly skipped: readonly { + readonly ref: string + readonly reason: string + }[] + } +> { + const refs = getUniqueTrimmedRefs(input.refs) + const question = input.question.trim() + + if (refs.length === 0 || question.length === 0) { + return { + ok: false, + message: "At least one image asset ref and a question are required.", + inspected: [], + skipped: [], + } + } + if (refs.length > imageInspectionRefLimit) { + return { + ok: false, + message: `inspectImage accepts at most ${imageInspectionRefLimit} refs per call.`, + inspected: [], + skipped: refs.map((ref) => ({ + ref, + reason: "Too many refs were requested in one inspectImage call.", + })), + } + } + + const snapshot = input.ledger.snapshot() + if (snapshot.retrievalCount === 0) { + return { + ok: false, + message: "retrieve must be called before inspectImage.", + inspected: [], + skipped: refs.map((ref) => ({ + ref, + reason: "No retrieval evidence is available yet.", + })), + } + } + if (!input.inspectImages) { + return { + ok: false, + message: "Image inspection is not available for this turn.", + inspected: [], + skipped: refs.map((ref) => ({ + ref, + reason: "No image inspection capability is configured.", + })), + } + } + + const assetsByRef = new Map( + snapshot.assets.map((asset) => [asset.ref, asset] as const), + ) + const skipped: { + readonly ref: string + readonly reason: string + }[] = [] + const selectedAssets: ImageInspectionAsset[] = [] + + for (const ref of refs) { + const asset = assetsByRef.get(ref) + if (!asset) { + skipped.push({ + ref, + reason: "Ref was not returned by retrieve as an asset.", + }) + continue + } + if (asset.type !== "image") { + skipped.push({ + ref, + reason: "Ref is not an image asset.", + }) + continue + } + + selectedAssets.push({ + ref: asset.ref, + label: asset.label, + ...(asset.assetUrl ? { assetUrl: asset.assetUrl } : {}), + ...(asset.sourcePath ? { sourcePath: asset.sourcePath } : {}), + ...(asset.revisionKey ? { revisionKey: asset.revisionKey } : {}), + source: asset.source, + }) + } + + if (selectedAssets.length === 0) { + return { + ok: false, + message: "No inspectable image asset refs were provided.", + inspected: [], + skipped, + } + } + + const inspectedImageRefs = input.state.inspectedImageRefs ?? [] + const inspectedCountAfterCall = + inspectedImageRefs.length + selectedAssets.length + if (inspectedCountAfterCall > imageInspectionRefLimit) { + return { + ok: false, + message: `inspectImage accepts at most ${imageInspectionRefLimit} image refs per turn.`, + inspected: [], + skipped: selectedAssets.map((asset) => ({ + ref: asset.ref, + reason: "The per-turn image inspection limit would be exceeded.", + })), + } + } + + input.state.inspectedImageRefs = [ + ...inspectedImageRefs, + ...selectedAssets.map((asset) => asset.ref), + ] + + try { + const response = await input.inspectImages({ + question, + assets: selectedAssets, + }) + return { + ok: true, + analysis: response.analysis, + inspected: response.inspected, + skipped: [...skipped, ...response.skipped], + } + } catch (error) { + return { + ok: false, + message: + error instanceof Error + ? `Image inspection failed: ${error.message}` + : "Image inspection failed.", + inspected: [], + skipped: selectedAssets.map((asset) => ({ + ref: asset.ref, + reason: "The image inspection request failed.", + })), + } + } +} + +function getUniqueTrimmedRefs(refs: readonly string[]): string[] { + const normalizedRefs: string[] = [] + for (const ref of refs) { + const normalizedRef = ref.trim() + if (!normalizedRef || normalizedRefs.includes(normalizedRef)) continue + normalizedRefs.push(normalizedRef) + } + return normalizedRefs +} + async function traceToolCall(input: { readonly toolCalls?: HarnessToolCallTrace[] }, call: { @@ -683,6 +928,30 @@ function summarizeReadEvidenceOutput(output: unknown): unknown { } } +function summarizeInspectImageRequest(request: { + readonly refs: readonly string[] + readonly question: string +}): unknown { + return { + refs: getUniqueTrimmedRefs(request.refs), + questionLength: request.question.trim().length, + } +} + +function summarizeInspectImageOutput(output: unknown): unknown { + if (!isRecord(output)) return output + return { + ok: output.ok, + analysisLength: + typeof output.analysis === "string" ? output.analysis.length : 0, + inspectedCount: Array.isArray(output.inspected) + ? output.inspected.length + : 0, + skippedCount: Array.isArray(output.skipped) ? output.skipped.length : 0, + message: output.message, + } +} + function summarizeReadPriorTurnOutput(output: unknown): unknown { if (!isRecord(output)) return output return { @@ -739,8 +1008,10 @@ export function buildHarnessSystemPrompt(turn: AgentTurnInput): string { "2. Call setContextPolicy next, deciding how prior turns should influence this turn.", "3. When the policy needs prior-turn detail (references or corrections), call readPriorTurn for the relevant ids.", "4. Call retrieve only when evidence is needed. The query must be concise and self-contained.", - "5. Use readEvidence only for chunk refs already in the evidence ledger.", - "6. Call finalize with text, citations, artifacts, and unresolved issues. finalize requires declareIntent and setContextPolicy first.", + "5. For pixel-level details, OCR, visual comparison, image verification, or when the likely answer is only visible on a retrieved page/image asset, call inspectImage only after retrieve returned image asset refs.", + `6. inspectImage accepts at most ${imageInspectionRefLimit} image asset refs per call and per turn.`, + "7. Use readEvidence only for chunk refs already in the evidence ledger.", + "8. Call finalize with text, citations, artifacts, and unresolved issues. finalize requires declareIntent and setContextPolicy first.", "", "Context rules:", "- If the current user request is unrelated to prior turns, set carryHistory to none and do not reuse prior topics.", @@ -752,6 +1023,8 @@ export function buildHarnessSystemPrompt(turn: AgentTurnInput): string { "- artifacts with display=true are the exact images/tables shown. Never display every candidate; honor constraints.desiredCount / maxCount.", "- Use type=derived_table only for tables you create from evidence; every derived_table.sourceRefs entry must reference evidence in the ledger.", "- citations and selected image/table artifact refs may only reference refs returned by retrieve (in the evidence ledger).", + "- inspectImage observations are inspection notes, not new source refs. Final citations and displayed image artifacts must use the original retrieved image asset refs.", + "- If text evidence identifies a relevant page/image but does not include the exact fact, inspect the returned image asset for OCR/detail before saying the answer is unavailable.", "- If evidence is insufficient, list it in unresolved instead of fabricating facts.", "- After a validation-feedback message, fix all listed issues and call finalize again.", `Surface: ${turn.surface}`, diff --git a/src/agent-harness/types.ts b/src/agent-harness/types.ts index c559623..4601f6f 100644 --- a/src/agent-harness/types.ts +++ b/src/agent-harness/types.ts @@ -107,6 +107,7 @@ export type EvidenceChunk = { readonly sourceFileName?: string | null readonly sectionPath?: string | null } + readonly revisionKey?: string | null readonly assetRef?: string readonly assetUrl?: string } @@ -115,11 +116,47 @@ export type EvidenceAsset = { readonly ref: string readonly chunkRef: string readonly type: "image" | "table" - readonly assetUrl: string + readonly assetUrl?: string + readonly sourcePath?: string + readonly revisionKey?: string | null readonly source: EvidenceChunk["source"] readonly label: string } +export type ImageInspectionAsset = { + readonly ref: string + readonly label: string + readonly assetUrl?: string | null + readonly sourcePath?: string | null + readonly revisionKey?: string | null + readonly source: EvidenceChunk["source"] +} + +export type ImageInspectionSkippedAsset = { + readonly ref: string + readonly reason: string +} + +export type ImageInspectionInspectedAsset = { + readonly ref: string + readonly label: string +} + +export type ImageInspectionRequest = { + readonly question: string + readonly assets: readonly ImageInspectionAsset[] +} + +export type ImageInspectionResponse = { + readonly analysis: string + readonly inspected: readonly ImageInspectionInspectedAsset[] + readonly skipped: readonly ImageInspectionSkippedAsset[] +} + +export type InspectImages = ( + input: ImageInspectionRequest, +) => Promise + export type EvidenceLedgerSnapshot = { readonly retrievalCount: number readonly chunks: readonly EvidenceChunk[] diff --git a/src/agent-harness/validator.test.ts b/src/agent-harness/validator.test.ts index 2815241..be8de3e 100644 --- a/src/agent-harness/validator.test.ts +++ b/src/agent-harness/validator.test.ts @@ -232,6 +232,149 @@ describe("validateOutputManifest", () => { "Typing compose output must be insertion-ready plain text.", ) }) + + it("rejects image inspection failure claims when inspectImage was not called", () => { + const validation = validateOutputManifest({ + manifest: makeManifest({ + text: "图像检查工具未能成功读取该页面的具体条款内容。", + citations: [ + { + ref: "r1:referenced:1", + label: "contract.pdf / page 8", + source: { + documentId: "doc_contract", + sourceFileName: "contract.pdf", + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + }, + }, + ], + }), + intent: makeIntent({}), + contextPolicy: unrelatedContextPolicy, + ledger: { + ...emptyLedger, + chunks: [ + { + ref: "r1:referenced:1", + kind: "referenced_chunk", + content: "", + contentPreview: "", + chunkType: "page", + score: null, + source: { + documentId: "doc_contract", + sourceFileName: "contract.pdf", + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + }, + assetRef: "asset:r1:referenced:1", + }, + ], + assets: [makeAsset("asset:r1:referenced:1")], + }, + toolCalls: [], + surface: "notebook_chat", + }) + + expect(validation.errors).toContain( + "Final output must not claim image/OCR inspection succeeded or failed unless inspectImage was called.", + ) + }) + + it("allows image inspection failure claims after inspectImage was called", () => { + const validation = validateOutputManifest({ + manifest: makeManifest({ + text: "对该页面的图像识别未成功提取具体数值。", + citations: [ + { + ref: "r1:referenced:1", + label: "contract.pdf / page 8", + source: { + documentId: "doc_contract", + sourceFileName: "contract.pdf", + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + }, + }, + ], + }), + intent: makeIntent({}), + contextPolicy: unrelatedContextPolicy, + ledger: { + ...emptyLedger, + chunks: [ + { + ref: "r1:referenced:1", + kind: "referenced_chunk", + content: "", + contentPreview: "", + chunkType: "page", + score: null, + source: { + documentId: "doc_contract", + sourceFileName: "contract.pdf", + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + }, + assetRef: "asset:r1:referenced:1", + }, + ], + assets: [makeAsset("asset:r1:referenced:1")], + }, + toolCalls: [ + { + tool: "inspectImage", + ok: false, + inputSummary: { refs: ["asset:r1:referenced:1"] }, + outputSummary: { inspectedCount: 0 }, + startedAt: "2026-07-06T00:00:00.000Z", + durationMs: 1, + }, + ], + surface: "notebook_chat", + }) + + expect(validation.errors).not.toContain( + "Final output must not claim image/OCR inspection succeeded or failed unless inspectImage was called.", + ) + }) + + it("rejects unreadable page claims when image assets are available but uninspected", () => { + const validation = validateOutputManifest({ + manifest: makeManifest({ + text: "相关的页面图像无法通过目前的检测工具获取。", + unresolved: ["由于无法直接读取该页面的详细条款内容，目前无法给出确切的赔偿金额。"], + }), + intent: makeIntent({}), + contextPolicy: unrelatedContextPolicy, + ledger: { + ...emptyLedger, + chunks: [ + { + ref: "r1:referenced:1", + kind: "referenced_chunk", + content: "", + contentPreview: "", + chunkType: "page", + score: null, + source: { + documentId: "doc_contract", + sourceFileName: "contract.pdf", + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + }, + assetRef: "asset:r1:referenced:1", + }, + ], + assets: [makeAsset("asset:r1:referenced:1")], + }, + toolCalls: [], + surface: "notebook_chat", + }) + + expect(validation.errors).toContain( + "Final output must inspect available image assets before claiming retrieved page/image content cannot be read.", + ) + expect(validation.errors).toContain( + "Final output must not claim image/OCR inspection succeeded or failed unless inspectImage was called.", + ) + }) }) const emptyLedger: EvidenceLedgerSnapshot = { diff --git a/src/agent-harness/validator.ts b/src/agent-harness/validator.ts index 89b5e43..dd67343 100644 --- a/src/agent-harness/validator.ts +++ b/src/agent-harness/validator.ts @@ -1,6 +1,7 @@ import type { ContextPolicy, EvidenceLedgerSnapshot, + HarnessToolCallTrace, IntentFrame, OutputManifest, } from "./types" @@ -11,6 +12,7 @@ export type ManifestValidationInput = { readonly contextPolicy?: ContextPolicy readonly finalized?: boolean readonly ledger: EvidenceLedgerSnapshot + readonly toolCalls?: readonly HarnessToolCallTrace[] readonly surface: "notebook_chat" | "typing_compose" | "typing_quick_ask" } @@ -34,6 +36,8 @@ export function validateOutputManifest( validateArtifactCounts(input, errors) validateGrounding(input, errors) validateTaskEvidence(input, errors) + validateImageInspectionClaims(input, errors) + validateUnavailableImageContentClaims(input, errors) validateTypingText(input, errors) return { @@ -201,3 +205,80 @@ function validateTypingText( errors.push("Typing compose output must be insertion-ready plain text.") } } + +function validateImageInspectionClaims( + input: ManifestValidationInput, + errors: string[], +): void { + const claimText = [input.manifest.text, ...input.manifest.unresolved].join("\n") + if (!mentionsImageInspectionResult(claimText)) return + + if (hasImageInspectionToolCall(input)) return + + errors.push( + "Final output must not claim image/OCR inspection succeeded or failed unless inspectImage was called.", + ) +} + +function validateUnavailableImageContentClaims( + input: ManifestValidationInput, + errors: string[], +): void { + const hasImageAssets = input.ledger.assets.some((asset) => asset.type === "image") + if (!hasImageAssets) return + if (hasImageInspectionToolCall(input)) return + + const claimText = [input.manifest.text, ...input.manifest.unresolved].join("\n") + if (!mentionsUnavailableImageContent(claimText)) return + + errors.push( + "Final output must inspect available image assets before claiming retrieved page/image content cannot be read.", + ) +} + +function hasImageInspectionToolCall(input: ManifestValidationInput): boolean { + return input.toolCalls?.some((call) => call.tool === "inspectImage") === true +} + +function mentionsImageInspectionResult(value: string): boolean { + const normalized = value.replace(/\s+/g, " ").trim() + if (!normalized) return false + + return ( + /\b(?:image|visual)\s+(?:inspection|recognition|analysis)\s+(?:failed|did not|could not|was unable|found|showed|confirmed)/iu.test( + normalized, + ) || + /\bOCR\s+(?:failed|did not|could not|was unable|found|showed|confirmed|extracted)/iu.test( + normalized, + ) || + /(?:图像|图片|视觉).{0,8}(?:识别|检查|检视|查看|分析).{0,12}(?:未|没|无法|不能|不成功|失败|成功|显示|发现|提取|读取)/u.test( + normalized, + ) || + /(?:图像|图片|视觉).{0,12}(?:未|没|无法|不能|不成功|失败).{0,16}(?:识别|检查|检视|查看|分析|检测|提取|读取|获取)/u.test( + normalized, + ) || + /(?:未|没|无法|不能|不成功|失败).{0,12}(?:图像|图片|视觉|OCR).{0,12}(?:识别|检查|检视|查看|分析|检测|提取|读取|获取)/u.test( + normalized, + ) + ) +} + +function mentionsUnavailableImageContent(value: string): boolean { + const normalized = value.replace(/\s+/g, " ").trim() + if (!normalized) return false + + return ( + /\b(?:cannot|can't|could not|unable to|was unable to|did not)\s+(?:directly\s+)?(?:read|inspect|access|extract|see|view)\b.{0,48}\b(?:page|image|visual|OCR|content|clause|details|text)\b/iu.test( + normalized, + ) || + /\b(?:page|image|visual|OCR|content|clause|details|text)\b.{0,48}\b(?:cannot|can't|could not|unable to|was unable to|did not)\s+(?:directly\s+)?(?:read|inspect|access|extract|see|view)\b/iu.test( + normalized, + ) || + /(?:无法|不能|未能|没法|没有办法).{0,8}(?:直接)?(?:读取|查看|识别|检测|提取|看清|访问|获取).{0,16}(?:页面|页|图片|图像|条款|内容|细节|文字)/u.test( + normalized, + ) || + /(?:页面|页|图片|图像|条款|内容|细节|文字).{0,16}(?:无法|不能|未能|没法|没有办法).{0,16}(?:直接)?(?:读取|查看|识别|检测|提取|看清|访问|获取)/u.test( + normalized, + ) + ) +} diff --git a/src/domains/chat/contracts.ts b/src/domains/chat/contracts.ts index 4752b86..3400af5 100644 --- a/src/domains/chat/contracts.ts +++ b/src/domains/chat/contracts.ts @@ -4,7 +4,7 @@ import type { } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" -import type { HarnessRunResult } from "@/agent-harness" +import type { HarnessRunResult, InspectImages } from "@/agent-harness" import type { ChatArtifactView, ChatCitationView, @@ -57,6 +57,7 @@ export type GenerateAnswer = (input: { sources: readonly Source[] excludedSourceIds: readonly string[] searchSources: SearchSources + inspectImages?: InspectImages }) => Promise export type AnswerQuestionInput = { @@ -69,6 +70,7 @@ export type AnswerQuestionInput = { generateAnswer: GenerateAnswer hardenChatAssetUrl?: HardenChatAssetUrl hardenMediaAssetUrls?: HardenMediaAssetUrls + inspectImages?: InspectImages messages: readonly ChatHistoryMessage[] } diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index d16f26e..8c239a9 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -1930,6 +1930,297 @@ describe("generateAgenticOutputManifest", () => { expect(JSON.stringify(capturedGenerateInput)).toContain("tax.pdf / deadline"); }); + it("lets the agent inspect retrieved image assets before finalizing cited image output", async () => { + process.env.AI_GATEWAY_API_KEY = "test_gateway_key"; + vi.spyOn(ToolLoopAgent.prototype, "generate").mockImplementation( + async function mockGenerate( + this: ToolLoopAgent, + ): ReturnType { + const tools = this.tools as unknown as Record< + string, + { execute: (input: unknown) => Promise } + >; + + await tools.declareIntent?.execute({ + task: "show_media", + dependsOnPreviousTurn: false, + retrievalNeeded: "yes", + targetModalities: ["image"], + constraints: { desiredCount: 1, maxCount: 1 }, + groundingPolicy: "must_use_sources", + }); + await tools.setContextPolicy?.execute({ + carryHistory: "none", + reason: "The current request is self-contained.", + activePriorTurnIds: [], + }); + await tools.retrieve?.execute({ + query: "identity card front image", + modalities: ["image"], + topK: 1, + purpose: "Find the ID card image to inspect.", + }); + await tools.inspectImage?.execute({ + refs: ["asset:r1:result:1"], + question: "What text is visible on the ID card?", + }); + await tools.finalize?.execute({ + text: "The inspected image appears to show the requested ID card.", + citations: [ + { + ref: "asset:r1:result:1", + label: "identity.pdf / images/id-front.png", + source: { + documentId: "doc_identity", + sourceFileName: "identity.pdf", + sectionPath: "images/id-front.png", + }, + }, + ], + artifacts: [ + { + type: "image", + ref: "asset:r1:result:1", + display: true, + reason: "Requested inspected ID card image.", + }, + ], + unresolved: [], + }); + + return { + text: "ignored", + } as Awaited>; + }, + ); + const searchSources = vi.fn().mockResolvedValue({ + results: [ + makeRetrievalResult({ + chunkType: "image", + assetUrl: "https://blob.example/images/id-front.png", + source: { + documentId: "doc_identity", + sourceFileName: "generated.pdf", + sectionPath: "images/id-front.png", + }, + }), + ], + evidenceText: "Identity image evidence.", + referencedChunks: [], + namespace: "notebook-workspace", + query: "identity card front image", + routerUsed: "workflow_single_step", + answerText: null, + stopReason: "answer_done", + failureReason: null, + }); + const inspectImages = vi.fn().mockResolvedValue({ + analysis: "The image contains a visible identity card number.", + inspected: [ + { + ref: "asset:r1:result:1", + label: "generated.pdf / images/id-front.png / image", + }, + ], + skipped: [], + }); + + const result = await generateAgenticOutputManifest({ + question: "Inspect and show the ID card image.", + messages: [], + sources: [ + makeSource({ title: "identity.pdf", knowhereDocumentId: "doc_identity" }), + ], + excludedSourceIds: [], + searchSources, + inspectImages, + }); + + expect(inspectImages).toHaveBeenCalledWith({ + question: "What text is visible on the ID card?", + assets: [ + { + ref: "asset:r1:result:1", + label: "generated.pdf / images/id-front.png / image", + assetUrl: "https://blob.example/images/id-front.png", + sourcePath: "images/id-front.png", + source: { + documentId: "doc_identity", + sourceFileName: "generated.pdf", + sectionPath: "images/id-front.png", + }, + }, + ], + }); + expect(result.manifest.citations.map((citation) => citation.ref)).toEqual([ + "asset:r1:result:1", + ]); + expect(result.manifest.artifacts).toEqual([ + { + type: "image", + ref: "asset:r1:result:1", + display: true, + reason: "Requested inspected ID card image.", + }, + ]); + expect(result.trace.toolCalls.map((call) => call.tool)).toContain( + "inspectImage", + ); + expect(result.trace.validationErrors).toEqual([]); + }); + + it("lets the agent inspect retrieved page assets before finalizing an OCR answer", async () => { + process.env.AI_GATEWAY_API_KEY = "test_gateway_key"; + vi.spyOn(ToolLoopAgent.prototype, "generate").mockImplementation( + async function mockGenerate( + this: ToolLoopAgent, + ): ReturnType { + const tools = this.tools as unknown as Record< + string, + { execute: (input: unknown) => Promise } + >; + + await tools.declareIntent?.execute({ + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "yes", + targetModalities: ["text"], + constraints: { citationRequired: true, language: "zh-CN" }, + groundingPolicy: "must_use_sources", + }); + await tools.setContextPolicy?.execute({ + carryHistory: "none", + reason: "The current request is self-contained.", + activePriorTurnIds: [], + }); + await tools.retrieve?.execute({ + query: "进度计划 违约金 承包人", + modalities: ["text"], + topK: 6, + purpose: "Find the contract clause and page for the liquidated damages amount.", + }); + await tools.inspectImage?.execute({ + refs: ["asset:r1:referenced:1"], + question: + "OCR this clause and identify the liquidated damages amount for unauthorized schedule changes.", + }); + await tools.finalize?.execute({ + text: "承包人自行修改发包人审批的进度计划，应按每次 5000 元赔偿违约金。", + citations: [ + { + ref: "asset:r1:referenced:1", + label: "投标书 / （6）现场工期进度管理方面的违约责任", + source: { + documentId: "doc_contract", + sourceFileName: "投标书.pdf", + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + }, + }, + ], + artifacts: [], + unresolved: [], + }); + + return { + text: "ignored", + } as Awaited>; + }, + ); + const searchSources = vi.fn().mockResolvedValue({ + results: [], + evidenceText: "Root / （6）现场工期进度管理方面的违约责任", + referencedChunks: [ + { + chunkId: "chunk_page_8", + documentId: "doc_contract", + chunkType: "page", + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + filePath: null, + jobId: "job_contract", + metadata: { + pageNums: [8], + pageAssets: [ + { + pageNum: 8, + artifactRef: "page_citation_assets/page-8.png", + assetUrl: "https://blob.example/page-8.png", + contentType: "image/png", + }, + ], + }, + }, + ], + namespace: "notebook-workspace", + query: "进度计划 违约金 承包人", + routerUsed: "workflow_single_step", + answerText: null, + stopReason: "answer_done", + failureReason: null, + }); + const inspectImages = vi.fn().mockResolvedValue({ + analysis: "The clause states 5000 yuan per occurrence.", + inspected: [ + { + ref: "asset:r1:referenced:1", + label: + "Root / （6）现场工期进度管理方面的违约责任 / page_citation_assets/page-8.png / page", + }, + ], + skipped: [], + }); + + const result = await generateAgenticOutputManifest({ + question: "承包人自行修改发包人审批的进度时需要赔偿多少违约金？", + messages: [], + sources: [ + makeSource({ + title: "投标书.pdf", + knowhereDocumentId: "doc_contract", + }), + ], + excludedSourceIds: [], + searchSources, + inspectImages, + }); + + expect(searchSources).toHaveBeenCalledWith({ + query: "进度计划 违约金 承包人", + targetContent: "text", + purpose: "Find the contract clause and page for the liquidated damages amount.", + topK: 6, + signalPaths: undefined, + filterMode: undefined, + threshold: undefined, + }); + expect(inspectImages).toHaveBeenCalledWith({ + question: + "OCR this clause and identify the liquidated damages amount for unauthorized schedule changes.", + assets: [ + { + ref: "asset:r1:referenced:1", + label: + "Root / （6）现场工期进度管理方面的违约责任 / page_citation_assets/page-8.png / page", + assetUrl: "https://blob.example/page-8.png", + sourcePath: "page_citation_assets/page-8.png", + revisionKey: "job_contract", + source: { + documentId: "doc_contract", + sourceFileName: null, + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + }, + }, + ], + }); + expect(result.manifest.text).toContain("5000 元"); + expect(result.manifest.citations.map((citation) => citation.ref)).toEqual([ + "asset:r1:referenced:1", + ]); + expect(result.trace.toolCalls.map((call) => call.tool)).toContain( + "inspectImage", + ); + expect(result.trace.validationErrors).toEqual([]); + }); + it("self-corrects an over-budget manifest via a validation-feedback revision", async () => { process.env.AI_GATEWAY_API_KEY = "test_gateway_key"; let generateCallCount = 0; diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index 21e6ef8..fa07579 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -214,6 +214,7 @@ export const answerQuestionWithRetrieval = ( sources: input.sources, excludedSourceIds: input.excludedSourceIds, searchSources, + ...(input.inspectImages ? { inspectImages: input.inspectImages } : {}), }), ) @@ -414,12 +415,12 @@ function toChatArtifactView(input: { ref: input.artifact.ref, display: input.artifact.display, reason: input.artifact.reason, - assetUrl: input.asset.assetUrl, + ...(input.asset.assetUrl ? { assetUrl: input.asset.assetUrl } : {}), label: input.asset.label, citation: { chunkType: input.asset.type, score: null, - assetUrl: input.asset.assetUrl, + ...(input.asset.assetUrl ? { assetUrl: input.asset.assetUrl } : {}), source, }, } diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index b13831e..efee008 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -10,6 +10,7 @@ import { type AgentTurnInput, type HarnessRetrievalRequest, type HarnessRunResult, + type InspectImages, type TargetModality, } from "@/agent-harness" import type { @@ -29,6 +30,7 @@ type GenerateAgenticOutputManifestInput = { sources: readonly Source[] excludedSourceIds: readonly string[] searchSources: SearchSources + inspectImages?: InspectImages } export const generateAgenticOutputManifestEffect = ( @@ -61,6 +63,7 @@ export const generateAgenticOutputManifestEffect = ( query: (request) => input.searchSources(toAgenticRetrievalQuery(request)), }, + ...(input.inspectImages ? { inspectImages: input.inspectImages } : {}), }), ) diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index 03c26ca..2f8ae6b 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -1,9 +1,17 @@ import { Cause, Effect, Either, Option } from "effect" +import { generateText } from "ai" import { generateAgenticOutputManifest, parseChatRequestBody, } from "@/domains/chat" +import type { + ImageInspectionAsset, + ImageInspectionRequest, + ImageInspectionResponse, + ImageInspectionSkippedAsset, + InspectImages, +} from "@/agent-harness" import { hardenChatMediaAssetUrls } from "@/domains/chat/media-asset-hardening" import { handleChatTurn, @@ -16,6 +24,7 @@ import { BlobParsedDocumentStorage } from "@/domains/sources/parsed-document-blo import { sourceWorkflowRuntime } from "@/domains/sources/workflow-runtime" import { notebookRequestContext } from "@/domains/workspace/request-context" import type { Source } from "@/infrastructure/db/schema" +import { CHAT_MODEL } from "@/lib/ai" import type { HardenChatAssetUrl } from "./media-assets" import { isAuthError } from "@/integrations/dashboard/api-key-service" import { summarizeUnknownError } from "@/lib/format-log-value" @@ -24,6 +33,23 @@ import { routeResult, type RouteResult } from "@/lib/route-result" type RouteResponse = RouteResult +const VISION_MODEL = process.env.VISION_MODEL ?? CHAT_MODEL +const IMAGE_INSPECTION_URL_REPLACEMENT = "[image URL hidden]" +const SUPPORTED_IMAGE_CONTENT_TYPES: Readonly> = { + ".jpeg": "image/jpeg", + ".jpg": "image/jpeg", + ".png": "image/png", + ".webp": "image/webp", +} as const + +type HardenImageInspectionAsset = (input: { + readonly documentId: string + readonly revisionKey: string + readonly sourcePath: string + readonly assetUrl?: string | null + readonly contentType?: string | null +}) => Promise + type MessageBody = { readonly message: string } @@ -86,6 +112,24 @@ const answerChatEffect = (input: AnswerChatInput) => assetUrl, contentType, }) + const hardenImageInspectionAsset: HardenImageInspectionAsset = (asset) => + hardenChatAssetByDocument({ + workspaceId: workspace.id, + parsedStorage, + documentId: asset.documentId, + revisionKey: asset.revisionKey, + sourcePath: asset.sourcePath, + assetUrl: asset.assetUrl, + contentType: asset.contentType, + }) + const inspectImages: InspectImages = (request) => + inspectChatImages({ + workspaceId: workspace.id, + sources, + hardenChatAssetUrl, + hardenImageInspectionAsset, + request, + }) const result: Either.Either = yield* Effect.tryPromise(() => @@ -106,6 +150,7 @@ const answerChatEffect = (input: AnswerChatInput) => artifacts, hardenChatAssetUrl, }), + inspectImages, repository: chatTurnPersistence.createRepository(), }), ).pipe( @@ -160,6 +205,355 @@ export const chatAnswerRouteService: ChatAnswerRouteService = { answerChat, } +async function inspectChatImages(input: { + readonly workspaceId: string + readonly sources: readonly Source[] + readonly hardenChatAssetUrl: HardenChatAssetUrl + readonly hardenImageInspectionAsset: HardenImageInspectionAsset + readonly request: ImageInspectionRequest +}): Promise { + const preparedAssets: PreparedImageInspectionAsset[] = [] + const skippedAssets: ImageInspectionSkippedAsset[] = [] + + for (const asset of input.request.assets) { + const preparedAsset = await prepareImageInspectionAsset({ + asset, + sources: input.sources, + hardenChatAssetUrl: input.hardenChatAssetUrl, + hardenImageInspectionAsset: input.hardenImageInspectionAsset, + }) + if (preparedAsset.ok) { + preparedAssets.push(preparedAsset.asset) + } else { + skippedAssets.push({ + ref: asset.ref, + reason: preparedAsset.reason, + }) + } + } + + if (preparedAssets.length === 0) { + return { + analysis: "", + inspected: [], + skipped: skippedAssets, + } + } + + logger.info("chat: image inspection request", { + workspaceId: input.workspaceId, + model: VISION_MODEL, + requestedCount: input.request.assets.length, + inspectedCount: preparedAssets.length, + skippedCount: skippedAssets.length, + refs: preparedAssets.map((asset) => asset.ref), + }) + + const response = await generateImageInspectionText({ + workspaceId: input.workspaceId, + question: input.request.question, + assets: preparedAssets, + }) + const analysis = removeImageInspectionUrls({ + text: response.text.trim(), + assets: preparedAssets, + }) + + logger.info("chat: image inspection response", { + workspaceId: input.workspaceId, + model: VISION_MODEL, + inspectedCount: preparedAssets.length, + skippedCount: skippedAssets.length, + analysisLength: analysis.length, + }) + + return { + analysis, + inspected: preparedAssets.map((asset) => ({ + ref: asset.ref, + label: asset.label, + })), + skipped: skippedAssets, + } +} + +type PreparedImageInspectionAsset = { + readonly ref: string + readonly label: string + readonly url: URL + readonly body: Uint8Array + readonly contentType: string + readonly fileName?: string +} + +type PrepareImageInspectionAssetResult = + | { + readonly ok: true + readonly asset: PreparedImageInspectionAsset + } + | { + readonly ok: false + readonly reason: string + } + +async function prepareImageInspectionAsset(input: { + readonly asset: ImageInspectionAsset + readonly sources: readonly Source[] + readonly hardenChatAssetUrl: HardenChatAssetUrl + readonly hardenImageInspectionAsset: HardenImageInspectionAsset +}): Promise { + const sourcePath = resolveImageInspectionSourcePath(input.asset) + if (!sourcePath) { + return { + ok: false, + reason: "The image asset path could not be resolved.", + } + } + + const contentType = getSupportedImageContentType(sourcePath) + if (!contentType) { + return { + ok: false, + reason: "Only PNG, JPEG, and WebP image assets can be inspected.", + } + } + + const source = resolveImageInspectionSource(input.asset, input.sources) + const durableUrlFromSource = source + ? await input.hardenChatAssetUrl({ + source, + sourcePath, + assetUrl: input.asset.assetUrl, + contentType, + }) + : null + const durableUrl = + durableUrlFromSource ?? + (await hardenImageInspectionAssetByRef({ + asset: input.asset, + sourcePath, + contentType, + hardenImageInspectionAsset: input.hardenImageInspectionAsset, + })) + if (!durableUrl) { + return { + ok: false, + reason: source + ? "The image asset was unavailable in Notebook storage." + : "No ready Notebook source matched the image asset.", + } + } + + const url = parseAbsoluteHttpUrl(durableUrl) + if (!url) { + return { + ok: false, + reason: "Notebook storage did not return a valid image URL.", + } + } + const image = await fetchPreparedInspectionImage({ + url, + contentType, + }) + if (!image.ok) { + return { + ok: false, + reason: image.reason, + } + } + + return { + ok: true, + asset: { + ref: input.asset.ref, + label: input.asset.label, + url, + body: image.body, + contentType, + fileName: getFileNameFromPath(sourcePath), + }, + } +} + +async function generateImageInspectionText(input: { + readonly workspaceId: string + readonly question: string + readonly assets: readonly PreparedImageInspectionAsset[] +}): Promise>> { + try { + return await generateText({ + model: VISION_MODEL, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: buildImageInspectionPrompt({ + question: input.question, + assets: input.assets, + }), + }, + ...input.assets.map((asset) => ({ + type: "image" as const, + image: asset.body, + mediaType: asset.contentType, + })), + ], + }, + ], + experimental_include: { + requestBody: false, + responseBody: false, + }, + }) + } catch (error) { + logger.warn("chat: image inspection model call failed", { + workspaceId: input.workspaceId, + model: VISION_MODEL, + inspectedCount: input.assets.length, + error: summarizeUnknownError(error), + }) + throw error + } +} + +async function fetchPreparedInspectionImage(input: { + readonly url: URL + readonly contentType: string +}): Promise< + | { + readonly ok: true + readonly body: Uint8Array + } + | { + readonly ok: false + readonly reason: string + } +> { + try { + const image = await fetchChatAsset({ + assetUrl: input.url.toString(), + fallbackContentType: input.contentType, + }) + return { + ok: true, + body: image.body, + } + } catch { + return { + ok: false, + reason: "The Notebook image asset could not be read for inspection.", + } + } +} + +async function hardenImageInspectionAssetByRef(input: { + readonly asset: ImageInspectionAsset + readonly sourcePath: string + readonly contentType: string + readonly hardenImageInspectionAsset: HardenImageInspectionAsset +}): Promise { + const documentId = getTrimmedString(input.asset.source.documentId) + const revisionKey = getTrimmedString(input.asset.revisionKey) + if (!documentId || !revisionKey) return null + + return input.hardenImageInspectionAsset({ + documentId, + revisionKey, + sourcePath: input.sourcePath, + assetUrl: input.asset.assetUrl, + contentType: input.contentType, + }) +} + +function resolveImageInspectionSource( + asset: ImageInspectionAsset, + sources: readonly Source[], +): Source | undefined { + const documentId = getTrimmedString(asset.source.documentId) + if (!documentId) return undefined + + return sources.find( + (source) => + source.status === "ready" && source.knowhereDocumentId === documentId, + ) +} + +function resolveImageInspectionSourcePath( + asset: ImageInspectionAsset, +): string | null { + const candidates = [ + asset.sourcePath, + asset.source.sectionPath, + getAssetUrlPathname(asset.assetUrl), + ] + for (const candidate of candidates) { + const sourcePath = getSupportedImageAssetPath(candidate) + if (sourcePath) return sourcePath + } + + return null +} + +function getSupportedImageAssetPath( + value: string | null | undefined, +): string | null { + const normalizedText = normalizeSourcePathCandidate(value) + if (!normalizedText) return null + + const match = + /(?:^|\/)((?:images|pages|page_citation_assets)\/[^?#]+)(?:[?#]|$)?/i.exec( + normalizedText, + ) + const matchedPath = match?.[1] + return matchedPath ? matchedPath.trim() : null +} + +function getSupportedImageContentType(sourcePath: string): string | null { + const lowerPath = sourcePath.toLowerCase() + const extension = Object.keys(SUPPORTED_IMAGE_CONTENT_TYPES).find((candidate) => + lowerPath.endsWith(candidate), + ) + return extension ? SUPPORTED_IMAGE_CONTENT_TYPES[extension] ?? null : null +} + +function buildImageInspectionPrompt(input: { + readonly question: string + readonly assets: readonly PreparedImageInspectionAsset[] +}): string { + return [ + "Inspect the attached Notebook image assets selected from retrieved Knowhere evidence.", + "Answer the inspection question using concise visual observations only.", + "Use the provided refs and labels to identify images. Do not include image URLs.", + "If OCR text is unclear, say it is unclear instead of guessing.", + "Do not create citations. The calling agent will cite the original retrieved asset refs.", + "", + "Inspection question:", + input.question, + "", + "Images:", + ...input.assets.map((asset) => `- ref=${asset.ref} label=${asset.label}`), + ].join("\n") +} + +function removeImageInspectionUrls(input: { + readonly text: string + readonly assets: readonly PreparedImageInspectionAsset[] +}): string { + const knownUrls = input.assets.map((asset) => asset.url.toString()) + const withoutKnownUrls = knownUrls.reduce( + (text, assetUrl) => + text.replaceAll(assetUrl, IMAGE_INSPECTION_URL_REPLACEMENT), + input.text, + ) + + return withoutKnownUrls + .replace(/https?:\/\/[^\s)\]}>"']+/g, IMAGE_INSPECTION_URL_REPLACEMENT) + .replace(/[ \t]{2,}/g, " ") + .trim() +} + async function hardenSingleChatAsset(input: { readonly workspaceId: string readonly parsedStorage: BlobParsedDocumentStorage @@ -176,10 +570,32 @@ async function hardenSingleChatAsset(input: { return null } + return hardenChatAssetByDocument({ + workspaceId: input.workspaceId, + parsedStorage: input.parsedStorage, + documentId: input.source.knowhereDocumentId, + revisionKey: input.source.knowhereJobId, + sourcePath: input.sourcePath, + assetUrl: input.assetUrl, + contentType: input.contentType, + sourceId: input.source.id, + }) +} + +async function hardenChatAssetByDocument(input: { + readonly workspaceId: string + readonly parsedStorage: BlobParsedDocumentStorage + readonly documentId: string + readonly revisionKey: string + readonly sourcePath: string + readonly assetUrl?: string | null + readonly contentType?: string | null + readonly sourceId?: string | null +}): Promise { try { const existingUrl = await input.parsedStorage.getAssetUrl({ - documentId: input.source.knowhereDocumentId, - revisionKey: input.source.knowhereJobId, + documentId: input.documentId, + revisionKey: input.revisionKey, sourcePath: input.sourcePath, }) if (existingUrl) return existingUrl @@ -194,8 +610,8 @@ async function hardenSingleChatAsset(input: { inferContentTypeFromPath(input.sourcePath), }) const writtenAsset = await input.parsedStorage.writeAsset({ - documentId: input.source.knowhereDocumentId, - revisionKey: input.source.knowhereJobId, + documentId: input.documentId, + revisionKey: input.revisionKey, sourcePath: input.sourcePath, body: fetchedAsset.body, contentType: fetchedAsset.contentType, @@ -204,16 +620,16 @@ async function hardenSingleChatAsset(input: { return ( writtenAsset.url ?? (await input.parsedStorage.getAssetUrl({ - documentId: input.source.knowhereDocumentId, - revisionKey: input.source.knowhereJobId, + documentId: input.documentId, + revisionKey: input.revisionKey, sourcePath: input.sourcePath, })) ) } catch (error) { logger.warn("chat: single parsed asset hardening failed", { workspaceId: input.workspaceId, - sourceId: input.source.id, - documentId: input.source.knowhereDocumentId, + sourceId: input.sourceId ?? null, + documentId: input.documentId, sourcePath: input.sourcePath, error: summarizeUnknownError(error), }) @@ -258,6 +674,54 @@ function inferContentTypeFromPath(sourcePath: string): string { return "application/octet-stream" } +function getAssetUrlPathname(assetUrl: string | null | undefined): string | null { + const normalizedAssetUrl = getTrimmedString(assetUrl) + if (!normalizedAssetUrl) return null + + try { + return new URL(normalizedAssetUrl).pathname + } catch { + return normalizedAssetUrl.split("?")[0] ?? normalizedAssetUrl + } +} + +function normalizeSourcePathCandidate( + value: string | null | undefined, +): string | null { + const trimmedValue = getTrimmedString(value) + if (!trimmedValue) return null + + const normalized = decodeUrlText(trimmedValue) + .replaceAll("\\", "/") + .replace(/\s*\/\s*/g, "/") + .replace(/\s+/g, " ") + .trim() + + return normalized.length > 0 ? normalized : null +} + +function decodeUrlText(value: string): string { + try { + return decodeURIComponent(value) + } catch { + return value + } +} + +function parseAbsoluteHttpUrl(assetUrl: string): URL | null { + try { + const url = new URL(assetUrl) + return url.protocol === "http:" || url.protocol === "https:" ? url : null + } catch { + return null + } +} + +function getFileNameFromPath(sourcePath: string): string | undefined { + const fileName = sourcePath.replaceAll("\\", "/").split("/").pop()?.trim() + return fileName && fileName.length > 0 ? fileName : undefined +} + function triggerBackgroundReconciliationForParsingSources(input: { readonly workspaceId: string readonly sources: readonly Source[] diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 6365edd..c771f12 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ ensureDefaultChatThread: vi.fn(), findChatThreadInWorkspace: vi.fn(), generateAgenticOutputManifest: vi.fn(), + generateText: vi.fn(), getAuthenticated: vi.fn(), getAuthenticatedWithClient: vi.fn(), handleChatTurn: vi.fn(), @@ -24,6 +25,14 @@ const mocks = vi.hoisted(() => ({ startBackgroundReconciliation: vi.fn(), })) +vi.mock("ai", async (importOriginal) => { + const original = await importOriginal() + return { + ...original, + generateText: mocks.generateText, + } +}) + vi.mock("@/domains/chat", async (importOriginal) => { const original = await importOriginal() return { @@ -90,6 +99,7 @@ describe("chat route services", () => { vi.clearAllMocks() mocks.parsedStorageGetAssetUrl.mockResolvedValue(null) mocks.parsedStorageWriteAsset.mockResolvedValue({ url: null }) + mocks.generateText.mockResolvedValue({ text: "The image shows a chart." }) vi.stubGlobal( "fetch", vi.fn(async () => new Response(new Uint8Array([1, 2, 3]), { @@ -228,6 +238,444 @@ describe("chat route services", () => { }) }) + it("hardens image inspection assets before sending Notebook URLs to Gemini", async () => { + const workspace = makeWorkspace() + const client = { retrieval: { query: vi.fn() } } + const readySource = makeSource({ + status: "ready", + knowhereDocumentId: "doc_identity", + knowhereJobId: "job_1", + }) + const rawUrl = + "https://knowhere-storage.example/results/job_1/images/id-front.png?AWSAccessKeyId=test" + const durableUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_identity/job_1/assets/images/id-front.png" + mocks.parsedStorageWriteAsset.mockResolvedValue({ url: durableUrl }) + mocks.generateText.mockResolvedValue({ + text: `The card number is visible. ${durableUrl} ${rawUrl}`, + }) + mocks.getAuthenticatedWithClient.mockResolvedValue({ + user: { id: "user_1" }, + workspace, + apiKey: "jwt_123", + client, + }) + mocks.listSourcesForWorkspace.mockResolvedValue([readySource]) + mocks.handleChatTurn.mockImplementation( + async (input: { + readonly inspectImages?: (request: { + readonly question: string + readonly assets: readonly { + readonly ref: string + readonly label: string + readonly assetUrl: string + readonly source: { + readonly documentId?: string | null + readonly sourceFileName?: string | null + readonly sectionPath?: string | null + } + }[] + }) => Promise<{ + readonly analysis: string + readonly inspected: readonly { + readonly ref: string + readonly label: string + }[] + readonly skipped: readonly { + readonly ref: string + readonly reason: string + }[] + }> + }) => { + const inspection = await input.inspectImages?.({ + question: "Read the ID card text.", + assets: [ + { + ref: "asset:r1:result:1", + label: "identity.pdf / images/id-front.png / image", + assetUrl: rawUrl, + source: { + documentId: "doc_identity", + sourceFileName: "identity.pdf", + sectionPath: "images/id-front.png", + }, + }, + ], + }) + expect(inspection).toEqual({ + analysis: "The card number is visible. [image URL hidden] [image URL hidden]", + inspected: [ + { + ref: "asset:r1:result:1", + label: "identity.pdf / images/id-front.png / image", + }, + ], + skipped: [], + }) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "message_user", role: "user", content: "Inspect it" }, + { id: "message_assistant", role: "assistant", content: "Answer" }, + ], + }) + }, + ) + + const result = await chatAnswerRouteService.answerChat({ + body: { message: "Inspect the ID card image" }, + }) + + expect(result.status).toBe(200) + expect(fetch).toHaveBeenCalledWith(rawUrl) + expect(fetch).toHaveBeenCalledWith(durableUrl) + expect(mocks.parsedStorageWriteAsset).toHaveBeenCalledWith({ + documentId: "doc_identity", + revisionKey: "job_1", + sourcePath: "images/id-front.png", + body: new Uint8Array([1, 2, 3]), + contentType: "image/png", + }) + const generateInput = mocks.generateText.mock.calls[0]?.[0] + expect(generateInput).toMatchObject({ + model: "google/gemini-3-flash", + experimental_include: { + requestBody: false, + responseBody: false, + }, + }) + expect(JSON.stringify(generateInput)).not.toContain( + "knowhere-storage.example", + ) + const content = generateInput.messages[0].content + const imagePart = content.find( + (part: { readonly type: string }) => part.type === "image", + ) + expect(imagePart.image).toEqual(new Uint8Array([1, 2, 3])) + expect(imagePart.mediaType).toBe("image/png") + }) + + it("uses image inspection source paths for retrieved page assets", async () => { + const workspace = makeWorkspace() + const client = { retrieval: { query: vi.fn() } } + const readySource = makeSource({ + status: "ready", + knowhereDocumentId: "doc_contract", + knowhereJobId: "job_1", + }) + const durableUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_contract/job_1/assets/page_citation_assets/page-8.png" + mocks.parsedStorageGetAssetUrl.mockResolvedValue(durableUrl) + mocks.generateText.mockResolvedValue({ + text: "The page states 5000 yuan per occurrence.", + }) + mocks.getAuthenticatedWithClient.mockResolvedValue({ + user: { id: "user_1" }, + workspace, + apiKey: "jwt_123", + client, + }) + mocks.listSourcesForWorkspace.mockResolvedValue([readySource]) + mocks.handleChatTurn.mockImplementation( + async (input: { + readonly inspectImages?: (request: { + readonly question: string + readonly assets: readonly { + readonly ref: string + readonly label: string + readonly assetUrl?: string | null + readonly sourcePath?: string | null + readonly source: { + readonly documentId?: string | null + readonly sourceFileName?: string | null + readonly sectionPath?: string | null + } + }[] + }) => Promise<{ + readonly analysis: string + readonly inspected: readonly { + readonly ref: string + readonly label: string + }[] + readonly skipped: readonly { + readonly ref: string + readonly reason: string + }[] + }> + }) => { + const inspection = await input.inspectImages?.({ + question: "Read the damages amount.", + assets: [ + { + ref: "asset:r1:referenced:1", + label: + "Root / （6）现场工期进度管理方面的违约责任 / page_citation_assets/page-8.png / page", + sourcePath: "page_citation_assets/page-8.png", + source: { + documentId: "doc_contract", + sourceFileName: "投标书.pdf", + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + }, + }, + ], + }) + expect(inspection).toEqual({ + analysis: "The page states 5000 yuan per occurrence.", + inspected: [ + { + ref: "asset:r1:referenced:1", + label: + "Root / （6）现场工期进度管理方面的违约责任 / page_citation_assets/page-8.png / page", + }, + ], + skipped: [], + }) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "message_user", role: "user", content: "Inspect it" }, + { id: "message_assistant", role: "assistant", content: "Answer" }, + ], + }) + }, + ) + + const result = await chatAnswerRouteService.answerChat({ + body: { message: "Inspect the clause page" }, + }) + + expect(result.status).toBe(200) + expect(mocks.parsedStorageGetAssetUrl).toHaveBeenCalledWith({ + documentId: "doc_contract", + revisionKey: "job_1", + sourcePath: "page_citation_assets/page-8.png", + }) + expect(fetch).toHaveBeenCalledWith(durableUrl) + expect(mocks.parsedStorageWriteAsset).not.toHaveBeenCalled() + const generateInput = mocks.generateText.mock.calls[0]?.[0] + const content = generateInput.messages[0].content + const imagePart = content.find( + (part: { readonly type: string }) => part.type === "image", + ) + expect(imagePart.image).toEqual(new Uint8Array([1, 2, 3])) + expect(imagePart.mediaType).toBe("image/png") + }) + + it("hardens remote page inspection assets by document revision when no local source matches", async () => { + const workspace = makeWorkspace() + const client = { retrieval: { query: vi.fn() } } + const rawUrl = + "https://knowhere-storage.example/results/job_remote/page_citation_assets/page-8.png?AWSAccessKeyId=test" + const durableUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_remote/job_remote/assets/page_citation_assets/page-8.png" + mocks.parsedStorageWriteAsset.mockResolvedValue({ url: durableUrl }) + mocks.generateText.mockResolvedValue({ + text: "The page states 5000 yuan per occurrence.", + }) + mocks.getAuthenticatedWithClient.mockResolvedValue({ + user: { id: "user_1" }, + workspace, + apiKey: "jwt_123", + client, + }) + mocks.listSourcesForWorkspace.mockResolvedValue([]) + mocks.handleChatTurn.mockImplementation( + async (input: { + readonly inspectImages?: (request: { + readonly question: string + readonly assets: readonly { + readonly ref: string + readonly label: string + readonly assetUrl?: string | null + readonly sourcePath?: string | null + readonly revisionKey?: string | null + readonly source: { + readonly documentId?: string | null + readonly sourceFileName?: string | null + readonly sectionPath?: string | null + } + }[] + }) => Promise<{ + readonly analysis: string + readonly inspected: readonly { + readonly ref: string + readonly label: string + }[] + readonly skipped: readonly { + readonly ref: string + readonly reason: string + }[] + }> + }) => { + const inspection = await input.inspectImages?.({ + question: "Read the damages amount.", + assets: [ + { + ref: "asset:r1:referenced:5", + label: + "Root / （6）现场工期进度管理方面的违约责任 / page_citation_assets/page-8.png / page", + assetUrl: rawUrl, + sourcePath: "page_citation_assets/page-8.png", + revisionKey: "job_remote", + source: { + documentId: "doc_remote", + sourceFileName: null, + sectionPath: "Root / （6）现场工期进度管理方面的违约责任", + }, + }, + ], + }) + expect(inspection).toEqual({ + analysis: "The page states 5000 yuan per occurrence.", + inspected: [ + { + ref: "asset:r1:referenced:5", + label: + "Root / （6）现场工期进度管理方面的违约责任 / page_citation_assets/page-8.png / page", + }, + ], + skipped: [], + }) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "message_user", role: "user", content: "Inspect it" }, + { id: "message_assistant", role: "assistant", content: "Answer" }, + ], + }) + }, + ) + + const result = await chatAnswerRouteService.answerChat({ + body: { message: "Inspect the remote clause page" }, + }) + + expect(result.status).toBe(200) + expect(mocks.parsedStorageGetAssetUrl).toHaveBeenCalledWith({ + documentId: "doc_remote", + revisionKey: "job_remote", + sourcePath: "page_citation_assets/page-8.png", + }) + expect(fetch).toHaveBeenCalledWith(rawUrl) + expect(fetch).toHaveBeenCalledWith(durableUrl) + expect(mocks.parsedStorageWriteAsset).toHaveBeenCalledWith({ + documentId: "doc_remote", + revisionKey: "job_remote", + sourcePath: "page_citation_assets/page-8.png", + body: new Uint8Array([1, 2, 3]), + contentType: "image/png", + }) + const generateInput = mocks.generateText.mock.calls[0]?.[0] + expect(JSON.stringify(generateInput)).not.toContain( + "knowhere-storage.example", + ) + const content = generateInput.messages[0].content + const imagePart = content.find( + (part: { readonly type: string }) => part.type === "image", + ) + expect(imagePart.image).toEqual(new Uint8Array([1, 2, 3])) + expect(imagePart.mediaType).toBe("image/png") + }) + + it("skips unsupported or unavailable image inspection assets without failing chat", async () => { + const workspace = makeWorkspace() + const client = { retrieval: { query: vi.fn() } } + const readySource = makeSource({ + status: "ready", + knowhereDocumentId: "doc_identity", + knowhereJobId: "job_1", + }) + mocks.getAuthenticatedWithClient.mockResolvedValue({ + user: { id: "user_1" }, + workspace, + apiKey: "jwt_123", + client, + }) + mocks.listSourcesForWorkspace.mockResolvedValue([readySource]) + mocks.handleChatTurn.mockImplementation( + async (input: { + readonly inspectImages?: (request: { + readonly question: string + readonly assets: readonly { + readonly ref: string + readonly label: string + readonly assetUrl: string + readonly source: { + readonly documentId?: string | null + readonly sourceFileName?: string | null + readonly sectionPath?: string | null + } + }[] + }) => Promise<{ + readonly analysis: string + readonly inspected: readonly { + readonly ref: string + readonly label: string + }[] + readonly skipped: readonly { + readonly ref: string + readonly reason: string + }[] + }> + }) => { + const inspection = await input.inspectImages?.({ + question: "Inspect these.", + assets: [ + { + ref: "asset:r1:result:1", + label: "identity.pdf / images/animated.gif / image", + assetUrl: "https://knowhere-storage.example/results/job_1/images/animated.gif", + source: { + documentId: "doc_identity", + sourceFileName: "identity.pdf", + sectionPath: "images/animated.gif", + }, + }, + { + ref: "asset:r1:result:2", + label: "missing.pdf / images/missing.png / image", + assetUrl: "https://knowhere-storage.example/results/job_2/images/missing.png", + source: { + documentId: "doc_missing", + sourceFileName: "missing.pdf", + sectionPath: "images/missing.png", + }, + }, + ], + }) + expect(inspection).toEqual({ + analysis: "", + inspected: [], + skipped: [ + { + ref: "asset:r1:result:1", + reason: "Only PNG, JPEG, and WebP image assets can be inspected.", + }, + { + ref: "asset:r1:result:2", + reason: "No ready Notebook source matched the image asset.", + }, + ], + }) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "message_user", role: "user", content: "Inspect it" }, + { id: "message_assistant", role: "assistant", content: "Answer" }, + ], + }) + }, + ) + + const result = await chatAnswerRouteService.answerChat({ + body: { message: "Inspect images" }, + }) + + expect(result.status).toBe(200) + expect(mocks.generateText).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + it("triggers background reconciliation for parsing sources without blocking chat", async () => { const workspace = makeWorkspace() const client = { retrieval: { query: vi.fn() } } diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index 8b67e7c..bfa7e42 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -69,6 +69,7 @@ type ChatTurnInput = { generateAnswer: GenerateAnswer hardenChatAssetUrl?: AnswerQuestionInput["hardenChatAssetUrl"] hardenMediaAssetUrls?: AnswerQuestionInput["hardenMediaAssetUrls"] + inspectImages?: AnswerQuestionInput["inspectImages"] repository: ChatRepository } @@ -128,6 +129,7 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => generateAnswer: input.generateAnswer, hardenChatAssetUrl: input.hardenChatAssetUrl, hardenMediaAssetUrls: input.hardenMediaAssetUrls, + inspectImages: input.inspectImages, messages: chatHistoryMessages, }).pipe(Effect.catchAllCause(Effect.die))