diff --git a/package-lock.json b/package-lock.json index f543f10..b8d1d91 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,9 +9,10 @@ "version": "0.1.0", "dependencies": { "@ai-sdk/react": "^3.0.177", + "@antv/chart-visualization-skills": "0.1.3", "@effect/platform": "^0.96.1", "@neondatabase/serverless": "^1.1.0", - "@ontos-ai/knowhere-sdk": "^0.6.0", + "@ontos-ai/knowhere-sdk": "^2.0.0", "@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-dialog": "^1.1.15", @@ -140,6 +141,28 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@antv/chart-visualization-skills": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@antv/chart-visualization-skills/-/chart-visualization-skills-0.1.3.tgz", + "integrity": "sha512-znoffbHkpo37zWAB+KjemVAk1arviP9tqlM5E0jZTp2NyYxspPiuCCMCFQNel9vNpn6g3mSErNt4s+BDpzXutg==", + "license": "MIT", + "dependencies": { + "commander": "^12.1.0", + "gray-matter": "^4.0.3" + }, + "bin": { + "antv": "dist/index.js" + } + }, + "node_modules/@antv/chart-visualization-skills/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@asamuzakjp/css-color": { "version": "5.1.11", "dev": true, @@ -2252,9 +2275,9 @@ } }, "node_modules/@ontos-ai/knowhere-sdk": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/@ontos-ai/knowhere-sdk/-/knowhere-sdk-0.6.0.tgz", - "integrity": "sha512-YqDgg+BKwGp7NepD50aquhEOmUgdDAQxpoyttqh0d+4tWjOgkSo74vEGiJVxt8v+LAcA/5CmxPeghX1Hx0v3wQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ontos-ai/knowhere-sdk/-/knowhere-sdk-2.0.0.tgz", + "integrity": "sha512-Zhx8mn/8pYyhBaO9BOdQYsVZybGXyq15YctjHSU8TwgNjJEzX5lEiBEkp0kIGFPh3S+/7X3rZGakefEPf5G2tQ==", "license": "MIT", "dependencies": { "axios": "^1.15.2", @@ -2262,7 +2285,8 @@ }, "engines": { "node": ">=20.19.0", - "npm": ">=10.0.0" + "npm": ">=10.0.0", + "pnpm": ">=9.0.0" } }, "node_modules/@open-draft/deferred-promise": { @@ -7260,7 +7284,6 @@ }, "node_modules/esprima": { "version": "4.0.1", - "dev": true, "license": "BSD-2-Clause", "bin": { "esparse": "bin/esparse.js", @@ -7469,6 +7492,18 @@ "version": "3.0.2", "license": "MIT" }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/fast-check": { "version": "3.23.2", "funding": [ @@ -8019,6 +8054,43 @@ "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "license": "MIT", + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/gray-matter/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/gray-matter/node_modules/js-yaml": { + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.0.tgz", + "integrity": "sha512-ttBQIIQPDeLjpPOohtUdXuXUVoA2uIB6fEH9HyJ7234s5mBJ5wTx20njxplLZQgLaOfpmPQA7X2t5AX6tIPbog==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/has-bigints": { "version": "1.1.0", "dev": true, @@ -8512,6 +8584,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "dev": true, @@ -9086,6 +9167,15 @@ "json-buffer": "3.0.1" } }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/kleur": { "version": "4.1.5", "dev": true, @@ -12497,6 +12587,19 @@ "version": "0.27.0", "license": "MIT" }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "license": "MIT", + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/semver": { "version": "6.3.1", "dev": true, @@ -13611,6 +13714,15 @@ "node": ">=4" } }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/strip-final-newline": { "version": "4.0.0", "dev": true, @@ -15341,6 +15453,111 @@ "type": "github", "url": "https://github.com/sponsors/wooorm" } + }, + "node_modules/@next/swc-darwin-x64": { + "version": "16.2.4", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.2.4.tgz", + "integrity": "sha512-XhpVnUfmYWvD3YrXu55XdcAkQtOnvaI6wtQa8fuF5fGoKoxIUZ0kWPtcOfqJEWngFF/lOS9l3+O9CcownhiQxQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-gnu": { + "version": "16.2.4", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.2.4.tgz", + "integrity": "sha512-Mx/tjlNA3G8kg14QvuGAJ4xBwPk1tUHq56JxZ8CXnZwz1Etz714soCEzGQQzVMz4bEnGPowzkV6Xrp6wAkEWOQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-musl": { + "version": "16.2.4", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.2.4.tgz", + "integrity": "sha512-iVMMp14514u7Nup2umQS03nT/bN9HurK8ufylC3FZNykrwjtx7V1A7+4kvhbDSCeonTVqV3Txnv0Lu+m2oDXNg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-gnu": { + "version": "16.2.4", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.2.4.tgz", + "integrity": "sha512-EZOvm1aQWgnI/N/xcWOlnS3RQBk0VtVav5Zo7n4p0A7UKyTDx047k8opDbXgBpHl4CulRqRfbw3QrX2w5UOXMQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-musl": { + "version": "16.2.4", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.2.4.tgz", + "integrity": "sha512-h9FxsngCm9cTBf71AR4fGznDEDx1hS7+kSEiIRjq5kO1oXWm07DxVGZjCvk0SGx7TSjlUqhI8oOyz7NfwAdPoA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-arm64-msvc": { + "version": "16.2.4", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.2.4.tgz", + "integrity": "sha512-3NdJV5OXMSOeJYijX+bjaLge3mJBlh4ybydbT4GFoB/2hAojWHtMhl3CYlYoMrjPuodp0nzFVi4Tj2+WaMg+Ow==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-x64-msvc": { + "version": "16.2.4", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.2.4.tgz", + "integrity": "sha512-kMVGgsqhO5YTYODD9IPGGhA6iprWidQckK3LmPeW08PIFENRmgfb4MjXHO+p//d+ts2rpjvK5gXWzXSMrPl9cw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } } } } diff --git a/package.json b/package.json index ee4221c..73b1e26 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "@antv/chart-visualization-skills": "0.1.3", "@effect/platform": "^0.96.1", "@neondatabase/serverless": "^1.1.0", - "@ontos-ai/knowhere-sdk": "^0.10.0", + "@ontos-ai/knowhere-sdk": "^2.0.0", "@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-dialog": "^1.1.15", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 66cd259..1c4223f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: ^1.1.0 version: 1.1.0 '@ontos-ai/knowhere-sdk': - specifier: ^0.10.0 - version: 0.10.0 + specifier: ^2.0.0 + version: 2.0.0 '@radix-ui/react-alert-dialog': specifier: ^1.1.15 version: 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -1409,8 +1409,8 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} - '@ontos-ai/knowhere-sdk@0.10.0': - resolution: {integrity: sha512-wb5wNnnp4YvdlZ6w2UOQ2gw/9anGsmn2hw2JD+uCya4VUyXK78HKMi1LCI6zNnLeZHyGAm0q6+d2wlqDXZ6haQ==} + '@ontos-ai/knowhere-sdk@2.0.0': + resolution: {integrity: sha512-Zhx8mn/8pYyhBaO9BOdQYsVZybGXyq15YctjHSU8TwgNjJEzX5lEiBEkp0kIGFPh3S+/7X3rZGakefEPf5G2tQ==} engines: {node: '>=20.19.0', npm: '>=10.0.0', pnpm: '>=9.0.0'} '@open-draft/deferred-promise@2.2.0': @@ -6397,7 +6397,7 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} - '@ontos-ai/knowhere-sdk@0.10.0': + '@ontos-ai/knowhere-sdk@2.0.0': dependencies: axios: 1.16.0 jszip: 3.10.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 581a9d5..d989c76 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,5 @@ ignoredBuiltDependencies: - sharp - unrs-resolver +minimumReleaseAgeExclude: + - '@ontos-ai/knowhere-sdk@2.0.0' diff --git a/src/components/chunks-panel-state.ts b/src/components/chunks-panel-state.ts index 16aaa33..dfd2ac7 100644 --- a/src/components/chunks-panel-state.ts +++ b/src/components/chunks-panel-state.ts @@ -272,6 +272,7 @@ function isAssetPath(sectionPath: string): boolean { function getFallbackSectionSegments( chunk: ParsedChunkView, ): readonly string[] { + if (chunk.type === "page") return ["Pages"] if (chunk.type === "image") return ["Assets", "Images"] if (chunk.type === "table") return ["Assets", "Tables"] return ["Unsectioned"] diff --git a/src/components/chunks-panel.tsx b/src/components/chunks-panel.tsx index 4a686a7..898ca39 100644 --- a/src/components/chunks-panel.tsx +++ b/src/components/chunks-panel.tsx @@ -935,7 +935,7 @@ function getChunkTreeLabel(chunk: ParsedChunkView): string { const summary = chunk.summary?.split(/\r?\n/, 1)[0]?.trim(); if (summary) return truncateTreeLabel(summary); - const content = chunk.content.replace(/\s+/g, " ").trim(); + const content = (chunk.readableContent ?? chunk.content).replace(/\s+/g, " ").trim(); if (content) return truncateTreeLabel(content); return `${getChunkTypeLabel(chunk.type)} chunk`; @@ -950,16 +950,43 @@ function getChunkTreeDetail(chunk: ParsedChunkView): string { function getChunkPageLabel(chunk: ParsedChunkView): string | null { const pageNumbers = chunk.pageNums ?? []; - const validPageNumbers = pageNumbers.filter( - (pageNumber) => Number.isFinite(pageNumber) && pageNumber > 0, - ); + const validPageNumbers = Array.from( + new Set( + pageNumbers.filter( + (pageNumber) => Number.isFinite(pageNumber) && pageNumber > 0, + ), + ), + ).sort((leftPageNumber, rightPageNumber) => leftPageNumber - rightPageNumber); if (validPageNumbers.length === 0) return null; + if (validPageNumbers.length === 1) return `Page ${validPageNumbers[0]}`; + + const ranges: string[] = []; + let rangeStart = validPageNumbers[0]!; + let previousPageNumber = rangeStart; + validPageNumbers.slice(1).forEach((pageNumber) => { + if (pageNumber === previousPageNumber + 1) { + previousPageNumber = pageNumber; + return; + } + ranges.push( + rangeStart === previousPageNumber + ? String(rangeStart) + : `${rangeStart}-${previousPageNumber}`, + ); + rangeStart = pageNumber; + previousPageNumber = pageNumber; + }); + ranges.push( + rangeStart === previousPageNumber + ? String(rangeStart) + : `${rangeStart}-${previousPageNumber}`, + ); - const firstPageNumber = Math.min(...validPageNumbers); - return `Page ${firstPageNumber}`; + return `Pages ${ranges.join(", ")}`; } function getChunkTypeLabel(type: ParsedChunkView["type"]): string { + if (type === "page") return "Page"; if (type === "image") return "Image"; if (type === "table") return "Table"; return "Text"; diff --git a/src/components/parsed-chunk-card-model.test.ts b/src/components/parsed-chunk-card-model.test.ts index 52d0a5e..57f7b2e 100644 --- a/src/components/parsed-chunk-card-model.test.ts +++ b/src/components/parsed-chunk-card-model.test.ts @@ -16,12 +16,28 @@ describe("parsedChunkCardModel", () => { ) expect(metadata).toEqual({ - pageLabel: "Pages 2, 4, 8...", + pageLabel: "Pages 2, 4, 8-9", sectionLabel: "FINANCIAL SUMMARY", typeLabel: "Text", }) }) + it("labels page chunks and formats continuous page ranges", () => { + const metadata = parsedChunkCardModel.getSourceMetadata( + makeChunk({ + type: "page", + pageNums: [6, 4, 5], + sectionPath: "Default_Root/manual.pdf-->pages/4-6", + }), + ) + + expect(metadata).toEqual({ + pageLabel: "Pages 4-6", + sectionLabel: "pages/4-6", + typeLabel: "Page", + }) + }) + it("splits text content into text and reference parts with display-ready labels", () => { const parts = parsedChunkCardModel.getTextContentParts( makeChunk({ diff --git a/src/components/parsed-chunk-card-model.ts b/src/components/parsed-chunk-card-model.ts index 1ca2c0e..08780f2 100644 --- a/src/components/parsed-chunk-card-model.ts +++ b/src/components/parsed-chunk-card-model.ts @@ -116,6 +116,7 @@ function getFocusCardClasses(isFocused: boolean): string { } function getChunkTypeLabel(type: ParsedChunkView["type"]): string { + if (type === "page") return "Page" if (type === "image") return "Image" if (type === "table") return "Table" return "Text" @@ -131,9 +132,31 @@ function formatPageNumbers( ) if (uniquePageNums.length === 1) return `Page ${uniquePageNums[0]}` - const visiblePageNums = uniquePageNums.slice(0, 3).join(", ") - const suffix = uniquePageNums.length > 3 ? "..." : "" - return `Pages ${visiblePageNums}${suffix}` + const ranges: string[] = [] + let rangeStart = uniquePageNums[0]! + let previousPageNum = rangeStart + + uniquePageNums.slice(1).forEach((pageNum) => { + if (pageNum === previousPageNum + 1) { + previousPageNum = pageNum + return + } + + ranges.push( + rangeStart === previousPageNum + ? String(rangeStart) + : `${rangeStart}-${previousPageNum}`, + ) + rangeStart = pageNum + previousPageNum = pageNum + }) + ranges.push( + rangeStart === previousPageNum + ? String(rangeStart) + : `${rangeStart}-${previousPageNum}`, + ) + + return `Pages ${ranges.join(", ")}` } export const parsedChunkCardModel: ParsedChunkCardModelModule = { diff --git a/src/components/parsed-chunk-card.test.ts b/src/components/parsed-chunk-card.test.ts index 2e34ab7..946cffb 100644 --- a/src/components/parsed-chunk-card.test.ts +++ b/src/components/parsed-chunk-card.test.ts @@ -45,6 +45,42 @@ describe("ParsedChunkCard", () => { ); }); + it("renders page chunks as page summaries with page citations", () => { + render( + React.createElement(ParsedChunkCard, { + chunk: { + chunkId: "page_1", + type: "page", + content: "The refund policy is summarized across these pages.", + contentSource: "summary", + readableContent: "The refund policy is summarized across these pages.", + sourceTitle: "manual.pdf", + sectionPath: "Default_Root/manual.pdf-->pages/4-6", + pageNums: [4, 5, 6], + entities: [{ text: "refund", label: "topic" }], + }, + isFocused: false, + isOriginalPreviewAvailable: true, + onChunkClick: vi.fn(), + onReferenceClick: vi.fn(), + }), + ); + + expect(screen.getByTestId("chunk-source-panel-page_1").textContent).toContain( + "Page", + ); + expect(screen.getByTestId("chunk-source-panel-page_1").textContent).toContain( + "Pages 4-6", + ); + expect(screen.getByTestId("chunk-content-panel-page_1").textContent).toContain( + "The refund policy is summarized", + ); + expect(screen.queryByTestId("chunk-summary-panel-page_1")).toBeNull(); + expect( + screen.getByRole("button", { name: "Open page 4 in original file" }), + ).toBeTruthy(); + }); + it("routes resolved artifact reference clicks to the target chunk", async () => { const user = userEvent.setup(); const onReferenceClick = vi.fn(); diff --git a/src/components/parsed-chunk-card.tsx b/src/components/parsed-chunk-card.tsx index 53178f5..1ba233d 100644 --- a/src/components/parsed-chunk-card.tsx +++ b/src/components/parsed-chunk-card.tsx @@ -35,6 +35,18 @@ export function ParsedChunkCard({ readonly onReferenceClick: (chunkId: string) => void; readonly sourceOriginalFile?: SourceOriginalFileView | null; }): ReactNode { + if (chunk.type === "page") { + return ( + + + + ); + } if (chunk.type === "image") { return ( @@ -257,9 +269,11 @@ function ChunkSummaryPanel({ function ChunkContentPanel({ chunk, + label = "Content", children, }: { readonly chunk: ParsedChunkView; + readonly label?: string; readonly children: ReactNode; }): ReactNode { return ( @@ -267,7 +281,7 @@ function ChunkContentPanel({ data-testid={`chunk-content-panel-${chunk.chunkId}`} className="rounded-lg border border-border/70 bg-card p-3" > - } label="Content" /> + } label={label} />
{children}
); @@ -361,6 +375,34 @@ function TextChunkCard({ ); } +function PageChunkCard({ + chunk, + isFocused, + isOriginalPreviewAvailable, + onChunkClick, +}: { + readonly chunk: ParsedChunkView; + readonly isFocused: boolean; + readonly isOriginalPreviewAvailable: boolean; + readonly onChunkClick?: (chunk: ParsedChunkView) => void; +}): ReactNode { + return ( + + +

+ {chunk.readableContent ?? chunk.content} +

+ + + + ); +} + function ImageChunkCard({ chunk, isFocused, @@ -525,12 +567,16 @@ function TableChunkCard({ } function renderChunkIcon(type: ParsedChunkView["type"]): ReactNode { + if (type === "page") return ; if (type === "image") return ; if (type === "table") return ; return ; } function getChunkIconClasses(type: ParsedChunkView["type"]): string { + if (type === "page") { + return "border-amber-500/20 bg-amber-500/10 text-amber-700 dark:text-amber-300"; + } if (type === "image") { return "border-violet-500/15 bg-violet-500/10 text-violet-600 dark:text-violet-300"; } diff --git a/src/domains/chunks/index.test.ts b/src/domains/chunks/index.test.ts index cfda6f7..0dfc22f 100644 --- a/src/domains/chunks/index.test.ts +++ b/src/domains/chunks/index.test.ts @@ -138,11 +138,51 @@ describe("toParsedChunkView", () => { }); }); - it("ignores legacy page number aliases outside metadata.page_nums", () => { + it("maps page-memory chunks to summary-first page chunk views", () => { + const chunk = { + ...makeDocumentChunk({ + id: "document_page_1", + chunkId: "parser_page_1", + chunkType: "page" as DocumentChunk["chunkType"], + content: null, + sectionPath: "Default_Root/manual.pdf-->pages/4-6", + metadata: { + summary: "Refund eligibility is summarized across pages 4 to 6.", + page_nums: [4, 5, 6], + entities: [{ text: "refund", label: "topic" }], + }, + assetUrl: "https://assets.example/crop.pdf", + }), + contentSource: "summary", + } as DocumentChunk & { readonly contentSource: string }; + + expect(toParsedChunkView(chunk, "manual.pdf", "doc_123")).toMatchObject({ + chunkId: "document_page_1", + parserChunkId: "parser_page_1", + type: "page", + contentSource: "summary", + content: "Refund eligibility is summarized across pages 4 to 6.", + readableContent: "Refund eligibility is summarized across pages 4 to 6.", + pageNums: [4, 5, 6], + entities: [{ text: "refund", label: "topic" }], + assetUrl: "https://assets.example/crop.pdf", + }); + }); + + it("maps SDK-normalized page number metadata", () => { + const chunk = makeDocumentChunk({ + metadata: { + pageNums: [4, 5, 6], + }, + }); + + expect(toParsedChunkView(chunk, "manual.pdf").pageNums).toEqual([4, 5, 6]); + }); + + it("ignores legacy page number aliases outside metadata", () => { const chunk = { ...makeDocumentChunk({ metadata: { - pageNums: [1, 2], pageNumbers: [3], page_numbers: [4], }, diff --git a/src/domains/chunks/index.ts b/src/domains/chunks/index.ts index 035d3ef..8a285d7 100644 --- a/src/domains/chunks/index.ts +++ b/src/domains/chunks/index.ts @@ -163,6 +163,9 @@ export function toParsedChunkView( parserChunkId: chunk.chunkId, sectionPath: chunk.sectionPath, chunkType: chunk.chunkType, + contentSource: + (chunk as DocumentChunk & { readonly contentSource?: unknown }).contentSource ?? + (chunk as DocumentChunk & { readonly content_source?: unknown }).content_source, content: chunk.content, metadata: chunk.metadata, filePathCandidates: [ diff --git a/src/domains/chunks/normalization.ts b/src/domains/chunks/normalization.ts index 4bdaa51..c6590b4 100644 --- a/src/domains/chunks/normalization.ts +++ b/src/domains/chunks/normalization.ts @@ -11,6 +11,7 @@ type ParsedChunkNormalizationInput = { readonly documentId?: string readonly sectionPath?: string | null readonly chunkType: unknown + readonly contentSource?: unknown readonly content?: unknown readonly metadata: Readonly> readonly filePathCandidates?: readonly unknown[] @@ -27,19 +28,31 @@ function createParsedChunkView( (filePath ? input.assetUrlsByFilePath?.[filePath] : undefined) ?? getString(input.assetUrl) const connections = getChunkConnections(input.metadata) + const type = normalizeChunkType(input.chunkType) + const content = normalizeChunkContent({ + type, + content: input.content, + metadata: input.metadata, + }) + const summary = getStringMetadata(input.metadata, "summary") return { chunkId: input.chunkId, documentId: input.documentId, parserChunkId: getString(input.parserChunkId), sectionPath: input.sectionPath, - type: normalizeChunkType(input.chunkType), - content: getString(input.content) ?? "", + type, + content, + contentSource: getContentSource(input), + readableContent: getReadableContent({ type, content, summary }), filePath, assetUrl, - summary: getStringMetadata(input.metadata, "summary"), + summary, keywords: getStringArrayMetadata(input.metadata, "keywords"), - pageNums: getPageNumbers(input.metadata["page_nums"]), + pageNums: getPageNumbers( + input.metadata["pageNums"] ?? input.metadata["page_nums"], + ), + entities: getEntities(input.metadata["entities"]), connections, sourceTitle: input.sourceTitle, } @@ -97,10 +110,41 @@ function resolveCitationChunkByContent( } function normalizeChunkType(value: unknown): ChunkType { - if (value === "image" || value === "table") return value + if (value === "image" || value === "table" || value === "page") return value return "text" } +function getContentSource( + input: ParsedChunkNormalizationInput, +): string | undefined { + return ( + getString(input.contentSource) ?? + getString(input.metadata["contentSource"]) ?? + getString(input.metadata["content_source"]) + ) +} + +function normalizeChunkContent(input: { + readonly type: ChunkType + readonly content?: unknown + readonly metadata: Readonly> +}): string { + const content = getString(input.content) + if (content) return content + if (input.type !== "page") return "" + + return getStringMetadata(input.metadata, "summary") ?? "" +} + +function getReadableContent(input: { + readonly type: ChunkType + readonly content: string + readonly summary?: string +}): string | undefined { + if (input.type !== "page") return undefined + return input.content || input.summary || undefined +} + function getChunkConnections( metadata: Readonly>, ): ParsedChunkConnection[] | undefined { @@ -214,6 +258,15 @@ function getStringArrayMetadata( return strings.length > 0 ? strings : undefined } +function getEntities( + value: unknown, +): readonly Readonly>[] | undefined { + if (!Array.isArray(value)) return undefined + + const entities = value.filter(isRecord) + return entities.length > 0 ? entities : undefined +} + function getPageNumbers(value: unknown): number[] | undefined { if (!Array.isArray(value)) return undefined diff --git a/src/domains/chunks/types.ts b/src/domains/chunks/types.ts index e99bcd7..2e84189 100644 --- a/src/domains/chunks/types.ts +++ b/src/domains/chunks/types.ts @@ -1,4 +1,4 @@ -export type ChunkType = "text" | "image" | "table" +export type ChunkType = "text" | "image" | "table" | "page" export type ParsedChunkConnection = { readonly targetParserChunkId: string @@ -23,7 +23,9 @@ export type ParsedChunkView = { /** Human-readable section path from Knowhere, used to focus citations. */ readonly sectionPath?: string | null readonly type: ChunkType + readonly contentSource?: string readonly content: string + readonly readableContent?: string /** ZIP-relative parsed artifact path, e.g. images/image-1.jpg. */ readonly filePath?: string /** Public Blob URL for parsed media/table artifacts when Notebook stored it. */ @@ -31,6 +33,7 @@ export type ParsedChunkView = { readonly summary?: string readonly keywords?: readonly string[] readonly pageNums?: readonly number[] + readonly entities?: readonly Readonly>[] readonly connections?: readonly ParsedChunkConnection[] /** Display-only attribution. */ readonly sourceTitle: string diff --git a/src/integrations/knowhere.ts b/src/integrations/knowhere.ts index ee4d5ff..81f6726 100644 --- a/src/integrations/knowhere.ts +++ b/src/integrations/knowhere.ts @@ -6,7 +6,11 @@ import { logger } from "@/lib/logger" * Use for per-request clients created from Dashboard-issued JWTs. */ export function makeKnowhereClient(apiKey: string): Knowhere { - const client = new Knowhere({ apiKey, baseURL: process.env.KNOWHERE_BASE_URL }) + const options: ConstructorParameters[0] = { + apiKey, + baseURL: process.env.KNOWHERE_BASE_URL, + } + const client = new Knowhere(options) return wrapKnowhereClient(client) }