From 9c1def15de797722737c8ba9da44429175014f65 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Fri, 3 Jul 2026 11:34:57 +0800 Subject: [PATCH 01/12] feat: add page citation asset support --- .gitignore | 1 + eslint.config.mjs | 1 + next.config.ts | 3 + package.json | 7 +- pnpm-lock.yaml | 672 +++++++++++++++++- pnpm-workspace.yaml | 4 + scripts/prepare-effect.sh | 13 + src/agent-harness/ledger.ts | 8 + src/agent-harness/types.ts | 4 + src/components/chat-message-list.test.ts | 50 ++ src/components/chat-message-list.tsx | 74 +- src/domains/chat/chat-citation-persistence.ts | 1 + src/domains/chat/citations.test.ts | 19 +- src/domains/chat/citations.ts | 6 +- src/domains/chat/index.ts | 28 +- src/domains/chat/page-citation-assets.test.ts | 216 ++++++ src/domains/chat/page-citation-assets.ts | 381 ++++++++++ src/domains/chat/types.ts | 2 +- src/domains/chat/view.ts | 4 + .../sources/page-citation-assets.test.ts | 82 +++ src/domains/sources/page-citation-assets.ts | 69 ++ .../source-reconcile-route-workflow.test.ts | 78 ++ .../source-reconcile-route-workflow.ts | 98 +++ .../sources/source-reconcile-workflow.test.ts | 32 + .../sources/source-reconcile-workflow.ts | 33 + src/domains/workspace/integration.test.ts | 2 + src/domains/workspace/persistence.test.ts | 2 + src/integrations/knowhere-sdk-storage.test.ts | 320 +++++++++ src/integrations/knowhere-sdk-storage.ts | 294 ++++++++ 29 files changed, 2440 insertions(+), 64 deletions(-) create mode 100755 scripts/prepare-effect.sh create mode 100644 src/domains/chat/page-citation-assets.test.ts create mode 100644 src/domains/chat/page-citation-assets.ts create mode 100644 src/domains/sources/page-citation-assets.test.ts create mode 100644 src/domains/sources/page-citation-assets.ts create mode 100644 src/integrations/knowhere-sdk-storage.test.ts create mode 100644 src/integrations/knowhere-sdk-storage.ts diff --git a/.gitignore b/.gitignore index 74889db..2a2a9f1 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ # misc .DS_Store *.pem +.repos/effect # debug npm-debug.log* diff --git a/eslint.config.mjs b/eslint.config.mjs index ac9d10f..6ed5798 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -13,6 +13,7 @@ const eslintConfig = defineConfig([ "build/**", "playwright-report/**", "test-results/**", + ".repos/effect/**", "next-env.d.ts", ]), ]); diff --git a/next.config.ts b/next.config.ts index e294c70..b73f520 100644 --- a/next.config.ts +++ b/next.config.ts @@ -7,6 +7,9 @@ const nextConfig: NextConfig = { "pg", "@neondatabase/serverless", "postgres", + "@ontos-ai/knowhere-sdk", + "@napi-rs/canvas", + "piscina", ], allowedDevOrigins: [ "127.0.0.1", diff --git a/package.json b/package.json index 73b1e26..c998de4 100644 --- a/package.json +++ b/package.json @@ -2,11 +2,14 @@ "name": "knowhere-notebook", "version": "0.1.0", "private": true, + "engines": { + "node": ">=22.13.0" + }, "scripts": { "dev": "pnpm run upstash:dev & next dev", "build": "next build", "typecheck": "tsc --noEmit", - "prepare": "effect-language-service patch", + "prepare": "./scripts/prepare-effect.sh && effect-language-service patch", "start": "next start", "lint": "eslint", "test": "vitest run", @@ -23,6 +26,7 @@ "@ai-sdk/react": "^3.0.177", "@antv/chart-visualization-skills": "0.1.3", "@effect/platform": "^0.96.1", + "@napi-rs/canvas": "^1.0.2", "@neondatabase/serverless": "^1.1.0", "@ontos-ai/knowhere-sdk": "^2.0.0", "@radix-ui/react-alert-dialog": "^1.1.15", @@ -51,6 +55,7 @@ "next": "16.2.4", "next-themes": "^0.4.6", "pdfjs-dist": "5.4.296", + "piscina": "^5.2.0", "postgres": "^3.4.9", "posthog-js": "^1.386.8", "react": "19.2.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1c4223f..978e912 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,9 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + '@ontos-ai/knowhere-sdk': file:../knowhere-node-sdk + importers: .: @@ -17,12 +20,15 @@ importers: '@effect/platform': specifier: ^0.96.1 version: 0.96.1(effect@3.21.2) + '@napi-rs/canvas': + specifier: ^1.0.2 + version: 1.0.2 '@neondatabase/serverless': specifier: ^1.1.0 version: 1.1.0 '@ontos-ai/knowhere-sdk': - specifier: ^2.0.0 - version: 2.0.0 + specifier: file:../knowhere-node-sdk + version: file:../knowhere-node-sdk(ioredis@5.11.1) '@radix-ui/react-alert-dialog': specifier: ^1.1.15 version: 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -101,6 +107,9 @@ importers: pdfjs-dist: specifier: 5.4.296 version: 5.4.296 + piscina: + specifier: ^5.2.0 + version: 5.2.0 postgres: specifier: ^3.4.9 version: 3.4.9 @@ -167,7 +176,7 @@ importers: version: 19.2.3(@types/react@19.2.14) '@vitejs/plugin-react': specifier: ^6.0.1 - version: 6.0.1(babel-plugin-react-compiler@1.0.0)(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)) + version: 6.0.1(babel-plugin-react-compiler@1.0.0)(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)) babel-plugin-react-compiler: specifier: ^1.0.0 version: 1.0.0 @@ -194,7 +203,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.1.5 - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@20.19.39)(jsdom@29.1.1(@noble/hashes@1.8.0))(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@20.19.39)(jsdom@29.1.1(@noble/hashes@1.8.0))(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)) packages: @@ -433,6 +442,19 @@ packages: resolution: {integrity: sha512-EXnJjIy6zQ3nUO/MZ+ynWUb8B895KZPotd1++oTs9JjDkplwM7cb6zo8Zq2zU6piwq+KflO7amXbEfj1UMpHkw==} hasBin: true + '@effect/platform-node-shared@4.0.0-beta.93': + resolution: {integrity: sha512-XUqZ2u5GglBqY8q2jj4Q7GjN5K/enedk8auZM9rY/l5a/myaQTrQp3QnvpIK4/Yg0WFjLGuctGPMKWRk3OLIrA==} + engines: {node: '>=18.0.0'} + peerDependencies: + effect: ^4.0.0-beta.93 + + '@effect/platform-node@4.0.0-beta.93': + resolution: {integrity: sha512-QagsCGR0ZOXaCQqS5qGR2mcDng4LiP2bYhiiX1D6UC8cT9vsusVVOHiJWn8CupeDx+yVnPcu81QmA/SDt6GM1w==} + engines: {node: '>=18.0.0'} + peerDependencies: + effect: ^4.0.0-beta.93 + ioredis: ^5.7.0 + '@effect/platform@0.96.1': resolution: {integrity: sha512-cjB1QZZYEP8JXCFNGvBLVi0T6YUBQTmOVEUA3SDbiQ6RUO+p6CE3eyD2vMWmrz5nE8yY5QSAuOV9v0boEcUv+A==} peerDependencies: @@ -1175,6 +1197,9 @@ packages: '@types/node': optional: true + '@ioredis/commands@1.10.0': + resolution: {integrity: sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==} + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -1206,31 +1231,61 @@ packages: cpu: [arm64] os: [darwin] + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': + resolution: {integrity: sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==} + cpu: [arm64] + os: [darwin] + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} cpu: [x64] os: [darwin] + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': + resolution: {integrity: sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==} + cpu: [x64] + os: [darwin] + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} cpu: [arm64] os: [linux] + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': + resolution: {integrity: sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==} + cpu: [arm64] + os: [linux] + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} cpu: [arm] os: [linux] + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': + resolution: {integrity: sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==} + cpu: [arm] + os: [linux] + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} cpu: [x64] os: [linux] + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': + resolution: {integrity: sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==} + cpu: [x64] + os: [linux] + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} cpu: [x64] os: [win32] + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': + resolution: {integrity: sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==} + cpu: [x64] + os: [win32] + '@mswjs/interceptors@0.41.8': resolution: {integrity: sha512-pRLMNKTSGRoLq+KnEB/7OY5vijw1XmcheAAOiv6pj7W1FG32kAGqj1C/RK/cqxRGr1Fh+zBi8sDur8kj3EQv6A==} engines: {node: '>=18'} @@ -1241,24 +1296,48 @@ packages: cpu: [arm64] os: [android] + '@napi-rs/canvas-android-arm64@1.0.2': + resolution: {integrity: sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + '@napi-rs/canvas-darwin-arm64@0.1.100': resolution: {integrity: sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] + '@napi-rs/canvas-darwin-arm64@1.0.2': + resolution: {integrity: sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + '@napi-rs/canvas-darwin-x64@0.1.100': resolution: {integrity: sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] + '@napi-rs/canvas-darwin-x64@1.0.2': + resolution: {integrity: sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.100': resolution: {integrity: sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==} engines: {node: '>= 10'} cpu: [arm] os: [linux] + '@napi-rs/canvas-linux-arm-gnueabihf@1.0.2': + resolution: {integrity: sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + '@napi-rs/canvas-linux-arm64-gnu@0.1.100': resolution: {integrity: sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==} engines: {node: '>= 10'} @@ -1266,6 +1345,13 @@ packages: os: [linux] libc: [glibc] + '@napi-rs/canvas-linux-arm64-gnu@1.0.2': + resolution: {integrity: sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + '@napi-rs/canvas-linux-arm64-musl@0.1.100': resolution: {integrity: sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==} engines: {node: '>= 10'} @@ -1273,6 +1359,13 @@ packages: os: [linux] libc: [musl] + '@napi-rs/canvas-linux-arm64-musl@1.0.2': + resolution: {integrity: sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + '@napi-rs/canvas-linux-riscv64-gnu@0.1.100': resolution: {integrity: sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==} engines: {node: '>= 10'} @@ -1280,6 +1373,13 @@ packages: os: [linux] libc: [glibc] + '@napi-rs/canvas-linux-riscv64-gnu@1.0.2': + resolution: {integrity: sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + '@napi-rs/canvas-linux-x64-gnu@0.1.100': resolution: {integrity: sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==} engines: {node: '>= 10'} @@ -1287,6 +1387,13 @@ packages: os: [linux] libc: [glibc] + '@napi-rs/canvas-linux-x64-gnu@1.0.2': + resolution: {integrity: sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + '@napi-rs/canvas-linux-x64-musl@0.1.100': resolution: {integrity: sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==} engines: {node: '>= 10'} @@ -1294,22 +1401,158 @@ packages: os: [linux] libc: [musl] + '@napi-rs/canvas-linux-x64-musl@1.0.2': + resolution: {integrity: sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + '@napi-rs/canvas-win32-arm64-msvc@0.1.100': resolution: {integrity: sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] + '@napi-rs/canvas-win32-arm64-msvc@1.0.2': + resolution: {integrity: sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + '@napi-rs/canvas-win32-x64-msvc@0.1.100': resolution: {integrity: sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==} engines: {node: '>= 10'} cpu: [x64] os: [win32] + '@napi-rs/canvas-win32-x64-msvc@1.0.2': + resolution: {integrity: sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + '@napi-rs/canvas@0.1.100': resolution: {integrity: sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==} engines: {node: '>= 10'} + '@napi-rs/canvas@1.0.2': + resolution: {integrity: sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ==} + engines: {node: '>= 10'} + + '@napi-rs/nice-android-arm-eabi@1.1.1': + resolution: {integrity: sha512-kjirL3N6TnRPv5iuHw36wnucNqXAO46dzK9oPb0wj076R5Xm8PfUVA9nAFB5ZNMmfJQJVKACAPd/Z2KYMppthw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@napi-rs/nice-android-arm64@1.1.1': + resolution: {integrity: sha512-blG0i7dXgbInN5urONoUCNf+DUEAavRffrO7fZSeoRMJc5qD+BJeNcpr54msPF6qfDD6kzs9AQJogZvT2KD5nw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/nice-darwin-arm64@1.1.1': + resolution: {integrity: sha512-s/E7w45NaLqTGuOjC2p96pct4jRfo61xb9bU1unM/MJ/RFkKlJyJDx7OJI/O0ll/hrfpqKopuAFDV8yo0hfT7A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/nice-darwin-x64@1.1.1': + resolution: {integrity: sha512-dGoEBnVpsdcC+oHHmW1LRK5eiyzLwdgNQq3BmZIav+9/5WTZwBYX7r5ZkQC07Nxd3KHOCkgbHSh4wPkH1N1LiQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/nice-freebsd-x64@1.1.1': + resolution: {integrity: sha512-kHv4kEHAylMYmlNwcQcDtXjklYp4FCf0b05E+0h6nDHsZ+F0bDe04U/tXNOqrx5CmIAth4vwfkjjUmp4c4JktQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/nice-linux-arm-gnueabihf@1.1.1': + resolution: {integrity: sha512-E1t7K0efyKXZDoZg1LzCOLxgolxV58HCkaEkEvIYQx12ht2pa8hoBo+4OB3qh7e+QiBlp1SRf+voWUZFxyhyqg==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/nice-linux-arm64-gnu@1.1.1': + resolution: {integrity: sha512-CIKLA12DTIZlmTaaKhQP88R3Xao+gyJxNWEn04wZwC2wmRapNnxCUZkVwggInMJvtVElA+D4ZzOU5sX4jV+SmQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@napi-rs/nice-linux-arm64-musl@1.1.1': + resolution: {integrity: sha512-+2Rzdb3nTIYZ0YJF43qf2twhqOCkiSrHx2Pg6DJaCPYhhaxbLcdlV8hCRMHghQ+EtZQWGNcS2xF4KxBhSGeutg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@napi-rs/nice-linux-ppc64-gnu@1.1.1': + resolution: {integrity: sha512-4FS8oc0GeHpwvv4tKciKkw3Y4jKsL7FRhaOeiPei0X9T4Jd619wHNe4xCLmN2EMgZoeGg+Q7GY7BsvwKpL22Tg==} + engines: {node: '>= 10'} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@napi-rs/nice-linux-riscv64-gnu@1.1.1': + resolution: {integrity: sha512-HU0nw9uD4FO/oGCCk409tCi5IzIZpH2agE6nN4fqpwVlCn5BOq0MS1dXGjXaG17JaAvrlpV5ZeyZwSon10XOXw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@napi-rs/nice-linux-s390x-gnu@1.1.1': + resolution: {integrity: sha512-2YqKJWWl24EwrX0DzCQgPLKQBxYDdBxOHot1KWEq7aY2uYeX+Uvtv4I8xFVVygJDgf6/92h9N3Y43WPx8+PAgQ==} + engines: {node: '>= 10'} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@napi-rs/nice-linux-x64-gnu@1.1.1': + resolution: {integrity: sha512-/gaNz3R92t+dcrfCw/96pDopcmec7oCcAQ3l/M+Zxr82KT4DljD37CpgrnXV+pJC263JkW572pdbP3hP+KjcIg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@napi-rs/nice-linux-x64-musl@1.1.1': + resolution: {integrity: sha512-xScCGnyj/oppsNPMnevsBe3pvNaoK7FGvMjT35riz9YdhB2WtTG47ZlbxtOLpjeO9SqqQ2J2igCmz6IJOD5JYw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@napi-rs/nice-openharmony-arm64@1.1.1': + resolution: {integrity: sha512-6uJPRVwVCLDeoOaNyeiW0gp2kFIM4r7PL2MczdZQHkFi9gVlgm+Vn+V6nTWRcu856mJ2WjYJiumEajfSm7arPQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [openharmony] + + '@napi-rs/nice-win32-arm64-msvc@1.1.1': + resolution: {integrity: sha512-uoTb4eAvM5B2aj/z8j+Nv8OttPf2m+HVx3UjA5jcFxASvNhQriyCQF1OB1lHL43ZhW+VwZlgvjmP5qF3+59atA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/nice-win32-ia32-msvc@1.1.1': + resolution: {integrity: sha512-CNQqlQT9MwuCsg1Vd/oKXiuH+TcsSPJmlAFc5frFyX/KkOh0UpBLEj7aoY656d5UKZQMQFP7vJNa1DNUNORvug==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/nice-win32-x64-msvc@1.1.1': + resolution: {integrity: sha512-vB+4G/jBQCAh0jelMTY3+kgFy00Hlx2f2/1zjMoH821IbplbWZOkLiTYXQkygNTzQJTq5cvwBDgn2ppHD+bglQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/nice@1.1.1': + resolution: {integrity: sha512-xJIPs+bYuc9ASBl+cvGsKbGrJmS6fAKaSZCnT0lhahT5rhA2VVy9/EcIgd2JhtEuFOJNx7UHNn/qiTPTY4nrQw==} + engines: {node: '>= 10'} + '@napi-rs/wasm-runtime@0.2.12': resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} @@ -1409,9 +1652,9 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} - '@ontos-ai/knowhere-sdk@2.0.0': - resolution: {integrity: sha512-Zhx8mn/8pYyhBaO9BOdQYsVZybGXyq15YctjHSU8TwgNjJEzX5lEiBEkp0kIGFPh3S+/7X3rZGakefEPf5G2tQ==} - engines: {node: '>=20.19.0', npm: '>=10.0.0', pnpm: '>=9.0.0'} + '@ontos-ai/knowhere-sdk@file:../knowhere-node-sdk': + resolution: {directory: ../knowhere-node-sdk, type: directory} + engines: {node: '>=22.13.0', npm: '>=10.0.0', pnpm: '>=9.0.0'} '@open-draft/deferred-promise@2.2.0': resolution: {integrity: sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==} @@ -2174,6 +2417,9 @@ packages: '@types/validate-npm-package-name@4.0.2': resolution: {integrity: sha512-lrpDziQipxCEeK5kWxvljWYhUvOiB2A9izZd9B2AFarYAkqZshb4lPbRs7zKEic6eGtH8V/2qJW+dPp9OtF6bw==} + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@typescript-eslint/eslint-plugin@8.59.2': resolution: {integrity: sha512-j/bwmkBvHUtPNxzuWe5z6BEk3q54YRyGlBXkSsmfoih7zNrBvl5A9A98anlp/7JbyZcWIJ8KXo/3Tq/DjFLtuQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -2415,6 +2661,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + agent-base@7.1.4: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} @@ -2533,8 +2783,8 @@ packages: resolution: {integrity: sha512-KunSNx+TVpkAw/6ULfhnx+HWRecjqZGTOyquAoWHYLRSdK1tB5Ihce1ZW+UY3fj33bYAFWPu7W/GRSmmrCGuxA==} engines: {node: '>=4'} - axios@1.16.0: - resolution: {integrity: sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==} + axios@1.18.1: + resolution: {integrity: sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==} axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} @@ -2670,6 +2920,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + cluster-key-slot@1.1.1: + resolution: {integrity: sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==} + engines: {node: '>=0.10.0'} + code-block-writer@13.0.3: resolution: {integrity: sha512-Oofo0pq3IKnsFtuHqSF7TqBfr71aeyZDVJ0HpmqB7FBM2qEigL0iPONSCZSO9pE9dZTAxANe5XHG9Uy0YMv8cg==} @@ -2852,6 +3106,10 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -3007,6 +3265,9 @@ packages: effect@3.21.2: resolution: {integrity: sha512-rXd2FGDM8KdjSIrc+mqEELo7ScW7xTVxEf1iInmPSpIde9/nyGuFM710cjTo7/EreGXiUX2MOonPpprbz2XHCg==} + effect@4.0.0-beta.93: + resolution: {integrity: sha512-wNS5MKFa3C42uBfIDik2oJ78lhpoYz2hN4oBR0229BeeDCIrkg/FiOvoiPGdCVlWa7MEKxEL5I0f8AILVHSD9A==} + electron-to-chromium@1.5.351: resolution: {integrity: sha512-9D7Iqx8RImSvCnOsj86rCH6eQjZFQoM04Jn6HnZVM0Nu/G58/gmKYQ1d12MZTbjQbQSTGI8nwEy07ErsA2slLA==} @@ -3279,6 +3540,10 @@ packages: resolution: {integrity: sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==} engines: {node: '>=8.0.0'} + fast-check@4.8.0: + resolution: {integrity: sha512-GOJ158CUMnN6cSahsv4+ExARvIDuzzinFjkp0E9WtiBa5zcVeLozVkWaE4IzFcc+Y48Wp1EDlUZsXRyAztQcSg==} + engines: {node: '>=12.17.0'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -3370,8 +3635,8 @@ packages: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} - form-data@4.0.5: - resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} engines: {node: '>= 6'} formdata-polyfill@4.0.10: @@ -3522,6 +3787,10 @@ packages: resolution: {integrity: sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==} engines: {node: '>= 0.4'} + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} @@ -3552,6 +3821,10 @@ packages: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + https-proxy-agent@7.0.6: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} @@ -3590,6 +3863,10 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + ini@7.0.0: + resolution: {integrity: sha512-ifK0CgjALofS5bkrcTy4RaQ9Vx2Knf/eLeIO+NaswQEpH1UblrtTSCIvN71qQDMq0PeQ/SSPojvEJp9vvvfr+w==} + engines: {node: ^22.22.2 || ^24.15.0 || >=26.0.0} + inline-style-parser@0.2.7: resolution: {integrity: sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==} @@ -3597,6 +3874,10 @@ packages: resolution: {integrity: sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==} engines: {node: '>= 0.4'} + ioredis@5.11.1: + resolution: {integrity: sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==} + engines: {node: '>=12.22.0'} + ip-address@10.1.0: resolution: {integrity: sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==} engines: {node: '>= 12'} @@ -3901,6 +4182,9 @@ packages: resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} engines: {node: '>=6'} + kubernetes-types@1.30.0: + resolution: {integrity: sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==} + language-subtag-registry@0.3.23: resolution: {integrity: sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==} @@ -4225,6 +4509,11 @@ packages: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} + mime@4.1.0: + resolution: {integrity: sha512-X5ju04+cAzsojXKes0B/S4tcYtFAJ6tTMuSPBEn9CPGlrWr8Fiw7qYeLT0XyH80HSoAoqWCaz+MWKh22P7G1cw==} + engines: {node: '>=16'} + hasBin: true + mimic-fn@2.1.0: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} @@ -4250,9 +4539,16 @@ packages: resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} hasBin: true + msgpackr-extract@3.0.4: + resolution: {integrity: sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==} + hasBin: true + msgpackr@1.11.12: resolution: {integrity: sha512-RBdJ1Un7yGlXWajrkxcSa93nvQ0w4zBf60c0yYv7YtBelP8H2FA7XsfBbMHtXKXUMUxH7zV3Zuozh+kUQWhHvg==} + msgpackr@2.0.4: + resolution: {integrity: sha512-o1C5KRmuRt+apqMr1HuGSqWStZoRBUpEsCsl15uM9VdAF1qHLtvMOU2En747EnTyEl6c4pzPewRMFF31s1CNbA==} + msw@2.14.3: resolution: {integrity: sha512-kk8G5cocVlJ4wsKMGZegn2H6XLOEKjbA+nSJE2354e/SRp4mDicCHUYnMXpymzVcVDCs+GUAsmNqSn+yHv4T2A==} engines: {node: '>=18'} @@ -4490,6 +4786,10 @@ packages: resolution: {integrity: sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==} engines: {node: '>=20.16.0 || >=22.3.0'} + pdfjs-dist@6.1.200: + resolution: {integrity: sha512-o8MolyzirkkLrcdsae/HEOiIcXWI7DS5zGpvqW8xTC2YUsW30rltFw2bDGvw/fskUdEMrQm2br68jzDS5BH2vw==} + engines: {node: '>=22.13.0 || >=24'} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -4501,6 +4801,10 @@ packages: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} + piscina@5.2.0: + resolution: {integrity: sha512-DszUCKeVN/5G5QKo6jAVHL8fmKnkJvQ0ACiVgY7YGCq3TUB2oznAOayvZPIAdEThvhczkXR+qm3IHsNXpFCYfA==} + engines: {node: '>=20.x'} + pkce-challenge@5.0.1: resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} engines: {node: '>=16.20.0'} @@ -4585,6 +4889,9 @@ packages: pure-rand@6.1.0: resolution: {integrity: sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==} + pure-rand@8.4.1: + resolution: {integrity: sha512-c58R2+SPFcSIPXoU834QN/KPDDOSd8sXcSrqf6e83Me6Rrp1EYkxukkjXMVrKvKaADs1SOyNkWdfvLf6zY8qLQ==} + qs@6.15.1: resolution: {integrity: sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==} engines: {node: '>=0.6'} @@ -4671,6 +4978,14 @@ packages: resolution: {integrity: sha512-YTUo+Flmw4ZXiWfQKGcwwc11KnoRAYgzAE2E7mXKCjSviTKShtxBsN6YUUBB2gtaBzKzeKunxhUwNHQuRryhWA==} engines: {node: '>= 4'} + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + reflect.getprototypeof@1.0.10: resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} engines: {node: '>= 0.4'} @@ -4880,6 +5195,9 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + statuses@2.0.2: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} @@ -5053,6 +5371,10 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + toml@4.1.2: + resolution: {integrity: sha512-m0vXfHODcw3gk+KONAOlVQ5yNHc3yS3B1ybM3HS1vqDoS0RWTDDVBVVTYi8hH0k+2OM1vmo9fb1WX9EVqjqfHA==} + engines: {node: '>=20'} + tough-cookie@6.0.1: resolution: {integrity: sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==} engines: {node: '>=16'} @@ -5152,6 +5474,10 @@ packages: resolution: {integrity: sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==} engines: {node: '>=20.18.1'} + undici@8.6.0: + resolution: {integrity: sha512-l2FlC6I510GawyEd1qgcE/okihKrzy+BRTEBlu6T0fdbM9m5yxtIH5Oa3ysRsH0zC4EhmWUEaSDsy2QngBeRlw==} + engines: {node: '>=22.19.0'} + unicorn-magic@0.3.0: resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} engines: {node: '>=18'} @@ -5225,6 +5551,10 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + uuid@14.0.1: + resolution: {integrity: sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==} + hasBin: true + validate-npm-package-name@7.0.2: resolution: {integrity: sha512-hVDIBwsRruT73PbK7uP5ebUt+ezEtCmzZz3F59BSr2F6OVFnJ/6h8liuvdLrQ88Xmnk6/+xGGuq+pG9WwTuy3A==} engines: {node: ^20.17.0 || >=22.9.0} @@ -5391,6 +5721,18 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + wsl-utils@0.3.1: resolution: {integrity: sha512-g/eziiSUNBSsdDJtCLB8bdYEUMj4jR7AGeUo96p/3dTafgjHhpF4RiCFPiRILwjQoDXx5MqkBr4fwWtR3Ky4Wg==} engines: {node: '>=20'} @@ -5413,6 +5755,11 @@ packages: yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + yargs-parser@21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} @@ -5747,6 +6094,26 @@ snapshots: '@effect/language-service@0.85.1': {} + '@effect/platform-node-shared@4.0.0-beta.93(effect@4.0.0-beta.93)': + dependencies: + '@types/ws': 8.18.1 + effect: 4.0.0-beta.93 + ws: 8.21.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@effect/platform-node@4.0.0-beta.93(effect@4.0.0-beta.93)(ioredis@5.11.1)': + dependencies: + '@effect/platform-node-shared': 4.0.0-beta.93(effect@4.0.0-beta.93) + effect: 4.0.0-beta.93 + ioredis: 5.11.1 + mime: 4.1.0 + undici: 8.6.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + '@effect/platform@0.96.1(effect@3.21.2)': dependencies: effect: 3.21.2 @@ -6213,6 +6580,8 @@ snapshots: optionalDependencies: '@types/node': 20.19.39 + '@ioredis/commands@1.10.0': {} + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -6257,21 +6626,39 @@ snapshots: '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': optional: true + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': + optional: true + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': optional: true + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': + optional: true + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': optional: true + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': + optional: true + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': optional: true + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': + optional: true + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': optional: true + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': + optional: true + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': optional: true + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': + optional: true + '@mswjs/interceptors@0.41.8': dependencies: '@open-draft/deferred-promise': 2.2.0 @@ -6284,36 +6671,69 @@ snapshots: '@napi-rs/canvas-android-arm64@0.1.100': optional: true + '@napi-rs/canvas-android-arm64@1.0.2': + optional: true + '@napi-rs/canvas-darwin-arm64@0.1.100': optional: true + '@napi-rs/canvas-darwin-arm64@1.0.2': + optional: true + '@napi-rs/canvas-darwin-x64@0.1.100': optional: true + '@napi-rs/canvas-darwin-x64@1.0.2': + optional: true + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.100': optional: true + '@napi-rs/canvas-linux-arm-gnueabihf@1.0.2': + optional: true + '@napi-rs/canvas-linux-arm64-gnu@0.1.100': optional: true + '@napi-rs/canvas-linux-arm64-gnu@1.0.2': + optional: true + '@napi-rs/canvas-linux-arm64-musl@0.1.100': optional: true + '@napi-rs/canvas-linux-arm64-musl@1.0.2': + optional: true + '@napi-rs/canvas-linux-riscv64-gnu@0.1.100': optional: true + '@napi-rs/canvas-linux-riscv64-gnu@1.0.2': + optional: true + '@napi-rs/canvas-linux-x64-gnu@0.1.100': optional: true + '@napi-rs/canvas-linux-x64-gnu@1.0.2': + optional: true + '@napi-rs/canvas-linux-x64-musl@0.1.100': optional: true + '@napi-rs/canvas-linux-x64-musl@1.0.2': + optional: true + '@napi-rs/canvas-win32-arm64-msvc@0.1.100': optional: true + '@napi-rs/canvas-win32-arm64-msvc@1.0.2': + optional: true + '@napi-rs/canvas-win32-x64-msvc@0.1.100': optional: true + '@napi-rs/canvas-win32-x64-msvc@1.0.2': + optional: true + '@napi-rs/canvas@0.1.100': optionalDependencies: '@napi-rs/canvas-android-arm64': 0.1.100 @@ -6329,6 +6749,92 @@ snapshots: '@napi-rs/canvas-win32-x64-msvc': 0.1.100 optional: true + '@napi-rs/canvas@1.0.2': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 1.0.2 + '@napi-rs/canvas-darwin-arm64': 1.0.2 + '@napi-rs/canvas-darwin-x64': 1.0.2 + '@napi-rs/canvas-linux-arm-gnueabihf': 1.0.2 + '@napi-rs/canvas-linux-arm64-gnu': 1.0.2 + '@napi-rs/canvas-linux-arm64-musl': 1.0.2 + '@napi-rs/canvas-linux-riscv64-gnu': 1.0.2 + '@napi-rs/canvas-linux-x64-gnu': 1.0.2 + '@napi-rs/canvas-linux-x64-musl': 1.0.2 + '@napi-rs/canvas-win32-arm64-msvc': 1.0.2 + '@napi-rs/canvas-win32-x64-msvc': 1.0.2 + + '@napi-rs/nice-android-arm-eabi@1.1.1': + optional: true + + '@napi-rs/nice-android-arm64@1.1.1': + optional: true + + '@napi-rs/nice-darwin-arm64@1.1.1': + optional: true + + '@napi-rs/nice-darwin-x64@1.1.1': + optional: true + + '@napi-rs/nice-freebsd-x64@1.1.1': + optional: true + + '@napi-rs/nice-linux-arm-gnueabihf@1.1.1': + optional: true + + '@napi-rs/nice-linux-arm64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-arm64-musl@1.1.1': + optional: true + + '@napi-rs/nice-linux-ppc64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-riscv64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-s390x-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-x64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-x64-musl@1.1.1': + optional: true + + '@napi-rs/nice-openharmony-arm64@1.1.1': + optional: true + + '@napi-rs/nice-win32-arm64-msvc@1.1.1': + optional: true + + '@napi-rs/nice-win32-ia32-msvc@1.1.1': + optional: true + + '@napi-rs/nice-win32-x64-msvc@1.1.1': + optional: true + + '@napi-rs/nice@1.1.1': + optionalDependencies: + '@napi-rs/nice-android-arm-eabi': 1.1.1 + '@napi-rs/nice-android-arm64': 1.1.1 + '@napi-rs/nice-darwin-arm64': 1.1.1 + '@napi-rs/nice-darwin-x64': 1.1.1 + '@napi-rs/nice-freebsd-x64': 1.1.1 + '@napi-rs/nice-linux-arm-gnueabihf': 1.1.1 + '@napi-rs/nice-linux-arm64-gnu': 1.1.1 + '@napi-rs/nice-linux-arm64-musl': 1.1.1 + '@napi-rs/nice-linux-ppc64-gnu': 1.1.1 + '@napi-rs/nice-linux-riscv64-gnu': 1.1.1 + '@napi-rs/nice-linux-s390x-gnu': 1.1.1 + '@napi-rs/nice-linux-x64-gnu': 1.1.1 + '@napi-rs/nice-linux-x64-musl': 1.1.1 + '@napi-rs/nice-openharmony-arm64': 1.1.1 + '@napi-rs/nice-win32-arm64-msvc': 1.1.1 + '@napi-rs/nice-win32-ia32-msvc': 1.1.1 + '@napi-rs/nice-win32-x64-msvc': 1.1.1 + optional: true + '@napi-rs/wasm-runtime@0.2.12': dependencies: '@emnapi/core': 1.10.0 @@ -6397,12 +6903,21 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} - '@ontos-ai/knowhere-sdk@2.0.0': + '@ontos-ai/knowhere-sdk@file:../knowhere-node-sdk(ioredis@5.11.1)': dependencies: - axios: 1.16.0 + '@effect/platform-node': 4.0.0-beta.93(effect@4.0.0-beta.93)(ioredis@5.11.1) + '@napi-rs/canvas': 1.0.2 + axios: 1.18.1 + effect: 4.0.0-beta.93 jszip: 3.10.1 + pdfjs-dist: 6.1.200 + piscina: 5.2.0 transitivePeerDependencies: + - bufferutil - debug + - ioredis + - supports-color + - utf-8-validate '@open-draft/deferred-promise@2.2.0': {} @@ -7065,6 +7580,10 @@ snapshots: '@types/validate-npm-package-name@4.0.2': {} + '@types/ws@8.18.1': + dependencies: + '@types/node': 20.19.39 + '@typescript-eslint/eslint-plugin@8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -7238,10 +7757,10 @@ snapshots: '@vercel/oidc@3.2.0': {} - '@vitejs/plugin-react@6.0.1(babel-plugin-react-compiler@1.0.0)(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0))': + '@vitejs/plugin-react@6.0.1(babel-plugin-react-compiler@1.0.0)(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0))': dependencies: '@rolldown/pluginutils': 1.0.0-rc.7 - vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0) + vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0) optionalDependencies: babel-plugin-react-compiler: 1.0.0 @@ -7254,14 +7773,14 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.5(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0))': + '@vitest/mocker@4.1.5(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: msw: 2.14.3(@types/node@20.19.39)(typescript@6.0.3) - vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0) + vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0) '@vitest/pretty-format@4.1.5': dependencies: @@ -7300,6 +7819,12 @@ snapshots: acorn@8.16.0: {} + agent-base@6.0.2: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + agent-base@7.1.4: {} ai@6.0.175(zod@4.4.3): @@ -7443,13 +7968,15 @@ snapshots: axe-core@4.11.4: {} - axios@1.16.0: + axios@1.18.1: dependencies: follow-redirects: 1.16.0 - form-data: 4.0.5 + form-data: 4.0.6 + https-proxy-agent: 5.0.1 proxy-from-env: 2.1.0 transitivePeerDependencies: - debug + - supports-color axobject-query@4.1.0: {} @@ -7578,6 +8105,8 @@ snapshots: clsx@2.1.1: {} + cluster-key-slot@1.1.1: {} + code-block-writer@13.0.3: {} color-convert@2.0.1: @@ -7721,6 +8250,8 @@ snapshots: delayed-stream@1.0.0: {} + denque@2.1.0: {} + depd@2.0.0: {} dequal@2.0.3: {} @@ -7790,6 +8321,19 @@ snapshots: '@standard-schema/spec': 1.1.0 fast-check: 3.23.2 + effect@4.0.0-beta.93: + dependencies: + '@standard-schema/spec': 1.1.0 + fast-check: 4.8.0 + find-my-way-ts: 0.1.6 + ini: 7.0.0 + kubernetes-types: 1.30.0 + msgpackr: 2.0.4 + multipasta: 0.2.7 + toml: 4.1.2 + uuid: 14.0.1 + yaml: 2.9.0 + electron-to-chromium@1.5.351: {} emoji-regex@10.6.0: {} @@ -8305,6 +8849,10 @@ snapshots: dependencies: pure-rand: 6.1.0 + fast-check@4.8.0: + dependencies: + pure-rand: 8.4.1 + fast-deep-equal@3.1.3: {} fast-glob@3.3.1: @@ -8397,12 +8945,12 @@ snapshots: dependencies: is-callable: 1.2.7 - form-data@4.0.5: + form-data@4.0.6: dependencies: asynckit: 0.4.0 combined-stream: 1.0.8 es-set-tostringtag: 2.1.0 - hasown: 2.0.3 + hasown: 2.0.4 mime-types: 2.1.35 formdata-polyfill@4.0.10: @@ -8539,6 +9087,10 @@ snapshots: dependencies: function-bind: 1.1.2 + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + hast-util-to-jsx-runtime@2.3.6: dependencies: '@types/estree': 1.0.8 @@ -8592,6 +9144,13 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 + https-proxy-agent@5.0.1: + dependencies: + agent-base: 6.0.2 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.4 @@ -8622,6 +9181,8 @@ snapshots: inherits@2.0.4: {} + ini@7.0.0: {} + inline-style-parser@0.2.7: {} internal-slot@1.1.0: @@ -8630,6 +9191,18 @@ snapshots: hasown: 2.0.3 side-channel: 1.1.0 + ioredis@5.11.1: + dependencies: + '@ioredis/commands': 1.10.0 + cluster-key-slot: 1.1.1 + debug: 4.4.3 + denque: 2.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + ip-address@10.1.0: {} ipaddr.js@1.9.1: {} @@ -8911,6 +9484,8 @@ snapshots: kleur@4.1.5: {} + kubernetes-types@1.30.0: {} + language-subtag-registry@0.3.23: {} language-tags@1.0.9: @@ -9412,6 +9987,8 @@ snapshots: dependencies: mime-db: 1.54.0 + mime@4.1.0: {} + mimic-fn@2.1.0: {} mimic-function@5.0.1: {} @@ -9440,10 +10017,26 @@ snapshots: '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 optional: true + msgpackr-extract@3.0.4: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.4 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.4 + optional: true + msgpackr@1.11.12: optionalDependencies: msgpackr-extract: 3.0.3 + msgpackr@2.0.4: + optionalDependencies: + msgpackr-extract: 3.0.4 + msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3): dependencies: '@inquirer/confirm': 6.0.12(@types/node@20.19.39) @@ -9709,12 +10302,20 @@ snapshots: optionalDependencies: '@napi-rs/canvas': 0.1.100 + pdfjs-dist@6.1.200: + optionalDependencies: + '@napi-rs/canvas': 1.0.2 + picocolors@1.1.1: {} picomatch@2.3.2: {} picomatch@4.0.4: {} + piscina@5.2.0: + optionalDependencies: + '@napi-rs/nice': 1.1.1 + pkce-challenge@5.0.1: {} playwright-core@1.58.2: {} @@ -9799,6 +10400,8 @@ snapshots: pure-rand@6.1.0: {} + pure-rand@8.4.1: {} + qs@6.15.1: dependencies: side-channel: 1.1.0 @@ -9905,6 +10508,12 @@ snapshots: tiny-invariant: 1.3.3 tslib: 2.8.1 + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + reflect.getprototypeof@1.0.10: dependencies: call-bind: 1.0.9 @@ -10251,6 +10860,8 @@ snapshots: stackback@0.0.2: {} + standard-as-callback@2.1.0: {} + statuses@2.0.2: {} std-env@4.1.0: {} @@ -10423,6 +11034,8 @@ snapshots: toidentifier@1.0.1: {} + toml@4.1.2: {} + tough-cookie@6.0.1: dependencies: tldts: 7.0.30 @@ -10543,6 +11156,8 @@ snapshots: undici@7.25.0: {} + undici@8.6.0: {} + unicorn-magic@0.3.0: {} unified@11.0.5: @@ -10639,6 +11254,8 @@ snapshots: util-deprecate@1.0.2: {} + uuid@14.0.1: {} + validate-npm-package-name@7.0.2: {} vary@1.1.2: {} @@ -10653,7 +11270,7 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0): + vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -10666,11 +11283,12 @@ snapshots: fsevents: 2.3.3 jiti: 2.7.0 tsx: 4.21.0 + yaml: 2.9.0 - vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@20.19.39)(jsdom@29.1.1(@noble/hashes@1.8.0))(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)): + vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@20.19.39)(jsdom@29.1.1(@noble/hashes@1.8.0))(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.5 - '@vitest/mocker': 4.1.5(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)) + '@vitest/mocker': 4.1.5(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.5 '@vitest/runner': 4.1.5 '@vitest/snapshot': 4.1.5 @@ -10687,7 +11305,7 @@ snapshots: tinyexec: 1.1.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0) + vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.0 @@ -10784,6 +11402,8 @@ snapshots: wrappy@1.0.2: {} + ws@8.21.0: {} + wsl-utils@0.3.1: dependencies: is-wsl: 3.1.1 @@ -10799,6 +11419,8 @@ snapshots: yallist@3.1.1: {} + yaml@2.9.0: {} + yargs-parser@21.1.1: {} yargs@17.7.2: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index d989c76..a49c9d9 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,5 +1,9 @@ ignoredBuiltDependencies: - sharp - unrs-resolver + minimumReleaseAgeExclude: - '@ontos-ai/knowhere-sdk@2.0.0' + +overrides: + '@ontos-ai/knowhere-sdk': file:../knowhere-node-sdk diff --git a/scripts/prepare-effect.sh b/scripts/prepare-effect.sh new file mode 100755 index 0000000..24a9d31 --- /dev/null +++ b/scripts/prepare-effect.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +set -eu + +repo_dir=".repos/effect" +repo_url="https://github.com/Effect-TS/effect-smol" + +if [ -d "$repo_dir/.git" ]; then + exit 0 +fi + +mkdir -p ".repos" +git clone "$repo_url" "$repo_dir" diff --git a/src/agent-harness/ledger.ts b/src/agent-harness/ledger.ts index da197e8..103f9c0 100644 --- a/src/agent-harness/ledger.ts +++ b/src/agent-harness/ledger.ts @@ -68,10 +68,14 @@ export function createEvidenceLedger() { chunk: { ref: `r${retrievalIndex}:referenced:${index + 1}`, kind: "referenced_chunk", + chunkId: chunk.chunkId, content, contentPreview: content, chunkType: chunk.chunkType, score: null, + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath, + metadata: chunk.metadata, source: { documentId: chunk.documentId, sourceFileName: null, @@ -146,6 +150,10 @@ function addChunkFromResult(input: { contentPreview: buildContentPreview(input.result.content), chunkType: input.result.chunkType, score: input.result.score, + chunkId: input.result.chunkId, + sourceChunkPath: input.result.sourceChunkPath, + filePath: input.result.filePath, + metadata: input.result.metadata, source: { documentId: input.result.source.documentId, sourceFileName: input.result.source.sourceFileName, diff --git a/src/agent-harness/types.ts b/src/agent-harness/types.ts index 1a1cce9..c559623 100644 --- a/src/agent-harness/types.ts +++ b/src/agent-harness/types.ts @@ -94,10 +94,14 @@ export type RetrievalCapability = { export type EvidenceChunk = { readonly ref: string readonly kind: "result" | "referenced_chunk" + readonly chunkId?: string readonly content: string readonly contentPreview: string readonly chunkType: string readonly score: number | null + readonly sourceChunkPath?: string | null + readonly filePath?: string | null + readonly metadata?: Readonly> readonly source: { readonly documentId?: string | null readonly sourceFileName?: string | null diff --git a/src/components/chat-message-list.test.ts b/src/components/chat-message-list.test.ts index 8d58630..f94478e 100644 --- a/src/components/chat-message-list.test.ts +++ b/src/components/chat-message-list.test.ts @@ -129,6 +129,56 @@ describe("ChatMessageList", () => { ); }); + it("renders a separate page image link without replacing source focus", async () => { + const user = userEvent.setup(); + const onCitationClick = vi.fn(); + + render( + React.createElement(ChatMessageList, { + messages: [ + { + id: "assistant_1", + role: "assistant", + content: "The referenced page discusses revenue.", + citations: [ + { + chunkType: "page", + score: 0.9, + pageCitationAssetUrl: + "https://blob.example/pages/page-000004.png", + source: { + documentId: "doc_1", + sourceFileName: "report.pdf", + sectionPath: "Page 4", + }, + }, + ], + }, + ], + onCitationClick, + }), + ); + + const citationButton = screen.getByRole("button", { + name: "Open source report.pdf", + }); + const pageImageLink = screen.getByRole("link", { + name: "Open page image for report.pdf", + }); + + expect(pageImageLink.getAttribute("href")).toBe( + "https://blob.example/pages/page-000004.png", + ); + + await user.click(citationButton); + expect(onCitationClick).toHaveBeenCalledWith( + expect.objectContaining({ + pageCitationAssetUrl: "https://blob.example/pages/page-000004.png", + }), + "assistant_1:0", + ); + }); + it("removes description-only source labels without changing other markdown whitespace", () => { render( React.createElement(ChatMessageList, { diff --git a/src/components/chat-message-list.tsx b/src/components/chat-message-list.tsx index e5e6460..531c71f 100644 --- a/src/components/chat-message-list.tsx +++ b/src/components/chat-message-list.tsx @@ -2,7 +2,7 @@ import { type CSSProperties, type ReactElement } from "react"; import { type VirtualItem } from "@tanstack/react-virtual"; -import { ImageIcon, MessageCircle } from "lucide-react"; +import { ExternalLink, ImageIcon, MessageCircle } from "lucide-react"; import ReactMarkdown, { defaultUrlTransform, type Components, @@ -633,28 +633,60 @@ function CitationChip({ citationId: string, ) => void; }): ReactElement { + const pageCitationAssetUrl = getTrimmedCitationField( + citation.pageCitationAssetUrl, + ); + return ( - - - + + - {label} - - - - {tooltipLabel} - - + {tooltipLabel} + + + {pageCitationAssetUrl ? ( + + + + + + + + Open page image + + + ) : null} + ); } diff --git a/src/domains/chat/chat-citation-persistence.ts b/src/domains/chat/chat-citation-persistence.ts index bf2a1a1..54e89de 100644 --- a/src/domains/chat/chat-citation-persistence.ts +++ b/src/domains/chat/chat-citation-persistence.ts @@ -86,6 +86,7 @@ function toCitationView( chunkType: citation.chunkType, score: citation.score, assetUrl: citation.assetUrl, + pageCitationAssetUrl: citation.pageCitationAssetUrl, description: "description" in citation ? citation.description : undefined, source: { documentId: citation.source.documentId, diff --git a/src/domains/chat/citations.test.ts b/src/domains/chat/citations.test.ts index ce681ec..b6daa0a 100644 --- a/src/domains/chat/citations.test.ts +++ b/src/domains/chat/citations.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest" -import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import { toChatCitationViews } from "./citations" +import type { PageCitationAssetRetrievalResult } from "./page-citation-assets" describe("toChatCitationViews", () => { it("extracts first citation description for each generated source label", () => { @@ -31,11 +31,24 @@ describe("toChatCitationViews", () => { { ...secondResult, description: "margin expansion" }, ]) }) + + it("preserves page citation asset URLs", () => { + const result = makeRetrievalResult({ + chunkType: "page", + pageCitationAssetUrl: "https://blob.example/pages/page-000004.png", + }) + + const citations = toChatCitationViews([result], "Grounded answer.") + + expect(citations[0]?.pageCitationAssetUrl).toBe( + "https://blob.example/pages/page-000004.png", + ) + }) }) function makeRetrievalResult( - overrides: Partial = {}, -): RetrievalResult { + overrides: Partial = {}, +): PageCitationAssetRetrievalResult { return { content: "Grounding content", chunkType: "text", diff --git a/src/domains/chat/citations.ts b/src/domains/chat/citations.ts index e010bdc..b6b4038 100644 --- a/src/domains/chat/citations.ts +++ b/src/domains/chat/citations.ts @@ -2,9 +2,10 @@ import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" import type { ChatCitationView } from "@/domains/chat/types" +import type { PageCitationAssetRetrievalResult } from "./page-citation-assets" export function toChatCitationViews( - results: readonly RetrievalResult[], + results: readonly PageCitationAssetRetrievalResult[], answer: string, ): ChatCitationView[] { const descriptionsBySourceNumber = getCitationDescriptions(answer) @@ -16,6 +17,9 @@ export function toChatCitationViews( chunkType: result.chunkType, score: result.score, ...(result.assetUrl ? { assetUrl: result.assetUrl } : {}), + ...(result.pageCitationAssetUrl + ? { pageCitationAssetUrl: result.pageCitationAssetUrl } + : {}), ...(description ? { description } : {}), source: { documentId: result.source.documentId, diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index 2d7245d..b0a95d9 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -38,6 +38,7 @@ import { enrichRetrievalResultsWithAssetUrls, removeRetrievedMediaAssetUrls, } from "./media-assets" +import { enrichRetrievalResultsWithPageCitationAssetUrls } from "./page-citation-assets" const DEFAULT_TOP_K = 8 const MAX_AGENTIC_TOP_K = 12 @@ -262,7 +263,12 @@ export const answerQuestionWithRetrieval = ( hardenedArtifacts: hardenedMedia.artifacts, }), }) - const citationResults = hardenedMedia.results + const citationResults = yield* Effect.tryPromise(() => + enrichRetrievalResultsWithPageCitationAssetUrls({ + results: hardenedMedia.results, + sources: input.sources, + }), + ) const displayArtifacts = hardenedMedia.artifacts ?? [] logger.info("chat-agent: answer complete", { answerLength: answer.length, @@ -792,17 +798,7 @@ function mapManifestCitationsToResults( resolveChunkForAssetRef(citation.ref, assetsByRef, chunksByRef) if (!chunk) continue - const retrievalResult: RetrievalResult = { - content: chunk.content, - chunkType: chunk.chunkType, - score: chunk.score, - ...(chunk.assetUrl ? { assetUrl: chunk.assetUrl } : {}), - source: { - documentId: chunk.source.documentId ?? undefined, - sourceFileName: chunk.source.sourceFileName ?? undefined, - sectionPath: chunk.source.sectionPath ?? undefined, - }, - } + const retrievalResult = toRetrievalResultFromEvidenceChunk(chunk) const key = getRetrievalResultKey(retrievalResult) if (seenKeys.has(key)) continue @@ -889,10 +885,14 @@ function toRetrievalResultFromEvidenceChunk( chunk: EvidenceChunk, ): RetrievalResult { return { + ...(chunk.chunkId ? { chunkId: chunk.chunkId } : {}), content: chunk.content, chunkType: chunk.chunkType, score: chunk.score, ...(chunk.assetUrl ? { assetUrl: chunk.assetUrl } : {}), + ...(chunk.sourceChunkPath ? { sourceChunkPath: chunk.sourceChunkPath } : {}), + ...(chunk.filePath ? { filePath: chunk.filePath } : {}), + ...(chunk.metadata ? { metadata: chunk.metadata } : {}), source: { documentId: chunk.source.documentId ?? undefined, sourceFileName: chunk.source.sourceFileName ?? undefined, @@ -921,10 +921,14 @@ function collectRetrievalResults( for (const result of [ ...response.results, ...response.referencedChunks.map((chunk): RetrievalResult => ({ + chunkId: chunk.chunkId, content: "", chunkType: chunk.chunkType, score: null, ...(chunk.assetUrl ? { assetUrl: chunk.assetUrl } : {}), + ...(chunk.sourceChunkPath ? { sourceChunkPath: chunk.sourceChunkPath } : {}), + ...(chunk.filePath ? { filePath: chunk.filePath } : {}), + ...(chunk.metadata ? { metadata: chunk.metadata } : {}), source: { documentId: chunk.documentId, sourceFileName: sourceTitlesByDocumentId.get(chunk.documentId), diff --git a/src/domains/chat/page-citation-assets.test.ts b/src/domains/chat/page-citation-assets.test.ts new file mode 100644 index 0000000..c76b385 --- /dev/null +++ b/src/domains/chat/page-citation-assets.test.ts @@ -0,0 +1,216 @@ +import { describe, expect, it } from "vitest" +import type { + KnowhereSdkStorage, + KnowhereSdkStorageHead, + KnowhereSdkStorageObject, + KnowhereSdkStorageReadResult, + KnowhereSdkStorageWriteResult, + RetrievalResult, +} from "@ontos-ai/knowhere-sdk" + +import type { Source } from "@/infrastructure/db/schema" +import { enrichRetrievalResultsWithPageCitationAssetUrls } from "./page-citation-assets" + +class MemorySdkStorage implements KnowhereSdkStorage { + private readonly objects = new Map() + + constructor(objects: Readonly>) { + for (const [key, value] of Object.entries(objects)) { + this.objects.set(key, new TextEncoder().encode(value)) + } + } + + headObject(): Promise { + return Promise.resolve(null) + } + + writeObject( + input: KnowhereSdkStorageObject, + ): Promise { + if (input.body instanceof Uint8Array) { + this.objects.set(input.key, new Uint8Array(input.body)) + } + return Promise.resolve({ key: input.key }) + } + + readObject(key: string): Promise { + const body = this.objects.get(key) + return Promise.resolve(body ? { body } : null) + } + + getObjectUrl(key: string): Promise { + return Promise.resolve(`https://blob.example/${key}`) + } +} + +describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { + it("adds a page citation asset URL for page results with matching metadata", async () => { + const storage = new MemorySdkStorage({ + "page-citation-assets/documents/doc_1/current.json": JSON.stringify({ + version: 1, + documentId: "doc_1", + jobId: "job_1", + variant: "default", + indexKey: + "page-citation-assets/documents/doc_1/jobs/job_1/variants/default/index.json", + updatedAt: "2026-07-03T00:00:00.000Z", + }), + "page-citation-assets/documents/doc_1/jobs/job_1/variants/default/index.json": + JSON.stringify({ + version: 1, + documentId: "doc_1", + jobId: "job_1", + variant: "default", + generatedAt: "2026-07-03T00:00:00.000Z", + assets: [ + { + pageNum: 4, + key: "page-citation-assets/documents/doc_1/jobs/job_1/variants/default/scale-1/page-000004.png", + assetUrl: "https://blob.example/pages/page-000004.png", + mimeType: "image/png", + width: 1200, + height: 1600, + source: "client-rendered-pdf-page", + variant: "default", + }, + ], + }), + }) + + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "page", + metadata: { pageNums: [4] }, + }), + ], + sources: [makeSource()], + storage, + }) + + expect(result?.pageCitationAssetUrl).toBe( + "https://blob.example/pages/page-000004.png", + ) + }) + + it("uses SDK-enriched page asset metadata before reading the index", async () => { + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "page", + metadata: { + pageAssets: [ + { + pageNum: 2, + key: "page-key", + assetUrl: "https://blob.example/pages/page-000002.png", + }, + ], + }, + }), + ], + sources: [makeSource()], + storage: new MemorySdkStorage({}), + }) + + expect(result?.pageCitationAssetUrl).toBe( + "https://blob.example/pages/page-000002.png", + ) + }) + + it("chooses the SDK-enriched asset matching the citation page metadata", async () => { + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "page", + metadata: { + pageNums: [4], + pageAssets: [ + { + pageNum: 2, + key: "page-2-key", + assetUrl: "https://blob.example/pages/page-000002.png", + }, + { + pageNum: 4, + key: "page-4-key", + assetUrl: "https://blob.example/pages/page-000004.png", + }, + ], + }, + }), + ], + sources: [makeSource()], + storage: new MemorySdkStorage({}), + }) + + expect(result?.pageCitationAssetUrl).toBe( + "https://blob.example/pages/page-000004.png", + ) + }) + + it("leaves non-page results unchanged even when they have page metadata", async () => { + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "text", + metadata: { + pageNums: [4], + pageAssets: [ + { + pageNum: 4, + key: "page-key", + assetUrl: "https://blob.example/pages/page-000004.png", + }, + ], + }, + }), + ], + sources: [makeSource()], + storage: new MemorySdkStorage({}), + }) + + expect(result?.pageCitationAssetUrl).toBeUndefined() + }) +}) + +function makeRetrievalResult( + overrides: Partial = {}, +): RetrievalResult { + return { + chunkId: "chunk_page_4", + content: "Page four summary", + chunkType: "page", + score: 0.8, + metadata: { pageNums: [4] }, + source: { + documentId: "doc_1", + sourceFileName: "source.pdf", + sectionPath: "Page 4", + }, + ...overrides, + } +} + +function makeSource(overrides: Partial = {}): Source { + return { + id: "source_1", + workspaceId: "workspace_1", + title: "source.pdf", + mimeType: "application/pdf", + sizeBytes: 1, + status: "ready", + failureReason: null, + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-07-03T00:00:00.000Z"), + updatedAt: new Date("2026-07-03T00:00:00.000Z"), + deletedAt: null, + ...overrides, + } +} diff --git a/src/domains/chat/page-citation-assets.ts b/src/domains/chat/page-citation-assets.ts new file mode 100644 index 0000000..dd30a8d --- /dev/null +++ b/src/domains/chat/page-citation-assets.ts @@ -0,0 +1,381 @@ +import "server-only" + +import { createHash } from "node:crypto" +import type { + KnowhereSdkStorage, + PageCitationAsset, + PageCitationAssetCurrentIndex, + PageCitationAssetIndex, + RetrievalResult, +} from "@ontos-ai/knowhere-sdk" + +import type { Source } from "@/infrastructure/db/schema" +import { createVercelBlobKnowhereSdkStorage } from "@/integrations/knowhere-sdk-storage" + +export type PageCitationAssetRetrievalResult = RetrievalResult & { + readonly pageCitationAssetUrl?: string +} + +type EnrichRetrievalResultsWithPageCitationAssetUrlsInput = { + readonly results: readonly RetrievalResult[] + readonly sources: readonly Source[] + readonly storage?: KnowhereSdkStorage +} + +type PageCitationAssetCandidate = { + readonly pageNum: number + readonly assetUrl?: string + readonly key?: string +} + +const defaultVariant = "default" + +export async function enrichRetrievalResultsWithPageCitationAssetUrls({ + results, + sources, + storage = createVercelBlobKnowhereSdkStorage(), +}: EnrichRetrievalResultsWithPageCitationAssetUrlsInput): Promise< + PageCitationAssetRetrievalResult[] +> { + if (results.length === 0) return [] + + const sourcesByDocumentId = new Map( + sources.flatMap((source): readonly [string, Source][] => + source.knowhereDocumentId ? [[source.knowhereDocumentId, source]] : [], + ), + ) + const indexByKey = new Map>() + + return Promise.all( + results.map((result) => + enrichRetrievalResultWithPageCitationAssetUrl({ + result, + storage, + source: result.source.documentId + ? sourcesByDocumentId.get(result.source.documentId) + : undefined, + indexByKey, + }), + ), + ) +} + +async function enrichRetrievalResultWithPageCitationAssetUrl(input: { + readonly result: RetrievalResult + readonly source?: Source + readonly storage: KnowhereSdkStorage + readonly indexByKey: Map> +}): Promise { + if (!isPageResult(input.result)) return input.result + + const pageNumbers = getPageNumbers(input.result.metadata) + const directAsset = getDirectPageCitationAsset(input.result, pageNumbers) + if (directAsset?.assetUrl) { + return { + ...input.result, + pageCitationAssetUrl: directAsset.assetUrl, + } + } + + if (pageNumbers.length === 0) return input.result + + const documentId = getTrimmedString(input.result.source.documentId) + if (!documentId) return input.result + + const index = await resolvePageCitationAssetIndex({ + storage: input.storage, + documentId, + jobId: getResultJobId(input.result, input.source), + indexByKey: input.indexByKey, + }) + if (!index) return input.result + + const asset = index.assets.find((candidate) => + pageNumbers.includes(candidate.pageNum), + ) + if (!asset) return input.result + + const assetUrl = + getTrimmedString(asset.assetUrl) ?? + (asset.key && input.storage.getObjectUrl + ? await input.storage.getObjectUrl(asset.key).catch(() => null) + : null) + if (!assetUrl) return input.result + + return { + ...input.result, + pageCitationAssetUrl: assetUrl, + } +} + +function isPageResult(result: RetrievalResult): boolean { + return result.chunkType.toLowerCase() === "page" +} + +function getDirectPageCitationAsset( + result: RetrievalResult, + pageNumbers: readonly number[], +): PageCitationAssetCandidate | null { + const candidates = [ + isRecord(result) ? result.pageAssets : undefined, + result.metadata?.pageAssets, + ].flatMap(parsePageCitationAssetCandidates) + + if (pageNumbers.length > 0) { + const matchingCandidates = candidates.filter((candidate) => + pageNumbers.includes(candidate.pageNum), + ) + const matchingAssetWithUrl = matchingCandidates.find( + (candidate) => candidate.assetUrl, + ) + if (matchingAssetWithUrl) return matchingAssetWithUrl + if (matchingCandidates[0]) return matchingCandidates[0] + } + + return candidates.find((candidate) => candidate.assetUrl) ?? candidates[0] ?? null +} + +function parsePageCitationAssetCandidates( + value: unknown, +): readonly PageCitationAssetCandidate[] { + if (!Array.isArray(value)) return [] + + return value.flatMap((item): PageCitationAssetCandidate[] => { + if (!isRecord(item)) return [] + const pageNum = getPositiveInteger(item.pageNum) + if (!pageNum) return [] + + return [ + { + pageNum, + assetUrl: getTrimmedString(item.assetUrl) ?? undefined, + key: getTrimmedString(item.key) ?? undefined, + }, + ] + }) +} + +async function resolvePageCitationAssetIndex(input: { + readonly storage: KnowhereSdkStorage + readonly documentId: string + readonly jobId: string | null + readonly indexByKey: Map> +}): Promise { + if (!input.storage.readObject) return null + + if (input.jobId) { + const jobIndexKey = createPageCitationAssetIndexKey( + input.documentId, + input.jobId, + ) + const jobIndex = await readCachedIndex( + input.storage, + jobIndexKey, + input.indexByKey, + ) + if (jobIndex) return jobIndex + } + + const current = await readCurrentIndexPointer(input.storage, input.documentId) + if (!current) return null + + return readCachedIndex(input.storage, current.indexKey, input.indexByKey) +} + +async function readCachedIndex( + storage: KnowhereSdkStorage, + key: string, + cache: Map>, +): Promise { + let cached = cache.get(key) + if (!cached) { + cached = readPageCitationAssetIndex(storage, key) + cache.set(key, cached) + } + return cached +} + +async function readCurrentIndexPointer( + storage: KnowhereSdkStorage, + documentId: string, +): Promise { + const key = createPageCitationAssetCurrentIndexKey(documentId) + const parsed = await readJsonObject(storage, key) + if (!isRecord(parsed)) return null + + const indexKey = getTrimmedString(parsed.indexKey) + const jobId = getTrimmedString(parsed.jobId) + const variant = getTrimmedString(parsed.variant) + if (!indexKey || !jobId || !variant) return null + + return { + version: 1, + documentId, + jobId, + variant, + indexKey, + namespace: getTrimmedString(parsed.namespace) ?? undefined, + jobResultId: getTrimmedString(parsed.jobResultId) ?? undefined, + updatedAt: getTrimmedString(parsed.updatedAt) ?? "", + } +} + +async function readPageCitationAssetIndex( + storage: KnowhereSdkStorage, + key: string, +): Promise { + const parsed = await readJsonObject(storage, key) + if (!isRecord(parsed) || !Array.isArray(parsed.assets)) return null + + const assets = parsed.assets.flatMap((asset): PageCitationAsset[] => { + if (!isRecord(asset)) return [] + const pageNum = getPositiveInteger(asset.pageNum) + const keyValue = getTrimmedString(asset.key) + const mimeType = getPageCitationMimeType(asset.mimeType) + const width = getPositiveInteger(asset.width) + const height = getPositiveInteger(asset.height) + if (!pageNum || !keyValue || !mimeType || !width || !height) return [] + + return [ + { + pageNum, + key: keyValue, + assetUrl: getTrimmedString(asset.assetUrl) ?? undefined, + mimeType, + width, + height, + source: "client-rendered-pdf-page", + variant: getTrimmedString(asset.variant) ?? defaultVariant, + }, + ] + }) + if (assets.length === 0) return null + + return { + version: 1, + documentId: getTrimmedString(parsed.documentId) ?? "", + jobId: getTrimmedString(parsed.jobId) ?? "", + variant: getTrimmedString(parsed.variant) ?? defaultVariant, + generatedAt: getTrimmedString(parsed.generatedAt) ?? "", + namespace: getTrimmedString(parsed.namespace) ?? undefined, + jobResultId: getTrimmedString(parsed.jobResultId) ?? undefined, + assets, + } +} + +async function readJsonObject( + storage: KnowhereSdkStorage, + key: string, +): Promise { + if (!storage.readObject) return null + + const object = await storage.readObject(key).catch(() => null) + if (!object) return null + + try { + const parsed: unknown = JSON.parse(new TextDecoder().decode(object.body)) + return parsed + } catch { + return null + } +} + +function getResultJobId( + result: RetrievalResult, + source: Source | undefined, +): string | null { + const metadata = result.metadata + return ( + getRecordString(metadata, "jobId") ?? + getRecordString(metadata, "job_id") ?? + getTrimmedString(source?.knowhereJobId) ?? + null + ) +} + +function getPageNumbers( + metadata: Readonly> | undefined, +): readonly number[] { + if (!metadata) return [] + + const values = [metadata.pageNums, metadata.page_nums, metadata.pageNum] + const pageNumbers = new Set() + + for (const value of values) { + if (Array.isArray(value)) { + for (const item of value) { + const pageNum = getPositiveInteger(item) + if (pageNum) pageNumbers.add(pageNum) + } + continue + } + + const pageNum = getPositiveInteger(value) + if (pageNum) pageNumbers.add(pageNum) + } + + return [...pageNumbers].sort((left, right) => left - right) +} + +function createPageCitationAssetIndexKey( + documentId: string, + jobId: string, +): string { + return [ + "page-citation-assets", + "documents", + toSafeKeySegment(documentId), + "jobs", + toSafeKeySegment(jobId), + "variants", + defaultVariant, + "index.json", + ].join("/") +} + +function createPageCitationAssetCurrentIndexKey(documentId: string): string { + return [ + "page-citation-assets", + "documents", + toSafeKeySegment(documentId), + "current.json", + ].join("/") +} + +function toSafeKeySegment(value: string): string { + const normalized = value.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^_+|_+$/g, "") + if (normalized.length > 0 && normalized === value) return normalized + + const hash = createHash("sha256").update(value).digest("hex").slice(0, 12) + return `${(normalized || "value").slice(0, 48)}-${hash}` +} + +function getPageCitationMimeType(value: unknown): PageCitationAsset["mimeType"] | null { + return value === "image/png" || value === "image/jpeg" ? value : null +} + +function getRecordString( + record: Readonly> | undefined, + key: string, +): string | null { + if (!record) return null + return getTrimmedString(record[key]) +} + +function getTrimmedString(value: unknown): string | null { + if (typeof value !== "string") return null + const trimmed = value.trim() + return trimmed.length > 0 ? trimmed : null +} + +function getPositiveInteger(value: unknown): number | null { + return typeof value === "number" && + Number.isSafeInteger(value) && + value > 0 + ? value + : null +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null +} diff --git a/src/domains/chat/types.ts b/src/domains/chat/types.ts index f3f987d..a0469e0 100644 --- a/src/domains/chat/types.ts +++ b/src/domains/chat/types.ts @@ -1,12 +1,12 @@ /** * Chat citation / retrieval hit. Mirrors RetrievalResult from the SDK. - * No chunkId here; retrieval does not expose one. */ export type RetrievalResultView = { readonly content: string readonly chunkType: string readonly score: number | null readonly assetUrl?: string + readonly pageCitationAssetUrl?: string readonly source: { readonly documentId?: string | null readonly sourceFileName?: string | null diff --git a/src/domains/chat/view.ts b/src/domains/chat/view.ts index 9afc4f6..ed39295 100644 --- a/src/domains/chat/view.ts +++ b/src/domains/chat/view.ts @@ -51,6 +51,7 @@ function toPersistedCitationViews(value: unknown): ChatCitationView[] | undefine chunkType: getString(item.chunkType) ?? "text", score: getNumber(item.score) ?? 0, assetUrl: getString(item.assetUrl), + pageCitationAssetUrl: getString(item.pageCitationAssetUrl), description: getString(item.description), source: { documentId: getString(item.source.documentId), @@ -81,6 +82,9 @@ function toPersistedArtifactViews(value: unknown): ChatArtifactView[] | undefine chunkType: getString(item.citation.chunkType) ?? "text", score: getNumber(item.citation.score) ?? 0, assetUrl: getString(item.citation.assetUrl), + pageCitationAssetUrl: getString( + item.citation.pageCitationAssetUrl, + ), description: getString(item.citation.description), source: { documentId: getString(item.citation.source.documentId), diff --git a/src/domains/sources/page-citation-assets.test.ts b/src/domains/sources/page-citation-assets.test.ts new file mode 100644 index 0000000..d748929 --- /dev/null +++ b/src/domains/sources/page-citation-assets.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it, vi } from "vitest" +import type { + KnowhereSdkStorage, + KnowhereSdkStorageHead, + KnowhereSdkStorageObject, + KnowhereSdkStorageReadResult, + KnowhereSdkStorageWriteResult, + PageRenderer, + RenderedPage, +} from "@ontos-ai/knowhere-sdk" + +import { prepareSourcePageCitationAssets } from "./page-citation-assets" + +class MemoryStorage implements KnowhereSdkStorage { + headObject(): Promise { + return Promise.resolve(null) + } + + writeObject( + input: KnowhereSdkStorageObject, + ): Promise { + return Promise.resolve({ key: input.key }) + } + + readObject(): Promise { + return Promise.resolve(null) + } +} + +describe("prepareSourcePageCitationAssets", () => { + it("calls SDK cacheJobResult with tolerant bounded page citation options", async () => { + const storage = new MemoryStorage() + const renderer: PageRenderer & { close: () => Promise } = { + renderPage(): Promise { + return Promise.resolve({ + body: new Uint8Array([1]), + mimeType: "image/png", + width: 1, + height: 1, + }) + }, + close: vi.fn(async () => undefined), + } + const cacheJobResult = vi.fn(async () => ({ + pageCitationAssetWarnings: [ + { + code: renderLimitExceededCode, + message: "limit reached", + }, + ], + })) + + const result = await prepareSourcePageCitationAssets({ + client: { + knowledge: { + cacheJobResult, + }, + }, + sourceId: "source_1", + jobId: "job_1", + documentId: "doc_1", + storage, + renderer, + maxPagesToRenderPerRun: 3, + }) + + expect(cacheJobResult).toHaveBeenCalledWith({ + jobId: "job_1", + localDocumentId: "source_1", + pageCitationAssets: { + storage, + renderer, + strict: false, + maxPagesToRenderPerRun: 3, + }, + }) + expect(result.warnings).toHaveLength(1) + expect(renderer.close).toHaveBeenCalled() + }) +}) + +const renderLimitExceededCode = "render_limit_exceeded" as const diff --git a/src/domains/sources/page-citation-assets.ts b/src/domains/sources/page-citation-assets.ts new file mode 100644 index 0000000..7349bc4 --- /dev/null +++ b/src/domains/sources/page-citation-assets.ts @@ -0,0 +1,69 @@ +import "server-only" + +import { createPdfJsPageRenderer } from "@ontos-ai/knowhere-sdk/page-renderer-pdfjs" +import type { + KnowhereSdkStorage, + PageCitationAssetWarning, + PageCitationAssetsOptions, + PageRenderer, +} from "@ontos-ai/knowhere-sdk" + +import { createVercelBlobKnowhereSdkStorage } from "@/integrations/knowhere-sdk-storage" + +type SourcePageCitationAssetClient = { + readonly knowledge: { + readonly cacheJobResult: (input: { + readonly jobId: string + readonly localDocumentId: string + readonly pageCitationAssets: PageCitationAssetsOptions + }) => Promise<{ + readonly pageCitationAssetWarnings?: readonly PageCitationAssetWarning[] + }> + } +} + +export type PrepareSourcePageCitationAssetsInput = { + readonly client: SourcePageCitationAssetClient + readonly sourceId: string + readonly jobId: string + readonly documentId: string + readonly storage?: KnowhereSdkStorage + readonly renderer?: PageRenderer & { close?: () => Promise } + readonly maxPagesToRenderPerRun?: number +} + +export type PrepareSourcePageCitationAssetsResult = { + readonly warnings: readonly PageCitationAssetWarning[] +} + +const defaultMaxPagesToRenderPerRun = 25 + +export async function prepareSourcePageCitationAssets({ + client, + sourceId, + jobId, + storage = createVercelBlobKnowhereSdkStorage(), + renderer, + maxPagesToRenderPerRun = defaultMaxPagesToRenderPerRun, +}: PrepareSourcePageCitationAssetsInput): Promise { + const pageRenderer = renderer ?? createPdfJsPageRenderer({ maxThreads: 1 }) + + try { + const result = await client.knowledge.cacheJobResult({ + jobId, + localDocumentId: sourceId, + pageCitationAssets: { + storage, + renderer: pageRenderer, + strict: false, + maxPagesToRenderPerRun, + }, + }) + + return { + warnings: result.pageCitationAssetWarnings ?? [], + } + } finally { + await pageRenderer.close?.().catch(() => undefined) + } +} diff --git a/src/domains/sources/source-reconcile-route-workflow.test.ts b/src/domains/sources/source-reconcile-route-workflow.test.ts index 0841c54..a9e8a17 100644 --- a/src/domains/sources/source-reconcile-route-workflow.test.ts +++ b/src/domains/sources/source-reconcile-route-workflow.test.ts @@ -5,16 +5,23 @@ const mocks = vi.hoisted(() => ({ loggerInfo: vi.fn(), loggerWarn: vi.fn(), makeKnowhereClient: vi.fn(), + markSourceFailedAfterReconciliation: vi.fn(), markFailed: vi.fn(), markSourceReadyAfterReconciliation: vi.fn(), pollSourceReconciliation: vi.fn(), + prepareSourcePageCitationAssets: vi.fn(), })) vi.mock("@/domains/sources/source-reconcile-workflow", () => ({ + markSourceFailedAfterReconciliation: mocks.markSourceFailedAfterReconciliation, markSourceReadyAfterReconciliation: mocks.markSourceReadyAfterReconciliation, pollSourceReconciliation: mocks.pollSourceReconciliation, })) +vi.mock("@/domains/sources/page-citation-assets", () => ({ + prepareSourcePageCitationAssets: mocks.prepareSourcePageCitationAssets, +})) + vi.mock("@/domains/sources/workflow-runtime", () => ({ sourceWorkflowRuntime: { markFailed: mocks.markFailed, @@ -76,6 +83,16 @@ describe("sourceReconcileRouteWorkflow", () => { mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ status: "ready", }) + mocks.prepareSourcePageCitationAssets.mockResolvedValue({ + warnings: [ + { + code: "render_limit_exceeded", + message: "One page was skipped.", + documentId: "doc_1", + jobId: "job_1", + }, + ], + }) try { await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ @@ -95,9 +112,70 @@ describe("sourceReconcileRouteWorkflow", () => { sourceId: "source_1", documentId: "doc_1", }) + expect(mocks.prepareSourcePageCitationAssets).toHaveBeenCalledWith({ + client, + sourceId: "source_1", + jobId: "job_1", + documentId: "doc_1", + }) + expect( + mocks.prepareSourcePageCitationAssets.mock.invocationCallOrder[0], + ).toBeLessThan( + mocks.markSourceReadyAfterReconciliation.mock.invocationCallOrder[0]!, + ) + expect(mocks.loggerWarn).toHaveBeenCalledWith( + "workflow: page citation asset warning", + expect.objectContaining({ + sourceId: "source_1", + jobId: "job_1", + documentId: "doc_1", + }), + ) expect(continuations).toEqual([]) }) + it("marks the source failed when page citation asset preparation throws", async () => { + const context = createWorkflowContext() + const restore = + sourceReconcileRouteWorkflow.setContinuationTriggerForTesting( + async () => undefined, + ) + const client = { jobs: {}, knowledge: {} } + mocks.makeKnowhereClient.mockReturnValue(client) + mocks.pollSourceReconciliation.mockResolvedValue({ + kind: "ready-to-prepare", + jobId: "job_1", + documentId: "doc_1", + }) + mocks.prepareSourcePageCitationAssets.mockRejectedValue( + new Error("renderer setup failed"), + ) + mocks.markSourceFailedAfterReconciliation.mockResolvedValue({ + status: "failed", + }) + + try { + await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ + context, + payload: sourceReconcileRouteWorkflow.normalizeReconcilePayload({ + workspaceId: "workspace_1", + sourceId: "source_1", + apiKey: "jwt_1", + }), + }) + } finally { + restore() + } + + expect(mocks.markSourceFailedAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + reason: + "Page citation asset preparation failed: renderer setup failed", + }) + expect(mocks.markSourceReadyAfterReconciliation).not.toHaveBeenCalled() + }) + it("triggers a fresh poll run when Knowhere is still running after the segment budget", async () => { const context = createWorkflowContext() const continuations: ContinuationTriggerInput[] = [] diff --git a/src/domains/sources/source-reconcile-route-workflow.ts b/src/domains/sources/source-reconcile-route-workflow.ts index a1e732a..f195250 100644 --- a/src/domains/sources/source-reconcile-route-workflow.ts +++ b/src/domains/sources/source-reconcile-route-workflow.ts @@ -3,11 +3,13 @@ import "server-only" import { Client, type WorkflowContext } from "@upstash/workflow" import { + markSourceFailedAfterReconciliation, markSourceReadyAfterReconciliation, pollSourceReconciliation, } from "@/domains/sources/source-reconcile-workflow" import { makeKnowhereClient } from "@/integrations/knowhere" import { logger } from "@/lib/logger" +import { prepareSourcePageCitationAssets } from "./page-citation-assets" import { sourceWorkflowRuntime } from "./workflow-runtime" type ReconcilePayload = { @@ -39,6 +41,19 @@ type ContinuationTriggerInput = { readonly workflowRunId: string } +type PageCitationAssetPreparationStepResult = + | { + readonly ok: true + readonly warnings: Awaited< + ReturnType + >["warnings"] + } + | { + readonly ok: false + readonly reason: string + readonly error: string + } + const maxPollAttempts = 25 const initialDelaySeconds = 3 const maxDelaySeconds = 30 @@ -118,6 +133,16 @@ async function runPollAndMirrorWorkflow(input: { return } + const preparation = await runPageCitationAssetPreparation({ + context, + workspaceId, + sourceId, + client, + jobId: jobToPrepare.jobId, + documentId: jobToPrepare.documentId, + }) + if (!preparation.ok) return + const ready = await context.run("source-ready", async () => markSourceReadyAfterReconciliation({ workspaceId, @@ -132,6 +157,79 @@ async function runPollAndMirrorWorkflow(input: { }) } +async function runPageCitationAssetPreparation(input: { + readonly context: ReconcileWorkflowContext + readonly workspaceId: string + readonly sourceId: string + readonly client: ReturnType + readonly jobId: string + readonly documentId: string +}): Promise<{ readonly ok: boolean }> { + const result = await input.context.run( + "prepare-page-citation-assets", + async (): Promise => { + try { + const preparation = await prepareSourcePageCitationAssets({ + client: input.client, + sourceId: input.sourceId, + jobId: input.jobId, + documentId: input.documentId, + }) + + return { + ok: true, + warnings: preparation.warnings, + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error) + return { + ok: false, + reason: `Page citation asset preparation failed: ${getSafeFailureReason( + errorMessage, + )}`, + error: errorMessage, + } + } + }, + ) + + if (!result.ok) { + const failed = await input.context.run( + "source-failed-page-citation-assets", + async () => + markSourceFailedAfterReconciliation({ + workspaceId: input.workspaceId, + sourceId: input.sourceId, + reason: result.reason, + }), + ) + logger.error("workflow: page citation asset preparation failed", { + sourceId: input.sourceId, + jobId: input.jobId, + documentId: input.documentId, + status: failed.status, + error: result.error, + }) + return { ok: false } + } + + for (const warning of result.warnings) { + logger.warn("workflow: page citation asset warning", { + sourceId: input.sourceId, + jobId: input.jobId, + documentId: input.documentId, + warning, + }) + } + logger.info("workflow: page citation assets prepared", { + sourceId: input.sourceId, + jobId: input.jobId, + documentId: input.documentId, + warningCount: result.warnings.length, + }) + return { ok: true } +} + function normalizeReconcilePayload( payload: ReconcilePayload, ): NormalizedReconcilePayload { diff --git a/src/domains/sources/source-reconcile-workflow.test.ts b/src/domains/sources/source-reconcile-workflow.test.ts index 3f0632e..f1907ef 100644 --- a/src/domains/sources/source-reconcile-workflow.test.ts +++ b/src/domains/sources/source-reconcile-workflow.test.ts @@ -3,6 +3,7 @@ import type { JobResult } from "@ontos-ai/knowhere-sdk" import type { Source, Workspace } from "@/infrastructure/db/schema" import { + markSourceFailedAfterReconciliation, markSourceReadyAfterReconciliation, pollSourceReconciliation, } from "./source-reconcile-workflow" @@ -197,3 +198,34 @@ describe("markSourceReadyAfterReconciliation", () => { ) }) }) + +describe("markSourceFailedAfterReconciliation", () => { + it("marks failed and cleans staged uploads after fatal preparation errors", async () => { + const source = makeSource({ + stagedBlobPathname: "source-uploads/upload_1/document.pdf", + }) + const repository = createRepository(source) + const blobStore = { + deleteStagedSourceBlob: vi.fn(async () => undefined), + } + + const result = await markSourceFailedAfterReconciliation({ + workspaceId: workspace.id, + sourceId: "source_1", + reason: "Page citation asset preparation failed.", + repository, + blobStore, + }) + + expect(result).toEqual({ status: "failed" }) + expect(repository.markFailed).toHaveBeenCalledWith( + workspace.id, + "source_1", + "Page citation asset preparation failed.", + "parsing", + ) + expect(blobStore.deleteStagedSourceBlob).toHaveBeenCalledWith( + "source-uploads/upload_1/document.pdf", + ) + }) +}) diff --git a/src/domains/sources/source-reconcile-workflow.ts b/src/domains/sources/source-reconcile-workflow.ts index 0355350..4571038 100644 --- a/src/domains/sources/source-reconcile-workflow.ts +++ b/src/domains/sources/source-reconcile-workflow.ts @@ -55,6 +55,14 @@ export type MarkSourceReadyAfterReconciliationInput = { readonly blobStore?: SourceReconcileWorkflowBlobStore } +export type MarkSourceFailedAfterReconciliationInput = { + readonly workspaceId: string + readonly sourceId: string + readonly reason: string + readonly repository?: SourceReconcileWorkflowRepository + readonly blobStore?: SourceReconcileWorkflowBlobStore +} + export type PollSourceReconciliationResult = | { readonly kind: "waiting" @@ -75,6 +83,10 @@ export type MarkSourceReadyAfterReconciliationResult = { readonly status: string } +export type MarkSourceFailedAfterReconciliationResult = { + readonly status: string +} + export async function pollSourceReconciliation({ workspaceId, sourceId, @@ -165,6 +177,27 @@ export async function markSourceReadyAfterReconciliation({ return { status: readySource.status } } +export async function markSourceFailedAfterReconciliation({ + workspaceId, + sourceId, + reason, + repository = sourceWorkflowRuntime, + blobStore = vercelBlobStore, +}: MarkSourceFailedAfterReconciliationInput): Promise { + const source = await repository.findInWorkspace(workspaceId, sourceId) + if (!source) return { status: "gone" } + if (source.status !== "parsing") return { status: source.status } + + await failSourceAndCleanup({ + workspaceId, + source, + reason, + repository, + blobStore, + }) + return { status: "failed" } +} + async function failSourceAndCleanup(input: { readonly workspaceId: string readonly source: Source diff --git a/src/domains/workspace/integration.test.ts b/src/domains/workspace/integration.test.ts index 797cdbf..9d27083 100644 --- a/src/domains/workspace/integration.test.ts +++ b/src/domains/workspace/integration.test.ts @@ -331,6 +331,7 @@ describeIfDb("workspace helpers — integration", () => { chunkType: "text", score: 0.91, assetUrl: "https://assets.example/doc.pdf", + pageCitationAssetUrl: "https://assets.example/page-4.png", description: "intro summary", source: { documentId: "doc_123", @@ -353,6 +354,7 @@ describeIfDb("workspace helpers — integration", () => { chunkType: "text", score: 0.91, assetUrl: "https://assets.example/doc.pdf", + pageCitationAssetUrl: "https://assets.example/page-4.png", description: "intro summary", source: { documentId: "doc_123", diff --git a/src/domains/workspace/persistence.test.ts b/src/domains/workspace/persistence.test.ts index 47534d1..1c9be69 100644 --- a/src/domains/workspace/persistence.test.ts +++ b/src/domains/workspace/persistence.test.ts @@ -207,6 +207,7 @@ describe("chatRepository", () => { chunkType: "text", score: 0.99, assetUrl: "https://assets.example/doc.pdf", + pageCitationAssetUrl: "https://assets.example/page-4.png", description: "intro", source: { documentId: "doc_1", @@ -225,6 +226,7 @@ describe("chatRepository", () => { chunkType: "text", score: 0.99, assetUrl: "https://assets.example/doc.pdf", + pageCitationAssetUrl: "https://assets.example/page-4.png", description: "intro", source: { documentId: "doc_1", diff --git a/src/integrations/knowhere-sdk-storage.test.ts b/src/integrations/knowhere-sdk-storage.test.ts new file mode 100644 index 0000000..c5a19b6 --- /dev/null +++ b/src/integrations/knowhere-sdk-storage.test.ts @@ -0,0 +1,320 @@ +import { BlobNotFoundError, type HeadBlobResult } from "@vercel/blob" +import { describe, expect, it } from "vitest" + +import { + createVercelBlobKnowhereSdkStorage, + type VercelBlobStore, +} from "./knowhere-sdk-storage" + +type StoredBlob = { + readonly pathname: string + readonly body: Uint8Array + readonly contentType?: string + readonly url: string +} + +class MemoryVercelBlobStore implements VercelBlobStore { + readonly objects = new Map() + + async put( + pathname: string, + body: Parameters[1], + options: Parameters[2], + ): Promise<{ readonly pathname: string; readonly url: string }> { + const url = `https://blob.example/${pathname}` + this.objects.set(pathname, { + pathname, + body: await readBody(body), + contentType: options.contentType, + url, + }) + return { pathname, url } + } + + async head(pathname: string): Promise { + const object = this.objects.get(pathname) + if (!object) throw new BlobNotFoundError() + + return { + size: object.body.byteLength, + uploadedAt: new Date("2026-07-03T00:00:00.000Z"), + pathname, + contentType: object.contentType ?? "application/octet-stream", + contentDisposition: "", + url: object.url, + downloadUrl: `${object.url}?download=1`, + cacheControl: "", + etag: `etag:${pathname}`, + } + } + + async get( + pathname: string, + ): Promise>> { + const object = this.objects.get(pathname) + if (!object) return null + + return { + statusCode: 200, + stream: toReadableStream(object.body), + headers: new Headers(), + blob: { + url: object.url, + downloadUrl: `${object.url}?download=1`, + pathname, + contentDisposition: "", + cacheControl: "", + uploadedAt: new Date("2026-07-03T00:00:00.000Z"), + etag: `etag:${pathname}`, + contentType: object.contentType ?? "application/octet-stream", + size: object.body.byteLength, + }, + } + } + + async list(input: { + readonly prefix: string + readonly limit: number + readonly cursor?: string + }): Promise>> { + const start = input.cursor ? Number(input.cursor) : 0 + const matches = [...this.objects.values()] + .filter((object) => object.pathname.startsWith(input.prefix)) + .sort((left, right) => left.pathname.localeCompare(right.pathname)) + const page = matches.slice(start, start + input.limit) + const nextIndex = start + page.length + const hasMore = nextIndex < matches.length + + return { + blobs: page.map((object) => ({ + url: object.url, + downloadUrl: `${object.url}?download=1`, + pathname: object.pathname, + size: object.body.byteLength, + uploadedAt: new Date("2026-07-03T00:00:00.000Z"), + etag: `etag:${object.pathname}`, + })), + hasMore, + ...(hasMore ? { cursor: String(nextIndex) } : {}), + } + } + + async del(pathname: string | readonly string[]): Promise { + const pathnames = typeof pathname === "string" ? [pathname] : [...pathname] + for (const item of pathnames) { + this.objects.delete(item) + } + } +} + +describe("createVercelBlobKnowhereSdkStorage", () => { + it("rejects unsafe storage keys", async () => { + const storage = createVercelBlobKnowhereSdkStorage( + new MemoryVercelBlobStore(), + ) + + for (const key of [ + "", + "/absolute/path.png", + "nested//empty.png", + "nested/../escape.png", + "nested/./dot.png", + "nested\\windows.png", + "nested/\0/null.png", + ]) { + await expect( + storage.writeObject({ + key, + body: new Uint8Array([1]), + contentType: "image/png", + }), + ).rejects.toThrow("Invalid Knowhere SDK storage key") + } + }) + + it("writes objects with metadata sidecars and returns blob URLs", async () => { + const blobStore = new MemoryVercelBlobStore() + const storage = createVercelBlobKnowhereSdkStorage(blobStore) + + const result = await storage.writeObject({ + key: "page-citation-assets/documents/doc_1/page.png", + body: new Uint8Array([1, 2, 3]), + contentType: "image/png", + metadata: { + width: "1200", + height: "1600", + mimeType: "image/png", + }, + }) + + expect(result).toEqual({ + key: "page-citation-assets/documents/doc_1/page.png", + url: "https://blob.example/page-citation-assets/documents/doc_1/page.png", + }) + expect( + blobStore.objects.has("page-citation-assets/documents/doc_1/page.png"), + ).toBe(true) + const metadataSidecar = blobStore.objects.get( + "page-citation-assets/documents/doc_1/page.png.metadata.json", + ) + expect(metadataSidecar).toBeDefined() + if (!metadataSidecar) throw new Error("Expected metadata sidecar.") + + expect(JSON.parse(decodeBody(metadataSidecar.body))).toEqual({ + metadata: { + width: "1200", + height: "1600", + mimeType: "image/png", + }, + }) + }) + + it("combines blob head data with sidecar metadata", async () => { + const blobStore = new MemoryVercelBlobStore() + const storage = createVercelBlobKnowhereSdkStorage(blobStore) + + await storage.writeObject({ + key: "page-citation-assets/documents/doc_1/page.png", + body: new Uint8Array([1, 2, 3]), + contentType: "image/png", + metadata: { + width: "1200", + height: "1600", + }, + }) + + await expect( + storage.headObject("page-citation-assets/documents/doc_1/page.png"), + ).resolves.toEqual({ + key: "page-citation-assets/documents/doc_1/page.png", + contentType: "image/png", + contentLength: 3, + metadata: { + width: "1200", + height: "1600", + }, + }) + }) + + it("reads objects, resolves URLs, deletes objects, and deletes prefixes", async () => { + const blobStore = new MemoryVercelBlobStore() + const storage = createVercelBlobKnowhereSdkStorage(blobStore) + + await storage.writeObject({ + key: "prefix/one.png", + body: new Uint8Array([1]), + contentType: "image/png", + }) + await storage.writeObject({ + key: "prefix/two.png", + body: new Uint8Array([2]), + contentType: "image/png", + }) + await storage.writeObject({ + key: "prefix-sibling/two.png", + body: new Uint8Array([4]), + contentType: "image/png", + }) + await storage.writeObject({ + key: "other/three.png", + body: new Uint8Array([3]), + contentType: "image/png", + }) + + const readObject = storage.readObject + const getObjectUrl = storage.getObjectUrl + const deleteObject = storage.deleteObject + const deletePrefix = storage.deletePrefix + expect(readObject).toBeDefined() + expect(getObjectUrl).toBeDefined() + expect(deleteObject).toBeDefined() + expect(deletePrefix).toBeDefined() + if (!readObject || !getObjectUrl || !deleteObject || !deletePrefix) { + throw new Error("Expected full storage adapter methods.") + } + + await expect(storage.readObject?.("prefix/one.png")).resolves.toMatchObject({ + body: new Uint8Array([1]), + contentType: "image/png", + }) + await expect(storage.getObjectUrl?.("prefix/one.png")).resolves.toBe( + "https://blob.example/prefix/one.png", + ) + + await storage.deleteObject?.("prefix/one.png") + expect(blobStore.objects.has("prefix/one.png")).toBe(false) + expect(blobStore.objects.has("prefix/one.png.metadata.json")).toBe(false) + + await storage.deletePrefix?.("prefix") + expect([...blobStore.objects.keys()].sort()).toEqual([ + "other/three.png", + "other/three.png.metadata.json", + "prefix-sibling/two.png", + "prefix-sibling/two.png.metadata.json", + ]) + }) + + it("accepts trailing slashes when deleting prefixes", async () => { + const blobStore = new MemoryVercelBlobStore() + const storage = createVercelBlobKnowhereSdkStorage(blobStore) + + await storage.writeObject({ + key: "prefix/one.png", + body: new Uint8Array([1]), + contentType: "image/png", + }) + + await storage.deletePrefix?.("prefix/") + + expect(blobStore.objects.size).toBe(0) + }) +}) + +async function readBody( + body: Parameters[1], +): Promise { + if (typeof body === "string") return new TextEncoder().encode(body) + if (body instanceof Uint8Array) return new Uint8Array(body) + if (body instanceof Blob) return new Uint8Array(await body.arrayBuffer()) + return readStream(body) +} + +async function readStream( + stream: ReadableStream, +): Promise { + const reader = stream.getReader() + const chunks: Uint8Array[] = [] + let totalLength = 0 + + try { + while (true) { + const result = await reader.read() + if (result.done) break + chunks.push(result.value) + totalLength += result.value.byteLength + } + } finally { + reader.releaseLock() + } + + const body = new Uint8Array(totalLength) + let offset = 0 + for (const chunk of chunks) { + body.set(chunk, offset) + offset += chunk.byteLength + } + return body +} + +function toReadableStream(body: Uint8Array): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(body) + controller.close() + }, + }) +} + +function decodeBody(body: Uint8Array): string { + return new TextDecoder().decode(body) +} diff --git a/src/integrations/knowhere-sdk-storage.ts b/src/integrations/knowhere-sdk-storage.ts new file mode 100644 index 0000000..387bc21 --- /dev/null +++ b/src/integrations/knowhere-sdk-storage.ts @@ -0,0 +1,294 @@ +import "server-only" + +import { Buffer } from "node:buffer" +import { + BlobNotFoundError, + del, + get, + head, + list, + put, + type GetBlobResult, + type HeadBlobResult, + type ListBlobResult, + type PutBlobResult, +} from "@vercel/blob" +import type { + KnowhereSdkStorage, + KnowhereSdkStorageBody, + KnowhereSdkStorageHead, + KnowhereSdkStorageObject, + KnowhereSdkStorageReadResult, + KnowhereSdkStorageWriteResult, +} from "@ontos-ai/knowhere-sdk" + +type VercelBlobBody = Buffer | Blob | ReadableStream + +export type VercelBlobStore = { + readonly put: ( + pathname: string, + body: VercelBlobBody | string, + options: { + readonly access: "public" + readonly addRandomSuffix: false + readonly allowOverwrite: true + readonly contentType?: string + }, + ) => Promise> + readonly head: (pathname: string) => Promise + readonly get: ( + pathname: string, + options: { readonly access: "public" }, + ) => Promise + readonly list: (options: { + readonly prefix: string + readonly limit: number + readonly cursor?: string + }) => Promise + readonly del: (pathname: string | readonly string[]) => Promise +} + +type StoredMetadataSidecar = { + readonly metadata?: Readonly> +} + +const metadataSuffix = ".metadata.json" +const deletePrefixPageSize = 1000 + +const vercelBlobStore: VercelBlobStore = { + put, + head, + get, + list, + del: (pathname) => + del(typeof pathname === "string" ? pathname : [...pathname]), +} + +class VercelBlobKnowhereSdkStorage implements KnowhereSdkStorage { + private readonly blobStore: VercelBlobStore + + constructor(blobStore: VercelBlobStore = vercelBlobStore) { + this.blobStore = blobStore + } + + async headObject(key: string): Promise { + validateBlobStorageKey(key) + + try { + const object = await this.blobStore.head(key) + const metadata = await this.readMetadata(key) + return { + key, + contentType: object.contentType, + contentLength: object.size, + metadata, + } + } catch (error) { + if (isBlobNotFoundError(error)) return null + throw error + } + } + + async writeObject( + input: KnowhereSdkStorageObject, + ): Promise { + validateBlobStorageKey(input.key) + + const object = await this.blobStore.put( + input.key, + toVercelBlobBody(input.body), + { + access: "public", + addRandomSuffix: false, + allowOverwrite: true, + contentType: input.contentType, + }, + ) + await this.writeMetadata(input.key, input.metadata ?? {}) + + return { + key: object.pathname, + url: object.url, + } + } + + async getObjectUrl(key: string): Promise { + validateBlobStorageKey(key) + + try { + const object = await this.blobStore.head(key) + return object.url + } catch (error) { + if (isBlobNotFoundError(error)) return null + throw error + } + } + + async readObject(key: string): Promise { + validateBlobStorageKey(key) + + const object = await this.blobStore.get(key, { access: "public" }) + if (!object || object.statusCode !== 200) return null + + return { + body: await readStream(object.stream), + contentType: object.blob.contentType, + metadata: await this.readMetadata(key), + } + } + + async deleteObject(key: string): Promise { + validateBlobStorageKey(key) + await this.deleteIgnoringMissing([key, getMetadataKey(key)]) + } + + async deletePrefix(prefix: string): Promise { + validateBlobStoragePrefix(prefix) + const blobPrefix = prefix.endsWith("/") ? prefix : `${prefix}/` + + let cursor: string | undefined + do { + const page = await this.blobStore.list({ + prefix: blobPrefix, + limit: deletePrefixPageSize, + ...(cursor ? { cursor } : {}), + }) + const pathnames = page.blobs.map((blob) => blob.pathname) + if (pathnames.length > 0) await this.deleteIgnoringMissing(pathnames) + cursor = page.cursor + if (page.hasMore && !cursor) break + } while (cursor) + } + + private async readMetadata( + key: string, + ): Promise>> { + const metadataObject = await this.blobStore.get(getMetadataKey(key), { + access: "public", + }) + if (!metadataObject || metadataObject.statusCode !== 200) return {} + + const body = await readStream(metadataObject.stream) + const parsed: unknown = JSON.parse(new TextDecoder().decode(body)) + return parseStoredMetadataSidecar(parsed).metadata ?? {} + } + + private async writeMetadata( + key: string, + metadata: Readonly>, + ): Promise { + const sidecar: StoredMetadataSidecar = { metadata } + await this.blobStore.put( + getMetadataKey(key), + JSON.stringify(sidecar, null, 2), + { + access: "public", + addRandomSuffix: false, + allowOverwrite: true, + contentType: "application/json", + }, + ) + } + + private async deleteIgnoringMissing( + pathnames: readonly string[], + ): Promise { + try { + await this.blobStore.del(pathnames) + } catch (error) { + if (!isBlobNotFoundError(error)) throw error + } + } +} + +export function createVercelBlobKnowhereSdkStorage( + blobStore?: VercelBlobStore, +): KnowhereSdkStorage { + return new VercelBlobKnowhereSdkStorage(blobStore) +} + +function validateBlobStorageKey(key: string): void { + if (!isSafeRelativePosixPath(key)) { + throw new Error(`Invalid Knowhere SDK storage key: ${key}`) + } +} + +function validateBlobStoragePrefix(prefix: string): void { + const normalizedPrefix = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix + if (!isSafeRelativePosixPath(normalizedPrefix)) { + throw new Error(`Invalid Knowhere SDK storage prefix: ${prefix}`) + } +} + +function isSafeRelativePosixPath(value: string): boolean { + if ( + value.length === 0 || + value.startsWith("/") || + value.includes("\\") || + value.includes("\0") + ) { + return false + } + + return value + .split("/") + .every((segment) => segment.length > 0 && segment !== "." && segment !== "..") +} + +function getMetadataKey(key: string): string { + return `${key}${metadataSuffix}` +} + +function toVercelBlobBody(body: KnowhereSdkStorageBody): VercelBlobBody { + if (body instanceof Uint8Array) return Buffer.from(body) + return body +} + +async function readStream( + stream: ReadableStream, +): Promise { + const reader = stream.getReader() + const chunks: Uint8Array[] = [] + let totalLength = 0 + + try { + while (true) { + const result = await reader.read() + if (result.done) break + chunks.push(result.value) + totalLength += result.value.byteLength + } + } finally { + reader.releaseLock() + } + + const body = new Uint8Array(totalLength) + let offset = 0 + for (const chunk of chunks) { + body.set(chunk, offset) + offset += chunk.byteLength + } + return body +} + +function parseStoredMetadataSidecar(value: unknown): StoredMetadataSidecar { + if (!isRecord(value)) return {} + const metadata = value.metadata + if (!isRecord(metadata)) return {} + + return { + metadata: Object.fromEntries( + Object.entries(metadata).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + ), + ), + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null +} + +function isBlobNotFoundError(error: unknown): boolean { + return error instanceof BlobNotFoundError +} From a44e5227651da7cf3b8c28b3287bd3de9a1a3a01 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Fri, 3 Jul 2026 17:40:37 +0800 Subject: [PATCH 02/12] Consume server page citation assets --- src/domains/chat/page-citation-assets.test.ts | 144 +++----- src/domains/chat/page-citation-assets.ts | 245 +------------- .../sources/page-citation-assets.test.ts | 82 ----- src/domains/sources/page-citation-assets.ts | 69 ---- .../source-reconcile-route-workflow.test.ts | 80 +---- .../source-reconcile-route-workflow.ts | 98 ------ src/integrations/knowhere-sdk-storage.test.ts | 320 ------------------ src/integrations/knowhere-sdk-storage.ts | 294 ---------------- 8 files changed, 55 insertions(+), 1277 deletions(-) delete mode 100644 src/domains/sources/page-citation-assets.test.ts delete mode 100644 src/domains/sources/page-citation-assets.ts delete mode 100644 src/integrations/knowhere-sdk-storage.test.ts delete mode 100644 src/integrations/knowhere-sdk-storage.ts diff --git a/src/domains/chat/page-citation-assets.test.ts b/src/domains/chat/page-citation-assets.test.ts index c76b385..f7a1812 100644 --- a/src/domains/chat/page-citation-assets.test.ts +++ b/src/domains/chat/page-citation-assets.test.ts @@ -1,124 +1,90 @@ import { describe, expect, it } from "vitest" -import type { - KnowhereSdkStorage, - KnowhereSdkStorageHead, - KnowhereSdkStorageObject, - KnowhereSdkStorageReadResult, - KnowhereSdkStorageWriteResult, - RetrievalResult, -} from "@ontos-ai/knowhere-sdk" +import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" import { enrichRetrievalResultsWithPageCitationAssetUrls } from "./page-citation-assets" -class MemorySdkStorage implements KnowhereSdkStorage { - private readonly objects = new Map() - - constructor(objects: Readonly>) { - for (const [key, value] of Object.entries(objects)) { - this.objects.set(key, new TextEncoder().encode(value)) - } - } - - headObject(): Promise { - return Promise.resolve(null) - } - - writeObject( - input: KnowhereSdkStorageObject, - ): Promise { - if (input.body instanceof Uint8Array) { - this.objects.set(input.key, new Uint8Array(input.body)) - } - return Promise.resolve({ key: input.key }) - } - - readObject(key: string): Promise { - const body = this.objects.get(key) - return Promise.resolve(body ? { body } : null) - } - - getObjectUrl(key: string): Promise { - return Promise.resolve(`https://blob.example/${key}`) - } -} - describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { - it("adds a page citation asset URL for page results with matching metadata", async () => { - const storage = new MemorySdkStorage({ - "page-citation-assets/documents/doc_1/current.json": JSON.stringify({ - version: 1, - documentId: "doc_1", - jobId: "job_1", - variant: "default", - indexKey: - "page-citation-assets/documents/doc_1/jobs/job_1/variants/default/index.json", - updatedAt: "2026-07-03T00:00:00.000Z", - }), - "page-citation-assets/documents/doc_1/jobs/job_1/variants/default/index.json": - JSON.stringify({ - version: 1, - documentId: "doc_1", - jobId: "job_1", - variant: "default", - generatedAt: "2026-07-03T00:00:00.000Z", - assets: [ - { - pageNum: 4, - key: "page-citation-assets/documents/doc_1/jobs/job_1/variants/default/scale-1/page-000004.png", - assetUrl: "https://blob.example/pages/page-000004.png", - mimeType: "image/png", - width: 1200, - height: 1600, - source: "client-rendered-pdf-page", - variant: "default", - }, - ], + it("uses server-provided page asset URLs from result metadata", async () => { + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "page", + metadata: { + pageNums: [2], + pageAssets: [ + { + pageNum: 2, + artifactRef: "page_citation_assets/page-2.png", + assetUrl: "https://assets.example/pages/page-2.png", + }, + ], + }, }), + ], + sources: [makeSource()], }) + expect(result?.pageCitationAssetUrl).toBe( + "https://assets.example/pages/page-2.png", + ) + }) + + it("uses server-provided top-level page assets", async () => { const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ results: [ makeRetrievalResult({ chunkType: "page", - metadata: { pageNums: [4] }, - }), + pageAssets: [ + { + pageNum: 4, + artifactRef: "page_citation_assets/page-4.png", + assetUrl: "https://assets.example/pages/page-4.png", + contentType: "image/png", + source: "knowhere-rendered-page-citation-source", + }, + ], + } as Partial), ], sources: [makeSource()], - storage, }) expect(result?.pageCitationAssetUrl).toBe( - "https://blob.example/pages/page-000004.png", + "https://assets.example/pages/page-4.png", ) }) - it("uses SDK-enriched page asset metadata before reading the index", async () => { + it("chooses the asset matching the citation page metadata", async () => { const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ results: [ makeRetrievalResult({ chunkType: "page", metadata: { + pageNums: [4], pageAssets: [ { pageNum: 2, - key: "page-key", - assetUrl: "https://blob.example/pages/page-000002.png", + artifactRef: "page_citation_assets/page-2.png", + assetUrl: "https://assets.example/pages/page-2.png", + }, + { + pageNum: 4, + artifactRef: "page_citation_assets/page-4.png", + assetUrl: "https://assets.example/pages/page-4.png", }, ], }, }), ], sources: [makeSource()], - storage: new MemorySdkStorage({}), }) expect(result?.pageCitationAssetUrl).toBe( - "https://blob.example/pages/page-000002.png", + "https://assets.example/pages/page-4.png", ) }) - it("chooses the SDK-enriched asset matching the citation page metadata", async () => { + it("does not invent a link when the server only provides an artifact ref", async () => { const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ results: [ makeRetrievalResult({ @@ -126,27 +92,18 @@ describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { metadata: { pageNums: [4], pageAssets: [ - { - pageNum: 2, - key: "page-2-key", - assetUrl: "https://blob.example/pages/page-000002.png", - }, { pageNum: 4, - key: "page-4-key", - assetUrl: "https://blob.example/pages/page-000004.png", + artifactRef: "page_citation_assets/page-4.png", }, ], }, }), ], sources: [makeSource()], - storage: new MemorySdkStorage({}), }) - expect(result?.pageCitationAssetUrl).toBe( - "https://blob.example/pages/page-000004.png", - ) + expect(result?.pageCitationAssetUrl).toBeUndefined() }) it("leaves non-page results unchanged even when they have page metadata", async () => { @@ -159,15 +116,14 @@ describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { pageAssets: [ { pageNum: 4, - key: "page-key", - assetUrl: "https://blob.example/pages/page-000004.png", + artifactRef: "page_citation_assets/page-4.png", + assetUrl: "https://assets.example/pages/page-4.png", }, ], }, }), ], sources: [makeSource()], - storage: new MemorySdkStorage({}), }) expect(result?.pageCitationAssetUrl).toBeUndefined() diff --git a/src/domains/chat/page-citation-assets.ts b/src/domains/chat/page-citation-assets.ts index dd30a8d..dccdd88 100644 --- a/src/domains/chat/page-citation-assets.ts +++ b/src/domains/chat/page-citation-assets.ts @@ -1,16 +1,8 @@ import "server-only" -import { createHash } from "node:crypto" -import type { - KnowhereSdkStorage, - PageCitationAsset, - PageCitationAssetCurrentIndex, - PageCitationAssetIndex, - RetrievalResult, -} from "@ontos-ai/knowhere-sdk" +import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" -import { createVercelBlobKnowhereSdkStorage } from "@/integrations/knowhere-sdk-storage" export type PageCitationAssetRetrievalResult = RetrievalResult & { readonly pageCitationAssetUrl?: string @@ -19,42 +11,26 @@ export type PageCitationAssetRetrievalResult = RetrievalResult & { type EnrichRetrievalResultsWithPageCitationAssetUrlsInput = { readonly results: readonly RetrievalResult[] readonly sources: readonly Source[] - readonly storage?: KnowhereSdkStorage } type PageCitationAssetCandidate = { readonly pageNum: number readonly assetUrl?: string - readonly key?: string } -const defaultVariant = "default" - export async function enrichRetrievalResultsWithPageCitationAssetUrls({ results, sources, - storage = createVercelBlobKnowhereSdkStorage(), }: EnrichRetrievalResultsWithPageCitationAssetUrlsInput): Promise< PageCitationAssetRetrievalResult[] > { + void sources if (results.length === 0) return [] - const sourcesByDocumentId = new Map( - sources.flatMap((source): readonly [string, Source][] => - source.knowhereDocumentId ? [[source.knowhereDocumentId, source]] : [], - ), - ) - const indexByKey = new Map>() - return Promise.all( results.map((result) => enrichRetrievalResultWithPageCitationAssetUrl({ result, - storage, - source: result.source.documentId - ? sourcesByDocumentId.get(result.source.documentId) - : undefined, - indexByKey, }), ), ) @@ -62,9 +38,6 @@ export async function enrichRetrievalResultsWithPageCitationAssetUrls({ async function enrichRetrievalResultWithPageCitationAssetUrl(input: { readonly result: RetrievalResult - readonly source?: Source - readonly storage: KnowhereSdkStorage - readonly indexByKey: Map> }): Promise { if (!isPageResult(input.result)) return input.result @@ -77,35 +50,7 @@ async function enrichRetrievalResultWithPageCitationAssetUrl(input: { } } - if (pageNumbers.length === 0) return input.result - - const documentId = getTrimmedString(input.result.source.documentId) - if (!documentId) return input.result - - const index = await resolvePageCitationAssetIndex({ - storage: input.storage, - documentId, - jobId: getResultJobId(input.result, input.source), - indexByKey: input.indexByKey, - }) - if (!index) return input.result - - const asset = index.assets.find((candidate) => - pageNumbers.includes(candidate.pageNum), - ) - if (!asset) return input.result - - const assetUrl = - getTrimmedString(asset.assetUrl) ?? - (asset.key && input.storage.getObjectUrl - ? await input.storage.getObjectUrl(asset.key).catch(() => null) - : null) - if (!assetUrl) return input.result - - return { - ...input.result, - pageCitationAssetUrl: assetUrl, - } + return input.result } function isPageResult(result: RetrievalResult): boolean { @@ -149,148 +94,9 @@ function parsePageCitationAssetCandidates( { pageNum, assetUrl: getTrimmedString(item.assetUrl) ?? undefined, - key: getTrimmedString(item.key) ?? undefined, - }, - ] - }) -} - -async function resolvePageCitationAssetIndex(input: { - readonly storage: KnowhereSdkStorage - readonly documentId: string - readonly jobId: string | null - readonly indexByKey: Map> -}): Promise { - if (!input.storage.readObject) return null - - if (input.jobId) { - const jobIndexKey = createPageCitationAssetIndexKey( - input.documentId, - input.jobId, - ) - const jobIndex = await readCachedIndex( - input.storage, - jobIndexKey, - input.indexByKey, - ) - if (jobIndex) return jobIndex - } - - const current = await readCurrentIndexPointer(input.storage, input.documentId) - if (!current) return null - - return readCachedIndex(input.storage, current.indexKey, input.indexByKey) -} - -async function readCachedIndex( - storage: KnowhereSdkStorage, - key: string, - cache: Map>, -): Promise { - let cached = cache.get(key) - if (!cached) { - cached = readPageCitationAssetIndex(storage, key) - cache.set(key, cached) - } - return cached -} - -async function readCurrentIndexPointer( - storage: KnowhereSdkStorage, - documentId: string, -): Promise { - const key = createPageCitationAssetCurrentIndexKey(documentId) - const parsed = await readJsonObject(storage, key) - if (!isRecord(parsed)) return null - - const indexKey = getTrimmedString(parsed.indexKey) - const jobId = getTrimmedString(parsed.jobId) - const variant = getTrimmedString(parsed.variant) - if (!indexKey || !jobId || !variant) return null - - return { - version: 1, - documentId, - jobId, - variant, - indexKey, - namespace: getTrimmedString(parsed.namespace) ?? undefined, - jobResultId: getTrimmedString(parsed.jobResultId) ?? undefined, - updatedAt: getTrimmedString(parsed.updatedAt) ?? "", - } -} - -async function readPageCitationAssetIndex( - storage: KnowhereSdkStorage, - key: string, -): Promise { - const parsed = await readJsonObject(storage, key) - if (!isRecord(parsed) || !Array.isArray(parsed.assets)) return null - - const assets = parsed.assets.flatMap((asset): PageCitationAsset[] => { - if (!isRecord(asset)) return [] - const pageNum = getPositiveInteger(asset.pageNum) - const keyValue = getTrimmedString(asset.key) - const mimeType = getPageCitationMimeType(asset.mimeType) - const width = getPositiveInteger(asset.width) - const height = getPositiveInteger(asset.height) - if (!pageNum || !keyValue || !mimeType || !width || !height) return [] - - return [ - { - pageNum, - key: keyValue, - assetUrl: getTrimmedString(asset.assetUrl) ?? undefined, - mimeType, - width, - height, - source: "client-rendered-pdf-page", - variant: getTrimmedString(asset.variant) ?? defaultVariant, }, ] }) - if (assets.length === 0) return null - - return { - version: 1, - documentId: getTrimmedString(parsed.documentId) ?? "", - jobId: getTrimmedString(parsed.jobId) ?? "", - variant: getTrimmedString(parsed.variant) ?? defaultVariant, - generatedAt: getTrimmedString(parsed.generatedAt) ?? "", - namespace: getTrimmedString(parsed.namespace) ?? undefined, - jobResultId: getTrimmedString(parsed.jobResultId) ?? undefined, - assets, - } -} - -async function readJsonObject( - storage: KnowhereSdkStorage, - key: string, -): Promise { - if (!storage.readObject) return null - - const object = await storage.readObject(key).catch(() => null) - if (!object) return null - - try { - const parsed: unknown = JSON.parse(new TextDecoder().decode(object.body)) - return parsed - } catch { - return null - } -} - -function getResultJobId( - result: RetrievalResult, - source: Source | undefined, -): string | null { - const metadata = result.metadata - return ( - getRecordString(metadata, "jobId") ?? - getRecordString(metadata, "job_id") ?? - getTrimmedString(source?.knowhereJobId) ?? - null - ) } function getPageNumbers( @@ -317,51 +123,6 @@ function getPageNumbers( return [...pageNumbers].sort((left, right) => left - right) } -function createPageCitationAssetIndexKey( - documentId: string, - jobId: string, -): string { - return [ - "page-citation-assets", - "documents", - toSafeKeySegment(documentId), - "jobs", - toSafeKeySegment(jobId), - "variants", - defaultVariant, - "index.json", - ].join("/") -} - -function createPageCitationAssetCurrentIndexKey(documentId: string): string { - return [ - "page-citation-assets", - "documents", - toSafeKeySegment(documentId), - "current.json", - ].join("/") -} - -function toSafeKeySegment(value: string): string { - const normalized = value.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^_+|_+$/g, "") - if (normalized.length > 0 && normalized === value) return normalized - - const hash = createHash("sha256").update(value).digest("hex").slice(0, 12) - return `${(normalized || "value").slice(0, 48)}-${hash}` -} - -function getPageCitationMimeType(value: unknown): PageCitationAsset["mimeType"] | null { - return value === "image/png" || value === "image/jpeg" ? value : null -} - -function getRecordString( - record: Readonly> | undefined, - key: string, -): string | null { - if (!record) return null - return getTrimmedString(record[key]) -} - function getTrimmedString(value: unknown): string | null { if (typeof value !== "string") return null const trimmed = value.trim() diff --git a/src/domains/sources/page-citation-assets.test.ts b/src/domains/sources/page-citation-assets.test.ts deleted file mode 100644 index d748929..0000000 --- a/src/domains/sources/page-citation-assets.test.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { describe, expect, it, vi } from "vitest" -import type { - KnowhereSdkStorage, - KnowhereSdkStorageHead, - KnowhereSdkStorageObject, - KnowhereSdkStorageReadResult, - KnowhereSdkStorageWriteResult, - PageRenderer, - RenderedPage, -} from "@ontos-ai/knowhere-sdk" - -import { prepareSourcePageCitationAssets } from "./page-citation-assets" - -class MemoryStorage implements KnowhereSdkStorage { - headObject(): Promise { - return Promise.resolve(null) - } - - writeObject( - input: KnowhereSdkStorageObject, - ): Promise { - return Promise.resolve({ key: input.key }) - } - - readObject(): Promise { - return Promise.resolve(null) - } -} - -describe("prepareSourcePageCitationAssets", () => { - it("calls SDK cacheJobResult with tolerant bounded page citation options", async () => { - const storage = new MemoryStorage() - const renderer: PageRenderer & { close: () => Promise } = { - renderPage(): Promise { - return Promise.resolve({ - body: new Uint8Array([1]), - mimeType: "image/png", - width: 1, - height: 1, - }) - }, - close: vi.fn(async () => undefined), - } - const cacheJobResult = vi.fn(async () => ({ - pageCitationAssetWarnings: [ - { - code: renderLimitExceededCode, - message: "limit reached", - }, - ], - })) - - const result = await prepareSourcePageCitationAssets({ - client: { - knowledge: { - cacheJobResult, - }, - }, - sourceId: "source_1", - jobId: "job_1", - documentId: "doc_1", - storage, - renderer, - maxPagesToRenderPerRun: 3, - }) - - expect(cacheJobResult).toHaveBeenCalledWith({ - jobId: "job_1", - localDocumentId: "source_1", - pageCitationAssets: { - storage, - renderer, - strict: false, - maxPagesToRenderPerRun: 3, - }, - }) - expect(result.warnings).toHaveLength(1) - expect(renderer.close).toHaveBeenCalled() - }) -}) - -const renderLimitExceededCode = "render_limit_exceeded" as const diff --git a/src/domains/sources/page-citation-assets.ts b/src/domains/sources/page-citation-assets.ts deleted file mode 100644 index 7349bc4..0000000 --- a/src/domains/sources/page-citation-assets.ts +++ /dev/null @@ -1,69 +0,0 @@ -import "server-only" - -import { createPdfJsPageRenderer } from "@ontos-ai/knowhere-sdk/page-renderer-pdfjs" -import type { - KnowhereSdkStorage, - PageCitationAssetWarning, - PageCitationAssetsOptions, - PageRenderer, -} from "@ontos-ai/knowhere-sdk" - -import { createVercelBlobKnowhereSdkStorage } from "@/integrations/knowhere-sdk-storage" - -type SourcePageCitationAssetClient = { - readonly knowledge: { - readonly cacheJobResult: (input: { - readonly jobId: string - readonly localDocumentId: string - readonly pageCitationAssets: PageCitationAssetsOptions - }) => Promise<{ - readonly pageCitationAssetWarnings?: readonly PageCitationAssetWarning[] - }> - } -} - -export type PrepareSourcePageCitationAssetsInput = { - readonly client: SourcePageCitationAssetClient - readonly sourceId: string - readonly jobId: string - readonly documentId: string - readonly storage?: KnowhereSdkStorage - readonly renderer?: PageRenderer & { close?: () => Promise } - readonly maxPagesToRenderPerRun?: number -} - -export type PrepareSourcePageCitationAssetsResult = { - readonly warnings: readonly PageCitationAssetWarning[] -} - -const defaultMaxPagesToRenderPerRun = 25 - -export async function prepareSourcePageCitationAssets({ - client, - sourceId, - jobId, - storage = createVercelBlobKnowhereSdkStorage(), - renderer, - maxPagesToRenderPerRun = defaultMaxPagesToRenderPerRun, -}: PrepareSourcePageCitationAssetsInput): Promise { - const pageRenderer = renderer ?? createPdfJsPageRenderer({ maxThreads: 1 }) - - try { - const result = await client.knowledge.cacheJobResult({ - jobId, - localDocumentId: sourceId, - pageCitationAssets: { - storage, - renderer: pageRenderer, - strict: false, - maxPagesToRenderPerRun, - }, - }) - - return { - warnings: result.pageCitationAssetWarnings ?? [], - } - } finally { - await pageRenderer.close?.().catch(() => undefined) - } -} diff --git a/src/domains/sources/source-reconcile-route-workflow.test.ts b/src/domains/sources/source-reconcile-route-workflow.test.ts index a9e8a17..14e9b03 100644 --- a/src/domains/sources/source-reconcile-route-workflow.test.ts +++ b/src/domains/sources/source-reconcile-route-workflow.test.ts @@ -5,23 +5,16 @@ const mocks = vi.hoisted(() => ({ loggerInfo: vi.fn(), loggerWarn: vi.fn(), makeKnowhereClient: vi.fn(), - markSourceFailedAfterReconciliation: vi.fn(), markFailed: vi.fn(), markSourceReadyAfterReconciliation: vi.fn(), pollSourceReconciliation: vi.fn(), - prepareSourcePageCitationAssets: vi.fn(), })) vi.mock("@/domains/sources/source-reconcile-workflow", () => ({ - markSourceFailedAfterReconciliation: mocks.markSourceFailedAfterReconciliation, markSourceReadyAfterReconciliation: mocks.markSourceReadyAfterReconciliation, pollSourceReconciliation: mocks.pollSourceReconciliation, })) -vi.mock("@/domains/sources/page-citation-assets", () => ({ - prepareSourcePageCitationAssets: mocks.prepareSourcePageCitationAssets, -})) - vi.mock("@/domains/sources/workflow-runtime", () => ({ sourceWorkflowRuntime: { markFailed: mocks.markFailed, @@ -83,16 +76,6 @@ describe("sourceReconcileRouteWorkflow", () => { mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ status: "ready", }) - mocks.prepareSourcePageCitationAssets.mockResolvedValue({ - warnings: [ - { - code: "render_limit_exceeded", - message: "One page was skipped.", - documentId: "doc_1", - jobId: "job_1", - }, - ], - }) try { await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ @@ -112,70 +95,11 @@ describe("sourceReconcileRouteWorkflow", () => { sourceId: "source_1", documentId: "doc_1", }) - expect(mocks.prepareSourcePageCitationAssets).toHaveBeenCalledWith({ - client, - sourceId: "source_1", - jobId: "job_1", - documentId: "doc_1", - }) - expect( - mocks.prepareSourcePageCitationAssets.mock.invocationCallOrder[0], - ).toBeLessThan( - mocks.markSourceReadyAfterReconciliation.mock.invocationCallOrder[0]!, - ) - expect(mocks.loggerWarn).toHaveBeenCalledWith( - "workflow: page citation asset warning", - expect.objectContaining({ - sourceId: "source_1", - jobId: "job_1", - documentId: "doc_1", - }), - ) + expect(client).toEqual({ jobs: {} }) + expect(mocks.loggerWarn).not.toHaveBeenCalled() expect(continuations).toEqual([]) }) - it("marks the source failed when page citation asset preparation throws", async () => { - const context = createWorkflowContext() - const restore = - sourceReconcileRouteWorkflow.setContinuationTriggerForTesting( - async () => undefined, - ) - const client = { jobs: {}, knowledge: {} } - mocks.makeKnowhereClient.mockReturnValue(client) - mocks.pollSourceReconciliation.mockResolvedValue({ - kind: "ready-to-prepare", - jobId: "job_1", - documentId: "doc_1", - }) - mocks.prepareSourcePageCitationAssets.mockRejectedValue( - new Error("renderer setup failed"), - ) - mocks.markSourceFailedAfterReconciliation.mockResolvedValue({ - status: "failed", - }) - - try { - await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ - context, - payload: sourceReconcileRouteWorkflow.normalizeReconcilePayload({ - workspaceId: "workspace_1", - sourceId: "source_1", - apiKey: "jwt_1", - }), - }) - } finally { - restore() - } - - expect(mocks.markSourceFailedAfterReconciliation).toHaveBeenCalledWith({ - workspaceId: "workspace_1", - sourceId: "source_1", - reason: - "Page citation asset preparation failed: renderer setup failed", - }) - expect(mocks.markSourceReadyAfterReconciliation).not.toHaveBeenCalled() - }) - it("triggers a fresh poll run when Knowhere is still running after the segment budget", async () => { const context = createWorkflowContext() const continuations: ContinuationTriggerInput[] = [] diff --git a/src/domains/sources/source-reconcile-route-workflow.ts b/src/domains/sources/source-reconcile-route-workflow.ts index f195250..a1e732a 100644 --- a/src/domains/sources/source-reconcile-route-workflow.ts +++ b/src/domains/sources/source-reconcile-route-workflow.ts @@ -3,13 +3,11 @@ import "server-only" import { Client, type WorkflowContext } from "@upstash/workflow" import { - markSourceFailedAfterReconciliation, markSourceReadyAfterReconciliation, pollSourceReconciliation, } from "@/domains/sources/source-reconcile-workflow" import { makeKnowhereClient } from "@/integrations/knowhere" import { logger } from "@/lib/logger" -import { prepareSourcePageCitationAssets } from "./page-citation-assets" import { sourceWorkflowRuntime } from "./workflow-runtime" type ReconcilePayload = { @@ -41,19 +39,6 @@ type ContinuationTriggerInput = { readonly workflowRunId: string } -type PageCitationAssetPreparationStepResult = - | { - readonly ok: true - readonly warnings: Awaited< - ReturnType - >["warnings"] - } - | { - readonly ok: false - readonly reason: string - readonly error: string - } - const maxPollAttempts = 25 const initialDelaySeconds = 3 const maxDelaySeconds = 30 @@ -133,16 +118,6 @@ async function runPollAndMirrorWorkflow(input: { return } - const preparation = await runPageCitationAssetPreparation({ - context, - workspaceId, - sourceId, - client, - jobId: jobToPrepare.jobId, - documentId: jobToPrepare.documentId, - }) - if (!preparation.ok) return - const ready = await context.run("source-ready", async () => markSourceReadyAfterReconciliation({ workspaceId, @@ -157,79 +132,6 @@ async function runPollAndMirrorWorkflow(input: { }) } -async function runPageCitationAssetPreparation(input: { - readonly context: ReconcileWorkflowContext - readonly workspaceId: string - readonly sourceId: string - readonly client: ReturnType - readonly jobId: string - readonly documentId: string -}): Promise<{ readonly ok: boolean }> { - const result = await input.context.run( - "prepare-page-citation-assets", - async (): Promise => { - try { - const preparation = await prepareSourcePageCitationAssets({ - client: input.client, - sourceId: input.sourceId, - jobId: input.jobId, - documentId: input.documentId, - }) - - return { - ok: true, - warnings: preparation.warnings, - } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error) - return { - ok: false, - reason: `Page citation asset preparation failed: ${getSafeFailureReason( - errorMessage, - )}`, - error: errorMessage, - } - } - }, - ) - - if (!result.ok) { - const failed = await input.context.run( - "source-failed-page-citation-assets", - async () => - markSourceFailedAfterReconciliation({ - workspaceId: input.workspaceId, - sourceId: input.sourceId, - reason: result.reason, - }), - ) - logger.error("workflow: page citation asset preparation failed", { - sourceId: input.sourceId, - jobId: input.jobId, - documentId: input.documentId, - status: failed.status, - error: result.error, - }) - return { ok: false } - } - - for (const warning of result.warnings) { - logger.warn("workflow: page citation asset warning", { - sourceId: input.sourceId, - jobId: input.jobId, - documentId: input.documentId, - warning, - }) - } - logger.info("workflow: page citation assets prepared", { - sourceId: input.sourceId, - jobId: input.jobId, - documentId: input.documentId, - warningCount: result.warnings.length, - }) - return { ok: true } -} - function normalizeReconcilePayload( payload: ReconcilePayload, ): NormalizedReconcilePayload { diff --git a/src/integrations/knowhere-sdk-storage.test.ts b/src/integrations/knowhere-sdk-storage.test.ts deleted file mode 100644 index c5a19b6..0000000 --- a/src/integrations/knowhere-sdk-storage.test.ts +++ /dev/null @@ -1,320 +0,0 @@ -import { BlobNotFoundError, type HeadBlobResult } from "@vercel/blob" -import { describe, expect, it } from "vitest" - -import { - createVercelBlobKnowhereSdkStorage, - type VercelBlobStore, -} from "./knowhere-sdk-storage" - -type StoredBlob = { - readonly pathname: string - readonly body: Uint8Array - readonly contentType?: string - readonly url: string -} - -class MemoryVercelBlobStore implements VercelBlobStore { - readonly objects = new Map() - - async put( - pathname: string, - body: Parameters[1], - options: Parameters[2], - ): Promise<{ readonly pathname: string; readonly url: string }> { - const url = `https://blob.example/${pathname}` - this.objects.set(pathname, { - pathname, - body: await readBody(body), - contentType: options.contentType, - url, - }) - return { pathname, url } - } - - async head(pathname: string): Promise { - const object = this.objects.get(pathname) - if (!object) throw new BlobNotFoundError() - - return { - size: object.body.byteLength, - uploadedAt: new Date("2026-07-03T00:00:00.000Z"), - pathname, - contentType: object.contentType ?? "application/octet-stream", - contentDisposition: "", - url: object.url, - downloadUrl: `${object.url}?download=1`, - cacheControl: "", - etag: `etag:${pathname}`, - } - } - - async get( - pathname: string, - ): Promise>> { - const object = this.objects.get(pathname) - if (!object) return null - - return { - statusCode: 200, - stream: toReadableStream(object.body), - headers: new Headers(), - blob: { - url: object.url, - downloadUrl: `${object.url}?download=1`, - pathname, - contentDisposition: "", - cacheControl: "", - uploadedAt: new Date("2026-07-03T00:00:00.000Z"), - etag: `etag:${pathname}`, - contentType: object.contentType ?? "application/octet-stream", - size: object.body.byteLength, - }, - } - } - - async list(input: { - readonly prefix: string - readonly limit: number - readonly cursor?: string - }): Promise>> { - const start = input.cursor ? Number(input.cursor) : 0 - const matches = [...this.objects.values()] - .filter((object) => object.pathname.startsWith(input.prefix)) - .sort((left, right) => left.pathname.localeCompare(right.pathname)) - const page = matches.slice(start, start + input.limit) - const nextIndex = start + page.length - const hasMore = nextIndex < matches.length - - return { - blobs: page.map((object) => ({ - url: object.url, - downloadUrl: `${object.url}?download=1`, - pathname: object.pathname, - size: object.body.byteLength, - uploadedAt: new Date("2026-07-03T00:00:00.000Z"), - etag: `etag:${object.pathname}`, - })), - hasMore, - ...(hasMore ? { cursor: String(nextIndex) } : {}), - } - } - - async del(pathname: string | readonly string[]): Promise { - const pathnames = typeof pathname === "string" ? [pathname] : [...pathname] - for (const item of pathnames) { - this.objects.delete(item) - } - } -} - -describe("createVercelBlobKnowhereSdkStorage", () => { - it("rejects unsafe storage keys", async () => { - const storage = createVercelBlobKnowhereSdkStorage( - new MemoryVercelBlobStore(), - ) - - for (const key of [ - "", - "/absolute/path.png", - "nested//empty.png", - "nested/../escape.png", - "nested/./dot.png", - "nested\\windows.png", - "nested/\0/null.png", - ]) { - await expect( - storage.writeObject({ - key, - body: new Uint8Array([1]), - contentType: "image/png", - }), - ).rejects.toThrow("Invalid Knowhere SDK storage key") - } - }) - - it("writes objects with metadata sidecars and returns blob URLs", async () => { - const blobStore = new MemoryVercelBlobStore() - const storage = createVercelBlobKnowhereSdkStorage(blobStore) - - const result = await storage.writeObject({ - key: "page-citation-assets/documents/doc_1/page.png", - body: new Uint8Array([1, 2, 3]), - contentType: "image/png", - metadata: { - width: "1200", - height: "1600", - mimeType: "image/png", - }, - }) - - expect(result).toEqual({ - key: "page-citation-assets/documents/doc_1/page.png", - url: "https://blob.example/page-citation-assets/documents/doc_1/page.png", - }) - expect( - blobStore.objects.has("page-citation-assets/documents/doc_1/page.png"), - ).toBe(true) - const metadataSidecar = blobStore.objects.get( - "page-citation-assets/documents/doc_1/page.png.metadata.json", - ) - expect(metadataSidecar).toBeDefined() - if (!metadataSidecar) throw new Error("Expected metadata sidecar.") - - expect(JSON.parse(decodeBody(metadataSidecar.body))).toEqual({ - metadata: { - width: "1200", - height: "1600", - mimeType: "image/png", - }, - }) - }) - - it("combines blob head data with sidecar metadata", async () => { - const blobStore = new MemoryVercelBlobStore() - const storage = createVercelBlobKnowhereSdkStorage(blobStore) - - await storage.writeObject({ - key: "page-citation-assets/documents/doc_1/page.png", - body: new Uint8Array([1, 2, 3]), - contentType: "image/png", - metadata: { - width: "1200", - height: "1600", - }, - }) - - await expect( - storage.headObject("page-citation-assets/documents/doc_1/page.png"), - ).resolves.toEqual({ - key: "page-citation-assets/documents/doc_1/page.png", - contentType: "image/png", - contentLength: 3, - metadata: { - width: "1200", - height: "1600", - }, - }) - }) - - it("reads objects, resolves URLs, deletes objects, and deletes prefixes", async () => { - const blobStore = new MemoryVercelBlobStore() - const storage = createVercelBlobKnowhereSdkStorage(blobStore) - - await storage.writeObject({ - key: "prefix/one.png", - body: new Uint8Array([1]), - contentType: "image/png", - }) - await storage.writeObject({ - key: "prefix/two.png", - body: new Uint8Array([2]), - contentType: "image/png", - }) - await storage.writeObject({ - key: "prefix-sibling/two.png", - body: new Uint8Array([4]), - contentType: "image/png", - }) - await storage.writeObject({ - key: "other/three.png", - body: new Uint8Array([3]), - contentType: "image/png", - }) - - const readObject = storage.readObject - const getObjectUrl = storage.getObjectUrl - const deleteObject = storage.deleteObject - const deletePrefix = storage.deletePrefix - expect(readObject).toBeDefined() - expect(getObjectUrl).toBeDefined() - expect(deleteObject).toBeDefined() - expect(deletePrefix).toBeDefined() - if (!readObject || !getObjectUrl || !deleteObject || !deletePrefix) { - throw new Error("Expected full storage adapter methods.") - } - - await expect(storage.readObject?.("prefix/one.png")).resolves.toMatchObject({ - body: new Uint8Array([1]), - contentType: "image/png", - }) - await expect(storage.getObjectUrl?.("prefix/one.png")).resolves.toBe( - "https://blob.example/prefix/one.png", - ) - - await storage.deleteObject?.("prefix/one.png") - expect(blobStore.objects.has("prefix/one.png")).toBe(false) - expect(blobStore.objects.has("prefix/one.png.metadata.json")).toBe(false) - - await storage.deletePrefix?.("prefix") - expect([...blobStore.objects.keys()].sort()).toEqual([ - "other/three.png", - "other/three.png.metadata.json", - "prefix-sibling/two.png", - "prefix-sibling/two.png.metadata.json", - ]) - }) - - it("accepts trailing slashes when deleting prefixes", async () => { - const blobStore = new MemoryVercelBlobStore() - const storage = createVercelBlobKnowhereSdkStorage(blobStore) - - await storage.writeObject({ - key: "prefix/one.png", - body: new Uint8Array([1]), - contentType: "image/png", - }) - - await storage.deletePrefix?.("prefix/") - - expect(blobStore.objects.size).toBe(0) - }) -}) - -async function readBody( - body: Parameters[1], -): Promise { - if (typeof body === "string") return new TextEncoder().encode(body) - if (body instanceof Uint8Array) return new Uint8Array(body) - if (body instanceof Blob) return new Uint8Array(await body.arrayBuffer()) - return readStream(body) -} - -async function readStream( - stream: ReadableStream, -): Promise { - const reader = stream.getReader() - const chunks: Uint8Array[] = [] - let totalLength = 0 - - try { - while (true) { - const result = await reader.read() - if (result.done) break - chunks.push(result.value) - totalLength += result.value.byteLength - } - } finally { - reader.releaseLock() - } - - const body = new Uint8Array(totalLength) - let offset = 0 - for (const chunk of chunks) { - body.set(chunk, offset) - offset += chunk.byteLength - } - return body -} - -function toReadableStream(body: Uint8Array): ReadableStream { - return new ReadableStream({ - start(controller) { - controller.enqueue(body) - controller.close() - }, - }) -} - -function decodeBody(body: Uint8Array): string { - return new TextDecoder().decode(body) -} diff --git a/src/integrations/knowhere-sdk-storage.ts b/src/integrations/knowhere-sdk-storage.ts deleted file mode 100644 index 387bc21..0000000 --- a/src/integrations/knowhere-sdk-storage.ts +++ /dev/null @@ -1,294 +0,0 @@ -import "server-only" - -import { Buffer } from "node:buffer" -import { - BlobNotFoundError, - del, - get, - head, - list, - put, - type GetBlobResult, - type HeadBlobResult, - type ListBlobResult, - type PutBlobResult, -} from "@vercel/blob" -import type { - KnowhereSdkStorage, - KnowhereSdkStorageBody, - KnowhereSdkStorageHead, - KnowhereSdkStorageObject, - KnowhereSdkStorageReadResult, - KnowhereSdkStorageWriteResult, -} from "@ontos-ai/knowhere-sdk" - -type VercelBlobBody = Buffer | Blob | ReadableStream - -export type VercelBlobStore = { - readonly put: ( - pathname: string, - body: VercelBlobBody | string, - options: { - readonly access: "public" - readonly addRandomSuffix: false - readonly allowOverwrite: true - readonly contentType?: string - }, - ) => Promise> - readonly head: (pathname: string) => Promise - readonly get: ( - pathname: string, - options: { readonly access: "public" }, - ) => Promise - readonly list: (options: { - readonly prefix: string - readonly limit: number - readonly cursor?: string - }) => Promise - readonly del: (pathname: string | readonly string[]) => Promise -} - -type StoredMetadataSidecar = { - readonly metadata?: Readonly> -} - -const metadataSuffix = ".metadata.json" -const deletePrefixPageSize = 1000 - -const vercelBlobStore: VercelBlobStore = { - put, - head, - get, - list, - del: (pathname) => - del(typeof pathname === "string" ? pathname : [...pathname]), -} - -class VercelBlobKnowhereSdkStorage implements KnowhereSdkStorage { - private readonly blobStore: VercelBlobStore - - constructor(blobStore: VercelBlobStore = vercelBlobStore) { - this.blobStore = blobStore - } - - async headObject(key: string): Promise { - validateBlobStorageKey(key) - - try { - const object = await this.blobStore.head(key) - const metadata = await this.readMetadata(key) - return { - key, - contentType: object.contentType, - contentLength: object.size, - metadata, - } - } catch (error) { - if (isBlobNotFoundError(error)) return null - throw error - } - } - - async writeObject( - input: KnowhereSdkStorageObject, - ): Promise { - validateBlobStorageKey(input.key) - - const object = await this.blobStore.put( - input.key, - toVercelBlobBody(input.body), - { - access: "public", - addRandomSuffix: false, - allowOverwrite: true, - contentType: input.contentType, - }, - ) - await this.writeMetadata(input.key, input.metadata ?? {}) - - return { - key: object.pathname, - url: object.url, - } - } - - async getObjectUrl(key: string): Promise { - validateBlobStorageKey(key) - - try { - const object = await this.blobStore.head(key) - return object.url - } catch (error) { - if (isBlobNotFoundError(error)) return null - throw error - } - } - - async readObject(key: string): Promise { - validateBlobStorageKey(key) - - const object = await this.blobStore.get(key, { access: "public" }) - if (!object || object.statusCode !== 200) return null - - return { - body: await readStream(object.stream), - contentType: object.blob.contentType, - metadata: await this.readMetadata(key), - } - } - - async deleteObject(key: string): Promise { - validateBlobStorageKey(key) - await this.deleteIgnoringMissing([key, getMetadataKey(key)]) - } - - async deletePrefix(prefix: string): Promise { - validateBlobStoragePrefix(prefix) - const blobPrefix = prefix.endsWith("/") ? prefix : `${prefix}/` - - let cursor: string | undefined - do { - const page = await this.blobStore.list({ - prefix: blobPrefix, - limit: deletePrefixPageSize, - ...(cursor ? { cursor } : {}), - }) - const pathnames = page.blobs.map((blob) => blob.pathname) - if (pathnames.length > 0) await this.deleteIgnoringMissing(pathnames) - cursor = page.cursor - if (page.hasMore && !cursor) break - } while (cursor) - } - - private async readMetadata( - key: string, - ): Promise>> { - const metadataObject = await this.blobStore.get(getMetadataKey(key), { - access: "public", - }) - if (!metadataObject || metadataObject.statusCode !== 200) return {} - - const body = await readStream(metadataObject.stream) - const parsed: unknown = JSON.parse(new TextDecoder().decode(body)) - return parseStoredMetadataSidecar(parsed).metadata ?? {} - } - - private async writeMetadata( - key: string, - metadata: Readonly>, - ): Promise { - const sidecar: StoredMetadataSidecar = { metadata } - await this.blobStore.put( - getMetadataKey(key), - JSON.stringify(sidecar, null, 2), - { - access: "public", - addRandomSuffix: false, - allowOverwrite: true, - contentType: "application/json", - }, - ) - } - - private async deleteIgnoringMissing( - pathnames: readonly string[], - ): Promise { - try { - await this.blobStore.del(pathnames) - } catch (error) { - if (!isBlobNotFoundError(error)) throw error - } - } -} - -export function createVercelBlobKnowhereSdkStorage( - blobStore?: VercelBlobStore, -): KnowhereSdkStorage { - return new VercelBlobKnowhereSdkStorage(blobStore) -} - -function validateBlobStorageKey(key: string): void { - if (!isSafeRelativePosixPath(key)) { - throw new Error(`Invalid Knowhere SDK storage key: ${key}`) - } -} - -function validateBlobStoragePrefix(prefix: string): void { - const normalizedPrefix = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix - if (!isSafeRelativePosixPath(normalizedPrefix)) { - throw new Error(`Invalid Knowhere SDK storage prefix: ${prefix}`) - } -} - -function isSafeRelativePosixPath(value: string): boolean { - if ( - value.length === 0 || - value.startsWith("/") || - value.includes("\\") || - value.includes("\0") - ) { - return false - } - - return value - .split("/") - .every((segment) => segment.length > 0 && segment !== "." && segment !== "..") -} - -function getMetadataKey(key: string): string { - return `${key}${metadataSuffix}` -} - -function toVercelBlobBody(body: KnowhereSdkStorageBody): VercelBlobBody { - if (body instanceof Uint8Array) return Buffer.from(body) - return body -} - -async function readStream( - stream: ReadableStream, -): Promise { - const reader = stream.getReader() - const chunks: Uint8Array[] = [] - let totalLength = 0 - - try { - while (true) { - const result = await reader.read() - if (result.done) break - chunks.push(result.value) - totalLength += result.value.byteLength - } - } finally { - reader.releaseLock() - } - - const body = new Uint8Array(totalLength) - let offset = 0 - for (const chunk of chunks) { - body.set(chunk, offset) - offset += chunk.byteLength - } - return body -} - -function parseStoredMetadataSidecar(value: unknown): StoredMetadataSidecar { - if (!isRecord(value)) return {} - const metadata = value.metadata - if (!isRecord(metadata)) return {} - - return { - metadata: Object.fromEntries( - Object.entries(metadata).filter( - (entry): entry is [string, string] => typeof entry[1] === "string", - ), - ), - } -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null -} - -function isBlobNotFoundError(error: unknown): boolean { - return error instanceof BlobNotFoundError -} From ce8357316b78b530270ed0047f6cdb663bd12af5 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Fri, 3 Jul 2026 18:51:32 +0800 Subject: [PATCH 03/12] Mirror page citation assets to Blob --- src/domains/chat/index.test.ts | 109 +++++++++++++++++- src/domains/chat/index.ts | 57 +++++++-- .../chat/media-asset-hardening.test.ts | 99 +++++++++++++++- src/domains/chat/media-asset-hardening.ts | 104 ++++++++++++----- 4 files changed, 326 insertions(+), 43 deletions(-) diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 0c8cfd6..9daadc3 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -9,7 +9,10 @@ import { generateAgenticOutputManifest, parseChatRequestBody, } from "." -import type { HardenMediaAssetUrlsInput } from "./media-asset-hardening" +import type { + HardenableRetrievalResult, + HardenMediaAssetUrlsInput, +} from "./media-asset-hardening" import type { Source } from "@/infrastructure/db/schema" import type { ChatArtifactView } from "@/domains/chat/types" @@ -665,6 +668,110 @@ describe("answerQuestionWithRetrieval", () => { expect(answer.artifacts?.[0]?.citation?.assetUrl).toBe(hardenedAssetUrl); }); + it("hardens page citation asset URLs before returning citations", async () => { + const rawPageAssetUrl = + "https://knowhere-storage.example/results/job_1/page_citation_assets/page-4.png?AWSAccessKeyId=test"; + const hardenedPageAssetUrl = + "https://blob.example/workspaces/workspace_1/chat-assets/source-source_pages/page-4.png"; + const result = makeRetrievalResult({ + chunkType: "page", + metadata: { + pageNums: [4], + pageAssets: [ + { + pageNum: 4, + artifactRef: "page_citation_assets/page-4.png", + assetUrl: rawPageAssetUrl, + contentType: "image/png", + }, + ], + }, + source: { + documentId: "doc_pages", + sourceFileName: "document-generated.pdf", + sectionPath: "Page 4", + }, + }); + const retrieval = { + query: vi.fn().mockResolvedValue({ + results: [result], + evidenceText: "Page four evidence.", + referencedChunks: [], + namespace: "notebook-workspace", + query: "page four evidence", + routerUsed: "workflow_single_step", + answerText: null, + }), + }; + const generateAnswer = vi.fn(async ({ searchSources }) => { + await searchSources({ query: "page four evidence" }); + return makeHarnessRunResult(`This page has the answer. ${rawPageAssetUrl}`); + }); + const hardenMediaAssetUrls = vi.fn( + async ({ + results, + artifacts, + }: HardenMediaAssetUrlsInput): Promise<{ + results: HardenableRetrievalResult[] + artifacts?: ChatArtifactView[] + }> => ({ + results: results.map( + (candidate): HardenableRetrievalResult => ({ + ...candidate, + pageCitationAssetUrl: + candidate.pageCitationAssetUrl === rawPageAssetUrl + ? hardenedPageAssetUrl + : candidate.pageCitationAssetUrl, + }), + ), + ...(artifacts ? { artifacts: [...artifacts] } : {}), + }), + ); + + const answer = await Effect.runPromise( + answerQuestionWithRetrieval({ + question: "What is on page four?", + namespace: "notebook-workspace", + sources: [ + makeSource({ + id: "source_pages", + title: "deck.pdf", + knowhereDocumentId: "doc_pages", + }), + ], + excludedSourceIds: [], + retrieval, + generateAnswer, + hardenMediaAssetUrls, + messages: [], + }), + ); + + expect(hardenMediaAssetUrls).toHaveBeenCalledWith({ + results: [ + expect.objectContaining({ + pageCitationAssetUrl: rawPageAssetUrl, + source: expect.objectContaining({ + sourceFileName: "deck.pdf", + }), + }), + ], + artifacts: undefined, + }); + expect(answer.answer).toBe("This page has the answer."); + expect(answer.answer).not.toContain("knowhere-storage.example"); + expect(answer.citations).toEqual([ + expect.objectContaining({ + chunkType: "page", + pageCitationAssetUrl: hardenedPageAssetUrl, + source: expect.objectContaining({ + sourceFileName: "deck.pdf", + }), + }), + ]); + expect(answer.citations[0]?.pageCitationAssetUrl).not.toBe(rawPageAssetUrl); + }); + it("returns only harness-selected artifacts when retrieval has extra media candidates", async () => { const frontAssetUrl = "https://blob.example/images/id-front.jpg"; const backAssetUrl = "https://blob.example/images/id-back.jpg"; diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index b0a95d9..ec19628 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -39,6 +39,7 @@ import { removeRetrievedMediaAssetUrls, } from "./media-assets" import { enrichRetrievalResultsWithPageCitationAssetUrls } from "./page-citation-assets" +import type { HardenableRetrievalResult } from "./media-asset-hardening" const DEFAULT_TOP_K = 8 const MAX_AGENTIC_TOP_K = 12 @@ -245,11 +246,17 @@ export const answerQuestionWithRetrieval = ( evidenceText: formatRetrievalEvidenceText(retrievalResponses), }), ) + const pageCitationResults = yield* Effect.tryPromise(() => + enrichRetrievalResultsWithPageCitationAssetUrls({ + results: enrichedResults, + sources: input.sources, + }), + ) const artifacts = toChatArtifactViewsFromHarness(generatedAnswer, input.sources) const hardenedMedia = yield* Effect.tryPromise(() => hardenAnswerMediaAssetUrls({ input, - results: enrichedResults, + results: pageCitationResults, artifacts, }), ) @@ -258,26 +265,21 @@ export const answerQuestionWithRetrieval = ( results: getGeneratedAnswerSanitizerResults({ rawResults, enrichedResults, + pageCitationResults, hardenedResults: hardenedMedia.results, artifacts, hardenedArtifacts: hardenedMedia.artifacts, }), }) - const citationResults = yield* Effect.tryPromise(() => - enrichRetrievalResultsWithPageCitationAssetUrls({ - results: hardenedMedia.results, - sources: input.sources, - }), - ) const displayArtifacts = hardenedMedia.artifacts ?? [] logger.info("chat-agent: answer complete", { answerLength: answer.length, - citationCount: citationResults.length, + citationCount: hardenedMedia.results.length, artifactCount: displayArtifacts.length, }) return { answer, - citations: toChatCitationViews(citationResults, answer), + citations: toChatCitationViews(hardenedMedia.results, answer), artifacts: displayArtifacts, } }) @@ -422,7 +424,7 @@ function normalizeHarnessSource( type AnswerMediaAssetHardeningInput = { readonly input: AnswerQuestionInput - readonly results: readonly RetrievalResult[] + readonly results: readonly HardenableRetrievalResult[] readonly artifacts?: readonly ChatArtifactView[] } @@ -431,7 +433,7 @@ async function hardenAnswerMediaAssetUrls({ results, artifacts, }: AnswerMediaAssetHardeningInput): Promise<{ - readonly results: RetrievalResult[] + readonly results: HardenableRetrievalResult[] readonly artifacts?: ChatArtifactView[] }> { if (!input.hardenMediaAssetUrls) { @@ -462,7 +464,8 @@ async function hardenAnswerMediaAssetUrls({ type GeneratedAnswerSanitizerResultsInput = { readonly rawResults: readonly RetrievalResult[] readonly enrichedResults: readonly RetrievalResult[] - readonly hardenedResults: readonly RetrievalResult[] + readonly pageCitationResults: readonly HardenableRetrievalResult[] + readonly hardenedResults: readonly HardenableRetrievalResult[] readonly artifacts?: readonly ChatArtifactView[] readonly hardenedArtifacts?: readonly ChatArtifactView[] } @@ -470,6 +473,7 @@ type GeneratedAnswerSanitizerResultsInput = { function getGeneratedAnswerSanitizerResults({ rawResults, enrichedResults, + pageCitationResults, hardenedResults, artifacts, hardenedArtifacts, @@ -477,12 +481,32 @@ function getGeneratedAnswerSanitizerResults({ return [ ...rawResults, ...enrichedResults, + ...toPageCitationSanitizerResults(pageCitationResults), ...hardenedResults, + ...toPageCitationSanitizerResults(hardenedResults), ...toArtifactSanitizerResults(artifacts), ...toArtifactSanitizerResults(hardenedArtifacts), ] } +function toPageCitationSanitizerResults( + results: readonly HardenableRetrievalResult[], +): RetrievalResult[] { + return results.flatMap((result): RetrievalResult[] => { + if (!result.pageCitationAssetUrl) return [] + + return [ + { + content: result.content, + chunkType: result.chunkType, + score: result.score, + assetUrl: result.pageCitationAssetUrl, + source: result.source, + }, + ] + }) +} + function toArtifactSanitizerResults( artifacts: readonly ChatArtifactView[] | undefined, ): RetrievalResult[] { @@ -506,6 +530,15 @@ function toArtifactSanitizerResults( }), ) } + if (artifact.citation?.pageCitationAssetUrl) { + results.push( + toArtifactSanitizerResult({ + assetUrl: artifact.citation.pageCitationAssetUrl, + artifact, + citation: artifact.citation, + }), + ) + } return results }) } diff --git a/src/domains/chat/media-asset-hardening.test.ts b/src/domains/chat/media-asset-hardening.test.ts index 70cf562..fe1f3df 100644 --- a/src/domains/chat/media-asset-hardening.test.ts +++ b/src/domains/chat/media-asset-hardening.test.ts @@ -1,11 +1,11 @@ import { afterEach, describe, expect, it, vi } from "vitest" -import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" import { hardenChatMediaAssetUrls, type ChatMediaAssetBlobStore, type FetchChatMediaAsset, + type HardenableRetrievalResult, } from "./media-asset-hardening" const loggerMock = vi.hoisted(() => ({ @@ -73,6 +73,54 @@ describe("hardenChatMediaAssetUrls", () => { ) }) + it("copies server page citation asset URLs into Notebook chat assets", async () => { + const rawPageAssetUrl = + "https://knowhere-storage.example/results/job_1/page_citation_assets/page-4.png?AWSAccessKeyId=test&Signature=secret" + const blobPageAssetUrl = + "https://blob.example/workspaces/workspace_1/chat-assets/source-source_1/page-4.png" + const blobStore = makeBlobStore(blobPageAssetUrl) + const fetchAsset = makeFetchAsset("page-image-bytes", "image/png") + + const result = await hardenChatMediaAssetUrls({ + workspaceId: "workspace_1", + sources: [ + makeSource({ + id: "source_1", + knowhereDocumentId: "doc_pages", + }), + ], + results: [ + makeRetrievalResult({ + chunkType: "page", + pageCitationAssetUrl: rawPageAssetUrl, + source: { + documentId: "doc_pages", + sourceFileName: "deck.pdf", + sectionPath: "Page 4", + }, + }), + ], + blobStore, + fetchAsset, + }) + + expect(fetchAsset).toHaveBeenCalledWith(rawPageAssetUrl) + expect(blobStore.put).toHaveBeenCalledWith( + expect.stringMatching( + /^workspaces\/workspace_1\/chat-assets\/source-source_1\/[a-f0-9]{24}-page-4\.png$/, + ), + expect.any(Buffer), + { + access: "public", + allowOverwrite: true, + contentType: "image/png", + multipart: true, + }, + ) + expect(result.results[0]?.pageCitationAssetUrl).toBe(blobPageAssetUrl) + expect(result.results[0]?.assetUrl).toBeUndefined() + }) + it("uses an existing parsed asset URL before fetching the upstream URL", async () => { const rawAssetUrl = "https://knowhere-storage.example/results/job_1/images/id-front.jpg?AWSAccessKeyId=test" @@ -234,6 +282,51 @@ describe("hardenChatMediaAssetUrls", () => { expect(artifact?.assetUrl).toBe(blobAssetUrl) expect(artifact?.citation?.assetUrl).toBe(blobAssetUrl) }) + + it("rewrites nested citation page citation asset URLs", async () => { + const rawPageAssetUrl = + "https://knowhere-storage.example/results/job_1/page_citation_assets/page-8.png?AWSAccessKeyId=test" + const blobPageAssetUrl = + "https://blob.example/workspaces/workspace_1/chat-assets/source-source_pages/page-8.png" + const blobStore = makeBlobStore(blobPageAssetUrl) + const fetchAsset = makeFetchAsset("page-eight-image", "image/png") + + const result = await hardenChatMediaAssetUrls({ + workspaceId: "workspace_1", + sources: [ + makeSource({ + id: "source_pages", + knowhereDocumentId: "doc_pages", + }), + ], + results: [], + artifacts: [ + { + type: "image", + ref: "asset:r1:result:1", + assetUrl: + "https://blob.example/workspaces/workspace_1/chat-assets/source-source_pages/figure.png", + label: "figure", + citation: { + chunkType: "page", + score: 0.9, + pageCitationAssetUrl: rawPageAssetUrl, + source: { + documentId: "doc_pages", + sourceFileName: "deck.pdf", + sectionPath: "Page 8", + }, + }, + }, + ], + blobStore, + fetchAsset, + }) + + const [artifact] = result.artifacts ?? [] + expect(fetchAsset).toHaveBeenCalledWith(rawPageAssetUrl) + expect(artifact?.citation?.pageCitationAssetUrl).toBe(blobPageAssetUrl) + }) }) function makeFetchAsset( @@ -257,8 +350,8 @@ function makeBlobStore(url: string): ChatMediaAssetBlobStore { } function makeRetrievalResult( - overrides: Partial = {}, -): RetrievalResult { + overrides: Partial = {}, +): HardenableRetrievalResult { return { content: "Asset evidence", chunkType: "text", diff --git a/src/domains/chat/media-asset-hardening.ts b/src/domains/chat/media-asset-hardening.ts index e0a813e..4161587 100644 --- a/src/domains/chat/media-asset-hardening.ts +++ b/src/domains/chat/media-asset-hardening.ts @@ -13,13 +13,17 @@ import { logger } from "@/lib/logger" import type { LoadSourceAssetUrls } from "./media-assets" import { resolveAssetUrlFromReferenceText } from "./media-assets" +export type HardenableRetrievalResult = RetrievalResult & { + readonly pageCitationAssetUrl?: string +} + export type HardenMediaAssetUrlsInput = { - readonly results: readonly RetrievalResult[] + readonly results: readonly HardenableRetrievalResult[] readonly artifacts?: readonly ChatArtifactView[] } export type HardenMediaAssetUrlsResult = { - readonly results: RetrievalResult[] + readonly results: HardenableRetrievalResult[] readonly artifacts?: ChatArtifactView[] } @@ -121,7 +125,7 @@ export async function hardenChatMediaAssetUrls({ } const hardenedResults = await Promise.all( - results.map((result): Promise => + results.map((result): Promise => hardenRetrievalResult(result, context), ), ) @@ -140,25 +144,48 @@ export async function hardenChatMediaAssetUrls({ } async function hardenRetrievalResult( - result: RetrievalResult, + result: HardenableRetrievalResult, context: HardeningContext, -): Promise { +): Promise { const assetUrl = getTrimmedString(result.assetUrl) - if (!assetUrl) return result - - const hardenedAssetUrl = await hardenAssetUrl( - { - assetUrl, - source: result.source, - content: result.content, - }, - context, + const pageCitationAssetUrl = getTrimmedString(result.pageCitationAssetUrl) + if (!assetUrl && !pageCitationAssetUrl) return result + + const hardenedAssetUrl = assetUrl + ? await hardenAssetUrl( + { + assetUrl, + source: result.source, + content: result.content, + }, + context, + ) + : undefined + const hardenedPageCitationAssetUrl = pageCitationAssetUrl + ? await hardenAssetUrl( + { + assetUrl: pageCitationAssetUrl, + source: result.source, + content: result.content, + }, + context, + ) + : undefined + const hasAssetUrlChange = Boolean( + assetUrl && hardenedAssetUrl !== result.assetUrl, + ) + const hasPageCitationAssetUrlChange = Boolean( + pageCitationAssetUrl && + hardenedPageCitationAssetUrl !== result.pageCitationAssetUrl, ) - if (hardenedAssetUrl === result.assetUrl) return result + if (!hasAssetUrlChange && !hasPageCitationAssetUrlChange) return result return { ...result, - assetUrl: hardenedAssetUrl, + ...(hardenedAssetUrl ? { assetUrl: hardenedAssetUrl } : {}), + ...(hardenedPageCitationAssetUrl + ? { pageCitationAssetUrl: hardenedPageCitationAssetUrl } + : {}), } } @@ -200,21 +227,44 @@ async function hardenCitation( context: HardeningContext, ): Promise { const assetUrl = getTrimmedString(citation.assetUrl) - if (!assetUrl) return citation - - const hardenedAssetUrl = await hardenAssetUrl( - { - assetUrl, - source: citation.source, - content: citation.content, - }, - context, + const pageCitationAssetUrl = getTrimmedString(citation.pageCitationAssetUrl) + if (!assetUrl && !pageCitationAssetUrl) return citation + + const hardenedAssetUrl = assetUrl + ? await hardenAssetUrl( + { + assetUrl, + source: citation.source, + content: citation.content, + }, + context, + ) + : undefined + const hardenedPageCitationAssetUrl = pageCitationAssetUrl + ? await hardenAssetUrl( + { + assetUrl: pageCitationAssetUrl, + source: citation.source, + content: citation.content, + }, + context, + ) + : undefined + const hasAssetUrlChange = Boolean( + assetUrl && hardenedAssetUrl !== citation.assetUrl, + ) + const hasPageCitationAssetUrlChange = Boolean( + pageCitationAssetUrl && + hardenedPageCitationAssetUrl !== citation.pageCitationAssetUrl, ) - if (hardenedAssetUrl === citation.assetUrl) return citation + if (!hasAssetUrlChange && !hasPageCitationAssetUrlChange) return citation return { ...citation, - assetUrl: hardenedAssetUrl, + ...(hardenedAssetUrl ? { assetUrl: hardenedAssetUrl } : {}), + ...(hardenedPageCitationAssetUrl + ? { pageCitationAssetUrl: hardenedPageCitationAssetUrl } + : {}), } } From 109bf3660a9f8fa1527940467a3145192cc51556 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Fri, 3 Jul 2026 21:14:10 +0800 Subject: [PATCH 04/12] Read page citation assets from metadata --- src/domains/chat/index.test.ts | 99 +++++++++++++++++++ src/domains/chat/page-citation-assets.test.ts | 24 ----- src/domains/chat/page-citation-assets.ts | 5 +- 3 files changed, 100 insertions(+), 28 deletions(-) diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 9daadc3..459add9 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -772,6 +772,105 @@ describe("answerQuestionWithRetrieval", () => { expect(answer.citations[0]?.pageCitationAssetUrl).not.toBe(rawPageAssetUrl); }); + it("hardens page citation asset URLs from referenced chunk metadata", async () => { + const rawPageAssetUrl = + "https://knowhere-storage.example/results/job_1/page_citation_assets/page-6.png?AWSAccessKeyId=test"; + const hardenedPageAssetUrl = + "https://blob.example/workspaces/workspace_1/chat-assets/source-source_pages/page-6.png"; + const retrieval = { + query: vi.fn().mockResolvedValue({ + results: [], + evidenceText: "Page six evidence.", + referencedChunks: [ + { + chunkId: "chunk_page_6", + documentId: "doc_pages", + chunkType: "page", + sectionPath: "Page 6", + filePath: null, + jobId: "job_1", + assetUrl: rawPageAssetUrl, + metadata: { + pageNums: [6], + pageAssets: [ + { + pageNum: 6, + artifactRef: "page_citation_assets/page-6.png", + assetUrl: rawPageAssetUrl, + contentType: "image/png", + }, + ], + }, + }, + ], + namespace: "notebook-workspace", + query: "page six evidence", + routerUsed: "workflow_single_step", + answerText: null, + }), + }; + const generateAnswer = vi.fn(async ({ searchSources }) => { + await searchSources({ query: "page six evidence" }); + return makeHarnessRunResult("This page has referenced evidence."); + }); + const hardenMediaAssetUrls = vi.fn( + async ({ + results, + artifacts, + }: HardenMediaAssetUrlsInput): Promise<{ + results: HardenableRetrievalResult[] + artifacts?: ChatArtifactView[] + }> => ({ + results: results.map( + (candidate): HardenableRetrievalResult => ({ + ...candidate, + pageCitationAssetUrl: + candidate.pageCitationAssetUrl === rawPageAssetUrl + ? hardenedPageAssetUrl + : candidate.pageCitationAssetUrl, + }), + ), + ...(artifacts ? { artifacts: [...artifacts] } : {}), + }), + ); + + const answer = await Effect.runPromise( + answerQuestionWithRetrieval({ + question: "What is on page six?", + namespace: "notebook-workspace", + sources: [ + makeSource({ + id: "source_pages", + title: "deck.pdf", + knowhereDocumentId: "doc_pages", + }), + ], + excludedSourceIds: [], + retrieval, + generateAnswer, + hardenMediaAssetUrls, + messages: [], + }), + ); + + expect(hardenMediaAssetUrls).toHaveBeenCalledWith({ + results: [ + expect.objectContaining({ + metadata: expect.objectContaining({ + pageAssets: [ + expect.objectContaining({ + assetUrl: rawPageAssetUrl, + }), + ], + }), + pageCitationAssetUrl: rawPageAssetUrl, + }), + ], + artifacts: undefined, + }); + expect(answer.citations[0]?.pageCitationAssetUrl).toBe(hardenedPageAssetUrl); + }); + it("returns only harness-selected artifacts when retrieval has extra media candidates", async () => { const frontAssetUrl = "https://blob.example/images/id-front.jpg"; const backAssetUrl = "https://blob.example/images/id-back.jpg"; diff --git a/src/domains/chat/page-citation-assets.test.ts b/src/domains/chat/page-citation-assets.test.ts index f7a1812..8a251cf 100644 --- a/src/domains/chat/page-citation-assets.test.ts +++ b/src/domains/chat/page-citation-assets.test.ts @@ -30,30 +30,6 @@ describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { ) }) - it("uses server-provided top-level page assets", async () => { - const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ - results: [ - makeRetrievalResult({ - chunkType: "page", - pageAssets: [ - { - pageNum: 4, - artifactRef: "page_citation_assets/page-4.png", - assetUrl: "https://assets.example/pages/page-4.png", - contentType: "image/png", - source: "knowhere-rendered-page-citation-source", - }, - ], - } as Partial), - ], - sources: [makeSource()], - }) - - expect(result?.pageCitationAssetUrl).toBe( - "https://assets.example/pages/page-4.png", - ) - }) - it("chooses the asset matching the citation page metadata", async () => { const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ results: [ diff --git a/src/domains/chat/page-citation-assets.ts b/src/domains/chat/page-citation-assets.ts index dccdd88..4f29a50 100644 --- a/src/domains/chat/page-citation-assets.ts +++ b/src/domains/chat/page-citation-assets.ts @@ -61,10 +61,7 @@ function getDirectPageCitationAsset( result: RetrievalResult, pageNumbers: readonly number[], ): PageCitationAssetCandidate | null { - const candidates = [ - isRecord(result) ? result.pageAssets : undefined, - result.metadata?.pageAssets, - ].flatMap(parsePageCitationAssetCandidates) + const candidates = parsePageCitationAssetCandidates(result.metadata?.pageAssets) if (pageNumbers.length > 0) { const matchingCandidates = candidates.filter((candidate) => From 3377032d6c7e5c414d35bb1b00640717481bc3d5 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sat, 4 Jul 2026 00:08:39 +0800 Subject: [PATCH 05/12] Sync page citation assets to Blob --- pnpm-lock.yaml | 265 +----------------- src/domains/chat/index.ts | 1 + src/domains/chat/page-citation-assets.test.ts | 36 ++- src/domains/chat/page-citation-assets.ts | 76 ++++- .../sources/parse-result-storage-adapter.ts | 99 +++++++ .../source-reconcile-route-workflow.test.ts | 58 +++- .../source-reconcile-route-workflow.ts | 28 ++ 7 files changed, 297 insertions(+), 266 deletions(-) create mode 100644 src/domains/sources/parse-result-storage-adapter.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 978e912..94e3f05 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -28,7 +28,7 @@ importers: version: 1.1.0 '@ontos-ai/knowhere-sdk': specifier: file:../knowhere-node-sdk - version: file:../knowhere-node-sdk(ioredis@5.11.1) + version: file:../knowhere-node-sdk '@radix-ui/react-alert-dialog': specifier: ^1.1.15 version: 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -442,19 +442,6 @@ packages: resolution: {integrity: sha512-EXnJjIy6zQ3nUO/MZ+ynWUb8B895KZPotd1++oTs9JjDkplwM7cb6zo8Zq2zU6piwq+KflO7amXbEfj1UMpHkw==} hasBin: true - '@effect/platform-node-shared@4.0.0-beta.93': - resolution: {integrity: sha512-XUqZ2u5GglBqY8q2jj4Q7GjN5K/enedk8auZM9rY/l5a/myaQTrQp3QnvpIK4/Yg0WFjLGuctGPMKWRk3OLIrA==} - engines: {node: '>=18.0.0'} - peerDependencies: - effect: ^4.0.0-beta.93 - - '@effect/platform-node@4.0.0-beta.93': - resolution: {integrity: sha512-QagsCGR0ZOXaCQqS5qGR2mcDng4LiP2bYhiiX1D6UC8cT9vsusVVOHiJWn8CupeDx+yVnPcu81QmA/SDt6GM1w==} - engines: {node: '>=18.0.0'} - peerDependencies: - effect: ^4.0.0-beta.93 - ioredis: ^5.7.0 - '@effect/platform@0.96.1': resolution: {integrity: sha512-cjB1QZZYEP8JXCFNGvBLVi0T6YUBQTmOVEUA3SDbiQ6RUO+p6CE3eyD2vMWmrz5nE8yY5QSAuOV9v0boEcUv+A==} peerDependencies: @@ -1197,9 +1184,6 @@ packages: '@types/node': optional: true - '@ioredis/commands@1.10.0': - resolution: {integrity: sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==} - '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -1231,61 +1215,31 @@ packages: cpu: [arm64] os: [darwin] - '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': - resolution: {integrity: sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==} - cpu: [arm64] - os: [darwin] - '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} cpu: [x64] os: [darwin] - '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': - resolution: {integrity: sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==} - cpu: [x64] - os: [darwin] - '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} cpu: [arm64] os: [linux] - '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': - resolution: {integrity: sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==} - cpu: [arm64] - os: [linux] - '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} cpu: [arm] os: [linux] - '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': - resolution: {integrity: sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==} - cpu: [arm] - os: [linux] - '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} cpu: [x64] os: [linux] - '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': - resolution: {integrity: sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==} - cpu: [x64] - os: [linux] - '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} cpu: [x64] os: [win32] - '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': - resolution: {integrity: sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==} - cpu: [x64] - os: [win32] - '@mswjs/interceptors@0.41.8': resolution: {integrity: sha512-pRLMNKTSGRoLq+KnEB/7OY5vijw1XmcheAAOiv6pj7W1FG32kAGqj1C/RK/cqxRGr1Fh+zBi8sDur8kj3EQv6A==} engines: {node: '>=18'} @@ -2417,9 +2371,6 @@ packages: '@types/validate-npm-package-name@4.0.2': resolution: {integrity: sha512-lrpDziQipxCEeK5kWxvljWYhUvOiB2A9izZd9B2AFarYAkqZshb4lPbRs7zKEic6eGtH8V/2qJW+dPp9OtF6bw==} - '@types/ws@8.18.1': - resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} - '@typescript-eslint/eslint-plugin@8.59.2': resolution: {integrity: sha512-j/bwmkBvHUtPNxzuWe5z6BEk3q54YRyGlBXkSsmfoih7zNrBvl5A9A98anlp/7JbyZcWIJ8KXo/3Tq/DjFLtuQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -2920,10 +2871,6 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} - cluster-key-slot@1.1.1: - resolution: {integrity: sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==} - engines: {node: '>=0.10.0'} - code-block-writer@13.0.3: resolution: {integrity: sha512-Oofo0pq3IKnsFtuHqSF7TqBfr71aeyZDVJ0HpmqB7FBM2qEigL0iPONSCZSO9pE9dZTAxANe5XHG9Uy0YMv8cg==} @@ -3106,10 +3053,6 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - denque@2.1.0: - resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} - engines: {node: '>=0.10'} - depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -3265,9 +3208,6 @@ packages: effect@3.21.2: resolution: {integrity: sha512-rXd2FGDM8KdjSIrc+mqEELo7ScW7xTVxEf1iInmPSpIde9/nyGuFM710cjTo7/EreGXiUX2MOonPpprbz2XHCg==} - effect@4.0.0-beta.93: - resolution: {integrity: sha512-wNS5MKFa3C42uBfIDik2oJ78lhpoYz2hN4oBR0229BeeDCIrkg/FiOvoiPGdCVlWa7MEKxEL5I0f8AILVHSD9A==} - electron-to-chromium@1.5.351: resolution: {integrity: sha512-9D7Iqx8RImSvCnOsj86rCH6eQjZFQoM04Jn6HnZVM0Nu/G58/gmKYQ1d12MZTbjQbQSTGI8nwEy07ErsA2slLA==} @@ -3540,10 +3480,6 @@ packages: resolution: {integrity: sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==} engines: {node: '>=8.0.0'} - fast-check@4.8.0: - resolution: {integrity: sha512-GOJ158CUMnN6cSahsv4+ExARvIDuzzinFjkp0E9WtiBa5zcVeLozVkWaE4IzFcc+Y48Wp1EDlUZsXRyAztQcSg==} - engines: {node: '>=12.17.0'} - fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -3863,10 +3799,6 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - ini@7.0.0: - resolution: {integrity: sha512-ifK0CgjALofS5bkrcTy4RaQ9Vx2Knf/eLeIO+NaswQEpH1UblrtTSCIvN71qQDMq0PeQ/SSPojvEJp9vvvfr+w==} - engines: {node: ^22.22.2 || ^24.15.0 || >=26.0.0} - inline-style-parser@0.2.7: resolution: {integrity: sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==} @@ -3874,10 +3806,6 @@ packages: resolution: {integrity: sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==} engines: {node: '>= 0.4'} - ioredis@5.11.1: - resolution: {integrity: sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==} - engines: {node: '>=12.22.0'} - ip-address@10.1.0: resolution: {integrity: sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==} engines: {node: '>= 12'} @@ -4182,9 +4110,6 @@ packages: resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} engines: {node: '>=6'} - kubernetes-types@1.30.0: - resolution: {integrity: sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==} - language-subtag-registry@0.3.23: resolution: {integrity: sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==} @@ -4509,11 +4434,6 @@ packages: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} - mime@4.1.0: - resolution: {integrity: sha512-X5ju04+cAzsojXKes0B/S4tcYtFAJ6tTMuSPBEn9CPGlrWr8Fiw7qYeLT0XyH80HSoAoqWCaz+MWKh22P7G1cw==} - engines: {node: '>=16'} - hasBin: true - mimic-fn@2.1.0: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} @@ -4539,16 +4459,9 @@ packages: resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} hasBin: true - msgpackr-extract@3.0.4: - resolution: {integrity: sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==} - hasBin: true - msgpackr@1.11.12: resolution: {integrity: sha512-RBdJ1Un7yGlXWajrkxcSa93nvQ0w4zBf60c0yYv7YtBelP8H2FA7XsfBbMHtXKXUMUxH7zV3Zuozh+kUQWhHvg==} - msgpackr@2.0.4: - resolution: {integrity: sha512-o1C5KRmuRt+apqMr1HuGSqWStZoRBUpEsCsl15uM9VdAF1qHLtvMOU2En747EnTyEl6c4pzPewRMFF31s1CNbA==} - msw@2.14.3: resolution: {integrity: sha512-kk8G5cocVlJ4wsKMGZegn2H6XLOEKjbA+nSJE2354e/SRp4mDicCHUYnMXpymzVcVDCs+GUAsmNqSn+yHv4T2A==} engines: {node: '>=18'} @@ -4786,10 +4699,6 @@ packages: resolution: {integrity: sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==} engines: {node: '>=20.16.0 || >=22.3.0'} - pdfjs-dist@6.1.200: - resolution: {integrity: sha512-o8MolyzirkkLrcdsae/HEOiIcXWI7DS5zGpvqW8xTC2YUsW30rltFw2bDGvw/fskUdEMrQm2br68jzDS5BH2vw==} - engines: {node: '>=22.13.0 || >=24'} - picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -4889,9 +4798,6 @@ packages: pure-rand@6.1.0: resolution: {integrity: sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==} - pure-rand@8.4.1: - resolution: {integrity: sha512-c58R2+SPFcSIPXoU834QN/KPDDOSd8sXcSrqf6e83Me6Rrp1EYkxukkjXMVrKvKaADs1SOyNkWdfvLf6zY8qLQ==} - qs@6.15.1: resolution: {integrity: sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==} engines: {node: '>=0.6'} @@ -4978,14 +4884,6 @@ packages: resolution: {integrity: sha512-YTUo+Flmw4ZXiWfQKGcwwc11KnoRAYgzAE2E7mXKCjSviTKShtxBsN6YUUBB2gtaBzKzeKunxhUwNHQuRryhWA==} engines: {node: '>= 4'} - redis-errors@1.2.0: - resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} - engines: {node: '>=4'} - - redis-parser@3.0.0: - resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} - engines: {node: '>=4'} - reflect.getprototypeof@1.0.10: resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} engines: {node: '>= 0.4'} @@ -5195,9 +5093,6 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - standard-as-callback@2.1.0: - resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} - statuses@2.0.2: resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} engines: {node: '>= 0.8'} @@ -5371,10 +5266,6 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} - toml@4.1.2: - resolution: {integrity: sha512-m0vXfHODcw3gk+KONAOlVQ5yNHc3yS3B1ybM3HS1vqDoS0RWTDDVBVVTYi8hH0k+2OM1vmo9fb1WX9EVqjqfHA==} - engines: {node: '>=20'} - tough-cookie@6.0.1: resolution: {integrity: sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==} engines: {node: '>=16'} @@ -5474,10 +5365,6 @@ packages: resolution: {integrity: sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==} engines: {node: '>=20.18.1'} - undici@8.6.0: - resolution: {integrity: sha512-l2FlC6I510GawyEd1qgcE/okihKrzy+BRTEBlu6T0fdbM9m5yxtIH5Oa3ysRsH0zC4EhmWUEaSDsy2QngBeRlw==} - engines: {node: '>=22.19.0'} - unicorn-magic@0.3.0: resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} engines: {node: '>=18'} @@ -5551,10 +5438,6 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} - uuid@14.0.1: - resolution: {integrity: sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==} - hasBin: true - validate-npm-package-name@7.0.2: resolution: {integrity: sha512-hVDIBwsRruT73PbK7uP5ebUt+ezEtCmzZz3F59BSr2F6OVFnJ/6h8liuvdLrQ88Xmnk6/+xGGuq+pG9WwTuy3A==} engines: {node: ^20.17.0 || >=22.9.0} @@ -5721,18 +5604,6 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - ws@8.21.0: - resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - wsl-utils@0.3.1: resolution: {integrity: sha512-g/eziiSUNBSsdDJtCLB8bdYEUMj4jR7AGeUo96p/3dTafgjHhpF4RiCFPiRILwjQoDXx5MqkBr4fwWtR3Ky4Wg==} engines: {node: '>=20'} @@ -6094,26 +5965,6 @@ snapshots: '@effect/language-service@0.85.1': {} - '@effect/platform-node-shared@4.0.0-beta.93(effect@4.0.0-beta.93)': - dependencies: - '@types/ws': 8.18.1 - effect: 4.0.0-beta.93 - ws: 8.21.0 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - '@effect/platform-node@4.0.0-beta.93(effect@4.0.0-beta.93)(ioredis@5.11.1)': - dependencies: - '@effect/platform-node-shared': 4.0.0-beta.93(effect@4.0.0-beta.93) - effect: 4.0.0-beta.93 - ioredis: 5.11.1 - mime: 4.1.0 - undici: 8.6.0 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - '@effect/platform@0.96.1(effect@3.21.2)': dependencies: effect: 3.21.2 @@ -6580,8 +6431,6 @@ snapshots: optionalDependencies: '@types/node': 20.19.39 - '@ioredis/commands@1.10.0': {} - '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -6626,39 +6475,21 @@ snapshots: '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': optional: true - '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4': - optional: true - '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': optional: true - '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4': - optional: true - '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': optional: true - '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4': - optional: true - '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': optional: true - '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4': - optional: true - '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': optional: true - '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4': - optional: true - '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': optional: true - '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4': - optional: true - '@mswjs/interceptors@0.41.8': dependencies: '@open-draft/deferred-promise': 2.2.0 @@ -6903,21 +6734,13 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} - '@ontos-ai/knowhere-sdk@file:../knowhere-node-sdk(ioredis@5.11.1)': + '@ontos-ai/knowhere-sdk@file:../knowhere-node-sdk': dependencies: - '@effect/platform-node': 4.0.0-beta.93(effect@4.0.0-beta.93)(ioredis@5.11.1) - '@napi-rs/canvas': 1.0.2 axios: 1.18.1 - effect: 4.0.0-beta.93 jszip: 3.10.1 - pdfjs-dist: 6.1.200 - piscina: 5.2.0 transitivePeerDependencies: - - bufferutil - debug - - ioredis - supports-color - - utf-8-validate '@open-draft/deferred-promise@2.2.0': {} @@ -7580,10 +7403,6 @@ snapshots: '@types/validate-npm-package-name@4.0.2': {} - '@types/ws@8.18.1': - dependencies: - '@types/node': 20.19.39 - '@typescript-eslint/eslint-plugin@8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -8105,8 +7924,6 @@ snapshots: clsx@2.1.1: {} - cluster-key-slot@1.1.1: {} - code-block-writer@13.0.3: {} color-convert@2.0.1: @@ -8250,8 +8067,6 @@ snapshots: delayed-stream@1.0.0: {} - denque@2.1.0: {} - depd@2.0.0: {} dequal@2.0.3: {} @@ -8321,19 +8136,6 @@ snapshots: '@standard-schema/spec': 1.1.0 fast-check: 3.23.2 - effect@4.0.0-beta.93: - dependencies: - '@standard-schema/spec': 1.1.0 - fast-check: 4.8.0 - find-my-way-ts: 0.1.6 - ini: 7.0.0 - kubernetes-types: 1.30.0 - msgpackr: 2.0.4 - multipasta: 0.2.7 - toml: 4.1.2 - uuid: 14.0.1 - yaml: 2.9.0 - electron-to-chromium@1.5.351: {} emoji-regex@10.6.0: {} @@ -8849,10 +8651,6 @@ snapshots: dependencies: pure-rand: 6.1.0 - fast-check@4.8.0: - dependencies: - pure-rand: 8.4.1 - fast-deep-equal@3.1.3: {} fast-glob@3.3.1: @@ -9181,8 +8979,6 @@ snapshots: inherits@2.0.4: {} - ini@7.0.0: {} - inline-style-parser@0.2.7: {} internal-slot@1.1.0: @@ -9191,18 +8987,6 @@ snapshots: hasown: 2.0.3 side-channel: 1.1.0 - ioredis@5.11.1: - dependencies: - '@ioredis/commands': 1.10.0 - cluster-key-slot: 1.1.1 - debug: 4.4.3 - denque: 2.1.0 - redis-errors: 1.2.0 - redis-parser: 3.0.0 - standard-as-callback: 2.1.0 - transitivePeerDependencies: - - supports-color - ip-address@10.1.0: {} ipaddr.js@1.9.1: {} @@ -9484,8 +9268,6 @@ snapshots: kleur@4.1.5: {} - kubernetes-types@1.30.0: {} - language-subtag-registry@0.3.23: {} language-tags@1.0.9: @@ -9987,8 +9769,6 @@ snapshots: dependencies: mime-db: 1.54.0 - mime@4.1.0: {} - mimic-fn@2.1.0: {} mimic-function@5.0.1: {} @@ -10017,26 +9797,10 @@ snapshots: '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 optional: true - msgpackr-extract@3.0.4: - dependencies: - node-gyp-build-optional-packages: 5.2.2 - optionalDependencies: - '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.4 - '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.4 - '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.4 - '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.4 - '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.4 - '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.4 - optional: true - msgpackr@1.11.12: optionalDependencies: msgpackr-extract: 3.0.3 - msgpackr@2.0.4: - optionalDependencies: - msgpackr-extract: 3.0.4 - msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3): dependencies: '@inquirer/confirm': 6.0.12(@types/node@20.19.39) @@ -10302,10 +10066,6 @@ snapshots: optionalDependencies: '@napi-rs/canvas': 0.1.100 - pdfjs-dist@6.1.200: - optionalDependencies: - '@napi-rs/canvas': 1.0.2 - picocolors@1.1.1: {} picomatch@2.3.2: {} @@ -10400,8 +10160,6 @@ snapshots: pure-rand@6.1.0: {} - pure-rand@8.4.1: {} - qs@6.15.1: dependencies: side-channel: 1.1.0 @@ -10508,12 +10266,6 @@ snapshots: tiny-invariant: 1.3.3 tslib: 2.8.1 - redis-errors@1.2.0: {} - - redis-parser@3.0.0: - dependencies: - redis-errors: 1.2.0 - reflect.getprototypeof@1.0.10: dependencies: call-bind: 1.0.9 @@ -10860,8 +10612,6 @@ snapshots: stackback@0.0.2: {} - standard-as-callback@2.1.0: {} - statuses@2.0.2: {} std-env@4.1.0: {} @@ -11034,8 +10784,6 @@ snapshots: toidentifier@1.0.1: {} - toml@4.1.2: {} - tough-cookie@6.0.1: dependencies: tldts: 7.0.30 @@ -11156,8 +10904,6 @@ snapshots: undici@7.25.0: {} - undici@8.6.0: {} - unicorn-magic@0.3.0: {} unified@11.0.5: @@ -11254,8 +11000,6 @@ snapshots: util-deprecate@1.0.2: {} - uuid@14.0.1: {} - validate-npm-package-name@7.0.2: {} vary@1.1.2: {} @@ -11402,8 +11146,6 @@ snapshots: wrappy@1.0.2: {} - ws@8.21.0: {} - wsl-utils@0.3.1: dependencies: is-wsl: 3.1.1 @@ -11419,7 +11161,8 @@ snapshots: yallist@3.1.1: {} - yaml@2.9.0: {} + yaml@2.9.0: + optional: true yargs-parser@21.1.1: {} diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index ec19628..bbf95b9 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -250,6 +250,7 @@ export const answerQuestionWithRetrieval = ( enrichRetrievalResultsWithPageCitationAssetUrls({ results: enrichedResults, sources: input.sources, + loadSourceAssetUrls: input.loadSourceAssetUrls, }), ) const artifacts = toChatArtifactViewsFromHarness(generatedAnswer, input.sources) diff --git a/src/domains/chat/page-citation-assets.test.ts b/src/domains/chat/page-citation-assets.test.ts index 8a251cf..6d0ee01 100644 --- a/src/domains/chat/page-citation-assets.test.ts +++ b/src/domains/chat/page-citation-assets.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest" +import { describe, expect, it, vi } from "vitest" import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" @@ -30,6 +30,40 @@ describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { ) }) + it("prefers stored Blob URLs over server-provided page asset URLs", async () => { + const loadSourceAssetUrls = vi.fn().mockResolvedValue({ + "page_citation_assets/page-2.png": + "https://blob.example/page_citation_assets/page-2.png", + }) + + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "page", + metadata: { + pageNums: [2], + pageAssets: [ + { + pageNum: 2, + artifactRef: "page_citation_assets/page-2.png", + assetUrl: "https://assets.example/pages/page-2.png", + }, + ], + }, + }), + ], + sources: [makeSource()], + loadSourceAssetUrls, + }) + + expect(loadSourceAssetUrls).toHaveBeenCalledWith( + expect.objectContaining({ id: "source_1" }), + ) + expect(result?.pageCitationAssetUrl).toBe( + "https://blob.example/page_citation_assets/page-2.png", + ) + }) + it("chooses the asset matching the citation page metadata", async () => { const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ results: [ diff --git a/src/domains/chat/page-citation-assets.ts b/src/domains/chat/page-citation-assets.ts index 4f29a50..dd1d34d 100644 --- a/src/domains/chat/page-citation-assets.ts +++ b/src/domains/chat/page-citation-assets.ts @@ -3,6 +3,7 @@ import "server-only" import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" +import type { LoadSourceAssetUrls } from "./media-assets" export type PageCitationAssetRetrievalResult = RetrievalResult & { readonly pageCitationAssetUrl?: string @@ -11,26 +12,41 @@ export type PageCitationAssetRetrievalResult = RetrievalResult & { type EnrichRetrievalResultsWithPageCitationAssetUrlsInput = { readonly results: readonly RetrievalResult[] readonly sources: readonly Source[] + readonly loadSourceAssetUrls?: LoadSourceAssetUrls } type PageCitationAssetCandidate = { readonly pageNum: number + readonly artifactRef?: string readonly assetUrl?: string } export async function enrichRetrievalResultsWithPageCitationAssetUrls({ results, sources, + loadSourceAssetUrls, }: EnrichRetrievalResultsWithPageCitationAssetUrlsInput): Promise< PageCitationAssetRetrievalResult[] > { - void sources if (results.length === 0) return [] + const sourcesByDocumentId = new Map( + sources.flatMap((source): readonly [string, Source][] => + source.knowhereDocumentId ? [[source.knowhereDocumentId, source]] : [], + ), + ) + const assetUrlsBySourceId = new Map< + string, + Promise>> + >() + return Promise.all( results.map((result) => enrichRetrievalResultWithPageCitationAssetUrl({ result, + sourcesByDocumentId, + loadSourceAssetUrls, + assetUrlsBySourceId, }), ), ) @@ -38,11 +54,30 @@ export async function enrichRetrievalResultsWithPageCitationAssetUrls({ async function enrichRetrievalResultWithPageCitationAssetUrl(input: { readonly result: RetrievalResult + readonly sourcesByDocumentId: ReadonlyMap + readonly loadSourceAssetUrls?: LoadSourceAssetUrls + readonly assetUrlsBySourceId: Map< + string, + Promise>> + > }): Promise { if (!isPageResult(input.result)) return input.result const pageNumbers = getPageNumbers(input.result.metadata) const directAsset = getDirectPageCitationAsset(input.result, pageNumbers) + const sourceAssetUrl = await getStoredPageCitationAssetUrl({ + result: input.result, + directAsset, + sourcesByDocumentId: input.sourcesByDocumentId, + loadSourceAssetUrls: input.loadSourceAssetUrls, + assetUrlsBySourceId: input.assetUrlsBySourceId, + }) + if (sourceAssetUrl) { + return { + ...input.result, + pageCitationAssetUrl: sourceAssetUrl, + } + } if (directAsset?.assetUrl) { return { ...input.result, @@ -53,6 +88,31 @@ async function enrichRetrievalResultWithPageCitationAssetUrl(input: { return input.result } +async function getStoredPageCitationAssetUrl(input: { + readonly result: RetrievalResult + readonly directAsset: PageCitationAssetCandidate | null + readonly sourcesByDocumentId: ReadonlyMap + readonly loadSourceAssetUrls?: LoadSourceAssetUrls + readonly assetUrlsBySourceId: Map< + string, + Promise>> + > +}): Promise { + const artifactRef = getTrimmedString(input.directAsset?.artifactRef) + const documentId = getTrimmedString(input.result.source.documentId) + if (!artifactRef || !documentId || !input.loadSourceAssetUrls) return null + + const source = input.sourcesByDocumentId.get(documentId) + if (!source) return null + + const assetUrls = await getCachedSourceAssetUrls( + source, + input.loadSourceAssetUrls, + input.assetUrlsBySourceId, + ) + return getTrimmedString(assetUrls[artifactRef]) +} + function isPageResult(result: RetrievalResult): boolean { return result.chunkType.toLowerCase() === "page" } @@ -90,12 +150,26 @@ function parsePageCitationAssetCandidates( return [ { pageNum, + artifactRef: getTrimmedString(item.artifactRef) ?? undefined, assetUrl: getTrimmedString(item.assetUrl) ?? undefined, }, ] }) } +async function getCachedSourceAssetUrls( + source: Source, + loadSourceAssetUrls: LoadSourceAssetUrls, + cache: Map>>>, +): Promise>> { + let cached = cache.get(source.id) + if (!cached) { + cached = loadSourceAssetUrls(source).catch(() => ({})) + cache.set(source.id, cached) + } + return cached +} + function getPageNumbers( metadata: Readonly> | undefined, ): readonly number[] { diff --git a/src/domains/sources/parse-result-storage-adapter.ts b/src/domains/sources/parse-result-storage-adapter.ts new file mode 100644 index 0000000..908dd34 --- /dev/null +++ b/src/domains/sources/parse-result-storage-adapter.ts @@ -0,0 +1,99 @@ +import "server-only" + +import { head, put, BlobNotFoundError } from "@vercel/blob" +import type { + KnowhereAssetStorageObject, + KnowhereAssetStorageOptions, +} from "@ontos-ai/knowhere-sdk" + +export type ParsedResultAssetStorageInput = { + readonly workspaceId: string + readonly sourceId: string +} + +export type ParsedResultAssetIndex = { + readonly workspaceId: string + readonly sourceId: string + readonly assetUrlsByFilePath: Readonly> + readonly updatedAt: string +} + +const parsedResultDirectoryName = "parsed-result" +const parsedResultAssetIndexFileName = "asset-index.json" + +export function createParsedResultStorageAdapter({ + workspaceId, + sourceId, +}: ParsedResultAssetStorageInput): KnowhereAssetStorageOptions { + return { + adapter: { + async headObject(key) { + try { + const blob = await head(key) + return { + key, + contentType: blob.contentType, + contentLength: blob.size, + url: blob.url, + } + } catch (error) { + if (error instanceof BlobNotFoundError) return null + throw error + } + }, + async getObjectUrl(key) { + try { + return (await head(key)).url + } catch (error) { + if (error instanceof BlobNotFoundError) return null + throw error + } + }, + async writeObject(input: KnowhereAssetStorageObject) { + const blob = await put(input.key, Buffer.from(input.body), { + access: "public", + addRandomSuffix: false, + allowOverwrite: true, + contentType: input.contentType, + multipart: true, + }) + + return { + key: blob.pathname, + url: blob.url, + } + }, + }, + keyPrefix: getParsedResultBlobPrefix(workspaceId, sourceId), + skipExisting: true, + } +} + +export async function writeParsedResultAssetIndex(input: { + readonly workspaceId: string + readonly sourceId: string + readonly assetUrlsByFilePath: Readonly> +}): Promise { + const index: ParsedResultAssetIndex = { + workspaceId: input.workspaceId, + sourceId: input.sourceId, + assetUrlsByFilePath: input.assetUrlsByFilePath, + updatedAt: new Date().toISOString(), + } + const pathname = `${getParsedResultBlobPrefix( + input.workspaceId, + input.sourceId, + )}/${parsedResultAssetIndexFileName}` + const blob = await put(pathname, JSON.stringify(index, null, 2), { + access: "public", + addRandomSuffix: false, + allowOverwrite: true, + contentType: "application/json; charset=utf-8", + }) + + return blob.url +} + +function getParsedResultBlobPrefix(workspaceId: string, sourceId: string): string { + return `workspaces/${workspaceId}/sources/${sourceId}/${parsedResultDirectoryName}` +} diff --git a/src/domains/sources/source-reconcile-route-workflow.test.ts b/src/domains/sources/source-reconcile-route-workflow.test.ts index 14e9b03..170941f 100644 --- a/src/domains/sources/source-reconcile-route-workflow.test.ts +++ b/src/domains/sources/source-reconcile-route-workflow.test.ts @@ -1,6 +1,9 @@ import { afterEach, describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ + createParsedResultStorageAdapter: vi.fn(), + saveParseResult: vi.fn(), + writeParsedResultAssetIndex: vi.fn(), loggerError: vi.fn(), loggerInfo: vi.fn(), loggerWarn: vi.fn(), @@ -18,9 +21,15 @@ vi.mock("@/domains/sources/source-reconcile-workflow", () => ({ vi.mock("@/domains/sources/workflow-runtime", () => ({ sourceWorkflowRuntime: { markFailed: mocks.markFailed, + saveParseResult: mocks.saveParseResult, }, })) +vi.mock("./parse-result-storage-adapter", () => ({ + createParsedResultStorageAdapter: mocks.createParsedResultStorageAdapter, + writeParsedResultAssetIndex: mocks.writeParsedResultAssetIndex, +})) + vi.mock("@/integrations/knowhere", () => ({ makeKnowhereClient: mocks.makeKnowhereClient, })) @@ -57,7 +66,7 @@ describe("sourceReconcileRouteWorkflow", () => { }) }) - it("marks the source ready when Knowhere publishes a document id", async () => { + it("caches parsed assets before marking the source ready", async () => { const context = createWorkflowContext() const continuations: ContinuationTriggerInput[] = [] const restore = @@ -66,8 +75,25 @@ describe("sourceReconcileRouteWorkflow", () => { continuations.push(input) }, ) - const client = { jobs: {} } + const storageAdapter = { + adapter: { + writeObject: vi.fn(), + }, + keyPrefix: "workspaces/workspace_1/sources/source_1/parsed-result", + } + const cacheJobResult = vi.fn().mockResolvedValue({ + assetUrlsByFilePath: { + "page_citation_assets/page-1.png": + "https://blob.example/page_citation_assets/page-1.png", + }, + }) + const client = { jobs: {}, knowledge: { cacheJobResult } } + mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) mocks.makeKnowhereClient.mockReturnValue(client) + mocks.writeParsedResultAssetIndex.mockResolvedValue( + "https://blob.example/asset-index.json", + ) + mocks.saveParseResult.mockResolvedValue({ id: "parse_result_1" }) mocks.pollSourceReconciliation.mockResolvedValue({ kind: "ready-to-prepare", jobId: "job_1", @@ -90,12 +116,38 @@ describe("sourceReconcileRouteWorkflow", () => { restore() } + expect(mocks.createParsedResultStorageAdapter).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + }) + expect(cacheJobResult).toHaveBeenCalledWith({ + jobId: "job_1", + storageAdapter, + }) + expect(mocks.writeParsedResultAssetIndex).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + assetUrlsByFilePath: { + "page_citation_assets/page-1.png": + "https://blob.example/page_citation_assets/page-1.png", + }, + }) + expect(mocks.saveParseResult).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { + resultBlobUrl: "https://blob.example/asset-index.json", + assetUrlsByFilePath: { + "page_citation_assets/page-1.png": + "https://blob.example/page_citation_assets/page-1.png", + }, + }, + ) expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ workspaceId: "workspace_1", sourceId: "source_1", documentId: "doc_1", }) - expect(client).toEqual({ jobs: {} }) expect(mocks.loggerWarn).not.toHaveBeenCalled() expect(continuations).toEqual([]) }) diff --git a/src/domains/sources/source-reconcile-route-workflow.ts b/src/domains/sources/source-reconcile-route-workflow.ts index a1e732a..cc03d40 100644 --- a/src/domains/sources/source-reconcile-route-workflow.ts +++ b/src/domains/sources/source-reconcile-route-workflow.ts @@ -8,6 +8,10 @@ import { } from "@/domains/sources/source-reconcile-workflow" import { makeKnowhereClient } from "@/integrations/knowhere" import { logger } from "@/lib/logger" +import { + createParsedResultStorageAdapter, + writeParsedResultAssetIndex, +} from "./parse-result-storage-adapter" import { sourceWorkflowRuntime } from "./workflow-runtime" type ReconcilePayload = { @@ -118,6 +122,30 @@ async function runPollAndMirrorWorkflow(input: { return } + const assetUrlsByFilePath = await context.run("cache-job-result-assets", async () => { + const cachedResult = await client.knowledge.cacheJobResult({ + jobId: jobToPrepare.jobId, + storageAdapter: createParsedResultStorageAdapter({ + workspaceId, + sourceId, + }), + }) + return cachedResult.assetUrlsByFilePath ?? {} + }) + const resultBlobUrl = await context.run("write-parse-result-asset-index", async () => + writeParsedResultAssetIndex({ + workspaceId, + sourceId, + assetUrlsByFilePath, + }), + ) + await context.run("save-parse-result-assets", async () => + sourceWorkflowRuntime.saveParseResult(workspaceId, sourceId, { + resultBlobUrl, + assetUrlsByFilePath, + }), + ) + const ready = await context.run("source-ready", async () => markSourceReadyAfterReconciliation({ workspaceId, From 907083e5bf4785b610539a71af9e4dd738ca19dd Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sat, 4 Jul 2026 01:23:08 +0800 Subject: [PATCH 06/12] Read parsed chunks from Blob snapshot --- drizzle/0010_minor_piledriver.sql | 2 + drizzle/meta/0010_snapshot.json | 728 ++++++++++++++++++ drizzle/meta/_journal.json | 7 + .../sources/[sourceId]/chunks/route.test.ts | 184 ++++- src/domains/chat/index.test.ts | 20 +- src/domains/chat/page-citation-assets.test.ts | 18 +- src/domains/chat/page-citation-assets.ts | 12 +- src/domains/chunks/server.test.ts | 123 +++ src/domains/chunks/server.ts | 176 ++++- .../sources/parse-result-storage-adapter.ts | 172 ++++- src/domains/sources/repository.ts | 3 + src/domains/sources/route-chunks.ts | 65 ++ src/domains/sources/route-dependencies.ts | 1 + src/domains/sources/route-types.ts | 9 + src/domains/sources/service.ts | 5 + .../sources/source-parse-result-repository.ts | 35 + .../source-reconcile-route-workflow.test.ts | 65 +- .../source-reconcile-route-workflow.ts | 40 +- src/domains/sources/workflow-runtime.ts | 18 + src/infrastructure/db/schema.ts | 9 +- 20 files changed, 1608 insertions(+), 84 deletions(-) create mode 100644 drizzle/0010_minor_piledriver.sql create mode 100644 drizzle/meta/0010_snapshot.json diff --git a/drizzle/0010_minor_piledriver.sql b/drizzle/0010_minor_piledriver.sql new file mode 100644 index 0000000..973029b --- /dev/null +++ b/drizzle/0010_minor_piledriver.sql @@ -0,0 +1,2 @@ +ALTER TABLE "source_parse_results" ADD COLUMN "snapshot_manifest_url" text;--> statement-breakpoint +ALTER TABLE "source_parse_results" ADD COLUMN "snapshot_manifest_key" text; \ No newline at end of file diff --git a/drizzle/meta/0010_snapshot.json b/drizzle/meta/0010_snapshot.json new file mode 100644 index 0000000..b9b5a4f --- /dev/null +++ b/drizzle/meta/0010_snapshot.json @@ -0,0 +1,728 @@ +{ + "id": "cc41c815-4e52-4d01-ac2e-abbc791331b7", + "prevId": "c59ee2f8-6b59-4a2c-98a3-d71a67388d47", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 568b77f..7174911 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -71,6 +71,13 @@ "when": 1782472980935, "tag": "0009_true_lila_cheney", "breakpoints": true + }, + { + "idx": 10, + "version": "7", + "when": 1783098230819, + "tag": "0010_minor_piledriver", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index 524a657..c41319c 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -11,6 +11,7 @@ const mocks = vi.hoisted(() => ({ findSourceInWorkspace: vi.fn(), getCurrentUser: vi.fn(), getSourceParseAssetUrls: vi.fn(), + getSourceParseSnapshotMetadata: vi.fn(), localizeRemoteDocument: vi.fn(), makeKnowhereClient: vi.fn(), requireUser: vi.fn(), @@ -51,6 +52,7 @@ vi.mock("@/domains/sources/service", () => ({ sourceService: { findInWorkspace: mocks.findSourceInWorkspace, getParseAssetUrls: mocks.getSourceParseAssetUrls, + getParseSnapshotMetadata: mocks.getSourceParseSnapshotMetadata, localizeRemoteDocument: mocks.localizeRemoteDocument, updateSourceRevisionKey: mocks.updateSourceRevisionKey, }, @@ -71,6 +73,16 @@ describe("GET /api/sources/[sourceId]/chunks", () => { mocks.blobPut.mockImplementation(async (pathname: string) => ({ url: `https://blob.example/${pathname}`, })) + mocks.getSourceParseAssetUrls.mockResolvedValue({}) + mocks.getSourceParseSnapshotMetadata.mockResolvedValue({ + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + assetUrlsByFilePath: {}, + }) mocks.updateSourceRevisionKey.mockResolvedValue(null) }) @@ -429,30 +441,10 @@ describe("GET /api/sources/[sourceId]/chunks", () => { } }) - it("loads authenticated workspace chunks without probing the demo endpoint first", async () => { + it("loads authenticated workspace chunks from the parsed Blob snapshot", async () => { const knowhereClient = { documents: { - listChunks: vi.fn(async () => ({ - chunks: [ - { - id: "dchk_1", - chunkId: "parser_1", - chunkType: "text", - content: "Workspace chunk", - sectionPath: "Summary", - sourceChunkPath: "Default_Root/notes.pdf/Summary", - filePath: null, - metadata: {}, - sortOrder: 0, - }, - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })), + listChunks: vi.fn(), }, } mocks.getCurrentUser.mockResolvedValue({ @@ -487,7 +479,81 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }) mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") mocks.makeKnowhereClient.mockReturnValue(knowhereClient) - mocks.getSourceParseAssetUrls.mockResolvedValue({}) + mocks.getSourceParseAssetUrls.mockResolvedValue({ + "images/chart.png": "https://blob.example/images/chart.png", + }) + mocks.getSourceParseSnapshotMetadata.mockResolvedValue({ + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", + assetUrlsByFilePath: { + "images/chart.png": "https://blob.example/images/chart.png", + }, + }) + mocks.blobGet.mockImplementation(async (pathname: string) => { + if (pathname.endsWith("/manifest/current.json")) { + return { + statusCode: 200, + stream: createTextStream( + JSON.stringify({ + version: 1, + kind: "knowhere-parsed-result-snapshot", + jobId: "job_1", + documentId: "doc_1", + sourceFileName: "notes.pdf", + totalChunks: 1, + chunkPageSize: 1, + chunkPages: [ + { + page: 1, + pageSize: 1, + chunkCount: 1, + key: "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/chunks/page-1.json", + }, + ], + assetUrlsByFilePath: { + "images/chart.png": "https://blob.example/images/chart.png", + }, + createdAt: "2026-07-03T00:00:00.000Z", + }), + ), + } + } + if (pathname.endsWith("/chunks/page-1.json")) { + return { + statusCode: 200, + stream: createTextStream( + JSON.stringify({ + version: 1, + jobId: "job_1", + documentId: "doc_1", + sourceFileName: "notes.pdf", + page: 1, + pageSize: 1, + total: 1, + totalPages: 1, + chunks: [ + { + id: "dchk_1", + chunkId: "parser_1", + chunkType: "image", + content: "Workspace chunk", + sectionPath: "Summary", + sourceChunkPath: "Default_Root/notes.pdf/Summary", + filePath: "images/chart.png", + metadata: {}, + sortOrder: 1, + }, + ], + }), + ), + } + } + return null + }) const response = await GET( new NextRequest( @@ -502,6 +568,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { chunkId: "dchk_1", parserChunkId: "parser_1", documentId: "doc_1", + assetUrl: "https://blob.example/images/chart.png", sourceTitle: "notes.pdf", }, ], @@ -513,11 +580,68 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }) expect(response.status).toBe(200) expect(mocks.fetchDemoChunkPage).not.toHaveBeenCalled() - expect(knowhereClient.documents.listChunks).toHaveBeenCalledWith("doc_1", { - page: 1, - pageSize: 1, - includeAssetUrls: true, + expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() + }) + + it("returns processing when a ready workspace source has no complete parsed snapshot", async () => { + const knowhereClient = { + documents: { + listChunks: vi.fn(), + }, + } + mocks.getCurrentUser.mockResolvedValue({ + id: "user_1", + email: null, + name: null, + }) + mocks.ensureWorkspace.mockResolvedValue({ + id: "workspace_1", + userId: "user_1", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + }) + mocks.findSourceInWorkspace.mockResolvedValue({ + id: "00000000-0000-0000-0000-000000000002", + workspaceId: "workspace_1", + title: "notes.pdf", + mimeType: "application/pdf", + sizeBytes: 1024, + status: "ready", + failureReason: null, + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + }) + mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") + mocks.makeKnowhereClient.mockReturnValue(knowhereClient) + mocks.getSourceParseSnapshotMetadata.mockResolvedValue(null) + + const response = await GET( + new NextRequest( + "http://localhost:3001/api/sources/00000000-0000-0000-0000-000000000002/chunks?page=1&pageSize=1", + ), + { params: Promise.resolve({ sourceId: "00000000-0000-0000-0000-000000000002" }) }, + ) + + await expect(response.json()).resolves.toEqual({ + chunks: [], + pagination: { + page: 1, + pageSize: 1, + total: 0, + totalPages: 0, + }, + message: "Source parsed snapshot is still being prepared.", }) + expect(response.status).toBe(202) + expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() }) it("materializes a remote source id on open before loading chunks", async () => { @@ -650,3 +774,9 @@ describe("GET /api/sources/[sourceId]/chunks", () => { ) }) }) + +function createTextStream(text: string): ReadableStream { + const stream = new Response(text).body + if (!stream) throw new Error("Response body stream was not created.") + return stream +} diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 459add9..71604f6 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -671,6 +671,8 @@ describe("answerQuestionWithRetrieval", () => { it("hardens page citation asset URLs before returning citations", async () => { const rawPageAssetUrl = "https://knowhere-storage.example/results/job_1/page_citation_assets/page-4.png?AWSAccessKeyId=test"; + const storedPageAssetUrl = + "https://blob.example/workspaces/workspace_1/sources/source_pages/parsed-result/page_citation_assets/page-4.png"; const hardenedPageAssetUrl = "https://blob.example/workspaces/workspace_1/chat-assets/source-source_pages/page-4.png"; const result = makeRetrievalResult({ @@ -705,7 +707,7 @@ describe("answerQuestionWithRetrieval", () => { }; const generateAnswer = vi.fn(async ({ searchSources }) => { await searchSources({ query: "page four evidence" }); - return makeHarnessRunResult(`This page has the answer. ${rawPageAssetUrl}`); + return makeHarnessRunResult(`This page has the answer. ${storedPageAssetUrl}`); }); const hardenMediaAssetUrls = vi.fn( async ({ @@ -720,6 +722,8 @@ describe("answerQuestionWithRetrieval", () => { ...candidate, pageCitationAssetUrl: candidate.pageCitationAssetUrl === rawPageAssetUrl + ? hardenedPageAssetUrl + : candidate.pageCitationAssetUrl === storedPageAssetUrl ? hardenedPageAssetUrl : candidate.pageCitationAssetUrl, }), @@ -743,6 +747,9 @@ describe("answerQuestionWithRetrieval", () => { retrieval, generateAnswer, hardenMediaAssetUrls, + loadSourceAssetUrls: vi.fn(async () => ({ + "page_citation_assets/page-4.png": storedPageAssetUrl, + })), messages: [], }), ); @@ -750,7 +757,7 @@ describe("answerQuestionWithRetrieval", () => { expect(hardenMediaAssetUrls).toHaveBeenCalledWith({ results: [ expect.objectContaining({ - pageCitationAssetUrl: rawPageAssetUrl, + pageCitationAssetUrl: storedPageAssetUrl, source: expect.objectContaining({ sourceFileName: "deck.pdf", }), @@ -775,6 +782,8 @@ describe("answerQuestionWithRetrieval", () => { it("hardens page citation asset URLs from referenced chunk metadata", async () => { const rawPageAssetUrl = "https://knowhere-storage.example/results/job_1/page_citation_assets/page-6.png?AWSAccessKeyId=test"; + const storedPageAssetUrl = + "https://blob.example/workspaces/workspace_1/sources/source_pages/parsed-result/page_citation_assets/page-6.png"; const hardenedPageAssetUrl = "https://blob.example/workspaces/workspace_1/chat-assets/source-source_pages/page-6.png"; const retrieval = { @@ -826,6 +835,8 @@ describe("answerQuestionWithRetrieval", () => { ...candidate, pageCitationAssetUrl: candidate.pageCitationAssetUrl === rawPageAssetUrl + ? hardenedPageAssetUrl + : candidate.pageCitationAssetUrl === storedPageAssetUrl ? hardenedPageAssetUrl : candidate.pageCitationAssetUrl, }), @@ -849,6 +860,9 @@ describe("answerQuestionWithRetrieval", () => { retrieval, generateAnswer, hardenMediaAssetUrls, + loadSourceAssetUrls: vi.fn(async () => ({ + "page_citation_assets/page-6.png": storedPageAssetUrl, + })), messages: [], }), ); @@ -863,7 +877,7 @@ describe("answerQuestionWithRetrieval", () => { }), ], }), - pageCitationAssetUrl: rawPageAssetUrl, + pageCitationAssetUrl: storedPageAssetUrl, }), ], artifacts: undefined, diff --git a/src/domains/chat/page-citation-assets.test.ts b/src/domains/chat/page-citation-assets.test.ts index 6d0ee01..ed62c66 100644 --- a/src/domains/chat/page-citation-assets.test.ts +++ b/src/domains/chat/page-citation-assets.test.ts @@ -5,7 +5,7 @@ import type { Source } from "@/infrastructure/db/schema" import { enrichRetrievalResultsWithPageCitationAssetUrls } from "./page-citation-assets" describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { - it("uses server-provided page asset URLs from result metadata", async () => { + it("does not use direct server-provided page asset URLs from result metadata", async () => { const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ results: [ makeRetrievalResult({ @@ -25,12 +25,10 @@ describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { sources: [makeSource()], }) - expect(result?.pageCitationAssetUrl).toBe( - "https://assets.example/pages/page-2.png", - ) + expect(result?.pageCitationAssetUrl).toBeUndefined() }) - it("prefers stored Blob URLs over server-provided page asset URLs", async () => { + it("uses stored Blob URLs for page citation assets", async () => { const loadSourceAssetUrls = vi.fn().mockResolvedValue({ "page_citation_assets/page-2.png": "https://blob.example/page_citation_assets/page-2.png", @@ -64,7 +62,12 @@ describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { ) }) - it("chooses the asset matching the citation page metadata", async () => { + it("chooses the stored asset matching the citation page metadata", async () => { + const loadSourceAssetUrls = vi.fn().mockResolvedValue({ + "page_citation_assets/page-4.png": + "https://blob.example/page_citation_assets/page-4.png", + }) + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ results: [ makeRetrievalResult({ @@ -87,10 +90,11 @@ describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { }), ], sources: [makeSource()], + loadSourceAssetUrls, }) expect(result?.pageCitationAssetUrl).toBe( - "https://assets.example/pages/page-4.png", + "https://blob.example/page_citation_assets/page-4.png", ) }) diff --git a/src/domains/chat/page-citation-assets.ts b/src/domains/chat/page-citation-assets.ts index dd1d34d..77b8d53 100644 --- a/src/domains/chat/page-citation-assets.ts +++ b/src/domains/chat/page-citation-assets.ts @@ -78,12 +78,6 @@ async function enrichRetrievalResultWithPageCitationAssetUrl(input: { pageCitationAssetUrl: sourceAssetUrl, } } - if (directAsset?.assetUrl) { - return { - ...input.result, - pageCitationAssetUrl: directAsset.assetUrl, - } - } return input.result } @@ -127,14 +121,10 @@ function getDirectPageCitationAsset( const matchingCandidates = candidates.filter((candidate) => pageNumbers.includes(candidate.pageNum), ) - const matchingAssetWithUrl = matchingCandidates.find( - (candidate) => candidate.assetUrl, - ) - if (matchingAssetWithUrl) return matchingAssetWithUrl if (matchingCandidates[0]) return matchingCandidates[0] } - return candidates.find((candidate) => candidate.assetUrl) ?? candidates[0] ?? null + return candidates[0] ?? null } function parsePageCitationAssetCandidates( diff --git a/src/domains/chunks/server.test.ts b/src/domains/chunks/server.test.ts index f58f5ea..2442b53 100644 --- a/src/domains/chunks/server.test.ts +++ b/src/domains/chunks/server.test.ts @@ -9,6 +9,107 @@ import { } from "./server" describe("server chunk cache", () => { + it("loads ready source chunks from the parsed Blob snapshot without calling Knowhere listChunks", async () => { + const listChunks = vi.fn(async () => ({ + documentId: "doc_1", + chunks: [], + pagination: { + page: 1, + pageSize: 1, + total: 0, + totalPages: 0, + }, + })) + const snapshotStore = createSnapshotStore({ + "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json": + JSON.stringify({ + version: 1, + kind: "knowhere-parsed-result-snapshot", + jobId: "job_1", + documentId: "doc_1", + sourceFileName: "notes.pdf", + totalChunks: 1, + chunkPageSize: 1, + chunkPages: [ + { + page: 1, + pageSize: 1, + chunkCount: 1, + key: "workspaces/workspace_1/sources/source_1/parsed-result/chunks/page-1.json", + }, + ], + assetUrlsByFilePath: { + "images/chart.png": "https://blob.example/images/chart.png", + }, + createdAt: "2026-07-03T00:00:00.000Z", + }), + "workspaces/workspace_1/sources/source_1/parsed-result/chunks/page-1.json": + JSON.stringify({ + version: 1, + jobId: "job_1", + documentId: "doc_1", + sourceFileName: "notes.pdf", + page: 1, + pageSize: 1, + total: 1, + totalPages: 1, + chunks: [ + { + id: "chunk_image_1", + chunkId: "parser_image_1", + chunkType: "image", + content: "Chart summary", + sectionPath: "Revenue", + sourceChunkPath: "notes.pdf/Revenue", + filePath: "images/chart.png", + sortOrder: 1, + metadata: { summary: "Chart summary" }, + }, + ], + }), + }) + + const page = await Effect.runPromise( + loadChunkPageForSource( + makeSource({ status: "ready", knowhereJobId: "job_1" }), + { documents: { listChunks } }, + { page: 1, pageSize: 1 }, + { + snapshot: { + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + assetUrlsByFilePath: {}, + }, + snapshotStore, + workspaceId: "workspace_1", + }, + ), + ) + + expect(listChunks).not.toHaveBeenCalled() + expect(page).toEqual({ + chunks: [ + expect.objectContaining({ + chunkId: "chunk_image_1", + parserChunkId: "parser_image_1", + type: "image", + assetUrl: "https://blob.example/images/chart.png", + sourceTitle: "notes.pdf", + }), + ], + pagination: { + page: 1, + pageSize: 1, + total: 1, + totalPages: 1, + }, + }) + }) + it("returns upstream chunks on a visible cache miss and warms mirrored assets in the background", async () => { const warmTasks: Array<() => Promise> = [] const cacheStore = createCacheStore() @@ -355,6 +456,13 @@ type TestCacheStore = { readonly putMock: TestCachePutMock } +type TestSnapshotStore = { + readonly get: ( + pathname: string, + options: { readonly access: "public" }, + ) => Promise +} + type TestCacheGetMock = Mock< ( pathname: string, @@ -391,6 +499,21 @@ function createCacheStore(overrides: Partial<{ } } +function createSnapshotStore( + entriesByPathname: Readonly>, +): TestSnapshotStore { + return { + get: vi.fn(async (pathname: string) => { + const entry = entriesByPathname[pathname] + if (!entry) return null + return { + statusCode: 200 as const, + stream: createTextStream(entry), + } + }), + } +} + function makeDocumentChunk( overrides: Partial = {}, ): DocumentChunk { diff --git a/src/domains/chunks/server.ts b/src/domains/chunks/server.ts index a28ee3b..64b7fe4 100644 --- a/src/domains/chunks/server.ts +++ b/src/domains/chunks/server.ts @@ -7,7 +7,6 @@ import { after } from "next/server" import { Effect } from "effect" import type { DocumentChunk, - DocumentChunkListResponse, } from "@ontos-ai/knowhere-sdk" import { @@ -18,6 +17,13 @@ import { type ChunkPageParams, type LoadChunksOptions, } from "@/domains/chunks" +import { + readParsedResultSnapshotChunkPage, + readParsedResultSnapshotManifest, + type ParsedResultSnapshotBlobStore, + type ParsedResultSnapshotChunk, + type ParsedResultSnapshotManifest, +} from "@/domains/sources/parse-result-storage-adapter" import type { ParsedChunkView } from "@/domains/chunks/types" import type { Source } from "@/infrastructure/db/schema" import { logger } from "@/lib/logger" @@ -59,6 +65,8 @@ type ChunkPageWarmScheduler = (task: () => Promise) => void type ServerLoadChunksOptions = LoadChunksOptions & { readonly workspaceId?: string + readonly snapshot?: SourceParsedSnapshotReadModel | null + readonly snapshotStore?: ParsedResultSnapshotBlobStore readonly cacheStore?: ChunkPageBlobStore readonly fetchAsset?: FetchChunkAsset readonly mode?: ChunkPageMode @@ -86,6 +94,13 @@ type MirrorableChunkAsset = { readonly filePath?: string } +export type SourceParsedSnapshotReadModel = { + readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null + readonly assetUrlsByFilePath: Readonly> +} + const documentChunkPageSize = 200 const visibleChunkPageMode: ChunkPageMode = "visible" const structureChunkPageMode: ChunkPageMode = "structure" @@ -124,6 +139,17 @@ export const loadChunksForSource = ( Effect.gen(function* () { if (source.status !== "ready" || !source.knowhereDocumentId) return [] + if (options.snapshot) { + const snapshotChunks = yield* Effect.promise(() => + loadAllSnapshotChunks({ + source, + snapshot: options.snapshot!, + blobStore: options.snapshotStore, + }), + ) + return resolveChunkConnectionTargets(snapshotChunks) + } + const chunks: ParsedChunkView[] = [] let page = 1 let totalPages = 1 @@ -156,6 +182,17 @@ export const loadChunkPageForSource = ( return emptyPage } + if (options.snapshot) { + return yield* Effect.promise(() => + loadSnapshotChunkPage({ + source, + params, + snapshot: options.snapshot!, + blobStore: options.snapshotStore, + }), + ) + } + const mode = options.mode ?? visibleChunkPageMode const workspaceId = options.workspaceId ?? source.workspaceId const cacheStore = options.cacheStore ?? defaultBlobStore @@ -236,6 +273,138 @@ export const loadChunkPageForSource = ( return chunkPage }) +async function loadAllSnapshotChunks(input: { + readonly source: Source + readonly snapshot: SourceParsedSnapshotReadModel + readonly blobStore?: ParsedResultSnapshotBlobStore +}): Promise { + const manifest = await loadSnapshotManifest(input) + if (!manifest) return [] + + const pages = await Promise.all( + manifest.chunkPages.map((page) => + readParsedResultSnapshotChunkPage({ + pageKey: page.key, + blobStore: input.blobStore, + }), + ), + ) + const chunks = pages.flatMap((page) => page?.chunks ?? []) + return chunks.map((chunk) => + toParsedChunkView( + toDocumentChunkFromSnapshot(chunk), + input.source.title, + input.source.knowhereDocumentId ?? undefined, + { + assetUrlsByFilePath: { + ...manifest.assetUrlsByFilePath, + ...input.snapshot.assetUrlsByFilePath, + }, + }, + ), + ) +} + +async function loadSnapshotChunkPage(input: { + readonly source: Source + readonly params: ChunkPageParams + readonly snapshot: SourceParsedSnapshotReadModel + readonly blobStore?: ParsedResultSnapshotBlobStore +}): Promise { + const manifest = await loadSnapshotManifest(input) + if (!manifest) return createEmptyChunkPage(input.params) + + const pageReference = manifest.chunkPages.find( + (page) => page.page === input.params.page && page.pageSize === input.params.pageSize, + ) + if (!pageReference) { + return createEmptyChunkPage({ + page: input.params.page, + pageSize: input.params.pageSize, + }) + } + + const page = await readParsedResultSnapshotChunkPage({ + pageKey: pageReference.key, + blobStore: input.blobStore, + }) + if (!page) return createEmptyChunkPage(input.params) + + const assetUrlsByFilePath = { + ...manifest.assetUrlsByFilePath, + ...input.snapshot.assetUrlsByFilePath, + } + const chunks = page.chunks.map((chunk) => + toParsedChunkView( + toDocumentChunkFromSnapshot(chunk), + input.source.title, + input.source.knowhereDocumentId ?? undefined, + { assetUrlsByFilePath }, + ), + ) + + return { + chunks, + pagination: { + page: page.page, + pageSize: page.pageSize, + total: page.total, + totalPages: page.totalPages, + }, + } +} + +async function loadSnapshotManifest(input: { + readonly source: Source + readonly snapshot: SourceParsedSnapshotReadModel + readonly blobStore?: ParsedResultSnapshotBlobStore +}): Promise { + const manifestKey = input.snapshot.snapshotManifestKey + if (!manifestKey) { + logger.warn("chunks: ready source missing parsed snapshot manifest", { + sourceId: input.source.id, + documentId: input.source.knowhereDocumentId, + }) + return null + } + + const manifest = await readParsedResultSnapshotManifest({ + workspaceId: input.source.workspaceId, + sourceId: input.source.id, + manifestKey, + blobStore: input.blobStore, + }) + if (!manifest) { + logger.warn("chunks: parsed snapshot manifest could not be read", { + sourceId: input.source.id, + documentId: input.source.knowhereDocumentId, + manifestKey, + }) + } + return manifest +} + +function toDocumentChunkFromSnapshot(chunk: ParsedResultSnapshotChunk): DocumentChunk { + return { + id: chunk.id, + chunkId: chunk.chunkId, + chunkType: normalizeDocumentChunkType(chunk.chunkType), + content: chunk.content, + sectionId: null, + sectionPath: chunk.sectionPath ?? null, + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath ?? null, + sortOrder: chunk.sortOrder, + metadata: chunk.metadata, + assetUrl: chunk.assetUrl ?? null, + } +} + +function normalizeDocumentChunkType(value: string): DocumentChunk["chunkType"] { + if (value === "image" || value === "table" || value === "page") return value + return "text" +} + export async function warmChunkPageCache( input: WarmChunkPageCacheInput, ): Promise { @@ -653,7 +822,10 @@ function getMirroredAssetContentType( } function getRevisionKey( - response: Pick, + response: { + readonly jobId?: string | null + readonly jobResultId?: string | null + }, source: Source, ): string | null { return ( diff --git a/src/domains/sources/parse-result-storage-adapter.ts b/src/domains/sources/parse-result-storage-adapter.ts index 908dd34..d195b8f 100644 --- a/src/domains/sources/parse-result-storage-adapter.ts +++ b/src/domains/sources/parse-result-storage-adapter.ts @@ -1,6 +1,6 @@ import "server-only" -import { head, put, BlobNotFoundError } from "@vercel/blob" +import { get, head, put, BlobNotFoundError } from "@vercel/blob" import type { KnowhereAssetStorageObject, KnowhereAssetStorageOptions, @@ -18,8 +18,56 @@ export type ParsedResultAssetIndex = { readonly updatedAt: string } +export type ParsedResultSnapshotChunk = { + readonly id: string + readonly chunkId: string + readonly chunkType: string + readonly contentSource?: string + readonly content: string + readonly sectionPath?: string + readonly sourceChunkPath: string + readonly filePath?: string + readonly sortOrder: number + readonly metadata: Record + readonly assetUrl?: string +} + +export type ParsedResultSnapshotChunkPage = { + readonly version: 1 + readonly jobId: string + readonly documentId?: string + readonly namespace?: string + readonly sourceFileName: string + readonly page: number + readonly pageSize: number + readonly total: number + readonly totalPages: number + readonly chunks: readonly ParsedResultSnapshotChunk[] +} + +export type ParsedResultSnapshotManifest = { + readonly version: 1 + readonly kind: "knowhere-parsed-result-snapshot" + readonly jobId: string + readonly documentId?: string + readonly namespace?: string + readonly sourceFileName: string + readonly totalChunks: number + readonly chunkPageSize: number + readonly chunkPages: readonly { + readonly page: number + readonly pageSize: number + readonly chunkCount: number + readonly key: string + readonly url?: string + }[] + readonly assetUrlsByFilePath: Readonly> + readonly createdAt: string +} + const parsedResultDirectoryName = "parsed-result" const parsedResultAssetIndexFileName = "asset-index.json" +const parsedResultSnapshotManifestPath = "manifest/current.json" export function createParsedResultStorageAdapter({ workspaceId, @@ -69,6 +117,42 @@ export function createParsedResultStorageAdapter({ } } +export function getParsedResultSnapshotManifestKey({ + workspaceId, + sourceId, +}: ParsedResultAssetStorageInput): string { + return `${getParsedResultBlobPrefix( + workspaceId, + sourceId, + )}/${parsedResultSnapshotManifestPath}` +} + +export async function readParsedResultSnapshotManifest(input: { + readonly workspaceId: string + readonly sourceId: string + readonly manifestKey?: string | null + readonly blobStore?: ParsedResultSnapshotBlobStore +}): Promise { + const key = + input.manifestKey ?? + getParsedResultSnapshotManifestKey({ + workspaceId: input.workspaceId, + sourceId: input.sourceId, + }) + const text = await readBlobText(key, input.blobStore ?? vercelSnapshotBlobStore) + if (!text) return null + return parseSnapshotManifest(text) +} + +export async function readParsedResultSnapshotChunkPage(input: { + readonly pageKey: string + readonly blobStore?: ParsedResultSnapshotBlobStore +}): Promise { + const text = await readBlobText(input.pageKey, input.blobStore ?? vercelSnapshotBlobStore) + if (!text) return null + return parseSnapshotChunkPage(text) +} + export async function writeParsedResultAssetIndex(input: { readonly workspaceId: string readonly sourceId: string @@ -97,3 +181,89 @@ export async function writeParsedResultAssetIndex(input: { function getParsedResultBlobPrefix(workspaceId: string, sourceId: string): string { return `workspaces/${workspaceId}/sources/${sourceId}/${parsedResultDirectoryName}` } + +type ParsedResultSnapshotBlobGetResult = + | { + readonly statusCode: 200 + readonly stream: ReadableStream + } + | { + readonly statusCode: 304 + readonly stream: null + } + +export type ParsedResultSnapshotBlobStore = { + readonly get: ( + pathname: string, + options: { readonly access: "public" }, + ) => Promise +} + +const vercelSnapshotBlobStore: ParsedResultSnapshotBlobStore = { + get: (pathname, options) => get(pathname, options), +} + +async function readBlobText( + key: string, + blobStore: ParsedResultSnapshotBlobStore, +): Promise { + try { + const result = await blobStore.get(key, { access: "public" }) + if (!result || result.statusCode !== 200) return null + return new Response(result.stream).text() + } catch (error) { + if (error instanceof BlobNotFoundError) return null + throw error + } +} + +function parseSnapshotManifest(text: string): ParsedResultSnapshotManifest | null { + try { + const value: unknown = JSON.parse(text) + return isSnapshotManifest(value) ? value : null + } catch { + return null + } +} + +function parseSnapshotChunkPage(text: string): ParsedResultSnapshotChunkPage | null { + try { + const value: unknown = JSON.parse(text) + return isSnapshotChunkPage(value) ? value : null + } catch { + return null + } +} + +function isSnapshotManifest(value: unknown): value is ParsedResultSnapshotManifest { + if (!isRecord(value)) return false + return ( + value["kind"] === "knowhere-parsed-result-snapshot" && + value["version"] === 1 && + typeof value["jobId"] === "string" && + typeof value["sourceFileName"] === "string" && + typeof value["totalChunks"] === "number" && + typeof value["chunkPageSize"] === "number" && + Array.isArray(value["chunkPages"]) && + isRecord(value["assetUrlsByFilePath"]) && + typeof value["createdAt"] === "string" + ) +} + +function isSnapshotChunkPage(value: unknown): value is ParsedResultSnapshotChunkPage { + if (!isRecord(value)) return false + return ( + value["version"] === 1 && + typeof value["jobId"] === "string" && + typeof value["sourceFileName"] === "string" && + typeof value["page"] === "number" && + typeof value["pageSize"] === "number" && + typeof value["total"] === "number" && + typeof value["totalPages"] === "number" && + Array.isArray(value["chunks"]) + ) +} + +function isRecord(value: unknown): value is Readonly> { + return typeof value === "object" && value !== null && !Array.isArray(value) +} diff --git a/src/domains/sources/repository.ts b/src/domains/sources/repository.ts index c7c4b6f..4d9c061 100644 --- a/src/domains/sources/repository.ts +++ b/src/domains/sources/repository.ts @@ -21,6 +21,7 @@ type SourceRepository = { readonly saveParseResultEffect: typeof sourceParseResultRepository.saveParseResultEffect readonly mergeParseAssetUrlsEffect: typeof sourceParseResultRepository.mergeParseAssetUrlsEffect readonly getParseResultProgressEffect: typeof sourceParseResultRepository.getParseResultProgressEffect + readonly getParseSnapshotMetadataEffect: typeof sourceParseResultRepository.getParseSnapshotMetadataEffect readonly getParseAssetUrlsEffect: typeof sourceParseResultRepository.getParseAssetUrlsEffect } @@ -45,5 +46,7 @@ export const sourceRepository: SourceRepository = { sourceParseResultRepository.mergeParseAssetUrlsEffect, getParseResultProgressEffect: sourceParseResultRepository.getParseResultProgressEffect, + getParseSnapshotMetadataEffect: + sourceParseResultRepository.getParseSnapshotMetadataEffect, getParseAssetUrlsEffect: sourceParseResultRepository.getParseAssetUrlsEffect, } diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index 733ccc1..fdbe0b1 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -88,8 +88,16 @@ const loadSourceChunksEffect = ( const client = yield* Effect.tryPromise(() => getClientForWorkspace(workspace.id, input.cookieHeader, deps), ) + const snapshot = yield* Effect.tryPromise(() => + deps.sourceService.getParseSnapshotMetadata(workspace.id, source.id), + ) + if (!isCompleteSnapshot(snapshot)) { + return sourceSnapshotProcessing(input) + } + if (input.shouldLoadAll) { const chunks = yield* deps.loadChunksForSource(source, client, { + snapshot, workspaceId: workspace.id, onRevisionKey: async (revisionKey) => { await deps.sourceService.updateSourceRevisionKey( @@ -111,6 +119,7 @@ const loadSourceChunksEffect = ( input.pageParams, { assetUrlsByFilePath, + snapshot, workspaceId: workspace.id, onRevisionKey: async (revisionKey) => { await deps.sourceService.updateSourceRevisionKey( @@ -294,4 +303,60 @@ function sourceNotFound(): JsonRouteResult<{ readonly message: string }> { return routeResult.error(404, "Source not found.") } +function sourceSnapshotProcessing( + input: LoadSourceChunksInput, +): JsonRouteResult<{ + readonly chunks: [] + readonly pagination?: { + readonly page: number + readonly pageSize: number + readonly total: 0 + readonly totalPages: 0 + } + readonly message: string +}> { + if (input.shouldLoadAll) { + return routeResult.ok( + { + chunks: [], + message: "Source parsed snapshot is still being prepared.", + }, + 202, + ) + } + + return routeResult.ok( + { + chunks: [], + pagination: { + page: input.pageParams.page, + pageSize: input.pageParams.pageSize, + total: 0, + totalPages: 0, + }, + message: "Source parsed snapshot is still being prepared.", + }, + 202, + ) +} + +function isCompleteSnapshot( + snapshot: + | { + readonly snapshotManifestKey?: string | null + readonly snapshotManifestUrl?: string | null + } + | null, +): snapshot is { + readonly snapshotManifestKey: string + readonly snapshotManifestUrl: string +} { + return ( + typeof snapshot?.snapshotManifestKey === "string" && + snapshot.snapshotManifestKey.length > 0 && + typeof snapshot.snapshotManifestUrl === "string" && + snapshot.snapshotManifestUrl.length > 0 + ) +} + export { createRouteChunks } diff --git a/src/domains/sources/route-dependencies.ts b/src/domains/sources/route-dependencies.ts index 25313ce..d48ede2 100644 --- a/src/domains/sources/route-dependencies.ts +++ b/src/domains/sources/route-dependencies.ts @@ -46,6 +46,7 @@ const defaultDependencies: SourceRouteServiceDependencies = { sourceService: { findInWorkspace: defaultSourceService.findInWorkspace, getParseAssetUrls: defaultSourceService.getParseAssetUrls, + getParseSnapshotMetadata: defaultSourceService.getParseSnapshotMetadata, hideDemoSource: defaultSourceService.hideDemoSource, listHiddenDemoSourceIds: defaultSourceService.listHiddenDemoSourceIds, localizeRemoteDocument: defaultSourceService.localizeRemoteDocument, diff --git a/src/domains/sources/route-types.ts b/src/domains/sources/route-types.ts index 7303ec3..79d0635 100644 --- a/src/domains/sources/route-types.ts +++ b/src/domains/sources/route-types.ts @@ -183,6 +183,15 @@ type SourceWorkflowService = { workspaceId: string, sourceId: string, ) => Promise>> + readonly getParseSnapshotMetadata: ( + workspaceId: string, + sourceId: string, + ) => Promise<{ + readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null + readonly assetUrlsByFilePath: Readonly> + } | null> readonly hideDemoSource: ( workspaceId: string, demoSourceId: string, diff --git a/src/domains/sources/service.ts b/src/domains/sources/service.ts index 0d7c7f6..57701e6 100644 --- a/src/domains/sources/service.ts +++ b/src/domains/sources/service.ts @@ -21,6 +21,10 @@ type SourceService = { workspaceId: string, sourceId: string, ) => Promise>> + readonly getParseSnapshotMetadata: ( + workspaceId: string, + sourceId: string, + ) => ReturnType readonly listForWorkspace: (workspaceId: string) => Promise readonly localizeRemoteDocument: ( workspaceId: string, @@ -106,6 +110,7 @@ const retrySourceToKnowhere: SourceService["retrySourceToKnowhere"] = ( export const sourceService: SourceService = { findInWorkspace: sourceWorkflowRuntime.findInWorkspace, getParseAssetUrls: sourceWorkflowRuntime.getParseAssetUrls, + getParseSnapshotMetadata: sourceWorkflowRuntime.getParseSnapshotMetadata, hideDemoSource: sourceWorkflowRuntime.hideDemoSource, listHiddenDemoSourceIds: sourceWorkflowRuntime.listHiddenDemoSourceIds, listForWorkspace: sourceWorkflowRuntime.listForWorkspace, diff --git a/src/domains/sources/source-parse-result-repository.ts b/src/domains/sources/source-parse-result-repository.ts index 8a5711e..c3e9859 100644 --- a/src/domains/sources/source-parse-result-repository.ts +++ b/src/domains/sources/source-parse-result-repository.ts @@ -12,11 +12,22 @@ import { sourceRowRepository } from "./source-row-repository" type SaveSourceParseResultInput = { readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string + readonly snapshotManifestKey?: string readonly assetUrlsByFilePath: Readonly> } type SourceParseResultProgress = { readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null + readonly assetUrlsByFilePath: Readonly> +} + +export type SourceParseSnapshotMetadata = { + readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null readonly assetUrlsByFilePath: Readonly> } @@ -35,6 +46,10 @@ type SourceParseResultRepository = { workspaceId: string, sourceId: string, ) => Effect.Effect + readonly getParseSnapshotMetadataEffect: ( + workspaceId: string, + sourceId: string, + ) => Effect.Effect readonly getParseAssetUrlsEffect: ( workspaceId: string, sourceId: string, @@ -62,12 +77,16 @@ const saveParseResultEffect: SourceParseResultRepository["saveParseResultEffect" .values({ sourceId, resultBlobUrl: input.resultBlobUrl, + snapshotManifestUrl: input.snapshotManifestUrl, + snapshotManifestKey: input.snapshotManifestKey, assetUrls: input.assetUrlsByFilePath, }) .onConflictDoUpdate({ target: sourceParseResults.sourceId, set: { resultBlobUrl: input.resultBlobUrl, + snapshotManifestUrl: input.snapshotManifestUrl, + snapshotManifestKey: input.snapshotManifestKey, assetUrls: input.assetUrlsByFilePath, updatedAt: sql`now()`, }, @@ -93,12 +112,16 @@ const mergeParseAssetUrlsEffect: SourceParseResultRepository["mergeParseAssetUrl .values({ sourceId, resultBlobUrl: input.resultBlobUrl, + snapshotManifestUrl: input.snapshotManifestUrl, + snapshotManifestKey: input.snapshotManifestKey, assetUrls: input.assetUrlsByFilePath, }) .onConflictDoUpdate({ target: sourceParseResults.sourceId, set: { resultBlobUrl: input.resultBlobUrl, + snapshotManifestUrl: input.snapshotManifestUrl, + snapshotManifestKey: input.snapshotManifestKey, assetUrls: buildAtomicAssetUrlsMergeSql( input.assetUrlsByFilePath, ), @@ -124,6 +147,8 @@ const getParseResultProgressEffect: SourceParseResultRepository["getParseResultP db .select({ resultBlobUrl: sourceParseResults.resultBlobUrl, + snapshotManifestUrl: sourceParseResults.snapshotManifestUrl, + snapshotManifestKey: sourceParseResults.snapshotManifestKey, assetUrls: sourceParseResults.assetUrls, }) .from(sourceParseResults) @@ -135,6 +160,8 @@ const getParseResultProgressEffect: SourceParseResultRepository["getParseResultP return { resultBlobUrl: progress.resultBlobUrl, + snapshotManifestUrl: progress.snapshotManifestUrl, + snapshotManifestKey: progress.snapshotManifestKey, assetUrlsByFilePath: progress.assetUrls, } }) @@ -159,9 +186,17 @@ const getParseAssetUrlsEffect: SourceParseResultRepository["getParseAssetUrlsEff return row[0]?.assetUrls ?? {} }) +const getParseSnapshotMetadataEffect: SourceParseResultRepository["getParseSnapshotMetadataEffect"] = + (workspaceId: string, sourceId: string) => + Effect.gen(function* () { + const progress = yield* getParseResultProgressEffect(workspaceId, sourceId) + return progress + }) + export const sourceParseResultRepository: SourceParseResultRepository = { saveParseResultEffect, mergeParseAssetUrlsEffect, getParseResultProgressEffect, + getParseSnapshotMetadataEffect, getParseAssetUrlsEffect, } diff --git a/src/domains/sources/source-reconcile-route-workflow.test.ts b/src/domains/sources/source-reconcile-route-workflow.test.ts index 170941f..4821229 100644 --- a/src/domains/sources/source-reconcile-route-workflow.test.ts +++ b/src/domains/sources/source-reconcile-route-workflow.test.ts @@ -3,7 +3,6 @@ import { afterEach, describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ createParsedResultStorageAdapter: vi.fn(), saveParseResult: vi.fn(), - writeParsedResultAssetIndex: vi.fn(), loggerError: vi.fn(), loggerInfo: vi.fn(), loggerWarn: vi.fn(), @@ -27,7 +26,6 @@ vi.mock("@/domains/sources/workflow-runtime", () => ({ vi.mock("./parse-result-storage-adapter", () => ({ createParsedResultStorageAdapter: mocks.createParsedResultStorageAdapter, - writeParsedResultAssetIndex: mocks.writeParsedResultAssetIndex, })) vi.mock("@/integrations/knowhere", () => ({ @@ -66,7 +64,7 @@ describe("sourceReconcileRouteWorkflow", () => { }) }) - it("caches parsed assets before marking the source ready", async () => { + it("writes a parsed snapshot before marking the source ready", async () => { const context = createWorkflowContext() const continuations: ContinuationTriggerInput[] = [] const restore = @@ -86,13 +84,16 @@ describe("sourceReconcileRouteWorkflow", () => { "page_citation_assets/page-1.png": "https://blob.example/page_citation_assets/page-1.png", }, + parsedSnapshot: { + manifestKey: + "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + manifestUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + }, }) const client = { jobs: {}, knowledge: { cacheJobResult } } mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) mocks.makeKnowhereClient.mockReturnValue(client) - mocks.writeParsedResultAssetIndex.mockResolvedValue( - "https://blob.example/asset-index.json", - ) mocks.saveParseResult.mockResolvedValue({ id: "parse_result_1" }) mocks.pollSourceReconciliation.mockResolvedValue({ kind: "ready-to-prepare", @@ -124,19 +125,16 @@ describe("sourceReconcileRouteWorkflow", () => { jobId: "job_1", storageAdapter, }) - expect(mocks.writeParsedResultAssetIndex).toHaveBeenCalledWith({ - workspaceId: "workspace_1", - sourceId: "source_1", - assetUrlsByFilePath: { - "page_citation_assets/page-1.png": - "https://blob.example/page_citation_assets/page-1.png", - }, - }) expect(mocks.saveParseResult).toHaveBeenCalledWith( "workspace_1", "source_1", { - resultBlobUrl: "https://blob.example/asset-index.json", + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", assetUrlsByFilePath: { "page_citation_assets/page-1.png": "https://blob.example/page_citation_assets/page-1.png", @@ -152,6 +150,43 @@ describe("sourceReconcileRouteWorkflow", () => { expect(continuations).toEqual([]) }) + it("does not mark ready when snapshot manifest storage is missing", async () => { + const context = createWorkflowContext() + const storageAdapter = { + adapter: { + writeObject: vi.fn(), + }, + keyPrefix: "workspaces/workspace_1/sources/source_1/parsed-result", + } + const cacheJobResult = vi.fn().mockResolvedValue({ + assetUrlsByFilePath: {}, + }) + mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) + mocks.makeKnowhereClient.mockReturnValue({ + jobs: {}, + knowledge: { cacheJobResult }, + }) + mocks.pollSourceReconciliation.mockResolvedValue({ + kind: "ready-to-prepare", + jobId: "job_1", + documentId: "doc_1", + }) + + await expect( + sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ + context, + payload: sourceReconcileRouteWorkflow.normalizeReconcilePayload({ + workspaceId: "workspace_1", + sourceId: "source_1", + apiKey: "jwt_1", + }), + }), + ).rejects.toThrow("Parsed result snapshot was not written") + + expect(mocks.saveParseResult).not.toHaveBeenCalled() + expect(mocks.markSourceReadyAfterReconciliation).not.toHaveBeenCalled() + }) + it("triggers a fresh poll run when Knowhere is still running after the segment budget", async () => { const context = createWorkflowContext() const continuations: ContinuationTriggerInput[] = [] diff --git a/src/domains/sources/source-reconcile-route-workflow.ts b/src/domains/sources/source-reconcile-route-workflow.ts index cc03d40..cb28c20 100644 --- a/src/domains/sources/source-reconcile-route-workflow.ts +++ b/src/domains/sources/source-reconcile-route-workflow.ts @@ -10,7 +10,6 @@ import { makeKnowhereClient } from "@/integrations/knowhere" import { logger } from "@/lib/logger" import { createParsedResultStorageAdapter, - writeParsedResultAssetIndex, } from "./parse-result-storage-adapter" import { sourceWorkflowRuntime } from "./workflow-runtime" @@ -24,6 +23,14 @@ type ReconcilePayload = { type ReconcilePhase = "poll-and-ready" | "poll-and-mirror" | "asset-batches" +type ParsedSnapshotResponse = { + readonly assetUrlsByFilePath?: Readonly> + readonly parsedSnapshot?: { + readonly manifestKey: string + readonly manifestUrl?: string + } +} + type NormalizedReconcilePayload = { readonly workspaceId: string readonly sourceId: string @@ -122,27 +129,32 @@ async function runPollAndMirrorWorkflow(input: { return } - const assetUrlsByFilePath = await context.run("cache-job-result-assets", async () => { - const cachedResult = await client.knowledge.cacheJobResult({ + const snapshot = await context.run("cache-job-result-snapshot", async () => { + const cachedResult = (await client.knowledge.cacheJobResult({ jobId: jobToPrepare.jobId, storageAdapter: createParsedResultStorageAdapter({ workspaceId, sourceId, }), - }) - return cachedResult.assetUrlsByFilePath ?? {} + })) as ParsedSnapshotResponse + const manifest = cachedResult.parsedSnapshot + if (!manifest?.manifestUrl || !manifest.manifestKey) { + throw new Error( + "Parsed result snapshot was not written; refusing to mark source ready.", + ) + } + return { + assetUrlsByFilePath: cachedResult.assetUrlsByFilePath ?? {}, + snapshotManifestUrl: manifest.manifestUrl, + snapshotManifestKey: manifest.manifestKey, + } }) - const resultBlobUrl = await context.run("write-parse-result-asset-index", async () => - writeParsedResultAssetIndex({ - workspaceId, - sourceId, - assetUrlsByFilePath, - }), - ) await context.run("save-parse-result-assets", async () => sourceWorkflowRuntime.saveParseResult(workspaceId, sourceId, { - resultBlobUrl, - assetUrlsByFilePath, + resultBlobUrl: snapshot.snapshotManifestUrl, + snapshotManifestUrl: snapshot.snapshotManifestUrl, + snapshotManifestKey: snapshot.snapshotManifestKey, + assetUrlsByFilePath: snapshot.assetUrlsByFilePath, }), ) diff --git a/src/domains/sources/workflow-runtime.ts b/src/domains/sources/workflow-runtime.ts index bbec158..8527fab 100644 --- a/src/domains/sources/workflow-runtime.ts +++ b/src/domains/sources/workflow-runtime.ts @@ -62,6 +62,17 @@ type SourceWorkflowRuntime = UploadRepositoryRuntime & { sourceId: string, ) => Promise<{ readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null + readonly assetUrlsByFilePath: Readonly> + } | null> + readonly getParseSnapshotMetadata: ( + workspaceId: string, + sourceId: string, + ) => Promise<{ + readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null readonly assetUrlsByFilePath: Readonly> } | null> readonly listForWorkspace: (workspaceId: string) => Promise @@ -248,6 +259,12 @@ const getParseAssetUrls: SourceWorkflowRuntime["getParseAssetUrls"] = ( sourceRepository.getParseAssetUrlsEffect(workspaceId, sourceId), ) +const getParseSnapshotMetadata: SourceWorkflowRuntime["getParseSnapshotMetadata"] = + (workspaceId: string, sourceId: string) => + databaseRuntime.runPromise( + sourceRepository.getParseSnapshotMetadataEffect(workspaceId, sourceId), + ) + function createUploadRepository( runtime: UploadRepositoryRuntime = sourceWorkflowRuntime, ): UploadSourceRepository { @@ -287,6 +304,7 @@ export const sourceWorkflowRuntime: SourceWorkflowRuntime = { findInWorkspace, getParseAssetUrls, getParseResultProgress, + getParseSnapshotMetadata, hideDemoSource, listForWorkspace, listHiddenDemoSourceIds, diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index 720a303..5e62a2c 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -161,10 +161,9 @@ export type NewDemoSourceVisibility = typeof demoSourceVisibilities.$inferInsert /** * Notebook-owned parse-result artifact index for one source. * - * Knowhere's chunk list currently may omit media asset URLs, while parsed chunk - * metadata still points to ZIP-relative files like `images/image-1.jpg`. - * This table stores the Notebook Blob copy of the result ZIP plus a - * file-path-to-public-URL map for those extracted parsed artifacts. + * Blob is the Notebook-owned read model for parsed chunks after source + * reconciliation completes. This row stores the current parsed snapshot + * manifest and the file-path-to-public-URL map for parsed media artifacts. */ export const sourceParseResults = pgTable( "source_parse_results", @@ -175,6 +174,8 @@ export const sourceParseResults = pgTable( .references(() => sources.id, { onDelete: "cascade" }) .unique(), resultBlobUrl: text("result_blob_url").notNull(), + snapshotManifestUrl: text("snapshot_manifest_url"), + snapshotManifestKey: text("snapshot_manifest_key"), assetUrls: jsonb("asset_urls") .$type>>() .notNull(), From 6b30c48d7cebe1baa6593181613ad4f1765b1bf0 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sat, 4 Jul 2026 03:24:37 +0800 Subject: [PATCH 07/12] Sync remote parsed snapshots into Notebook Blob --- .../sources/[sourceId]/chunks/route.test.ts | 361 ++++++++++++++++-- .../remote-parsed-snapshot-sync.test.ts | 316 +++++++++++++++ .../sources/remote-parsed-snapshot-sync.ts | 210 ++++++++++ src/domains/sources/route-chunks.ts | 30 +- src/domains/sources/route-dependencies.ts | 1 + src/domains/sources/route-types.ts | 7 + src/domains/sources/service.ts | 3 + 7 files changed, 897 insertions(+), 31 deletions(-) create mode 100644 src/domains/sources/remote-parsed-snapshot-sync.test.ts create mode 100644 src/domains/sources/remote-parsed-snapshot-sync.ts diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index c41319c..f9b9cbd 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -15,6 +15,7 @@ const mocks = vi.hoisted(() => ({ localizeRemoteDocument: vi.fn(), makeKnowhereClient: vi.fn(), requireUser: vi.fn(), + syncRemoteParsedSnapshot: vi.fn(), updateSourceRevisionKey: vi.fn(), })) @@ -54,6 +55,7 @@ vi.mock("@/domains/sources/service", () => ({ getParseAssetUrls: mocks.getSourceParseAssetUrls, getParseSnapshotMetadata: mocks.getSourceParseSnapshotMetadata, localizeRemoteDocument: mocks.localizeRemoteDocument, + syncRemoteParsedSnapshot: mocks.syncRemoteParsedSnapshot, updateSourceRevisionKey: mocks.updateSourceRevisionKey, }, })) @@ -84,6 +86,15 @@ describe("GET /api/sources/[sourceId]/chunks", () => { assetUrlsByFilePath: {}, }) mocks.updateSourceRevisionKey.mockResolvedValue(null) + mocks.syncRemoteParsedSnapshot.mockResolvedValue({ + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + assetUrlsByFilePath: {}, + }) }) it("serves API-owned demo chunks for anonymous canonical demo sources", async () => { @@ -644,7 +655,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() }) - it("materializes a remote source id on open before loading chunks", async () => { + it("materializes a remote source id and loads chunks from the parsed Blob snapshot", async () => { const knowhereClient = { documents: { list: vi.fn(async () => ({ @@ -661,29 +672,10 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }, ], })), - listChunks: vi.fn(async () => ({ - documentId: "doc_remote", - jobResultId: "job_result_1", - chunks: [ - { - id: "dchk_remote", - chunkId: "parser_remote", - chunkType: "text", - content: "Remote chunk", - sectionPath: "Summary", - sourceChunkPath: "Default_Root/remote.pdf/Summary", - filePath: null, - metadata: {}, - sortOrder: 0, - }, - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })), + listChunks: vi.fn(), + }, + knowledge: { + cacheJobResult: vi.fn(), }, } mocks.getCurrentUser.mockResolvedValue({ @@ -700,7 +692,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { mocks.fetchDemoChunkPage.mockRejectedValue(new Error("not a demo")) mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") mocks.makeKnowhereClient.mockReturnValue(knowhereClient) - mocks.localizeRemoteDocument.mockResolvedValue({ + const localizedSource = { id: "00000000-0000-0000-0000-000000000009", workspaceId: "workspace_1", title: "remote.pdf", @@ -718,6 +710,79 @@ describe("GET /api/sources/[sourceId]/chunks", () => { createdAt: new Date("2026-05-10T00:00:00.000Z"), updatedAt: new Date("2026-05-10T00:00:00.000Z"), deletedAt: null, + } + mocks.localizeRemoteDocument.mockResolvedValue(localizedSource) + mocks.syncRemoteParsedSnapshot.mockResolvedValue({ + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + assetUrlsByFilePath: { + "pages/page-1.png": "https://blob.example/pages/page-1.png", + }, + }) + mocks.blobGet.mockImplementation(async (pathname: string) => { + if (pathname.endsWith("/manifest/current.json")) { + return { + statusCode: 200, + stream: createTextStream( + JSON.stringify({ + version: 1, + kind: "knowhere-parsed-result-snapshot", + jobId: "job_result_1", + documentId: "doc_remote", + sourceFileName: "remote.pdf", + totalChunks: 1, + chunkPageSize: 1, + chunkPages: [ + { + page: 1, + pageSize: 1, + chunkCount: 1, + key: "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/chunks/page-1.json", + }, + ], + assetUrlsByFilePath: { + "pages/page-1.png": "https://blob.example/pages/page-1.png", + }, + createdAt: "2026-07-03T00:00:00.000Z", + }), + ), + } + } + if (pathname.endsWith("/chunks/page-1.json")) { + return { + statusCode: 200, + stream: createTextStream( + JSON.stringify({ + version: 1, + jobId: "job_result_1", + documentId: "doc_remote", + sourceFileName: "remote.pdf", + page: 1, + pageSize: 1, + total: 1, + totalPages: 1, + chunks: [ + { + id: "dchk_remote", + chunkId: "parser_remote", + chunkType: "page", + content: "Remote chunk", + sectionPath: "Summary", + sourceChunkPath: "Default_Root/remote.pdf/Summary", + filePath: "pages/page-1.png", + metadata: {}, + sortOrder: 0, + }, + ], + }), + ), + } + } + return null }) const response = await GET( @@ -737,6 +802,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { chunkId: "dchk_remote", parserChunkId: "parser_remote", documentId: "doc_remote", + assetUrl: "https://blob.example/pages/page-1.png", sourceTitle: "remote.pdf", }, ], @@ -764,14 +830,253 @@ describe("GET /api/sources/[sourceId]/chunks", () => { revisionKey: "job_result_1", }, ) - expect(knowhereClient.documents.listChunks).toHaveBeenCalledWith( - "doc_remote", + expect(mocks.syncRemoteParsedSnapshot).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + source: localizedSource, + client: { + documents: knowhereClient.documents, + knowledge: knowhereClient.knowledge, + }, + }) + expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() + expect(knowhereClient.knowledge.cacheJobResult).not.toHaveBeenCalled() + }) + + it("returns processing for a remote source when the parsed Blob snapshot is incomplete", async () => { + const knowhereClient = { + documents: { + list: vi.fn(async () => ({ + documents: [ + { + documentId: "doc_remote", + namespace: "default", + status: "active", + currentJobResultId: "job_result_1", + sourceFileName: "remote.pdf", + documentMetadata: { + mimeType: "application/pdf", + }, + }, + ], + })), + listChunks: vi.fn(), + }, + knowledge: { + cacheJobResult: vi.fn(), + }, + } + mocks.getCurrentUser.mockResolvedValue({ + id: "user_1", + email: null, + name: null, + }) + mocks.ensureWorkspace.mockResolvedValue({ + id: "workspace_1", + userId: "user_1", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + }) + mocks.fetchDemoChunkPage.mockRejectedValue(new Error("not a demo")) + mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") + mocks.makeKnowhereClient.mockReturnValue(knowhereClient) + mocks.localizeRemoteDocument.mockResolvedValue({ + id: "00000000-0000-0000-0000-000000000009", + workspaceId: "workspace_1", + title: "remote.pdf", + mimeType: "application/pdf", + sizeBytes: 0, + status: "ready", + failureReason: null, + knowhereJobId: "job_result_1", + knowhereDocumentId: "doc_remote", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + }) + mocks.syncRemoteParsedSnapshot.mockResolvedValue(null) + + const response = await GET( + new NextRequest( + "http://localhost:3001/api/sources/knowhere-doc:default:doc_remote/chunks?page=1&pageSize=1", + ), { + params: Promise.resolve({ + sourceId: "knowhere-doc:default:doc_remote", + }), + }, + ) + + await expect(response.json()).resolves.toEqual({ + chunks: [], + pagination: { page: 1, pageSize: 1, - includeAssetUrls: true, + total: 0, + totalPages: 0, + }, + message: "Source parsed snapshot is still being prepared.", + }) + expect(response.status).toBe(202) + expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() + }) + + it("loads remote snapshot chunks after an interrupted sync left the source parsing", async () => { + const knowhereClient = { + documents: { + list: vi.fn(async () => ({ + documents: [ + { + documentId: "doc_remote", + namespace: "default", + status: "active", + currentJobResultId: "job_result_1", + sourceFileName: "remote.pdf", + documentMetadata: { + mimeType: "application/pdf", + }, + }, + ], + })), + listChunks: vi.fn(), + }, + knowledge: { + cacheJobResult: vi.fn(), + }, + } + mocks.getCurrentUser.mockResolvedValue({ + id: "user_1", + email: null, + name: null, + }) + mocks.ensureWorkspace.mockResolvedValue({ + id: "workspace_1", + userId: "user_1", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + }) + mocks.fetchDemoChunkPage.mockRejectedValue(new Error("not a demo")) + mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") + mocks.makeKnowhereClient.mockReturnValue(knowhereClient) + mocks.localizeRemoteDocument.mockResolvedValue({ + id: "00000000-0000-0000-0000-000000000009", + workspaceId: "workspace_1", + title: "remote.pdf", + mimeType: "application/pdf", + sizeBytes: 0, + status: "parsing", + failureReason: null, + knowhereJobId: "job_result_1", + knowhereDocumentId: "doc_remote", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + }) + mocks.syncRemoteParsedSnapshot.mockResolvedValue({ + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + assetUrlsByFilePath: {}, + }) + mocks.blobGet.mockImplementation(async (pathname: string) => { + if (pathname.endsWith("/manifest/current.json")) { + return { + statusCode: 200, + stream: createTextStream( + JSON.stringify({ + version: 1, + kind: "knowhere-parsed-result-snapshot", + jobId: "job_result_1", + documentId: "doc_remote", + sourceFileName: "remote.pdf", + totalChunks: 1, + chunkPageSize: 1, + chunkPages: [ + { + page: 1, + pageSize: 1, + chunkCount: 1, + key: "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/chunks/page-1.json", + }, + ], + assetUrlsByFilePath: {}, + createdAt: "2026-07-03T00:00:00.000Z", + }), + ), + } + } + if (pathname.endsWith("/chunks/page-1.json")) { + return { + statusCode: 200, + stream: createTextStream( + JSON.stringify({ + version: 1, + jobId: "job_result_1", + documentId: "doc_remote", + sourceFileName: "remote.pdf", + page: 1, + pageSize: 1, + total: 1, + totalPages: 1, + chunks: [ + { + id: "dchk_remote", + chunkId: "parser_remote", + chunkType: "text", + content: "Remote chunk", + sectionPath: "Summary", + sourceChunkPath: "Default_Root/remote.pdf/Summary", + metadata: {}, + sortOrder: 0, + }, + ], + }), + ), + } + } + return null + }) + + const response = await GET( + new NextRequest( + "http://localhost:3001/api/sources/knowhere-doc:default:doc_remote/chunks?page=1&pageSize=1", + ), + { + params: Promise.resolve({ + sourceId: "knowhere-doc:default:doc_remote", + }), }, ) + + await expect(response.json()).resolves.toMatchObject({ + chunks: [ + { + chunkId: "dchk_remote", + parserChunkId: "parser_remote", + documentId: "doc_remote", + sourceTitle: "remote.pdf", + }, + ], + pagination: { + page: 1, + pageSize: 1, + total: 1, + }, + }) + expect(response.status).toBe(200) + expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() }) }) diff --git a/src/domains/sources/remote-parsed-snapshot-sync.test.ts b/src/domains/sources/remote-parsed-snapshot-sync.test.ts new file mode 100644 index 0000000..8b0d7f9 --- /dev/null +++ b/src/domains/sources/remote-parsed-snapshot-sync.test.ts @@ -0,0 +1,316 @@ +import { afterEach, describe, expect, it, vi } from "vitest" + +const mocks = vi.hoisted(() => ({ + createParsedResultStorageAdapter: vi.fn(), + loggerInfo: vi.fn(), +})) + +vi.mock("./parse-result-storage-adapter", () => ({ + createParsedResultStorageAdapter: mocks.createParsedResultStorageAdapter, +})) + +vi.mock("@/lib/logger", () => ({ + logger: { + info: mocks.loggerInfo, + }, +})) + +import type { Source, SourceParseResult } from "@/infrastructure/db/schema" +import { syncRemoteParsedSnapshot } from "./remote-parsed-snapshot-sync" + +describe("syncRemoteParsedSnapshot", () => { + afterEach(() => { + vi.clearAllMocks() + }) + + it("caches an already-parsed remote document into the Notebook Blob snapshot", async () => { + const storageAdapter = { + adapter: { + writeObject: vi.fn(), + }, + keyPrefix: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result", + } + const listChunks = vi.fn(async () => ({ + jobResultId: "job_result_1", + })) + const cacheJobResult = vi.fn(async () => ({ + assetUrlsByFilePath: { + "pages/page-1.png": "https://blob.example/pages/page-1.png", + }, + parsedSnapshot: { + manifestKey: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + manifestUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + }, + })) + const repository = createRepository() + mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) + + const snapshot = await syncRemoteParsedSnapshot({ + workspaceId: "workspace_1", + source: makeSource({ + knowhereJobId: null, + }), + client: { + documents: { + listChunks, + }, + knowledge: { + cacheJobResult, + }, + }, + repository, + }) + + expect(listChunks).toHaveBeenCalledWith("doc_remote", { + page: 1, + pageSize: 1, + includeAssetUrls: false, + }) + expect(mocks.createParsedResultStorageAdapter).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "00000000-0000-0000-0000-000000000009", + }) + expect(cacheJobResult).toHaveBeenCalledWith({ + jobId: "job_result_1", + storageAdapter, + }) + expect(repository.markParsing).toHaveBeenCalledWith( + "workspace_1", + "00000000-0000-0000-0000-000000000009", + "job_result_1", + "doc_remote", + ) + expect(repository.saveParseResult).toHaveBeenCalledWith( + "workspace_1", + "00000000-0000-0000-0000-000000000009", + { + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + assetUrlsByFilePath: { + "pages/page-1.png": "https://blob.example/pages/page-1.png", + }, + }, + ) + expect(repository.markReady).toHaveBeenCalledWith( + "workspace_1", + "00000000-0000-0000-0000-000000000009", + "doc_remote", + ) + expect(snapshot).toEqual({ + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + assetUrlsByFilePath: { + "pages/page-1.png": "https://blob.example/pages/page-1.png", + }, + }) + }) + + it("returns the existing complete snapshot without calling Knowhere", async () => { + const existingSnapshot = { + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + assetUrlsByFilePath: {}, + } + const repository = createRepository({ + getParseSnapshotMetadata: vi.fn(async () => existingSnapshot), + }) + const listChunks = vi.fn() + const cacheJobResult = vi.fn() + + const snapshot = await syncRemoteParsedSnapshot({ + workspaceId: "workspace_1", + source: makeSource(), + client: { + documents: { + listChunks, + }, + knowledge: { + cacheJobResult, + }, + }, + repository, + }) + + expect(snapshot).toEqual(existingSnapshot) + expect(listChunks).not.toHaveBeenCalled() + expect(cacheJobResult).not.toHaveBeenCalled() + expect(repository.markParsing).not.toHaveBeenCalled() + expect(repository.markReady).not.toHaveBeenCalled() + }) + + it("marks a parsing source ready when a complete snapshot already exists", async () => { + const existingSnapshot = { + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + assetUrlsByFilePath: {}, + } + const repository = createRepository({ + getParseSnapshotMetadata: vi.fn(async () => existingSnapshot), + }) + const listChunks = vi.fn() + const cacheJobResult = vi.fn() + + const snapshot = await syncRemoteParsedSnapshot({ + workspaceId: "workspace_1", + source: makeSource({ + status: "parsing", + knowhereDocumentId: "doc_remote", + }), + client: { + documents: { + listChunks, + }, + knowledge: { + cacheJobResult, + }, + }, + repository, + }) + + expect(snapshot).toEqual(existingSnapshot) + expect(listChunks).not.toHaveBeenCalled() + expect(cacheJobResult).not.toHaveBeenCalled() + expect(repository.markReady).toHaveBeenCalledWith( + "workspace_1", + "00000000-0000-0000-0000-000000000009", + "doc_remote", + ) + }) + + it("retries an interrupted remote snapshot sync from a parsing source row", async () => { + const storageAdapter = { + adapter: { + writeObject: vi.fn(), + }, + keyPrefix: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result", + } + const listChunks = vi.fn() + const cacheJobResult = vi.fn(async () => ({ + assetUrlsByFilePath: {}, + parsedSnapshot: { + manifestKey: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + manifestUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + }, + })) + const repository = createRepository() + mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) + + await syncRemoteParsedSnapshot({ + workspaceId: "workspace_1", + source: makeSource({ + status: "parsing", + knowhereJobId: "job_result_1", + }), + client: { + documents: { + listChunks, + }, + knowledge: { + cacheJobResult, + }, + }, + repository, + }) + + expect(listChunks).not.toHaveBeenCalled() + expect(cacheJobResult).toHaveBeenCalledWith({ + jobId: "job_result_1", + storageAdapter, + }) + expect(repository.markReady).toHaveBeenCalledWith( + "workspace_1", + "00000000-0000-0000-0000-000000000009", + "doc_remote", + ) + }) +}) + +type SyncRemoteParsedSnapshotInput = Parameters< + typeof syncRemoteParsedSnapshot +>[0] + +type TestRemoteParsedSnapshotRepository = NonNullable< + SyncRemoteParsedSnapshotInput["repository"] +> + +function createRepository( + overrides: Partial = {}, +): TestRemoteParsedSnapshotRepository { + return { + ...createRepositoryShape(), + ...overrides, + } +} + +function createRepositoryShape(): TestRemoteParsedSnapshotRepository { + return { + getParseSnapshotMetadata: vi.fn(async () => null), + markParsing: vi.fn(async () => makeSource({ status: "parsing" })), + markReady: vi.fn(async () => makeSource()), + saveParseResult: vi.fn(async () => makeSourceParseResult()), + } +} + +function makeSource(overrides: Partial = {}): Source { + return { + id: "00000000-0000-0000-0000-000000000009", + workspaceId: "workspace_1", + title: "remote.pdf", + mimeType: "application/pdf", + sizeBytes: 0, + status: "ready", + failureReason: null, + knowhereJobId: "job_result_1", + knowhereDocumentId: "doc_remote", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + ...overrides, + } +} + +function makeSourceParseResult( + overrides: Partial = {}, +): SourceParseResult { + return { + id: "parse_result_1", + sourceId: "00000000-0000-0000-0000-000000000009", + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", + assetUrls: {}, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + ...overrides, + } +} diff --git a/src/domains/sources/remote-parsed-snapshot-sync.ts b/src/domains/sources/remote-parsed-snapshot-sync.ts new file mode 100644 index 0000000..fb7edac --- /dev/null +++ b/src/domains/sources/remote-parsed-snapshot-sync.ts @@ -0,0 +1,210 @@ +import "server-only" + +import { Effect } from "effect" + +import type { Source } from "@/infrastructure/db/schema" +import { logger } from "@/lib/logger" +import { createParsedResultStorageAdapter } from "./parse-result-storage-adapter" +import { sourceWorkflowRuntime } from "./workflow-runtime" + +type RemoteParsedSnapshotSource = Pick< + Source, + "id" | "workspaceId" | "knowhereDocumentId" | "knowhereJobId" | "status" +> + +type RemoteParsedSnapshotClient = { + readonly documents: { + listChunks( + documentId: string, + params: { + readonly page: number + readonly pageSize: number + readonly includeAssetUrls: boolean + }, + ): Promise<{ + readonly jobId?: string | null + readonly jobResultId?: string | null + }> + } + readonly knowledge: { + cacheJobResult(params: { + readonly jobId: string + readonly storageAdapter: ReturnType + }): Promise + } +} + +type RemoteParsedSnapshotResponse = { + readonly assetUrlsByFilePath?: Readonly> + readonly parsedSnapshot?: { + readonly manifestKey: string + readonly manifestUrl?: string + } +} + +type RemoteParsedSnapshotRepository = { + readonly getParseSnapshotMetadata: typeof sourceWorkflowRuntime.getParseSnapshotMetadata + readonly markParsing: typeof sourceWorkflowRuntime.markParsing + readonly markReady: typeof sourceWorkflowRuntime.markReady + readonly saveParseResult: typeof sourceWorkflowRuntime.saveParseResult +} + +type SyncRemoteParsedSnapshotInput = { + readonly workspaceId: string + readonly source: RemoteParsedSnapshotSource + readonly client: RemoteParsedSnapshotClient + readonly repository?: RemoteParsedSnapshotRepository +} + +type RemoteParsedSnapshotReadModel = { + readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null + readonly assetUrlsByFilePath: Readonly> +} + +const jobProbePage = 1 +const jobProbePageSize = 1 + +const syncRemoteParsedSnapshotEffect = Effect.fn( + "syncRemoteParsedSnapshot", +)(function* ({ + workspaceId, + source, + client, + repository = sourceWorkflowRuntime, +}: SyncRemoteParsedSnapshotInput) { + const existingSnapshot = yield* Effect.tryPromise(() => + repository.getParseSnapshotMetadata(workspaceId, source.id), + ) + if (isCompleteSnapshot(existingSnapshot)) { + if (source.status === "parsing" && source.knowhereDocumentId) { + yield* Effect.tryPromise(() => + repository.markReady(workspaceId, source.id, source.knowhereDocumentId!), + ) + } + return existingSnapshot + } + + if (source.status !== "ready" && source.status !== "parsing") { + return existingSnapshot + } + + const documentId = source.knowhereDocumentId + if (!documentId) { + return yield* Effect.die( + new Error("Remote source is missing a Knowhere document id."), + ) + } + + const jobId = yield* resolveRemoteDocumentJobId({ + client, + documentId, + source, + }) + yield* Effect.tryPromise(() => + repository.markParsing(workspaceId, source.id, jobId, documentId), + ) + + const snapshot = yield* Effect.tryPromise(async () => { + const cachedResult = await client.knowledge.cacheJobResult({ + jobId, + storageAdapter: createParsedResultStorageAdapter({ + workspaceId, + sourceId: source.id, + }), + }) + const manifest = cachedResult.parsedSnapshot + if (!manifest?.manifestUrl || !manifest.manifestKey) { + throw new Error( + "Remote parsed snapshot was not written; refusing to mark source ready.", + ) + } + return { + assetUrlsByFilePath: cachedResult.assetUrlsByFilePath ?? {}, + snapshotManifestUrl: manifest.manifestUrl, + snapshotManifestKey: manifest.manifestKey, + } + }) + + const savedSnapshot = yield* Effect.tryPromise(() => + repository.saveParseResult(workspaceId, source.id, { + resultBlobUrl: snapshot.snapshotManifestUrl, + snapshotManifestUrl: snapshot.snapshotManifestUrl, + snapshotManifestKey: snapshot.snapshotManifestKey, + assetUrlsByFilePath: snapshot.assetUrlsByFilePath, + }), + ) + if (!savedSnapshot) { + return yield* Effect.die( + new Error("Remote parsed snapshot could not be saved for the source."), + ) + } + + yield* Effect.tryPromise(() => + repository.markReady(workspaceId, source.id, documentId), + ) + + logger.info("sources: remote parsed snapshot synced", { + sourceId: source.id, + documentId, + jobId, + snapshotManifestKey: snapshot.snapshotManifestKey, + }) + + return { + resultBlobUrl: snapshot.snapshotManifestUrl, + snapshotManifestUrl: snapshot.snapshotManifestUrl, + snapshotManifestKey: snapshot.snapshotManifestKey, + assetUrlsByFilePath: snapshot.assetUrlsByFilePath, + } +}) + +export async function syncRemoteParsedSnapshot( + input: SyncRemoteParsedSnapshotInput, +): Promise { + return Effect.runPromise(syncRemoteParsedSnapshotEffect(input)) +} + +const resolveRemoteDocumentJobId = Effect.fn("resolveRemoteDocumentJobId")( + function* ({ + client, + documentId, + source, + }: { + readonly client: RemoteParsedSnapshotClient + readonly documentId: string + readonly source: RemoteParsedSnapshotSource + }) { + if (source.knowhereJobId) return source.knowhereJobId + + const response = yield* Effect.tryPromise(() => + client.documents.listChunks(documentId, { + page: jobProbePage, + pageSize: jobProbePageSize, + includeAssetUrls: false, + }), + ) + const jobId = response.jobId ?? response.jobResultId + if (!jobId) { + return yield* Effect.die( + new Error("Remote document chunk metadata did not include a job id."), + ) + } + return jobId + }, +) + +function isCompleteSnapshot( + snapshot: RemoteParsedSnapshotReadModel | null, +): snapshot is RemoteParsedSnapshotReadModel & { + readonly snapshotManifestKey: string + readonly snapshotManifestUrl: string +} { + return ( + typeof snapshot?.snapshotManifestKey === "string" && + snapshot.snapshotManifestKey.length > 0 && + typeof snapshot.snapshotManifestUrl === "string" && + snapshot.snapshotManifestUrl.length > 0 + ) +} diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index fdbe0b1..ff9a174 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -94,7 +94,6 @@ const loadSourceChunksEffect = ( if (!isCompleteSnapshot(snapshot)) { return sourceSnapshotProcessing(input) } - if (input.shouldLoadAll) { const chunks = yield* deps.loadChunksForSource(source, client, { snapshot, @@ -168,9 +167,32 @@ const loadRemoteChunkPageEffect = ( revisionKey: remoteDocument.revisionKey ?? null, }), ) + const knowledge = client.knowledge + if (!knowledge) { + throw new Error("Knowhere client does not support parsed snapshot caching.") + } + const snapshot = yield* Effect.tryPromise(() => + deps.sourceService.syncRemoteParsedSnapshot({ + workspaceId: workspace.id, + source, + client: { + documents: client.documents, + knowledge, + }, + }), + ) + if (!isCompleteSnapshot(snapshot)) { + return sourceSnapshotProcessing(input) + } + const readySource = { + ...source, + status: "ready" as const, + knowhereDocumentId: source.knowhereDocumentId ?? remoteDocument.documentId, + } if (input.shouldLoadAll) { - const chunks = yield* deps.loadChunksForSource(source, client, { + const chunks = yield* deps.loadChunksForSource(readySource, client, { + snapshot, workspaceId: workspace.id, onRevisionKey: async (revisionKey) => { await deps.sourceService.updateSourceRevisionKey( @@ -184,10 +206,12 @@ const loadRemoteChunkPageEffect = ( } const chunkPage = yield* deps.loadChunkPageForSource( - source, + readySource, client, input.pageParams, { + assetUrlsByFilePath: snapshot.assetUrlsByFilePath, + snapshot, workspaceId: workspace.id, onRevisionKey: async (revisionKey) => { await deps.sourceService.updateSourceRevisionKey( diff --git a/src/domains/sources/route-dependencies.ts b/src/domains/sources/route-dependencies.ts index d48ede2..cdc5751 100644 --- a/src/domains/sources/route-dependencies.ts +++ b/src/domains/sources/route-dependencies.ts @@ -55,6 +55,7 @@ const defaultDependencies: SourceRouteServiceDependencies = { upsertMaterializedDemoSource: defaultSourceService.upsertMaterializedDemoSource, retrySourceToKnowhere: defaultSourceService.retrySourceToKnowhere, + syncRemoteParsedSnapshot: defaultSourceService.syncRemoteParsedSnapshot, uploadSourceBlobToKnowhere: defaultSourceService.uploadSourceBlobToKnowhere, uploadSourceToKnowhere: defaultSourceService.uploadSourceToKnowhere, }, diff --git a/src/domains/sources/route-types.ts b/src/domains/sources/route-types.ts index 79d0635..a8deed9 100644 --- a/src/domains/sources/route-types.ts +++ b/src/domains/sources/route-types.ts @@ -18,8 +18,13 @@ import type { import type { RouteResult } from "@/lib/route-result" import type { SourceBlobUploadInput } from "./blob-upload" import type { sourceViewOptionsBySourceId } from "./counts" +import type { syncRemoteParsedSnapshot } from "./remote-parsed-snapshot-sync" import type { UploadKnowhereClient } from "./upload" +type SyncRemoteParsedSnapshotInput = Parameters< + typeof syncRemoteParsedSnapshot +>[0] + type SourceRouteKnowhereClient = UploadKnowhereClient & ChunkKnowhereClient & { readonly documents: ChunkKnowhereClient["documents"] & { @@ -54,6 +59,7 @@ type SourceRouteKnowhereClient = UploadKnowhereClient & }> archive(documentId: string): Promise } + readonly knowledge?: SyncRemoteParsedSnapshotInput["client"]["knowledge"] } type SourceUploadRequest = @@ -171,6 +177,7 @@ type SourceWorkflowService = { source: Source, knowhere: UploadKnowhereClient, ) => Promise + readonly syncRemoteParsedSnapshot: typeof syncRemoteParsedSnapshot readonly findInWorkspace: ( workspaceId: string, sourceId: string, diff --git a/src/domains/sources/service.ts b/src/domains/sources/service.ts index 57701e6..2e25596 100644 --- a/src/domains/sources/service.ts +++ b/src/domains/sources/service.ts @@ -9,6 +9,7 @@ import { uploadSourceBlobToKnowhereEffect, uploadSourceToKnowhereEffect, } from "./upload" +import { syncRemoteParsedSnapshot } from "./remote-parsed-snapshot-sync" import { retrySourceToKnowhereEffect } from "./retry" import { sourceWorkflowRuntime } from "./workflow-runtime" @@ -65,6 +66,7 @@ type SourceService = { source: Source, knowhere: UploadKnowhereClient, ) => Promise + readonly syncRemoteParsedSnapshot: typeof syncRemoteParsedSnapshot } const uploadSourceToKnowhere: SourceService["uploadSourceToKnowhere"] = ( @@ -122,4 +124,5 @@ export const sourceService: SourceService = { uploadSourceToKnowhere, uploadSourceBlobToKnowhere, retrySourceToKnowhere, + syncRemoteParsedSnapshot, } From 95a879698315e26f2336389df45f347325281627 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sat, 4 Jul 2026 15:50:01 +0800 Subject: [PATCH 08/12] Use server-safe parsed snapshot loads --- .../sources/[sourceId]/chunks/route.test.ts | 8 +++--- .../remote-parsed-snapshot-sync.test.ts | 26 +++++++++---------- .../sources/remote-parsed-snapshot-sync.ts | 8 +++--- .../source-reconcile-route-workflow.test.ts | 10 +++---- .../source-reconcile-route-workflow.ts | 8 +++--- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index f9b9cbd..f00d531 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -675,7 +675,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { listChunks: vi.fn(), }, knowledge: { - cacheJobResult: vi.fn(), + loadJobResult: vi.fn(), }, } mocks.getCurrentUser.mockResolvedValue({ @@ -839,7 +839,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }, }) expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() - expect(knowhereClient.knowledge.cacheJobResult).not.toHaveBeenCalled() + expect(knowhereClient.knowledge.loadJobResult).not.toHaveBeenCalled() }) it("returns processing for a remote source when the parsed Blob snapshot is incomplete", async () => { @@ -862,7 +862,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { listChunks: vi.fn(), }, knowledge: { - cacheJobResult: vi.fn(), + loadJobResult: vi.fn(), }, } mocks.getCurrentUser.mockResolvedValue({ @@ -945,7 +945,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { listChunks: vi.fn(), }, knowledge: { - cacheJobResult: vi.fn(), + loadJobResult: vi.fn(), }, } mocks.getCurrentUser.mockResolvedValue({ diff --git a/src/domains/sources/remote-parsed-snapshot-sync.test.ts b/src/domains/sources/remote-parsed-snapshot-sync.test.ts index 8b0d7f9..6a91582 100644 --- a/src/domains/sources/remote-parsed-snapshot-sync.test.ts +++ b/src/domains/sources/remote-parsed-snapshot-sync.test.ts @@ -23,7 +23,7 @@ describe("syncRemoteParsedSnapshot", () => { vi.clearAllMocks() }) - it("caches an already-parsed remote document into the Notebook Blob snapshot", async () => { + it("loads an already-parsed remote document into the Notebook Blob snapshot", async () => { const storageAdapter = { adapter: { writeObject: vi.fn(), @@ -34,7 +34,7 @@ describe("syncRemoteParsedSnapshot", () => { const listChunks = vi.fn(async () => ({ jobResultId: "job_result_1", })) - const cacheJobResult = vi.fn(async () => ({ + const loadJobResult = vi.fn(async () => ({ assetUrlsByFilePath: { "pages/page-1.png": "https://blob.example/pages/page-1.png", }, @@ -58,7 +58,7 @@ describe("syncRemoteParsedSnapshot", () => { listChunks, }, knowledge: { - cacheJobResult, + loadJobResult, }, }, repository, @@ -73,7 +73,7 @@ describe("syncRemoteParsedSnapshot", () => { workspaceId: "workspace_1", sourceId: "00000000-0000-0000-0000-000000000009", }) - expect(cacheJobResult).toHaveBeenCalledWith({ + expect(loadJobResult).toHaveBeenCalledWith({ jobId: "job_result_1", storageAdapter, }) @@ -130,7 +130,7 @@ describe("syncRemoteParsedSnapshot", () => { getParseSnapshotMetadata: vi.fn(async () => existingSnapshot), }) const listChunks = vi.fn() - const cacheJobResult = vi.fn() + const loadJobResult = vi.fn() const snapshot = await syncRemoteParsedSnapshot({ workspaceId: "workspace_1", @@ -140,7 +140,7 @@ describe("syncRemoteParsedSnapshot", () => { listChunks, }, knowledge: { - cacheJobResult, + loadJobResult, }, }, repository, @@ -148,7 +148,7 @@ describe("syncRemoteParsedSnapshot", () => { expect(snapshot).toEqual(existingSnapshot) expect(listChunks).not.toHaveBeenCalled() - expect(cacheJobResult).not.toHaveBeenCalled() + expect(loadJobResult).not.toHaveBeenCalled() expect(repository.markParsing).not.toHaveBeenCalled() expect(repository.markReady).not.toHaveBeenCalled() }) @@ -167,7 +167,7 @@ describe("syncRemoteParsedSnapshot", () => { getParseSnapshotMetadata: vi.fn(async () => existingSnapshot), }) const listChunks = vi.fn() - const cacheJobResult = vi.fn() + const loadJobResult = vi.fn() const snapshot = await syncRemoteParsedSnapshot({ workspaceId: "workspace_1", @@ -180,7 +180,7 @@ describe("syncRemoteParsedSnapshot", () => { listChunks, }, knowledge: { - cacheJobResult, + loadJobResult, }, }, repository, @@ -188,7 +188,7 @@ describe("syncRemoteParsedSnapshot", () => { expect(snapshot).toEqual(existingSnapshot) expect(listChunks).not.toHaveBeenCalled() - expect(cacheJobResult).not.toHaveBeenCalled() + expect(loadJobResult).not.toHaveBeenCalled() expect(repository.markReady).toHaveBeenCalledWith( "workspace_1", "00000000-0000-0000-0000-000000000009", @@ -205,7 +205,7 @@ describe("syncRemoteParsedSnapshot", () => { "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result", } const listChunks = vi.fn() - const cacheJobResult = vi.fn(async () => ({ + const loadJobResult = vi.fn(async () => ({ assetUrlsByFilePath: {}, parsedSnapshot: { manifestKey: @@ -228,14 +228,14 @@ describe("syncRemoteParsedSnapshot", () => { listChunks, }, knowledge: { - cacheJobResult, + loadJobResult, }, }, repository, }) expect(listChunks).not.toHaveBeenCalled() - expect(cacheJobResult).toHaveBeenCalledWith({ + expect(loadJobResult).toHaveBeenCalledWith({ jobId: "job_result_1", storageAdapter, }) diff --git a/src/domains/sources/remote-parsed-snapshot-sync.ts b/src/domains/sources/remote-parsed-snapshot-sync.ts index fb7edac..3e939c4 100644 --- a/src/domains/sources/remote-parsed-snapshot-sync.ts +++ b/src/domains/sources/remote-parsed-snapshot-sync.ts @@ -27,7 +27,7 @@ type RemoteParsedSnapshotClient = { }> } readonly knowledge: { - cacheJobResult(params: { + loadJobResult(params: { readonly jobId: string readonly storageAdapter: ReturnType }): Promise @@ -107,21 +107,21 @@ const syncRemoteParsedSnapshotEffect = Effect.fn( ) const snapshot = yield* Effect.tryPromise(async () => { - const cachedResult = await client.knowledge.cacheJobResult({ + const loadedResult = await client.knowledge.loadJobResult({ jobId, storageAdapter: createParsedResultStorageAdapter({ workspaceId, sourceId: source.id, }), }) - const manifest = cachedResult.parsedSnapshot + const manifest = loadedResult.parsedSnapshot if (!manifest?.manifestUrl || !manifest.manifestKey) { throw new Error( "Remote parsed snapshot was not written; refusing to mark source ready.", ) } return { - assetUrlsByFilePath: cachedResult.assetUrlsByFilePath ?? {}, + assetUrlsByFilePath: loadedResult.assetUrlsByFilePath ?? {}, snapshotManifestUrl: manifest.manifestUrl, snapshotManifestKey: manifest.manifestKey, } diff --git a/src/domains/sources/source-reconcile-route-workflow.test.ts b/src/domains/sources/source-reconcile-route-workflow.test.ts index 4821229..ea68cbe 100644 --- a/src/domains/sources/source-reconcile-route-workflow.test.ts +++ b/src/domains/sources/source-reconcile-route-workflow.test.ts @@ -79,7 +79,7 @@ describe("sourceReconcileRouteWorkflow", () => { }, keyPrefix: "workspaces/workspace_1/sources/source_1/parsed-result", } - const cacheJobResult = vi.fn().mockResolvedValue({ + const loadJobResult = vi.fn().mockResolvedValue({ assetUrlsByFilePath: { "page_citation_assets/page-1.png": "https://blob.example/page_citation_assets/page-1.png", @@ -91,7 +91,7 @@ describe("sourceReconcileRouteWorkflow", () => { "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", }, }) - const client = { jobs: {}, knowledge: { cacheJobResult } } + const client = { jobs: {}, knowledge: { loadJobResult } } mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) mocks.makeKnowhereClient.mockReturnValue(client) mocks.saveParseResult.mockResolvedValue({ id: "parse_result_1" }) @@ -121,7 +121,7 @@ describe("sourceReconcileRouteWorkflow", () => { workspaceId: "workspace_1", sourceId: "source_1", }) - expect(cacheJobResult).toHaveBeenCalledWith({ + expect(loadJobResult).toHaveBeenCalledWith({ jobId: "job_1", storageAdapter, }) @@ -158,13 +158,13 @@ describe("sourceReconcileRouteWorkflow", () => { }, keyPrefix: "workspaces/workspace_1/sources/source_1/parsed-result", } - const cacheJobResult = vi.fn().mockResolvedValue({ + const loadJobResult = vi.fn().mockResolvedValue({ assetUrlsByFilePath: {}, }) mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) mocks.makeKnowhereClient.mockReturnValue({ jobs: {}, - knowledge: { cacheJobResult }, + knowledge: { loadJobResult }, }) mocks.pollSourceReconciliation.mockResolvedValue({ kind: "ready-to-prepare", diff --git a/src/domains/sources/source-reconcile-route-workflow.ts b/src/domains/sources/source-reconcile-route-workflow.ts index cb28c20..16f9054 100644 --- a/src/domains/sources/source-reconcile-route-workflow.ts +++ b/src/domains/sources/source-reconcile-route-workflow.ts @@ -129,22 +129,22 @@ async function runPollAndMirrorWorkflow(input: { return } - const snapshot = await context.run("cache-job-result-snapshot", async () => { - const cachedResult = (await client.knowledge.cacheJobResult({ + const snapshot = await context.run("load-job-result-snapshot", async () => { + const loadedResult = (await client.knowledge.loadJobResult({ jobId: jobToPrepare.jobId, storageAdapter: createParsedResultStorageAdapter({ workspaceId, sourceId, }), })) as ParsedSnapshotResponse - const manifest = cachedResult.parsedSnapshot + const manifest = loadedResult.parsedSnapshot if (!manifest?.manifestUrl || !manifest.manifestKey) { throw new Error( "Parsed result snapshot was not written; refusing to mark source ready.", ) } return { - assetUrlsByFilePath: cachedResult.assetUrlsByFilePath ?? {}, + assetUrlsByFilePath: loadedResult.assetUrlsByFilePath ?? {}, snapshotManifestUrl: manifest.manifestUrl, snapshotManifestKey: manifest.manifestKey, } From 0304aeee051ff36f5bf1cd572237475668f3bc74 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sat, 4 Jul 2026 17:22:26 +0800 Subject: [PATCH 09/12] Sync legacy parsed sources before chunk reads --- .../sources/[sourceId]/chunks/route.test.ts | 23 +++ src/components/chunks-panel.tsx | 14 +- src/components/source-row.tsx | 7 +- src/components/workspace-citation-focus.ts | 3 + .../workspace-selected-chunks.test.ts | 32 +++ src/components/workspace-selected-chunks.ts | 30 ++- src/components/workspace-shell-layout.tsx | 3 + src/components/workspace-shell.tsx | 1 + src/domains/chat/route-answer.ts | 28 ++- src/domains/chat/route-service.test.ts | 63 ++++++ .../demo/workspace-source-resolution.ts | 2 +- src/domains/sources/counts.test.ts | 70 ++++++- src/domains/sources/counts.ts | 80 ++++++-- src/domains/sources/route-chunks.ts | 28 ++- src/domains/sources/route-dependencies.ts | 2 + src/domains/sources/route-listing.ts | 8 +- src/domains/sources/route-service.test.ts | 184 ++++++++++++++++++ src/domains/sources/route-types.ts | 10 + src/domains/sources/service.ts | 60 ++++++ src/domains/sources/types.ts | 2 +- src/domains/workspace/client.test.ts | 27 +++ src/domains/workspace/client.ts | 4 + src/domains/workspace/initial-state.ts | 8 +- src/infrastructure/db/schema.ts | 10 +- 24 files changed, 645 insertions(+), 54 deletions(-) diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index f00d531..911646a 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ ensureWorkspace: vi.fn(), fetchDemoChunkPage: vi.fn(), findSourceInWorkspace: vi.fn(), + ensureParsedSnapshotForRead: vi.fn(), getCurrentUser: vi.fn(), getSourceParseAssetUrls: vi.fn(), getSourceParseSnapshotMetadata: vi.fn(), @@ -51,6 +52,7 @@ vi.mock("@vercel/blob", () => ({ vi.mock("@/domains/sources/service", () => ({ sourceService: { + ensureParsedSnapshotForRead: mocks.ensureParsedSnapshotForRead, findInWorkspace: mocks.findSourceInWorkspace, getParseAssetUrls: mocks.getSourceParseAssetUrls, getParseSnapshotMetadata: mocks.getSourceParseSnapshotMetadata, @@ -85,6 +87,15 @@ describe("GET /api/sources/[sourceId]/chunks", () => { "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", assetUrlsByFilePath: {}, }) + mocks.ensureParsedSnapshotForRead.mockResolvedValue({ + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + assetUrlsByFilePath: {}, + }) mocks.updateSourceRevisionKey.mockResolvedValue(null) mocks.syncRemoteParsedSnapshot.mockResolvedValue({ resultBlobUrl: @@ -504,6 +515,17 @@ describe("GET /api/sources/[sourceId]/chunks", () => { "images/chart.png": "https://blob.example/images/chart.png", }, }) + mocks.ensureParsedSnapshotForRead.mockResolvedValue({ + resultBlobUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", + snapshotManifestUrl: + "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", + assetUrlsByFilePath: { + "images/chart.png": "https://blob.example/images/chart.png", + }, + }) mocks.blobGet.mockImplementation(async (pathname: string) => { if (pathname.endsWith("/manifest/current.json")) { return { @@ -633,6 +655,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") mocks.makeKnowhereClient.mockReturnValue(knowhereClient) mocks.getSourceParseSnapshotMetadata.mockResolvedValue(null) + mocks.ensureParsedSnapshotForRead.mockResolvedValue(null) const response = await GET( new NextRequest( diff --git a/src/components/chunks-panel.tsx b/src/components/chunks-panel.tsx index 898ca39..78cd170 100644 --- a/src/components/chunks-panel.tsx +++ b/src/components/chunks-panel.tsx @@ -57,6 +57,7 @@ export type ChunksPanelProps = { isLoadingMore?: boolean; isLoadingAllChunks?: boolean; hasMoreChunks?: boolean; + processingMessage?: string | null; onLoadMore?: () => void; onLoadAllChunks?: () => void; onLoginClick?: () => void; @@ -84,6 +85,7 @@ export function ChunksPanel({ isLoadingMore = false, isLoadingAllChunks = false, hasMoreChunks = false, + processingMessage = null, onLoadMore, onLoadAllChunks, onLoginClick, @@ -360,7 +362,7 @@ export function ChunksPanel({ className="mx-auto flex w-[90%] min-w-0 max-w-[1600px] flex-col items-center p-3 sm:p-6" > {isLoading ? ( - + ) : chunks.length === 0 ? ( selectedSource ? ( @@ -1138,13 +1140,19 @@ function EmptyChunks(): ReactNode { ); } -function LoadingChunks(): ReactNode { +function LoadingChunks({ + message, +}: { + readonly message?: string | null; +}): ReactNode { return (
-

Loading parsed chunks...

+

+ {message ?? "Loading parsed chunks..."} +

); } diff --git a/src/components/source-row.tsx b/src/components/source-row.tsx index abfe8cb..e964c93 100644 --- a/src/components/source-row.tsx +++ b/src/components/source-row.tsx @@ -102,7 +102,7 @@ export function SourceRow({ }`} > {isReady - ? `${getReadySourceLabel(source)} · ${source.chunkCount ?? 0} chunks` + ? getReadySourceStatusText(source) : source.status === "parsing" ? "Preparing" : source.status === "uploading" @@ -197,6 +197,11 @@ function getReadySourceLabel(source: SourceView): string { return "Processed"; } +function getReadySourceStatusText(source: SourceView): string { + if (typeof source.chunkCount !== "number") return getReadySourceLabel(source); + return `${getReadySourceLabel(source)} · ${source.chunkCount} chunks`; +} + function fileIconTint(title: string): { bg: string; fg: string } { const ext = title.split(".").pop()?.toLowerCase(); switch (ext) { diff --git a/src/components/workspace-citation-focus.ts b/src/components/workspace-citation-focus.ts index b0d0f66..3485dd2 100644 --- a/src/components/workspace-citation-focus.ts +++ b/src/components/workspace-citation-focus.ts @@ -43,6 +43,7 @@ type WorkspaceCitationFocus = { readonly requestChunkFocus: (chunkId: string | null) => void readonly isSelectedChunksLoading: boolean readonly isSelectedChunksLoadingMore: boolean + readonly selectedChunksMessage: string | null readonly selectedChunks: ParsedChunkView[] readonly selectedSource: SourceView | undefined } @@ -80,6 +81,7 @@ export function useWorkspaceCitationFocus({ handleLoadMoreChunks, isSelectedChunksLoading, isSelectedChunksLoadingMore, + selectedChunksMessage, selectedChunks, selectedSource, } = useWorkspaceSelectedChunks({ @@ -277,6 +279,7 @@ export function useWorkspaceCitationFocus({ pendingCitationId, prefetchedChunksBySourceId, requestChunkFocus, + selectedChunksMessage, selectedChunks, selectedSource, } diff --git a/src/components/workspace-selected-chunks.test.ts b/src/components/workspace-selected-chunks.test.ts index 2772c85..fa08df5 100644 --- a/src/components/workspace-selected-chunks.test.ts +++ b/src/components/workspace-selected-chunks.test.ts @@ -129,6 +129,38 @@ describe("useWorkspaceSelectedChunks", () => { }); }); + it("treats a processing chunk page as a loading state", async () => { + fetchChunkPageMock.mockResolvedValue({ + chunks: [], + isProcessing: true, + message: "Source parsed snapshot is still being prepared.", + pagination: { + page: 1, + pageSize: 50, + total: 0, + totalPages: 0, + }, + }); + + const { result } = renderHook( + () => + useWorkspaceSelectedChunks({ + selectedSourceId: "source_1", + sources: [readySource], + prefetchedChunksBySourceId: {}, + }), + { wrapper: createSWRWrapper }, + ); + + await waitFor(() => + expect(result.current.selectedChunksMessage).toBe( + "Source parsed snapshot is still being prepared.", + ), + ); + expect(result.current.isSelectedChunksLoading).toBe(true); + expect(result.current.selectedChunks).toEqual([]); + }); + it("returns an empty chunk list when no source is selected", () => { const { result } = renderHook( () => diff --git a/src/components/workspace-selected-chunks.ts b/src/components/workspace-selected-chunks.ts index 4ec9f97..6d79175 100644 --- a/src/components/workspace-selected-chunks.ts +++ b/src/components/workspace-selected-chunks.ts @@ -24,6 +24,7 @@ type WorkspaceSelectedChunks = { readonly handleLoadMoreChunks: () => void readonly isSelectedChunksLoading: boolean readonly isSelectedChunksLoadingMore: boolean + readonly selectedChunksMessage: string | null readonly selectedChunks: ParsedChunkView[] readonly selectedSource: SourceView | undefined } @@ -57,8 +58,11 @@ export function useWorkspaceSelectedChunks({ { revalidateIfStale: false, keepPreviousData: false, + refreshInterval: (pages: readonly SourceChunksResponse[] | undefined) => + hasProcessingChunkPage(pages) ? 2_000 : 0, }, ) + const selectedChunksMessage = getSelectedChunksMessage(selectedChunkPages) const pagedSelectedChunks = useMemo( () => resolveChunkConnectionTargets( @@ -90,10 +94,11 @@ export function useWorkspaceSelectedChunks({ typeof selectedChunkPages[selectedChunkPageCount - 1] === "undefined", ) const isSelectedChunksLoading = - selectedChunkSourceId !== null && - !prefetchedSelectedChunks && - !selectedChunkPages && - isChunksLoading + Boolean(selectedChunksMessage) || + (selectedChunkSourceId !== null && + !prefetchedSelectedChunks && + !selectedChunkPages && + isChunksLoading) function handleLoadMoreChunks(): void { if (!hasMoreSelectedChunks || isSelectedChunksLoadingMore) return @@ -105,6 +110,7 @@ export function useWorkspaceSelectedChunks({ handleLoadMoreChunks, isSelectedChunksLoading, isSelectedChunksLoadingMore, + selectedChunksMessage, selectedChunks, selectedSource, } @@ -118,6 +124,22 @@ function fetchChunksByKey([ return workspaceClient.fetchChunkPage(sourceId, page) } +function hasProcessingChunkPage( + pages: readonly SourceChunksResponse[] | undefined, +): boolean { + return pages?.some((page) => page.isProcessing) ?? false +} + +function getSelectedChunksMessage( + pages: readonly SourceChunksResponse[] | undefined, +): string | null { + const page = pages?.find( + (candidate) => + candidate.isProcessing && typeof candidate.message === "string", + ) + return page?.message ?? null +} + function mergeVisibleChunkAssetUrls( chunks: readonly ParsedChunkView[], visibleChunks: readonly ParsedChunkView[], diff --git a/src/components/workspace-shell-layout.tsx b/src/components/workspace-shell-layout.tsx index 235eb07..3b9e997 100644 --- a/src/components/workspace-shell-layout.tsx +++ b/src/components/workspace-shell-layout.tsx @@ -80,6 +80,7 @@ export type WorkspaceShellLayoutProps = { readonly pendingCitationId: string | null readonly readySourceCount: number readonly selectedChunks: readonly ParsedChunkView[] + readonly selectedChunksMessage: string | null readonly selectedSourceFile: SourceOriginalFileView | null readonly selectedSourceId: string | null readonly selectedSourceTitle: string | null @@ -271,6 +272,7 @@ export function WorkspaceShellLayout( isLoading={props.isSelectedChunksLoading} isLoadingAllChunks={props.isSelectedAllChunksLoading} isLoadingMore={props.isSelectedChunksLoadingMore} + processingMessage={props.selectedChunksMessage} hasMoreChunks={props.hasMoreSelectedChunks} onLoadAllChunks={props.onLoadAllChunks} onLoadMore={props.onLoadMoreChunks} @@ -412,6 +414,7 @@ export function WorkspaceShellLayout( isLoading={props.isSelectedChunksLoading} isLoadingAllChunks={props.isSelectedAllChunksLoading} isLoadingMore={props.isSelectedChunksLoadingMore} + processingMessage={props.selectedChunksMessage} hasMoreChunks={props.hasMoreSelectedChunks} onLoadAllChunks={props.onLoadAllChunks} onLoadMore={props.onLoadMoreChunks} diff --git a/src/components/workspace-shell.tsx b/src/components/workspace-shell.tsx index 254c91e..4dd8b9c 100644 --- a/src/components/workspace-shell.tsx +++ b/src/components/workspace-shell.tsx @@ -236,6 +236,7 @@ function WorkspaceShellContent({ pendingCitationId={citationFocus.pendingCitationId} readySourceCount={sourceWorkflow.readySourceCount} selectedChunks={citationFocus.selectedChunks} + selectedChunksMessage={citationFocus.selectedChunksMessage} selectedSourceFile={citationFocus.selectedSource?.originalFile ?? null} selectedSourceId={sourceWorkflow.selectedSourceId} selectedSourceTitle={selectedSourceTitle} diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index ea75943..3df55e0 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -68,8 +68,32 @@ const answerChatEffect = (input: AnswerChatInput) => apiKey, }), ) - const loadSourceAssetUrls = (source: (typeof sources)[number]) => - sourceService.getParseAssetUrls(workspace.id, source.id) + const loadSourceAssetUrls = async ( + source: (typeof sources)[number], + ): Promise>> => { + try { + const snapshot = await sourceService.ensureParsedSnapshotForRead({ + workspaceId: workspace.id, + source, + client: client.knowledge + ? { + documents: client.documents, + knowledge: client.knowledge, + } + : null, + }) + if (snapshot) return snapshot.assetUrlsByFilePath + } catch (error) { + logger.warn("chat: parsed snapshot sync for assets failed", { + workspaceId: workspace.id, + sourceId: source.id, + documentId: source.knowhereDocumentId, + error: summarizeUnknownError(error), + }) + } + + return sourceService.getParseAssetUrls(workspace.id, source.id) + } const result: Either.Either = yield* Effect.tryPromise(() => diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 4358e8a..189e862 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -73,6 +73,7 @@ vi.mock("@/lib/logger", () => ({ import { chatAnswerRouteService } from "./route-answer" import { chatThreadRouteService } from "./route-threads" +import { sourceService } from "@/domains/sources/service" describe("chat route services", () => { beforeEach(() => { @@ -140,6 +141,68 @@ describe("chat route services", () => { ) }) + it("ensures a legacy ready source snapshot before chat loads citation assets", async () => { + const workspace = makeWorkspace() + const documents = { listChunks: vi.fn() } + const knowledge = { loadJobResult: vi.fn() } + const client = { documents, knowledge, retrieval: { query: vi.fn() } } + const readySource = makeSource({ + status: "ready", + knowhereDocumentId: "doc_legacy", + knowhereJobId: null, + }) + const snapshot = { + resultBlobUrl: "https://blob.example/manifest/current.json", + snapshotManifestUrl: "https://blob.example/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", + assetUrlsByFilePath: { + "pages/page-1.png": "https://blob.example/pages/page-1.png", + }, + } + const ensureParsedSnapshotForRead = vi + .spyOn(sourceService, "ensureParsedSnapshotForRead") + .mockResolvedValue(snapshot) + mocks.getAuthenticatedWithClient.mockResolvedValue({ + user: { id: "user_1" }, + workspace, + apiKey: "jwt_123", + client, + }) + mocks.listSourcesForWorkspace.mockResolvedValue([readySource]) + mocks.handleChatTurn.mockImplementation( + async (input: { + readonly loadSourceAssetUrls?: ( + source: Source, + ) => Promise>> + }) => { + const assetUrls = await input.loadSourceAssetUrls?.(readySource) + expect(assetUrls).toEqual(snapshot.assetUrlsByFilePath) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "message_user", role: "user", content: "Show the page" }, + { id: "message_assistant", role: "assistant", content: "Answer" }, + ], + }) + }, + ) + + const result = await chatAnswerRouteService.answerChat({ + body: { message: "Show the page" }, + }) + + expect(result.status).toBe(200) + expect(ensureParsedSnapshotForRead).toHaveBeenCalledWith({ + workspaceId: workspace.id, + source: readySource, + client: { + documents, + knowledge, + }, + }) + }) + it("triggers background reconciliation for parsing sources without blocking chat", async () => { const workspace = makeWorkspace() const client = { retrieval: { query: vi.fn() } } diff --git a/src/domains/demo/workspace-source-resolution.ts b/src/domains/demo/workspace-source-resolution.ts index 39abc5a..d0918af 100644 --- a/src/domains/demo/workspace-source-resolution.ts +++ b/src/domains/demo/workspace-source-resolution.ts @@ -39,7 +39,7 @@ export function resolveWorkspaceDemoSources( } } -export function getWorkspaceSourcesNeedingKnowhereChunkCount( +export function getWorkspaceSourcesNeedingChunkCount( sources: readonly Source[], ): Source[] { return sources.filter((source) => !source.demoKey) diff --git a/src/domains/sources/counts.test.ts b/src/domains/sources/counts.test.ts index 22cdda9..e520eff 100644 --- a/src/domains/sources/counts.test.ts +++ b/src/domains/sources/counts.test.ts @@ -29,13 +29,36 @@ function makeSource(overrides: Partial = {}): Source { } describe("countChunksBySourceId", () => { - it("counts chunks only for ready sources with a Knowhere document id", async () => { - const listChunks = vi.fn().mockResolvedValue({ - pagination: { total: 12 }, - }) + it("counts ready source chunks from the parsed snapshot manifest", async () => { + const listChunks = vi.fn() const mockClient = { documents: { listChunks }, } as unknown as Knowhere + const repository = { + getParseSnapshotMetadata: vi.fn(async (workspaceId: string, sourceId: string) => + sourceId === "ready" + ? { + resultBlobUrl: "https://blob.example/manifest/current.json", + snapshotManifestUrl: "https://blob.example/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/ready/parsed-result/manifest/current.json", + assetUrlsByFilePath: {}, + } + : null, + ), + } + const readSnapshotManifest = vi.fn(async () => ({ + version: 1 as const, + kind: "knowhere-parsed-result-snapshot" as const, + jobId: "job_1", + documentId: "doc_ready", + sourceFileName: "notes.pdf", + totalChunks: 12, + chunkPageSize: 50, + chunkPages: [], + assetUrlsByFilePath: {}, + createdAt: "2026-07-04T00:00:00.000Z", + })) const { countChunksBySourceId } = await import("./counts") @@ -47,22 +70,44 @@ describe("countChunksBySourceId", () => { makeSource({ id: "missing-doc", knowhereDocumentId: null }), ], mockClient, + { + repository, + readSnapshotManifest, + }, ), ) - expect(listChunks).toHaveBeenCalledOnce() - expect(listChunks).toHaveBeenCalledWith("doc_ready", { - page: 1, - pageSize: 1, + expect(listChunks).not.toHaveBeenCalled() + expect(repository.getParseSnapshotMetadata).toHaveBeenCalledWith( + "workspace_1", + "ready", + ) + expect(readSnapshotManifest).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "ready", + manifestKey: + "workspaces/workspace_1/sources/ready/parsed-result/manifest/current.json", }) expect(counts).toEqual(new Map([["ready", 12]])) }) - it("skips a source count when Knowhere chunks lookup fails", async () => { - const listChunks = vi.fn().mockRejectedValue(new Error("temporary outage")) + it("skips a source count when snapshot manifest lookup fails", async () => { + const listChunks = vi.fn() const mockClient = { documents: { listChunks }, } as unknown as Knowhere + const repository = { + getParseSnapshotMetadata: vi.fn(async () => ({ + resultBlobUrl: "https://blob.example/manifest/current.json", + snapshotManifestUrl: "https://blob.example/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/ready/parsed-result/manifest/current.json", + assetUrlsByFilePath: {}, + })), + } + const readSnapshotManifest = vi.fn(async () => { + throw new Error("temporary outage") + }) const { countChunksBySourceId } = await import("./counts") @@ -70,10 +115,15 @@ describe("countChunksBySourceId", () => { countChunksBySourceId( [makeSource({ id: "ready", knowhereDocumentId: "doc_ready" })], mockClient, + { + repository, + readSnapshotManifest, + }, ), ) expect(counts.size).toBe(0) + expect(listChunks).not.toHaveBeenCalled() }) it("does not count materialized demo sources through their copied document id", async () => { diff --git a/src/domains/sources/counts.ts b/src/domains/sources/counts.ts index 310a694..8f934bb 100644 --- a/src/domains/sources/counts.ts +++ b/src/domains/sources/counts.ts @@ -1,15 +1,32 @@ import "server-only" -import { Effect, Either } from "effect" +import { Effect } from "effect" import type Knowhere from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" +import { + readParsedResultSnapshotManifest, + type ParsedResultSnapshotManifest, +} from "./parse-result-storage-adapter" +import { sourceWorkflowRuntime } from "./workflow-runtime" + +type ChunkCountRepository = Pick< + typeof sourceWorkflowRuntime, + "getParseSnapshotMetadata" +> + +type ChunkCountOptions = { + readonly readSnapshotManifest?: typeof readParsedResultSnapshotManifest + readonly repository?: ChunkCountRepository +} export const countChunksBySourceId = ( sources: readonly Source[], client: Knowhere, + options: ChunkCountOptions = {}, ) => Effect.gen(function* () { + void client const readySources = sources.filter( (source) => !source.demoKey && @@ -21,23 +38,18 @@ export const countChunksBySourceId = ( const entries = yield* Effect.all( readySources.map((source) => Effect.gen(function* () { - const documentId = source.knowhereDocumentId - if (!documentId) return [source.id, undefined] as const - - const result = yield* Effect.either( - Effect.tryPromise(() => - client.documents.listChunks(documentId, { - page: 1, - pageSize: 1, - }), + const manifest = yield* Effect.tryPromise(() => + loadSourceSnapshotManifest(source, options), + ).pipe( + Effect.catchAll(() => + Effect.succeed(null), ), ) - - if (Either.isLeft(result)) return [source.id, undefined] as const + if (!manifest) return [source.id, undefined] as const return [ source.id, - result.right.pagination.total, + manifest.totalChunks, ] as const }), ), @@ -55,9 +67,10 @@ export const countChunksBySourceId = ( export const sourceViewOptionsBySourceId = ( sources: readonly Source[], client: Knowhere, + options: ChunkCountOptions = {}, ) => Effect.gen(function* () { - const counts = yield* countChunksBySourceId(sources, client) + const counts = yield* countChunksBySourceId(sources, client, options) return new Map( sources.map((source) => [ source.id, @@ -65,3 +78,42 @@ export const sourceViewOptionsBySourceId = ( ]), ) }) + +async function loadSourceSnapshotManifest( + source: Source, + options: ChunkCountOptions, +): Promise { + const repository = options.repository ?? sourceWorkflowRuntime + const readSnapshotManifest = + options.readSnapshotManifest ?? readParsedResultSnapshotManifest + const snapshot = await repository.getParseSnapshotMetadata( + source.workspaceId, + source.id, + ) + if (!isCompleteSnapshot(snapshot)) return null + + return readSnapshotManifest({ + workspaceId: source.workspaceId, + sourceId: source.id, + manifestKey: snapshot.snapshotManifestKey, + }) +} + +function isCompleteSnapshot( + snapshot: + | { + readonly snapshotManifestKey?: string | null + readonly snapshotManifestUrl?: string | null + } + | null, +): snapshot is { + readonly snapshotManifestKey: string + readonly snapshotManifestUrl: string +} { + return ( + typeof snapshot?.snapshotManifestKey === "string" && + snapshot.snapshotManifestKey.length > 0 && + typeof snapshot.snapshotManifestUrl === "string" && + snapshot.snapshotManifestUrl.length > 0 + ) +} diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index ff9a174..b8fbca2 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -89,7 +89,28 @@ const loadSourceChunksEffect = ( getClientForWorkspace(workspace.id, input.cookieHeader, deps), ) const snapshot = yield* Effect.tryPromise(() => - deps.sourceService.getParseSnapshotMetadata(workspace.id, source.id), + deps.sourceService.ensureParsedSnapshotForRead({ + workspaceId: workspace.id, + source, + client: client.knowledge + ? { + documents: client.documents, + knowledge: client.knowledge, + } + : null, + }), + ).pipe( + Effect.catchAll((error) => + Effect.sync(() => { + logger.warn("sources: parsed snapshot sync for read failed", { + workspaceId: workspace.id, + sourceId: source.id, + documentId: source.knowhereDocumentId, + error: getErrorMessage(error), + }) + return null + }), + ), ) if (!isCompleteSnapshot(snapshot)) { return sourceSnapshotProcessing(input) @@ -109,15 +130,12 @@ const loadSourceChunksEffect = ( return routeResult.ok({ chunks }) } - const assetUrlsByFilePath = yield* Effect.tryPromise(() => - deps.sourceService.getParseAssetUrls(workspace.id, source.id), - ) const chunkPage = yield* deps.loadChunkPageForSource( source, client, input.pageParams, { - assetUrlsByFilePath, + assetUrlsByFilePath: snapshot.assetUrlsByFilePath, snapshot, workspaceId: workspace.id, onRevisionKey: async (revisionKey) => { diff --git a/src/domains/sources/route-dependencies.ts b/src/domains/sources/route-dependencies.ts index cdc5751..af81a3c 100644 --- a/src/domains/sources/route-dependencies.ts +++ b/src/domains/sources/route-dependencies.ts @@ -44,6 +44,8 @@ const defaultDependencies: SourceRouteServiceDependencies = { ), requireUser, sourceService: { + ensureParsedSnapshotForRead: + defaultSourceService.ensureParsedSnapshotForRead, findInWorkspace: defaultSourceService.findInWorkspace, getParseAssetUrls: defaultSourceService.getParseAssetUrls, getParseSnapshotMetadata: defaultSourceService.getParseSnapshotMetadata, diff --git a/src/domains/sources/route-listing.ts b/src/domains/sources/route-listing.ts index 9476285..35c0c72 100644 --- a/src/domains/sources/route-listing.ts +++ b/src/domains/sources/route-listing.ts @@ -3,7 +3,7 @@ import { Effect } from "effect" import { demoView } from "@/domains/demo/view" import { getMaterializedDemoSourceViewOptionsBySourceId, - getWorkspaceSourcesNeedingKnowhereChunkCount, + getWorkspaceSourcesNeedingChunkCount, resolveWorkspaceDemoSources, } from "@/domains/demo/workspace-source-resolution" import { routeResult } from "@/lib/route-result" @@ -96,8 +96,8 @@ const listSourcesEffect = ( client, localSources: demoSourceResolution.workspaceSources, }) - const sourcesNeedingKnowhereChunkCount = - getWorkspaceSourcesNeedingKnowhereChunkCount(workspaceSources) + const sourcesNeedingChunkCount = + getWorkspaceSourcesNeedingChunkCount(workspaceSources) const materializedDemoSourceOptions = getMaterializedDemoSourceViewOptionsBySourceId(workspaceSources, catalog) yield* Effect.sync(() => @@ -111,7 +111,7 @@ const listSourcesEffect = ( }), ) const sourceOptions = yield* deps.getSourceViewOptionsBySourceId( - sourcesNeedingKnowhereChunkCount, + sourcesNeedingChunkCount, client, ) const hiddenDemoSourceIds = new Set( diff --git a/src/domains/sources/route-service.test.ts b/src/domains/sources/route-service.test.ts index 3d11b96..ca37390 100644 --- a/src/domains/sources/route-service.test.ts +++ b/src/domains/sources/route-service.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { Effect } from "effect"; import type { Job } from "@ontos-ai/knowhere-sdk"; +import type { ChunkPage } from "@/domains/chunks"; import type { Source, Workspace } from "@/infrastructure/db/schema"; import { createRouteListing } from "./route-listing"; import { createSourceRouteService } from "./route-service"; @@ -119,6 +120,189 @@ describe("source route service", () => { expect(listHiddenDemoSourceIds).toHaveBeenCalledWith(workspace.id); }); + it("syncs a ready legacy source snapshot before loading displayed chunks", async () => { + const readySource: Source = { + ...source, + id: "00000000-0000-4000-8000-000000000001", + knowhereJobId: null, + knowhereDocumentId: "doc_legacy", + status: "ready", + }; + const snapshot = { + resultBlobUrl: "https://blob.example/manifest/current.json", + snapshotManifestUrl: "https://blob.example/manifest/current.json", + snapshotManifestKey: + "workspaces/workspace_1/sources/00000000-0000-4000-8000-000000000001/parsed-result/manifest/current.json", + assetUrlsByFilePath: { + "images/chart.png": "https://blob.example/images/chart.png", + }, + }; + const listChunks = vi.fn(async () => { + throw new Error("display path must not call Knowhere listChunks"); + }); + const knowledge = { + loadJobResult: vi.fn(), + }; + const knowhereClient = { + documents: { + archive: vi.fn(async () => undefined), + listChunks, + }, + jobs: { + create: vi.fn(), + get: vi.fn(), + upload: vi.fn(), + }, + knowledge, + }; + const loadedPage: ChunkPage = { + chunks: [ + { + chunkId: "chunk_1", + type: "text", + content: "Chunk body", + sourceTitle: "notes.pdf", + }, + ], + pagination: { + page: 1, + pageSize: 50, + total: 1, + totalPages: 1, + }, + }; + const loadChunkPageForSource = vi.fn(() => Effect.succeed(loadedPage)); + const syncRemoteParsedSnapshot = vi.fn(async () => snapshot); + const service = createSourceRouteService({ + ensureApiKeyForWorkspace: vi.fn(async () => "jwt_123"), + ensureWorkspace: vi.fn(async () => workspace), + getCurrentUser: vi.fn(async () => ({ + id: "user_1", + email: null, + name: null, + })), + loadChunkPageForSource, + makeKnowhereClient: vi.fn(() => knowhereClient), + sourceService: { + findInWorkspace: vi.fn(async () => readySource), + ensureParsedSnapshotForRead: syncRemoteParsedSnapshot, + updateSourceRevisionKey: vi.fn(async () => readySource), + }, + }); + + const result = await service.loadSourceChunks({ + cookieHeader: "session=abc", + sourceId: readySource.id, + shouldLoadAll: false, + pageParams: { page: 1, pageSize: 50 }, + }); + + expect(result).toEqual({ + status: 200, + body: { + chunks: [ + { + chunkId: "chunk_1", + type: "text", + content: "Chunk body", + sourceTitle: "notes.pdf", + }, + ], + pagination: { + page: 1, + pageSize: 50, + total: 1, + totalPages: 1, + }, + }, + }); + expect(syncRemoteParsedSnapshot).toHaveBeenCalledWith({ + workspaceId: workspace.id, + source: readySource, + client: { + documents: knowhereClient.documents, + knowledge, + }, + }); + expect(loadChunkPageForSource).toHaveBeenCalledWith( + readySource, + knowhereClient, + { page: 1, pageSize: 50 }, + expect.objectContaining({ + assetUrlsByFilePath: snapshot.assetUrlsByFilePath, + snapshot, + workspaceId: workspace.id, + }), + ); + expect(listChunks).not.toHaveBeenCalled(); + }); + + it("returns processing when a ready legacy source snapshot is not synced yet", async () => { + const readySource: Source = { + ...source, + id: "00000000-0000-4000-8000-000000000002", + knowhereDocumentId: "doc_legacy", + knowhereJobId: null, + status: "ready", + }; + const knowledge = { + loadJobResult: vi.fn(), + }; + const knowhereClient = { + documents: { + archive: vi.fn(async () => undefined), + listChunks: vi.fn(), + }, + jobs: { + create: vi.fn(), + get: vi.fn(), + upload: vi.fn(), + }, + knowledge, + }; + const loadChunkPageForSource = vi.fn(); + const service = createSourceRouteService({ + ensureApiKeyForWorkspace: vi.fn(async () => "jwt_123"), + ensureWorkspace: vi.fn(async () => workspace), + getCurrentUser: vi.fn(async () => ({ + id: "user_1", + email: null, + name: null, + })), + loadChunkPageForSource, + makeKnowhereClient: vi.fn(() => knowhereClient), + sourceService: { + findInWorkspace: vi.fn(async () => readySource), + ensureParsedSnapshotForRead: vi.fn(async () => { + throw new Error("snapshot import still running"); + }), + }, + }); + + const result = await service.loadSourceChunks({ + cookieHeader: "session=abc", + sourceId: readySource.id, + shouldLoadAll: false, + pageParams: { page: 1, pageSize: 50 }, + }); + + expect(result).toEqual({ + status: 202, + body: { + chunks: [], + pagination: { + page: 1, + pageSize: 50, + total: 0, + totalPages: 0, + }, + message: "Source parsed snapshot is still being prepared.", + }, + }); + expect(loadChunkPageForSource).not.toHaveBeenCalled(); + expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled(); + }); + it("lists shared default and legacy namespace documents as lightweight remote sources", async () => { const localReadySource: Source = { ...source, diff --git a/src/domains/sources/route-types.ts b/src/domains/sources/route-types.ts index a8deed9..a32cad2 100644 --- a/src/domains/sources/route-types.ts +++ b/src/domains/sources/route-types.ts @@ -162,6 +162,16 @@ type SourceRouteService = { } type SourceWorkflowService = { + readonly ensureParsedSnapshotForRead: (input: { + readonly workspaceId: string + readonly source: Source + readonly client?: SyncRemoteParsedSnapshotInput["client"] | null + }) => Promise<{ + readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null + readonly assetUrlsByFilePath: Readonly> + } | null> readonly uploadSourceToKnowhere: ( workspace: Workspace, file: File, diff --git a/src/domains/sources/service.ts b/src/domains/sources/service.ts index 2e25596..11e9199 100644 --- a/src/domains/sources/service.ts +++ b/src/domains/sources/service.ts @@ -14,6 +14,9 @@ import { retrySourceToKnowhereEffect } from "./retry" import { sourceWorkflowRuntime } from "./workflow-runtime" type SourceService = { + readonly ensureParsedSnapshotForRead: ( + input: EnsureParsedSnapshotForReadInput, + ) => Promise readonly findInWorkspace: ( workspaceId: string, sourceId: string, @@ -69,6 +72,19 @@ type SourceService = { readonly syncRemoteParsedSnapshot: typeof syncRemoteParsedSnapshot } +type EnsureParsedSnapshotForReadInput = { + readonly workspaceId: string + readonly source: Source + readonly client?: Parameters[0]["client"] | null +} + +type EnsureParsedSnapshotForReadResult = { + readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null + readonly assetUrlsByFilePath: Readonly> +} + const uploadSourceToKnowhere: SourceService["uploadSourceToKnowhere"] = ( workspace: Workspace, file: File, @@ -109,7 +125,25 @@ const retrySourceToKnowhere: SourceService["retrySourceToKnowhere"] = ( }), ) +const ensureParsedSnapshotForRead: SourceService["ensureParsedSnapshotForRead"] = + async ({ workspaceId, source, client }: EnsureParsedSnapshotForReadInput) => { + const existingSnapshot = + await sourceWorkflowRuntime.getParseSnapshotMetadata(workspaceId, source.id) + if (isCompleteParsedSnapshot(existingSnapshot)) return existingSnapshot + + if (!canSyncParsedSnapshotForRead(source, client)) { + return existingSnapshot + } + + return syncRemoteParsedSnapshot({ + workspaceId, + source, + client, + }) + } + export const sourceService: SourceService = { + ensureParsedSnapshotForRead, findInWorkspace: sourceWorkflowRuntime.findInWorkspace, getParseAssetUrls: sourceWorkflowRuntime.getParseAssetUrls, getParseSnapshotMetadata: sourceWorkflowRuntime.getParseSnapshotMetadata, @@ -126,3 +160,29 @@ export const sourceService: SourceService = { retrySourceToKnowhere, syncRemoteParsedSnapshot, } + +function canSyncParsedSnapshotForRead( + source: Source, + client: EnsureParsedSnapshotForReadInput["client"], +): client is Parameters[0]["client"] { + return ( + Boolean(client) && + (source.status === "ready" || source.status === "parsing") && + typeof source.knowhereDocumentId === "string" && + source.knowhereDocumentId.length > 0 + ) +} + +function isCompleteParsedSnapshot( + snapshot: EnsureParsedSnapshotForReadResult | null, +): snapshot is EnsureParsedSnapshotForReadResult & { + readonly snapshotManifestKey: string + readonly snapshotManifestUrl: string +} { + return ( + typeof snapshot?.snapshotManifestKey === "string" && + snapshot.snapshotManifestKey.length > 0 && + typeof snapshot.snapshotManifestUrl === "string" && + snapshot.snapshotManifestUrl.length > 0 + ) +} diff --git a/src/domains/sources/types.ts b/src/domains/sources/types.ts index e1abe9e..8479f87 100644 --- a/src/domains/sources/types.ts +++ b/src/domains/sources/types.ts @@ -48,7 +48,7 @@ export type SourceView = { readonly originalFile?: SourceOriginalFileView /** Official Library metadata when this row is an API-owned catalog item. */ readonly officialLibrary?: SourceOfficialLibraryView - /** Count from the Knowhere chunks API, not a local aggregate. */ + /** Count from the Notebook parsed snapshot manifest when available. */ readonly chunkCount?: number /** User opt-out for this query session. Drives excludeDocumentIds. */ readonly excludedFromQuery?: boolean diff --git a/src/domains/workspace/client.test.ts b/src/domains/workspace/client.test.ts index 42e8c65..bc19bba 100644 --- a/src/domains/workspace/client.test.ts +++ b/src/domains/workspace/client.test.ts @@ -63,6 +63,33 @@ describe("workspaceClient", () => { }) }) + it("preserves source chunk processing messages", async () => { + mockRouteClient.getJson.mockResolvedValue({ + chunks: [], + message: "Source parsed snapshot is still being prepared.", + pagination: { + page: 1, + pageSize: 50, + total: 0, + totalPages: 0, + }, + }) + + const page = await workspaceClient.fetchChunkPage("source_1", 1) + + expect(page).toEqual({ + chunks: [], + isProcessing: true, + message: "Source parsed snapshot is still being prepared.", + pagination: { + page: 1, + pageSize: 50, + total: 0, + totalPages: 0, + }, + }) + }) + it("throws materialization route errors instead of treating them as empty sources", async () => { mockRouteClient.postJsonWithStatus.mockResolvedValue({ status: 502, diff --git a/src/domains/workspace/client.ts b/src/domains/workspace/client.ts index 9a7b3cd..a650b6f 100644 --- a/src/domains/workspace/client.ts +++ b/src/domains/workspace/client.ts @@ -24,6 +24,8 @@ const workspaceClientConfig = { type SourceChunksResponse = { chunks?: ParsedChunkView[] + isProcessing?: boolean + message?: string pagination?: { page: number pageSize: number @@ -128,6 +130,8 @@ async function fetchChunkPage( return { chunks: Array.isArray(body.chunks) ? body.chunks : [], + ...(typeof body.message === "string" ? { message: body.message } : {}), + ...(typeof body.message === "string" ? { isProcessing: true } : {}), pagination: body.pagination, } } diff --git a/src/domains/workspace/initial-state.ts b/src/domains/workspace/initial-state.ts index 368d43b..851a34b 100644 --- a/src/domains/workspace/initial-state.ts +++ b/src/domains/workspace/initial-state.ts @@ -7,7 +7,7 @@ import type { ParsedChunkView } from "@/domains/chunks/types" import { demoView } from "@/domains/demo/view" import { getMaterializedDemoSourceViewOptionsBySourceId, - getWorkspaceSourcesNeedingKnowhereChunkCount, + getWorkspaceSourcesNeedingChunkCount, resolveWorkspaceDemoSources, } from "@/domains/demo/workspace-source-resolution" import { chatThreadService } from "@/domains/chat/thread-service" @@ -331,8 +331,8 @@ export const loadWorkspaceShellInitialStateEffect = ( localSources: demoSourceResolution.workspaceSources, }), ) - const sourcesNeedingKnowhereChunkCount = - getWorkspaceSourcesNeedingKnowhereChunkCount(workspaceSources) + const sourcesNeedingChunkCount = + getWorkspaceSourcesNeedingChunkCount(workspaceSources) const materializedDemoSourceOptions = getMaterializedDemoSourceViewOptionsBySourceId( workspaceSources, @@ -354,7 +354,7 @@ export const loadWorkspaceShellInitialStateEffect = ( operation: "sourceViewOptionsBySourceId", }, deps.sourceViewOptionsBySourceId( - sourcesNeedingKnowhereChunkCount, + sourcesNeedingChunkCount, client, ), ) diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index 5e62a2c..8c909a9 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -17,8 +17,8 @@ import { * - Postgres stores only metadata, status, Knowhere IDs, and chat * threads/messages. * - It does NOT store file bytes or chunk copies in Postgres. Original - * uploads and parsed media artifacts live in Blob storage; chunks are - * fetched on demand from Knowhere's chunks API. + * uploads and parsed-source snapshots live in Blob storage; retrieval + * stays upstream in Knowhere. * * Soft delete: * - Every user-visible resource has a nullable `deleted_at` timestamp. @@ -67,9 +67,9 @@ export type NewWorkspace = typeof workspaces.$inferInsert; * - `status` — lifecycle: uploading | parsing | ready | failed * - `failure_reason` — human-readable error text when status=failed * - `knowhere_job_id` — set once the parse job is created - * - `knowhere_document_id` — set when parsing completes; the sole handle - * used to fetch chunks and to exclude a source - * from a retrieval query + * - `knowhere_document_id` — set when parsing completes; used to import + * parsed snapshots and to exclude a source from a + * retrieval query * - `original_blob_*` — public Blob pointer for the original upload preview * and download path * - `staged_blob_*` — legacy temporary Blob staging pointer retained for From 1269fb22fe74df4a0b7e9a6ba844bc91bc6509f8 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sun, 5 Jul 2026 02:18:47 +0800 Subject: [PATCH 10/12] Move parsed chunk reads and sync into the SDK storage model Replace the hand-rolled Blob snapshot machinery with the SDK's ParsedDocumentStorage: a Vercel-Blob adapter, a QStash-backed resumable sync scheduler + /api/sources/parsed-sync route, and chunk reads routed through knowledge.readChunks/getDocumentOutline (assetUrlPolicy:"durable"). Parse flow syncs to storage before markReady and stages sync failures as failure_stage="storage_sync"; retry resumes sync without reparsing. Chat citation hardening is unified onto the durable read path. Adds migration 0011 (revision_key/sync_status/sync_error on source_parse_results, failure_stage on sources). Deletes chunks/server, remote-parsed-snapshot-sync, and parse-result-storage-adapter. Co-Authored-By: Claude Opus 4.8 --- drizzle/0011_rare_deadpool.sql | 5 + drizzle/meta/0011_snapshot.json | 752 +++++++++++++++ drizzle/meta/_journal.json | 7 + .../demo-sources/materialize/route.test.ts | 1 + .../sources/[sourceId]/chunks/route.test.ts | 682 +++---------- src/app/api/sources/parsed-sync/route.ts | 27 + src/app/api/sources/route.test.ts | 1 + src/domains/chat/index.test.ts | 1 + .../chat/media-asset-hardening.test.ts | 283 ++---- src/domains/chat/media-asset-hardening.ts | 470 +++------ src/domains/chat/media-assets.test.ts | 1 + src/domains/chat/page-citation-assets.test.ts | 1 + src/domains/chat/route-answer.ts | 27 +- src/domains/chat/route-service.test.ts | 83 +- src/domains/chat/service.test.ts | 1 + src/domains/chunks/index.test.ts | 1 + src/domains/chunks/index.ts | 31 +- src/domains/chunks/read.test.ts | 123 +++ src/domains/chunks/read.ts | 142 +++ src/domains/chunks/server.test.ts | 563 ----------- src/domains/chunks/server.ts | 911 ------------------ src/domains/sources/counts.test.ts | 78 +- src/domains/sources/counts.ts | 90 +- .../sources/parse-result-storage-adapter.ts | 269 ------ .../parsed-document-blob-storage.test.ts | 223 +++++ .../sources/parsed-document-blob-storage.ts | 291 ++++++ .../sources/parsed-document-sync-scheduler.ts | 154 +++ .../parsed-sync-route-workflow.test.ts | 201 ++++ .../sources/parsed-sync-route-workflow.ts | 226 +++++ src/domains/sources/reconcile.test.ts | 1 + .../remote-parsed-snapshot-sync.test.ts | 316 ------ .../sources/remote-parsed-snapshot-sync.ts | 210 ---- src/domains/sources/repository.ts | 2 + src/domains/sources/retry.test.ts | 2 + src/domains/sources/route-chunks.ts | 194 ++-- src/domains/sources/route-dependencies.ts | 52 +- src/domains/sources/route-retry.test.ts | 107 ++ src/domains/sources/route-retry.ts | 73 +- src/domains/sources/route-service.test.ts | 157 +-- src/domains/sources/route-types.ts | 41 - src/domains/sources/service.ts | 79 -- .../sources/source-parse-result-repository.ts | 85 +- .../source-reconcile-route-workflow.test.ts | 173 ++-- .../source-reconcile-route-workflow.ts | 121 ++- .../sources/source-reconcile-workflow.test.ts | 1 + src/domains/sources/source-row-repository.ts | 6 + src/domains/sources/upload.test.ts | 1 + src/domains/sources/view.test.ts | 1 + src/domains/sources/workflow-runtime.test.ts | 1 + src/domains/sources/workflow-runtime.ts | 44 +- src/domains/workspace/initial-state.test.ts | 1 + src/domains/workspace/integration.test.ts | 2 +- src/infrastructure/db/schema.ts | 22 +- src/integrations/knowhere.ts | 53 + 54 files changed, 3276 insertions(+), 4114 deletions(-) create mode 100644 drizzle/0011_rare_deadpool.sql create mode 100644 drizzle/meta/0011_snapshot.json create mode 100644 src/app/api/sources/parsed-sync/route.ts create mode 100644 src/domains/chunks/read.test.ts create mode 100644 src/domains/chunks/read.ts delete mode 100644 src/domains/chunks/server.test.ts delete mode 100644 src/domains/chunks/server.ts delete mode 100644 src/domains/sources/parse-result-storage-adapter.ts create mode 100644 src/domains/sources/parsed-document-blob-storage.test.ts create mode 100644 src/domains/sources/parsed-document-blob-storage.ts create mode 100644 src/domains/sources/parsed-document-sync-scheduler.ts create mode 100644 src/domains/sources/parsed-sync-route-workflow.test.ts create mode 100644 src/domains/sources/parsed-sync-route-workflow.ts delete mode 100644 src/domains/sources/remote-parsed-snapshot-sync.test.ts delete mode 100644 src/domains/sources/remote-parsed-snapshot-sync.ts create mode 100644 src/domains/sources/route-retry.test.ts diff --git a/drizzle/0011_rare_deadpool.sql b/drizzle/0011_rare_deadpool.sql new file mode 100644 index 0000000..0cf5000 --- /dev/null +++ b/drizzle/0011_rare_deadpool.sql @@ -0,0 +1,5 @@ +ALTER TABLE "source_parse_results" ALTER COLUMN "result_blob_url" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "source_parse_results" ADD COLUMN "revision_key" text;--> statement-breakpoint +ALTER TABLE "source_parse_results" ADD COLUMN "sync_status" text;--> statement-breakpoint +ALTER TABLE "source_parse_results" ADD COLUMN "sync_error" text;--> statement-breakpoint +ALTER TABLE "sources" ADD COLUMN "failure_stage" text; \ No newline at end of file diff --git a/drizzle/meta/0011_snapshot.json b/drizzle/meta/0011_snapshot.json new file mode 100644 index 0000000..0f8448c --- /dev/null +++ b/drizzle/meta/0011_snapshot.json @@ -0,0 +1,752 @@ +{ + "id": "bab84445-8eb2-4d15-856e-b2f59edcaf47", + "prevId": "cc41c815-4e52-4d01-ac2e-abbc791331b7", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 7174911..fc814c6 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -78,6 +78,13 @@ "when": 1783098230819, "tag": "0010_minor_piledriver", "breakpoints": true + }, + { + "idx": 11, + "version": "7", + "when": 1783175614191, + "tag": "0011_rare_deadpool", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/app/api/demo-sources/materialize/route.test.ts b/src/app/api/demo-sources/materialize/route.test.ts index 9b41725..0818dca 100644 --- a/src/app/api/demo-sources/materialize/route.test.ts +++ b/src/app/api/demo-sources/materialize/route.test.ts @@ -232,6 +232,7 @@ function makeSource( sizeBytes: 5648867, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: null, knowhereDocumentId: "doc_user_copy", stagedBlobPathname: null, diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index 911646a..85fb1bd 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -2,22 +2,17 @@ import { NextRequest } from "next/server" import { beforeEach, describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ - blobGet: vi.fn(), - blobPut: vi.fn(), deleteBlob: vi.fn(), ensureApiKeyForWorkspace: vi.fn(), ensureWorkspace: vi.fn(), fetchDemoChunkPage: vi.fn(), findSourceInWorkspace: vi.fn(), - ensureParsedSnapshotForRead: vi.fn(), getCurrentUser: vi.fn(), - getSourceParseAssetUrls: vi.fn(), - getSourceParseSnapshotMetadata: vi.fn(), localizeRemoteDocument: vi.fn(), makeKnowhereClient: vi.fn(), + makeKnowhereClientWithParsedStorage: vi.fn(), + readChunks: vi.fn(), requireUser: vi.fn(), - syncRemoteParsedSnapshot: vi.fn(), - updateSourceRevisionKey: vi.fn(), })) vi.mock("next/headers", () => ({ @@ -42,23 +37,18 @@ vi.mock("@/infrastructure/auth", () => ({ vi.mock("@/integrations/knowhere", () => ({ makeKnowhereClient: mocks.makeKnowhereClient, + makeKnowhereClientWithParsedStorage: + mocks.makeKnowhereClientWithParsedStorage, })) vi.mock("@vercel/blob", () => ({ del: mocks.deleteBlob, - get: mocks.blobGet, - put: mocks.blobPut, })) vi.mock("@/domains/sources/service", () => ({ sourceService: { - ensureParsedSnapshotForRead: mocks.ensureParsedSnapshotForRead, findInWorkspace: mocks.findSourceInWorkspace, - getParseAssetUrls: mocks.getSourceParseAssetUrls, - getParseSnapshotMetadata: mocks.getSourceParseSnapshotMetadata, localizeRemoteDocument: mocks.localizeRemoteDocument, - syncRemoteParsedSnapshot: mocks.syncRemoteParsedSnapshot, - updateSourceRevisionKey: mocks.updateSourceRevisionKey, }, })) @@ -73,38 +63,9 @@ import { GET } from "./route" describe("GET /api/sources/[sourceId]/chunks", () => { beforeEach(() => { vi.clearAllMocks() - mocks.blobGet.mockResolvedValue(null) - mocks.blobPut.mockImplementation(async (pathname: string) => ({ - url: `https://blob.example/${pathname}`, - })) - mocks.getSourceParseAssetUrls.mockResolvedValue({}) - mocks.getSourceParseSnapshotMetadata.mockResolvedValue({ - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - assetUrlsByFilePath: {}, - }) - mocks.ensureParsedSnapshotForRead.mockResolvedValue({ - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - assetUrlsByFilePath: {}, - }) - mocks.updateSourceRevisionKey.mockResolvedValue(null) - mocks.syncRemoteParsedSnapshot.mockResolvedValue({ - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - assetUrlsByFilePath: {}, + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: { documents: { listChunks: vi.fn() } }, + knowledge: { readChunks: mocks.readChunks }, }) }) @@ -166,7 +127,6 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }) expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() - expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() }) it("loads every API-owned demo chunk page for full anonymous chunk requests", async () => { @@ -321,7 +281,6 @@ describe("GET /api/sources/[sourceId]/chunks", () => { expect(mocks.findSourceInWorkspace).not.toHaveBeenCalled() expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() - expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() }) it("serves demo chunks for authenticated materialized demo sources", async () => { @@ -411,7 +370,6 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }) expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() - expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() }) it("logs the demo chunk load failure before returning 404", async () => { @@ -463,12 +421,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { } }) - it("loads authenticated workspace chunks from the parsed Blob snapshot", async () => { - const knowhereClient = { - documents: { - listChunks: vi.fn(), - }, - } + it("loads authenticated workspace chunks through the SDK durable read", async () => { mocks.getCurrentUser.mockResolvedValue({ id: "user_1", email: null, @@ -480,128 +433,55 @@ describe("GET /api/sources/[sourceId]/chunks", () => { namespace: "notebook-workspace_1", createdAt: new Date("2026-05-10T00:00:00.000Z"), }) - mocks.findSourceInWorkspace.mockResolvedValue({ - id: "00000000-0000-0000-0000-000000000002", - workspaceId: "workspace_1", - title: "notes.pdf", - mimeType: "application/pdf", - sizeBytes: 1024, - status: "ready", - failureReason: null, - knowhereJobId: "job_1", - knowhereDocumentId: "doc_1", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-10T00:00:00.000Z"), - updatedAt: new Date("2026-05-10T00:00:00.000Z"), - deletedAt: null, - }) + mocks.findSourceInWorkspace.mockResolvedValue( + makeReadySource({ + id: "00000000-0000-0000-0000-000000000002", + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + }), + ) mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") - mocks.makeKnowhereClient.mockReturnValue(knowhereClient) - mocks.getSourceParseAssetUrls.mockResolvedValue({ - "images/chart.png": "https://blob.example/images/chart.png", - }) - mocks.getSourceParseSnapshotMetadata.mockResolvedValue({ - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", - assetUrlsByFilePath: { - "images/chart.png": "https://blob.example/images/chart.png", - }, - }) - mocks.ensureParsedSnapshotForRead.mockResolvedValue({ - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/manifest/current.json", - assetUrlsByFilePath: { - "images/chart.png": "https://blob.example/images/chart.png", - }, - }) - mocks.blobGet.mockImplementation(async (pathname: string) => { - if (pathname.endsWith("/manifest/current.json")) { - return { - statusCode: 200, - stream: createTextStream( - JSON.stringify({ - version: 1, - kind: "knowhere-parsed-result-snapshot", - jobId: "job_1", - documentId: "doc_1", - sourceFileName: "notes.pdf", - totalChunks: 1, - chunkPageSize: 1, - chunkPages: [ - { - page: 1, - pageSize: 1, - chunkCount: 1, - key: "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000002/parsed-result/chunks/page-1.json", - }, - ], - assetUrlsByFilePath: { - "images/chart.png": "https://blob.example/images/chart.png", - }, - createdAt: "2026-07-03T00:00:00.000Z", - }), - ), - } - } - if (pathname.endsWith("/chunks/page-1.json")) { - return { - statusCode: 200, - stream: createTextStream( - JSON.stringify({ - version: 1, - jobId: "job_1", - documentId: "doc_1", - sourceFileName: "notes.pdf", - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - chunks: [ - { - id: "dchk_1", - chunkId: "parser_1", - chunkType: "image", - content: "Workspace chunk", - sectionPath: "Summary", - sourceChunkPath: "Default_Root/notes.pdf/Summary", - filePath: "images/chart.png", - metadata: {}, - sortOrder: 1, - }, - ], - }), - ), - } - } - return null + mocks.readChunks.mockResolvedValue({ + document: { localDocumentId: "doc_1" }, + chunks: [ + { + position: 1, + chunkId: "parser_1", + chunkType: "image", + content: "Workspace chunk", + readableContent: "Workspace chunk", + sectionPath: "Summary", + sourceChunkPath: "Summary", + filePath: "images/chart.png", + assetUrl: + "https://fake.public.blob.vercel-storage.com/images/chart.png", + metadata: {}, + }, + ], + page: 1, + pageSize: 1, + totalChunks: 1, + totalPages: 1, }) const response = await GET( new NextRequest( "http://localhost:3001/api/sources/00000000-0000-0000-0000-000000000002/chunks?page=1&pageSize=1", ), - { params: Promise.resolve({ sourceId: "00000000-0000-0000-0000-000000000002" }) }, + { + params: Promise.resolve({ + sourceId: "00000000-0000-0000-0000-000000000002", + }), + }, ) await expect(response.json()).resolves.toMatchObject({ chunks: [ { - chunkId: "dchk_1", parserChunkId: "parser_1", documentId: "doc_1", - assetUrl: "https://blob.example/images/chart.png", + assetUrl: + "https://fake.public.blob.vercel-storage.com/images/chart.png", sourceTitle: "notes.pdf", }, ], @@ -613,15 +493,16 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }) expect(response.status).toBe(200) expect(mocks.fetchDemoChunkPage).not.toHaveBeenCalled() - expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() + expect(mocks.readChunks).toHaveBeenCalledWith({ + documentId: "doc_1", + revisionKey: "job_1", + page: 1, + pageSize: 1, + assetUrlPolicy: "durable", + }) }) - it("returns processing when a ready workspace source has no complete parsed snapshot", async () => { - const knowhereClient = { - documents: { - listChunks: vi.fn(), - }, - } + it("returns processing when a workspace source is not ready", async () => { mocks.getCurrentUser.mockResolvedValue({ id: "user_1", email: null, @@ -633,35 +514,24 @@ describe("GET /api/sources/[sourceId]/chunks", () => { namespace: "notebook-workspace_1", createdAt: new Date("2026-05-10T00:00:00.000Z"), }) - mocks.findSourceInWorkspace.mockResolvedValue({ - id: "00000000-0000-0000-0000-000000000002", - workspaceId: "workspace_1", - title: "notes.pdf", - mimeType: "application/pdf", - sizeBytes: 1024, - status: "ready", - failureReason: null, - knowhereJobId: "job_1", - knowhereDocumentId: "doc_1", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-10T00:00:00.000Z"), - updatedAt: new Date("2026-05-10T00:00:00.000Z"), - deletedAt: null, - }) - mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") - mocks.makeKnowhereClient.mockReturnValue(knowhereClient) - mocks.getSourceParseSnapshotMetadata.mockResolvedValue(null) - mocks.ensureParsedSnapshotForRead.mockResolvedValue(null) + mocks.findSourceInWorkspace.mockResolvedValue( + makeReadySource({ + id: "00000000-0000-0000-0000-000000000002", + status: "parsing", + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + }), + ) const response = await GET( new NextRequest( "http://localhost:3001/api/sources/00000000-0000-0000-0000-000000000002/chunks?page=1&pageSize=1", ), - { params: Promise.resolve({ sourceId: "00000000-0000-0000-0000-000000000002" }) }, + { + params: Promise.resolve({ + sourceId: "00000000-0000-0000-0000-000000000002", + }), + }, ) await expect(response.json()).resolves.toEqual({ @@ -672,13 +542,13 @@ describe("GET /api/sources/[sourceId]/chunks", () => { total: 0, totalPages: 0, }, - message: "Source parsed snapshot is still being prepared.", + message: "Source is still being prepared.", }) expect(response.status).toBe(202) - expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() + expect(mocks.readChunks).not.toHaveBeenCalled() }) - it("materializes a remote source id and loads chunks from the parsed Blob snapshot", async () => { + it("materializes a remote source id and reads chunks through the SDK", async () => { const knowhereClient = { documents: { list: vi.fn(async () => ({ @@ -697,9 +567,6 @@ describe("GET /api/sources/[sourceId]/chunks", () => { })), listChunks: vi.fn(), }, - knowledge: { - loadJobResult: vi.fn(), - }, } mocks.getCurrentUser.mockResolvedValue({ id: "user_1", @@ -715,97 +582,35 @@ describe("GET /api/sources/[sourceId]/chunks", () => { mocks.fetchDemoChunkPage.mockRejectedValue(new Error("not a demo")) mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") mocks.makeKnowhereClient.mockReturnValue(knowhereClient) - const localizedSource = { - id: "00000000-0000-0000-0000-000000000009", - workspaceId: "workspace_1", - title: "remote.pdf", - mimeType: "application/pdf", - sizeBytes: 0, - status: "ready", - failureReason: null, - knowhereJobId: "job_result_1", - knowhereDocumentId: "doc_remote", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-10T00:00:00.000Z"), - updatedAt: new Date("2026-05-10T00:00:00.000Z"), - deletedAt: null, - } - mocks.localizeRemoteDocument.mockResolvedValue(localizedSource) - mocks.syncRemoteParsedSnapshot.mockResolvedValue({ - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - assetUrlsByFilePath: { - "pages/page-1.png": "https://blob.example/pages/page-1.png", - }, - }) - mocks.blobGet.mockImplementation(async (pathname: string) => { - if (pathname.endsWith("/manifest/current.json")) { - return { - statusCode: 200, - stream: createTextStream( - JSON.stringify({ - version: 1, - kind: "knowhere-parsed-result-snapshot", - jobId: "job_result_1", - documentId: "doc_remote", - sourceFileName: "remote.pdf", - totalChunks: 1, - chunkPageSize: 1, - chunkPages: [ - { - page: 1, - pageSize: 1, - chunkCount: 1, - key: "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/chunks/page-1.json", - }, - ], - assetUrlsByFilePath: { - "pages/page-1.png": "https://blob.example/pages/page-1.png", - }, - createdAt: "2026-07-03T00:00:00.000Z", - }), - ), - } - } - if (pathname.endsWith("/chunks/page-1.json")) { - return { - statusCode: 200, - stream: createTextStream( - JSON.stringify({ - version: 1, - jobId: "job_result_1", - documentId: "doc_remote", - sourceFileName: "remote.pdf", - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - chunks: [ - { - id: "dchk_remote", - chunkId: "parser_remote", - chunkType: "page", - content: "Remote chunk", - sectionPath: "Summary", - sourceChunkPath: "Default_Root/remote.pdf/Summary", - filePath: "pages/page-1.png", - metadata: {}, - sortOrder: 0, - }, - ], - }), - ), - } - } - return null + mocks.localizeRemoteDocument.mockResolvedValue( + makeReadySource({ + id: "00000000-0000-0000-0000-000000000009", + title: "remote.pdf", + knowhereJobId: "job_result_1", + knowhereDocumentId: "doc_remote", + }), + ) + mocks.readChunks.mockResolvedValue({ + document: { localDocumentId: "doc_remote" }, + chunks: [ + { + position: 1, + chunkId: "parser_remote", + chunkType: "page", + content: "Remote chunk", + readableContent: "Remote chunk", + sectionPath: "Summary", + sourceChunkPath: "Summary", + filePath: "pages/page-1.png", + assetUrl: + "https://fake.public.blob.vercel-storage.com/pages/page-1.png", + metadata: {}, + }, + ], + page: 1, + pageSize: 1, + totalChunks: 1, + totalPages: 1, }) const response = await GET( @@ -822,10 +627,10 @@ describe("GET /api/sources/[sourceId]/chunks", () => { await expect(response.json()).resolves.toMatchObject({ chunks: [ { - chunkId: "dchk_remote", parserChunkId: "parser_remote", documentId: "doc_remote", - assetUrl: "https://blob.example/pages/page-1.png", + assetUrl: + "https://fake.public.blob.vercel-storage.com/pages/page-1.png", sourceTitle: "remote.pdf", }, ], @@ -841,270 +646,45 @@ describe("GET /api/sources/[sourceId]/chunks", () => { "workspace_1", "session=abc", ) - expect(mocks.localizeRemoteDocument).toHaveBeenCalledWith( - "workspace_1", - { - documentId: "doc_remote", - namespace: "default", - status: "ready", - title: "remote.pdf", - mimeType: "application/pdf", - sizeBytes: undefined, - revisionKey: "job_result_1", - }, - ) - expect(mocks.syncRemoteParsedSnapshot).toHaveBeenCalledWith({ - workspaceId: "workspace_1", - source: localizedSource, - client: { - documents: knowhereClient.documents, - knowledge: knowhereClient.knowledge, - }, - }) - expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() - expect(knowhereClient.knowledge.loadJobResult).not.toHaveBeenCalled() - }) - - it("returns processing for a remote source when the parsed Blob snapshot is incomplete", async () => { - const knowhereClient = { - documents: { - list: vi.fn(async () => ({ - documents: [ - { - documentId: "doc_remote", - namespace: "default", - status: "active", - currentJobResultId: "job_result_1", - sourceFileName: "remote.pdf", - documentMetadata: { - mimeType: "application/pdf", - }, - }, - ], - })), - listChunks: vi.fn(), - }, - knowledge: { - loadJobResult: vi.fn(), - }, - } - mocks.getCurrentUser.mockResolvedValue({ - id: "user_1", - email: null, - name: null, - }) - mocks.ensureWorkspace.mockResolvedValue({ - id: "workspace_1", - userId: "user_1", - namespace: "notebook-workspace_1", - createdAt: new Date("2026-05-10T00:00:00.000Z"), - }) - mocks.fetchDemoChunkPage.mockRejectedValue(new Error("not a demo")) - mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") - mocks.makeKnowhereClient.mockReturnValue(knowhereClient) - mocks.localizeRemoteDocument.mockResolvedValue({ - id: "00000000-0000-0000-0000-000000000009", - workspaceId: "workspace_1", - title: "remote.pdf", - mimeType: "application/pdf", - sizeBytes: 0, + expect(mocks.localizeRemoteDocument).toHaveBeenCalledWith("workspace_1", { + documentId: "doc_remote", + namespace: "default", status: "ready", - failureReason: null, - knowhereJobId: "job_result_1", - knowhereDocumentId: "doc_remote", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-10T00:00:00.000Z"), - updatedAt: new Date("2026-05-10T00:00:00.000Z"), - deletedAt: null, - }) - mocks.syncRemoteParsedSnapshot.mockResolvedValue(null) - - const response = await GET( - new NextRequest( - "http://localhost:3001/api/sources/knowhere-doc:default:doc_remote/chunks?page=1&pageSize=1", - ), - { - params: Promise.resolve({ - sourceId: "knowhere-doc:default:doc_remote", - }), - }, - ) - - await expect(response.json()).resolves.toEqual({ - chunks: [], - pagination: { - page: 1, - pageSize: 1, - total: 0, - totalPages: 0, - }, - message: "Source parsed snapshot is still being prepared.", - }) - expect(response.status).toBe(202) - expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() - }) - - it("loads remote snapshot chunks after an interrupted sync left the source parsing", async () => { - const knowhereClient = { - documents: { - list: vi.fn(async () => ({ - documents: [ - { - documentId: "doc_remote", - namespace: "default", - status: "active", - currentJobResultId: "job_result_1", - sourceFileName: "remote.pdf", - documentMetadata: { - mimeType: "application/pdf", - }, - }, - ], - })), - listChunks: vi.fn(), - }, - knowledge: { - loadJobResult: vi.fn(), - }, - } - mocks.getCurrentUser.mockResolvedValue({ - id: "user_1", - email: null, - name: null, - }) - mocks.ensureWorkspace.mockResolvedValue({ - id: "workspace_1", - userId: "user_1", - namespace: "notebook-workspace_1", - createdAt: new Date("2026-05-10T00:00:00.000Z"), - }) - mocks.fetchDemoChunkPage.mockRejectedValue(new Error("not a demo")) - mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") - mocks.makeKnowhereClient.mockReturnValue(knowhereClient) - mocks.localizeRemoteDocument.mockResolvedValue({ - id: "00000000-0000-0000-0000-000000000009", - workspaceId: "workspace_1", title: "remote.pdf", mimeType: "application/pdf", - sizeBytes: 0, - status: "parsing", - failureReason: null, - knowhereJobId: "job_result_1", - knowhereDocumentId: "doc_remote", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-10T00:00:00.000Z"), - updatedAt: new Date("2026-05-10T00:00:00.000Z"), - deletedAt: null, + sizeBytes: undefined, + revisionKey: "job_result_1", }) - mocks.syncRemoteParsedSnapshot.mockResolvedValue({ - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - assetUrlsByFilePath: {}, - }) - mocks.blobGet.mockImplementation(async (pathname: string) => { - if (pathname.endsWith("/manifest/current.json")) { - return { - statusCode: 200, - stream: createTextStream( - JSON.stringify({ - version: 1, - kind: "knowhere-parsed-result-snapshot", - jobId: "job_result_1", - documentId: "doc_remote", - sourceFileName: "remote.pdf", - totalChunks: 1, - chunkPageSize: 1, - chunkPages: [ - { - page: 1, - pageSize: 1, - chunkCount: 1, - key: "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/chunks/page-1.json", - }, - ], - assetUrlsByFilePath: {}, - createdAt: "2026-07-03T00:00:00.000Z", - }), - ), - } - } - if (pathname.endsWith("/chunks/page-1.json")) { - return { - statusCode: 200, - stream: createTextStream( - JSON.stringify({ - version: 1, - jobId: "job_result_1", - documentId: "doc_remote", - sourceFileName: "remote.pdf", - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - chunks: [ - { - id: "dchk_remote", - chunkId: "parser_remote", - chunkType: "text", - content: "Remote chunk", - sectionPath: "Summary", - sourceChunkPath: "Default_Root/remote.pdf/Summary", - metadata: {}, - sortOrder: 0, - }, - ], - }), - ), - } - } - return null - }) - - const response = await GET( - new NextRequest( - "http://localhost:3001/api/sources/knowhere-doc:default:doc_remote/chunks?page=1&pageSize=1", - ), - { - params: Promise.resolve({ - sourceId: "knowhere-doc:default:doc_remote", - }), - }, - ) - - await expect(response.json()).resolves.toMatchObject({ - chunks: [ - { - chunkId: "dchk_remote", - parserChunkId: "parser_remote", - documentId: "doc_remote", - sourceTitle: "remote.pdf", - }, - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - }, + expect(mocks.readChunks).toHaveBeenCalledWith({ + documentId: "doc_remote", + revisionKey: "job_result_1", + page: 1, + pageSize: 1, + assetUrlPolicy: "durable", }) - expect(response.status).toBe(200) - expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled() }) }) -function createTextStream(text: string): ReadableStream { - const stream = new Response(text).body - if (!stream) throw new Error("Response body stream was not created.") - return stream +function makeReadySource(overrides: Record) { + return { + id: "00000000-0000-0000-0000-000000000002", + workspaceId: "workspace_1", + title: "notes.pdf", + mimeType: "application/pdf", + sizeBytes: 1024, + status: "ready", + failureReason: null, + failureStage: null, + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + ...overrides, + } } diff --git a/src/app/api/sources/parsed-sync/route.ts b/src/app/api/sources/parsed-sync/route.ts new file mode 100644 index 0000000..5272176 --- /dev/null +++ b/src/app/api/sources/parsed-sync/route.ts @@ -0,0 +1,27 @@ +import { serve } from "@upstash/workflow/nextjs" + +import { parsedSyncRouteWorkflow } from "@/domains/sources/parsed-sync-route-workflow" + +type ParsedSyncPayload = Parameters< + typeof parsedSyncRouteWorkflow.normalizeParsedSyncPayload +>[0] + +export const { POST } = serve( + async (context) => { + const payload = parsedSyncRouteWorkflow.normalizeParsedSyncPayload( + context.requestPayload, + ) + await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ + context, + payload, + }) + }, + { + failureFunction: async ({ context, failResponse }) => { + await parsedSyncRouteWorkflow.markSyncFailedAfterWorkflowFailure( + context.requestPayload, + failResponse, + ) + }, + }, +) diff --git a/src/app/api/sources/route.test.ts b/src/app/api/sources/route.test.ts index a78639b..a7b10c6 100644 --- a/src/app/api/sources/route.test.ts +++ b/src/app/api/sources/route.test.ts @@ -67,6 +67,7 @@ const source: Source = { sizeBytes: 5, status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: null, stagedBlobPathname: null, diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 71604f6..1b51180 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -1993,6 +1993,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_123", knowhereDocumentId: "doc_included", stagedBlobPathname: null, diff --git a/src/domains/chat/media-asset-hardening.test.ts b/src/domains/chat/media-asset-hardening.test.ts index fe1f3df..d3c87b2 100644 --- a/src/domains/chat/media-asset-hardening.test.ts +++ b/src/domains/chat/media-asset-hardening.test.ts @@ -3,8 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest" import type { Source } from "@/infrastructure/db/schema" import { hardenChatMediaAssetUrls, - type ChatMediaAssetBlobStore, - type FetchChatMediaAsset, + isNotebookOwnedAssetUrl, type HardenableRetrievalResult, } from "./media-asset-hardening" @@ -20,116 +19,42 @@ vi.mock("@/lib/logger", () => ({ afterEach(() => { vi.clearAllMocks() - delete process.env.KNOWHERE_BASE_URL }) describe("hardenChatMediaAssetUrls", () => { - it("copies upstream absolute asset URLs into Notebook chat assets", async () => { - const rawAssetUrl = - "https://knowhere-storage.example/results/job_1/images/image-6-%E6%83%85%E6%84%9F%E5%88%86%E7%B1%BB%E6%A8%A1%E5%9E%8B.jpg?AWSAccessKeyId=test&Signature=secret" - const blobStore = makeBlobStore( - "https://blob.example/workspaces/workspace_1/chat-assets/source-source_1/image-6.jpg", - ) - const fetchAsset = makeFetchAsset("image-bytes", "image/jpeg") + it("keeps an already Notebook-owned asset URL without loading the asset map", async () => { + const ownedUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_1/rev_1/assets/images/a.png" + const loadSourceAssetUrls = vi.fn(async () => ({})) const result = await hardenChatMediaAssetUrls({ workspaceId: "workspace_1", - sources: [ - makeSource({ - id: "source_1", - knowhereDocumentId: "doc_model", - }), - ], + sources: [makeSource({ id: "source_1", knowhereDocumentId: "doc_1" })], results: [ makeRetrievalResult({ chunkType: "image", - assetUrl: rawAssetUrl, + assetUrl: ownedUrl, source: { - documentId: "doc_model", + documentId: "doc_1", sourceFileName: "model.pdf", sectionPath: "Root", }, }), ], - blobStore, - fetchAsset, - }) - - expect(fetchAsset).toHaveBeenCalledWith(rawAssetUrl) - expect(blobStore.put).toHaveBeenCalledWith( - expect.stringMatching( - /^workspaces\/workspace_1\/chat-assets\/source-source_1\/[a-f0-9]{24}-image-6\.jpg$/, - ), - expect.any(Buffer), - { - access: "public", - allowOverwrite: true, - contentType: "image/jpeg", - multipart: true, - }, - ) - expect(result.results[0]?.assetUrl).toBe( - "https://blob.example/workspaces/workspace_1/chat-assets/source-source_1/image-6.jpg", - ) - }) - - it("copies server page citation asset URLs into Notebook chat assets", async () => { - const rawPageAssetUrl = - "https://knowhere-storage.example/results/job_1/page_citation_assets/page-4.png?AWSAccessKeyId=test&Signature=secret" - const blobPageAssetUrl = - "https://blob.example/workspaces/workspace_1/chat-assets/source-source_1/page-4.png" - const blobStore = makeBlobStore(blobPageAssetUrl) - const fetchAsset = makeFetchAsset("page-image-bytes", "image/png") - - const result = await hardenChatMediaAssetUrls({ - workspaceId: "workspace_1", - sources: [ - makeSource({ - id: "source_1", - knowhereDocumentId: "doc_pages", - }), - ], - results: [ - makeRetrievalResult({ - chunkType: "page", - pageCitationAssetUrl: rawPageAssetUrl, - source: { - documentId: "doc_pages", - sourceFileName: "deck.pdf", - sectionPath: "Page 4", - }, - }), - ], - blobStore, - fetchAsset, + loadSourceAssetUrls, }) - expect(fetchAsset).toHaveBeenCalledWith(rawPageAssetUrl) - expect(blobStore.put).toHaveBeenCalledWith( - expect.stringMatching( - /^workspaces\/workspace_1\/chat-assets\/source-source_1\/[a-f0-9]{24}-page-4\.png$/, - ), - expect.any(Buffer), - { - access: "public", - allowOverwrite: true, - contentType: "image/png", - multipart: true, - }, - ) - expect(result.results[0]?.pageCitationAssetUrl).toBe(blobPageAssetUrl) - expect(result.results[0]?.assetUrl).toBeUndefined() + expect(result.results[0]?.assetUrl).toBe(ownedUrl) + expect(loadSourceAssetUrls).not.toHaveBeenCalled() }) - it("uses an existing parsed asset URL before fetching the upstream URL", async () => { + it("resolves a raw asset URL to the durable parsed asset URL", async () => { const rawAssetUrl = "https://knowhere-storage.example/results/job_1/images/id-front.jpg?AWSAccessKeyId=test" - const parsedAssetUrl = - "https://blob.example/workspaces/workspace_1/sources/source_identity/parsed-result/images/id-front.jpg" - const blobStore = makeBlobStore("https://blob.example/should-not-upload.jpg") - const fetchAsset = makeFetchAsset("should-not-fetch", "image/jpeg") + const durableUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_identity/rev_1/assets/images/id-front.jpg" const loadSourceAssetUrls = vi.fn().mockResolvedValue({ - "images/id-front.jpg": parsedAssetUrl, + "images/id-front.jpg": durableUrl, }) const result = await hardenChatMediaAssetUrls({ @@ -152,99 +77,49 @@ describe("hardenChatMediaAssetUrls", () => { }), ], loadSourceAssetUrls, - blobStore, - fetchAsset, }) expect(loadSourceAssetUrls).toHaveBeenCalledWith( expect.objectContaining({ id: "source_identity" }), ) - expect(fetchAsset).not.toHaveBeenCalled() - expect(blobStore.put).not.toHaveBeenCalled() - expect(result.results[0]?.assetUrl).toBe(parsedAssetUrl) + expect(result.results[0]?.assetUrl).toBe(durableUrl) }) - it("fetches demo asset routes from the upstream demo API", async () => { - process.env.KNOWHERE_BASE_URL = "https://knowhere.example" - const demoAssetUrl = - "/api/demo-sources/demo_source_1/assets/images/demo%20chart.png" - const blobStore = makeBlobStore( - "https://blob.example/workspaces/workspace_1/chat-assets/demo-demo_source_1/demo-chart.png", - ) - const fetchAsset = makeFetchAsset("demo-image", "image/png") - - const result = await hardenChatMediaAssetUrls({ - workspaceId: "workspace_1", - sources: [], - results: [ - makeRetrievalResult({ - chunkType: "image", - assetUrl: demoAssetUrl, - source: { - documentId: "demo_doc", - sourceFileName: "demo.pdf", - sectionPath: "images/demo chart.png", - }, - }), - ], - blobStore, - fetchAsset, - }) - - expect(fetchAsset).toHaveBeenCalledWith( - "https://knowhere.example/api/v1/demo/sources/demo_source_1/assets/images/demo%20chart.png", - ) - expect(fetchAsset).not.toHaveBeenCalledWith(demoAssetUrl) - expect(blobStore.put).toHaveBeenCalledWith( - expect.stringContaining("/chat-assets/demo-demo_source_1/"), - expect.any(Buffer), - expect.objectContaining({ contentType: "image/png" }), - ) - expect(result.results[0]?.assetUrl).toBe( - "https://blob.example/workspaces/workspace_1/chat-assets/demo-demo_source_1/demo-chart.png", - ) - }) - - it("falls back to the raw URL when hardening fails", async () => { + it("omits an asset URL that cannot be resolved to a durable URL", async () => { const rawAssetUrl = "https://knowhere-storage.example/results/job_1/tables/table-1.html?AWSAccessKeyId=test" - const blobStore = makeBlobStore("https://blob.example/should-not-exist.html") - const fetchAsset: FetchChatMediaAsset = vi - .fn() - .mockRejectedValue(new Error("expired URL")) + const loadSourceAssetUrls = vi.fn().mockResolvedValue({}) const result = await hardenChatMediaAssetUrls({ workspaceId: "workspace_1", - sources: [], + sources: [ + makeSource({ id: "source_1", knowhereDocumentId: "doc_1" }), + ], results: [ makeRetrievalResult({ chunkType: "table", assetUrl: rawAssetUrl, + source: { + documentId: "doc_1", + sourceFileName: "source.pdf", + sectionPath: "tables/table-1.html", + }, }), ], - blobStore, - fetchAsset, + loadSourceAssetUrls, }) - expect(result.results[0]?.assetUrl).toBe(rawAssetUrl) - expect(blobStore.put).not.toHaveBeenCalled() - expect(loggerMock.warn).toHaveBeenCalledWith( - "chat-agent: media asset hardening failed; keeping raw URL", - expect.objectContaining({ - assetUrl: - "https://knowhere-storage.example/results/job_1/tables/table-1.html", - error: "expired URL", - }), - ) + expect(result.results[0]?.assetUrl).toBeUndefined() }) - it("rewrites artifact asset URLs and nested citation asset URLs", async () => { + it("resolves an artifact asset URL and its nested citation URL", async () => { const rawAssetUrl = "https://knowhere-storage.example/results/job_1/images/front.jpg?AWSAccessKeyId=test" - const blobAssetUrl = - "https://blob.example/workspaces/workspace_1/chat-assets/source-source_identity/front.jpg" - const blobStore = makeBlobStore(blobAssetUrl) - const fetchAsset = makeFetchAsset("front-image", "image/jpeg") + const durableUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_identity/rev_1/assets/images/front.jpg" + const loadSourceAssetUrls = vi.fn().mockResolvedValue({ + "images/front.jpg": durableUrl, + }) const result = await hardenChatMediaAssetUrls({ workspaceId: "workspace_1", @@ -273,82 +148,35 @@ describe("hardenChatMediaAssetUrls", () => { }, }, ], - blobStore, - fetchAsset, + loadSourceAssetUrls, }) const [artifact] = result.artifacts ?? [] - expect(fetchAsset).toHaveBeenCalledTimes(1) - expect(artifact?.assetUrl).toBe(blobAssetUrl) - expect(artifact?.citation?.assetUrl).toBe(blobAssetUrl) + expect(artifact?.assetUrl).toBe(durableUrl) + expect(artifact?.citation?.assetUrl).toBe(durableUrl) }) +}) - it("rewrites nested citation page citation asset URLs", async () => { - const rawPageAssetUrl = - "https://knowhere-storage.example/results/job_1/page_citation_assets/page-8.png?AWSAccessKeyId=test" - const blobPageAssetUrl = - "https://blob.example/workspaces/workspace_1/chat-assets/source-source_pages/page-8.png" - const blobStore = makeBlobStore(blobPageAssetUrl) - const fetchAsset = makeFetchAsset("page-eight-image", "image/png") - - const result = await hardenChatMediaAssetUrls({ - workspaceId: "workspace_1", - sources: [ - makeSource({ - id: "source_pages", - knowhereDocumentId: "doc_pages", - }), - ], - results: [], - artifacts: [ - { - type: "image", - ref: "asset:r1:result:1", - assetUrl: - "https://blob.example/workspaces/workspace_1/chat-assets/source-source_pages/figure.png", - label: "figure", - citation: { - chunkType: "page", - score: 0.9, - pageCitationAssetUrl: rawPageAssetUrl, - source: { - documentId: "doc_pages", - sourceFileName: "deck.pdf", - sectionPath: "Page 8", - }, - }, - }, - ], - blobStore, - fetchAsset, - }) - - const [artifact] = result.artifacts ?? [] - expect(fetchAsset).toHaveBeenCalledWith(rawPageAssetUrl) - expect(artifact?.citation?.pageCitationAssetUrl).toBe(blobPageAssetUrl) +describe("isNotebookOwnedAssetUrl", () => { + it("treats Vercel Blob hosts and parsed-document paths as owned", () => { + expect( + isNotebookOwnedAssetUrl( + "https://fake.public.blob.vercel-storage.com/x/y.png", + ), + ).toBe(true) + expect( + isNotebookOwnedAssetUrl( + "https://cdn.example/workspaces/w/parsed-documents/d/r/assets/a.png", + ), + ).toBe(true) + expect( + isNotebookOwnedAssetUrl( + "https://knowhere-storage.example/results/job_1/images/a.png?sig=x", + ), + ).toBe(false) }) }) -function makeFetchAsset( - body: string, - contentType: string, -): FetchChatMediaAsset { - return vi.fn().mockResolvedValue( - new Response(Buffer.from(body), { - status: 200, - headers: { - "content-type": contentType, - }, - }), - ) -} - -function makeBlobStore(url: string): ChatMediaAssetBlobStore { - return { - put: vi.fn().mockResolvedValue({ url }), - } -} - function makeRetrievalResult( overrides: Partial = {}, ): HardenableRetrievalResult { @@ -374,6 +202,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_123", knowhereDocumentId: "doc_1", stagedBlobPathname: null, diff --git a/src/domains/chat/media-asset-hardening.ts b/src/domains/chat/media-asset-hardening.ts index 4161587..f768515 100644 --- a/src/domains/chat/media-asset-hardening.ts +++ b/src/domains/chat/media-asset-hardening.ts @@ -1,6 +1,3 @@ -import path from "node:path" -import { createHash } from "node:crypto" -import { put } from "@vercel/blob" import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import type { @@ -8,7 +5,6 @@ import type { ChatCitationView, } from "@/domains/chat/types" import type { Source } from "@/infrastructure/db/schema" -import { knowhereDemoApi } from "@/integrations/knowhere-demo" import { logger } from "@/lib/logger" import type { LoadSourceAssetUrls } from "./media-assets" import { resolveAssetUrlFromReferenceText } from "./media-assets" @@ -31,30 +27,11 @@ export type HardenMediaAssetUrls = ( input: HardenMediaAssetUrlsInput, ) => Promise -export type ChatMediaAssetBlobStore = { - readonly put: ( - pathname: string, - body: Buffer, - options: ChatMediaAssetBlobPutOptions, - ) => Promise<{ readonly url: string }> -} - -export type ChatMediaAssetBlobPutOptions = { - readonly access?: "public" - readonly allowOverwrite?: boolean - readonly contentType: string - readonly multipart?: boolean -} - -export type FetchChatMediaAsset = (url: string) => Promise - export type HardenChatMediaAssetUrlsForWorkspaceInput = HardenMediaAssetUrlsInput & { readonly workspaceId: string readonly sources: readonly Source[] - readonly loadSourceAssetUrls?: LoadSourceAssetUrls - readonly blobStore?: ChatMediaAssetBlobStore - readonly fetchAsset?: FetchChatMediaAsset + readonly loadSourceAssetUrls: LoadSourceAssetUrls } type AssetReferenceSource = ChatCitationView["source"] @@ -65,63 +42,38 @@ type AssetUrlReference = { readonly content?: string } -type AssetFetchRequest = { - readonly fetchUrl: string - readonly canonicalKey: string - readonly sourceSegment: string - readonly suggestedFileName: string -} - type HardeningContext = { - readonly workspaceId: string readonly sourcesByDocumentId: ReadonlyMap - readonly loadSourceAssetUrls?: LoadSourceAssetUrls + readonly loadSourceAssetUrls: LoadSourceAssetUrls readonly assetUrlsBySourceId: Map< string, Promise>> > - readonly hardenedAssetUrlByKey: Map> - readonly blobStore: ChatMediaAssetBlobStore - readonly fetchAsset: FetchChatMediaAsset } -type DemoAssetRoute = { - readonly demoSourceId: string - readonly encodedAssetPath: string - readonly decodedAssetPath: string -} - -const chatAssetsDirectoryName = "chat-assets" const parsedResultDirectoryName = "parsed-result" -const fallbackContentType = "application/octet-stream" -const defaultFetchAsset: FetchChatMediaAsset = (url) => fetch(url) -const defaultBlobStore: ChatMediaAssetBlobStore = { - put: (pathname, body, options) => - put(pathname, body, { - access: options.access ?? "public", - allowOverwrite: options.allowOverwrite, - contentType: options.contentType, - multipart: options.multipart, - }), -} +const chatAssetsDirectoryName = "chat-assets" +/** + * Resolve chat citation/media asset URLs to durable Notebook Blob URLs. The + * single hardening path is the SDK's `assetUrlPolicy: "durable"` read that + * `loadSourceAssetUrls` performs; here we only map a retrieval result's + * reference text to the durable URL that read produced. + * + * An asset that is already Notebook-owned is kept as-is. An asset that cannot + * be resolved to a durable URL is omitted rather than exposing a presigned + * Knowhere URL to the client. + */ export async function hardenChatMediaAssetUrls({ results, artifacts, - workspaceId, sources, loadSourceAssetUrls, - blobStore = defaultBlobStore, - fetchAsset = defaultFetchAsset, }: HardenChatMediaAssetUrlsForWorkspaceInput): Promise { const context: HardeningContext = { - workspaceId, sourcesByDocumentId: createSourcesByDocumentId(sources), loadSourceAssetUrls, assetUrlsBySourceId: new Map(), - hardenedAssetUrlByKey: new Map(), - blobStore, - fetchAsset, } const hardenedResults = await Promise.all( @@ -152,17 +104,13 @@ async function hardenRetrievalResult( if (!assetUrl && !pageCitationAssetUrl) return result const hardenedAssetUrl = assetUrl - ? await hardenAssetUrl( - { - assetUrl, - source: result.source, - content: result.content, - }, + ? await resolveDurableAssetUrl( + { assetUrl, source: result.source, content: result.content }, context, ) : undefined const hardenedPageCitationAssetUrl = pageCitationAssetUrl - ? await hardenAssetUrl( + ? await resolveDurableAssetUrl( { assetUrl: pageCitationAssetUrl, source: result.source, @@ -171,22 +119,13 @@ async function hardenRetrievalResult( context, ) : undefined - const hasAssetUrlChange = Boolean( - assetUrl && hardenedAssetUrl !== result.assetUrl, - ) - const hasPageCitationAssetUrlChange = Boolean( - pageCitationAssetUrl && - hardenedPageCitationAssetUrl !== result.pageCitationAssetUrl, - ) - if (!hasAssetUrlChange && !hasPageCitationAssetUrlChange) return result - return { - ...result, - ...(hardenedAssetUrl ? { assetUrl: hardenedAssetUrl } : {}), - ...(hardenedPageCitationAssetUrl - ? { pageCitationAssetUrl: hardenedPageCitationAssetUrl } - : {}), - } + return applyAssetUrls(result, { + hadAssetUrl: Boolean(assetUrl), + hadPageCitationAssetUrl: Boolean(pageCitationAssetUrl), + assetUrl: hardenedAssetUrl, + pageCitationAssetUrl: hardenedPageCitationAssetUrl, + }) } async function hardenArtifact( @@ -197,27 +136,26 @@ async function hardenArtifact( ? await hardenCitation(artifact.citation, context) : undefined const assetUrl = getTrimmedString(artifact.assetUrl) - if (!assetUrl) { - return citation && citation !== artifact.citation - ? { ...artifact, citation } - : artifact - } + const hardenedAssetUrl = assetUrl + ? await resolveDurableAssetUrl( + { + assetUrl, + source: artifact.citation?.source, + content: artifact.label, + }, + context, + ) + : undefined - const hardenedAssetUrl = await hardenAssetUrl( - { - assetUrl, - source: artifact.citation?.source, - content: artifact.label, - }, - context, - ) - const hasAssetUrlChange = hardenedAssetUrl !== artifact.assetUrl - const hasCitationChange = citation && citation !== artifact.citation - if (!hasAssetUrlChange && !hasCitationChange) return artifact + const citationChanged = citation && citation !== artifact.citation + const assetUrlChanged = assetUrl + ? hardenedAssetUrl !== artifact.assetUrl + : false + if (!citationChanged && !assetUrlChanged) return artifact return { ...artifact, - assetUrl: hardenedAssetUrl, + ...(assetUrl ? { assetUrl: hardenedAssetUrl } : {}), ...(citation ? { citation } : {}), } } @@ -231,17 +169,13 @@ async function hardenCitation( if (!assetUrl && !pageCitationAssetUrl) return citation const hardenedAssetUrl = assetUrl - ? await hardenAssetUrl( - { - assetUrl, - source: citation.source, - content: citation.content, - }, + ? await resolveDurableAssetUrl( + { assetUrl, source: citation.source, content: citation.content }, context, ) : undefined const hardenedPageCitationAssetUrl = pageCitationAssetUrl - ? await hardenAssetUrl( + ? await resolveDurableAssetUrl( { assetUrl: pageCitationAssetUrl, source: citation.source, @@ -250,70 +184,34 @@ async function hardenCitation( context, ) : undefined - const hasAssetUrlChange = Boolean( - assetUrl && hardenedAssetUrl !== citation.assetUrl, - ) - const hasPageCitationAssetUrlChange = Boolean( - pageCitationAssetUrl && - hardenedPageCitationAssetUrl !== citation.pageCitationAssetUrl, - ) - if (!hasAssetUrlChange && !hasPageCitationAssetUrlChange) return citation - return { - ...citation, - ...(hardenedAssetUrl ? { assetUrl: hardenedAssetUrl } : {}), - ...(hardenedPageCitationAssetUrl - ? { pageCitationAssetUrl: hardenedPageCitationAssetUrl } - : {}), - } + return applyAssetUrls(citation, { + hadAssetUrl: Boolean(assetUrl), + hadPageCitationAssetUrl: Boolean(pageCitationAssetUrl), + assetUrl: hardenedAssetUrl, + pageCitationAssetUrl: hardenedPageCitationAssetUrl, + }) } -async function hardenAssetUrl( +/** + * Return a durable Notebook-owned URL for a reference: keep already-owned URLs, + * otherwise resolve against the source's durable parsed asset map. Returns + * `undefined` when no durable URL is available so callers omit the URL rather + * than leak a presigned Knowhere URL. + */ +async function resolveDurableAssetUrl( reference: AssetUrlReference, context: HardeningContext, -): Promise { +): Promise { if (isNotebookOwnedAssetUrl(reference.assetUrl)) { return reference.assetUrl } - const parsedAssetUrl = await resolveParsedAssetUrl(reference, context) - if (parsedAssetUrl) return parsedAssetUrl - - const fetchRequest = resolveAssetFetchRequest(reference.assetUrl) - if (!fetchRequest) return reference.assetUrl - - const source = resolveSourceForReference(reference, context) - const sourceSegment = source - ? `source-${toSafePathSegment(source.id)}` - : fetchRequest.sourceSegment - const hardeningKey = [ - context.workspaceId, - source?.id ?? reference.source?.documentId ?? "", - fetchRequest.canonicalKey, - ].join("\0") - const cached = context.hardenedAssetUrlByKey.get(hardeningKey) - if (cached) return cached - - const hardenedAssetUrl = copyAssetToBlob({ - reference, - fetchRequest, - context, - sourceSegment, - hardeningKey, - }) - context.hardenedAssetUrlByKey.set(hardeningKey, hardenedAssetUrl) - return hardenedAssetUrl -} - -async function resolveParsedAssetUrl( - reference: AssetUrlReference, - context: HardeningContext, -): Promise { const source = resolveSourceForReference(reference, context) - if (!source || !context.loadSourceAssetUrls) return null + if (!source) return undefined const assetUrlsByFilePath = await getCachedSourceAssetUrls(source, context) - return resolveAssetUrlFromReferenceText({ + const durableUrl = resolveAssetUrlFromReferenceText({ values: [ reference.source?.sectionPath, reference.content, @@ -321,6 +219,46 @@ async function resolveParsedAssetUrl( ], assetUrlsByFilePath, }) + return durableUrl ?? undefined +} + +function applyAssetUrls< + T extends { + readonly assetUrl?: string | null + readonly pageCitationAssetUrl?: string | null + }, +>( + value: T, + hardened: { + readonly hadAssetUrl: boolean + readonly hadPageCitationAssetUrl: boolean + readonly assetUrl: string | undefined + readonly pageCitationAssetUrl: string | undefined + }, +): T { + const assetUrlChanged = + hardened.hadAssetUrl && hardened.assetUrl !== value.assetUrl + const pageCitationChanged = + hardened.hadPageCitationAssetUrl && + hardened.pageCitationAssetUrl !== value.pageCitationAssetUrl + if (!assetUrlChanged && !pageCitationChanged) return value + + const next: Record = { ...value } + if (hardened.hadAssetUrl) { + if (hardened.assetUrl) { + next["assetUrl"] = hardened.assetUrl + } else { + delete next["assetUrl"] + } + } + if (hardened.hadPageCitationAssetUrl) { + if (hardened.pageCitationAssetUrl) { + next["pageCitationAssetUrl"] = hardened.pageCitationAssetUrl + } else { + delete next["pageCitationAssetUrl"] + } + } + return next as T } async function getCachedSourceAssetUrls( @@ -330,116 +268,25 @@ async function getCachedSourceAssetUrls( const cached = context.assetUrlsBySourceId.get(source.id) if (cached) return cached - const loaded = context.loadSourceAssetUrls - ? context.loadSourceAssetUrls(source).catch((error: unknown) => { - logger.warn("chat-agent: failed to load parsed asset map", { - sourceId: source.id, - error: formatUnknownError(error), - }) - return {} + const loaded = context + .loadSourceAssetUrls(source) + .catch((error: unknown) => { + logger.warn("chat: failed to load durable parsed asset map", { + sourceId: source.id, + error: formatUnknownError(error), }) - : Promise.resolve({}) + return {} + }) context.assetUrlsBySourceId.set(source.id, loaded) return loaded } -async function copyAssetToBlob(input: { - readonly reference: AssetUrlReference - readonly fetchRequest: AssetFetchRequest - readonly context: HardeningContext - readonly sourceSegment: string - readonly hardeningKey: string -}): Promise { - try { - const response = await input.context.fetchAsset(input.fetchRequest.fetchUrl) - if (!response.ok) { - logger.warn("chat-agent: media asset hardening fetch failed", { - assetUrl: redactAssetUrl(input.reference.assetUrl), - status: response.status, - }) - return input.reference.assetUrl - } - - const body = Buffer.from(await response.arrayBuffer()) - const contentType = normalizeContentType( - response.headers.get("content-type"), - input.fetchRequest.suggestedFileName, - ) - const blobPathname = getChatAssetBlobPathname({ - workspaceId: input.context.workspaceId, - sourceSegment: input.sourceSegment, - hardeningKey: input.hardeningKey, - suggestedFileName: input.fetchRequest.suggestedFileName, - contentType, - }) - const blob = await input.context.blobStore.put(blobPathname, body, { - access: "public", - allowOverwrite: true, - contentType, - multipart: true, - }) - return blob.url - } catch (error) { - logger.warn("chat-agent: media asset hardening failed; keeping raw URL", { - assetUrl: redactAssetUrl(input.reference.assetUrl), - error: formatUnknownError(error), - }) - return input.reference.assetUrl - } -} - -function resolveAssetFetchRequest(assetUrl: string): AssetFetchRequest | null { - const demoAsset = parseDemoAssetRoute(assetUrl) - if (demoAsset) { - return { - fetchUrl: knowhereDemoApi.resolveApiURL( - `/api/v1/demo/sources/${encodeURIComponent( - demoAsset.demoSourceId, - )}/assets/${demoAsset.encodedAssetPath}`, - ), - canonicalKey: `demo:${demoAsset.demoSourceId}:${demoAsset.decodedAssetPath}`, - sourceSegment: `demo-${toSafePathSegment(demoAsset.demoSourceId)}`, - suggestedFileName: getPathBasename(demoAsset.decodedAssetPath), - } - } - - const absoluteUrl = parseAbsoluteHttpUrl(assetUrl) - if (!absoluteUrl) return null - - return { - fetchUrl: assetUrl, - canonicalKey: `url:${absoluteUrl.origin}${absoluteUrl.pathname}`, - sourceSegment: `external-${hashText(absoluteUrl.origin).slice(0, 16)}`, - suggestedFileName: getPathBasename(absoluteUrl.pathname), - } -} - -function parseDemoAssetRoute(assetUrl: string): DemoAssetRoute | null { - const pathname = getAssetUrlPathname(assetUrl) - const match = /^\/api\/demo-sources\/([^/]+)\/assets\/(.+)$/.exec(pathname) - const encodedDemoSourceId = match?.[1] - const encodedAssetPath = match?.[2] - if (!encodedDemoSourceId || !encodedAssetPath) return null - - const demoSourceId = decodeUrlComponent(encodedDemoSourceId) - const assetPathSegments = encodedAssetPath - .split("/") - .map(decodeUrlComponent) - .filter((segment): boolean => segment.length > 0) - if (!demoSourceId || assetPathSegments.length === 0) return null - - return { - demoSourceId, - encodedAssetPath: assetPathSegments.map(encodeURIComponent).join("/"), - decodedAssetPath: assetPathSegments.join("/"), - } -} - -function isNotebookOwnedAssetUrl(assetUrl: string): boolean { +export function isNotebookOwnedAssetUrl(assetUrl: string): boolean { const pathname = getAssetUrlPathname(assetUrl).toLowerCase() if ( pathname.includes(`/${parsedResultDirectoryName}/`) || - pathname.includes(`/${chatAssetsDirectoryName}/`) + pathname.includes(`/${chatAssetsDirectoryName}/`) || + pathname.includes("/parsed-documents/") ) { return true } @@ -449,80 +296,6 @@ function isNotebookOwnedAssetUrl(assetUrl: string): boolean { return hostname?.endsWith(".blob.vercel-storage.com") === true } -function getChatAssetBlobPathname(input: { - readonly workspaceId: string - readonly sourceSegment: string - readonly hardeningKey: string - readonly suggestedFileName: string - readonly contentType: string -}): string { - const hash = hashText(input.hardeningKey).slice(0, 24) - const fileName = toSafeFileName(input.suggestedFileName, input.contentType) - return [ - "workspaces", - toSafePathSegment(input.workspaceId), - chatAssetsDirectoryName, - input.sourceSegment, - `${hash}-${fileName}`, - ].join("/") -} - -function normalizeContentType( - value: string | null, - fileName: string, -): string { - const normalized = value?.replace(/\s+/g, " ").trim() - if (normalized) return normalized - return getContentTypeForPath(fileName) -} - -function getContentTypeForPath(filePath: string): string { - const extension = path.extname(filePath).toLowerCase() - if (extension === ".jpg" || extension === ".jpeg") return "image/jpeg" - if (extension === ".png") return "image/png" - if (extension === ".gif") return "image/gif" - if (extension === ".webp") return "image/webp" - if (extension === ".svg") return "image/svg+xml" - if (extension === ".html" || extension === ".htm") { - return "text/html; charset=utf-8" - } - if (extension === ".csv") return "text/csv; charset=utf-8" - if (extension === ".pdf") return "application/pdf" - return fallbackContentType -} - -function getExtensionForContentType(contentType: string): string { - const normalized = contentType.split(";")[0]?.trim().toLowerCase() - if (normalized === "image/jpeg") return ".jpg" - if (normalized === "image/png") return ".png" - if (normalized === "image/gif") return ".gif" - if (normalized === "image/webp") return ".webp" - if (normalized === "image/svg+xml") return ".svg" - if (normalized === "text/html") return ".html" - if (normalized === "text/csv") return ".csv" - if (normalized === "application/pdf") return ".pdf" - return ".bin" -} - -function toSafeFileName(fileName: string, contentType: string): string { - const extension = getSafeFileExtension(fileName, contentType) - const stem = path.basename(fileName, path.extname(fileName)) - const safeStem = toSafePathSegment(stem) - return `${safeStem}${extension}` -} - -function getSafeFileExtension(fileName: string, contentType: string): string { - const extension = path.extname(fileName).toLowerCase() - if (/^\.[a-z0-9]{1,12}$/.test(extension)) return extension - return getExtensionForContentType(contentType) -} - -function getPathBasename(value: string): string { - const decodedPath = decodeUrlComponent(value) - const basename = decodedPath.replaceAll("\\", "/").split("/").pop() - return basename && basename.trim().length > 0 ? basename : "asset" -} - function getAssetUrlPathname(assetUrl: string): string { try { return new URL(assetUrl, "http://notebook.local").pathname @@ -558,33 +331,6 @@ function createSourcesByDocumentId( ) } -function toSafePathSegment(value: string): string { - const decoded = decodeUrlComponent(value) - const normalized = decoded - .replace(/[^A-Za-z0-9._-]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 80) - return normalized || hashText(value).slice(0, 16) -} - -function hashText(value: string): string { - return createHash("sha256").update(value).digest("hex") -} - -function decodeUrlComponent(value: string): string { - try { - return decodeURIComponent(value) - } catch { - return value - } -} - -function redactAssetUrl(assetUrl: string): string { - const absoluteUrl = parseAbsoluteHttpUrl(assetUrl) - if (absoluteUrl) return `${absoluteUrl.origin}${absoluteUrl.pathname}` - return getAssetUrlPathname(assetUrl) -} - function formatUnknownError(error: unknown): string { if (error instanceof Error) return error.message return String(error) diff --git a/src/domains/chat/media-assets.test.ts b/src/domains/chat/media-assets.test.ts index f79338b..2fc84cb 100644 --- a/src/domains/chat/media-assets.test.ts +++ b/src/domains/chat/media-assets.test.ts @@ -298,6 +298,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: "doc_1", stagedBlobPathname: null, diff --git a/src/domains/chat/page-citation-assets.test.ts b/src/domains/chat/page-citation-assets.test.ts index ed62c66..d99dbcb 100644 --- a/src/domains/chat/page-citation-assets.test.ts +++ b/src/domains/chat/page-citation-assets.test.ts @@ -171,6 +171,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 1, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: "doc_1", stagedBlobPathname: null, diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index 3df55e0..9582b77 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -11,9 +11,10 @@ import { type ChatTurnValue, } from "@/domains/chat/service" import { chatTurnPersistence } from "@/domains/chat/chat-turn-persistence" +import { readSourceAssetUrls } from "@/domains/chunks/read" import { startBackgroundReconciliation } from "@/domains/sources/background-reconcile" -import { sourceService } from "@/domains/sources/service" import { sourceWorkflowRuntime } from "@/domains/sources/workflow-runtime" +import { makeKnowhereClientWithParsedStorage } from "@/integrations/knowhere" import { notebookRequestContext } from "@/domains/workspace/request-context" import type { Source } from "@/infrastructure/db/schema" import { isAuthError } from "@/integrations/dashboard/api-key-service" @@ -68,31 +69,29 @@ const answerChatEffect = (input: AnswerChatInput) => apiKey, }), ) + const { knowledge } = makeKnowhereClientWithParsedStorage(apiKey, { + workspaceId: workspace.id, + }) const loadSourceAssetUrls = async ( source: (typeof sources)[number], ): Promise>> => { + if (source.status !== "ready" || !source.knowhereDocumentId) return {} + try { - const snapshot = await sourceService.ensureParsedSnapshotForRead({ - workspaceId: workspace.id, - source, - client: client.knowledge - ? { - documents: client.documents, - knowledge: client.knowledge, - } - : null, + return await readSourceAssetUrls({ + knowledge, + documentId: source.knowhereDocumentId, + revisionKey: source.knowhereJobId, }) - if (snapshot) return snapshot.assetUrlsByFilePath } catch (error) { - logger.warn("chat: parsed snapshot sync for assets failed", { + logger.warn("chat: durable parsed asset read failed", { workspaceId: workspace.id, sourceId: source.id, documentId: source.knowhereDocumentId, error: summarizeUnknownError(error), }) + return {} } - - return sourceService.getParseAssetUrls(workspace.id, source.id) } const result: Either.Either = diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 189e862..f4df92a 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -18,6 +18,8 @@ const mocks = vi.hoisted(() => ({ loggerInfo: vi.fn(), loggerWarn: vi.fn(), listSourcesForWorkspace: vi.fn(), + makeKnowhereClientWithParsedStorage: vi.fn(), + readChunks: vi.fn(), softDeleteChatThread: vi.fn(), startBackgroundReconciliation: vi.fn(), })) @@ -51,6 +53,11 @@ vi.mock("@/domains/workspace/request-context", () => ({ }, })) +vi.mock("@/integrations/knowhere", () => ({ + makeKnowhereClientWithParsedStorage: + mocks.makeKnowhereClientWithParsedStorage, +})) + vi.mock("@/domains/chat/thread-service", () => ({ chatThreadService: { appendMessage: mocks.appendMessageToThread, @@ -73,11 +80,22 @@ vi.mock("@/lib/logger", () => ({ import { chatAnswerRouteService } from "./route-answer" import { chatThreadRouteService } from "./route-threads" -import { sourceService } from "@/domains/sources/service" describe("chat route services", () => { beforeEach(() => { vi.clearAllMocks() + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: { documents: { listChunks: vi.fn() } }, + knowledge: { readChunks: mocks.readChunks }, + }) + mocks.readChunks.mockResolvedValue({ + document: { localDocumentId: "doc" }, + chunks: [], + page: 1, + pageSize: 200, + totalChunks: 0, + totalPages: 1, + }) }) it("orchestrates a chat turn from request body to response body", async () => { @@ -141,28 +159,37 @@ describe("chat route services", () => { ) }) - it("ensures a legacy ready source snapshot before chat loads citation assets", async () => { + it("builds durable citation asset URLs from SDK reads for a ready source", async () => { const workspace = makeWorkspace() - const documents = { listChunks: vi.fn() } - const knowledge = { loadJobResult: vi.fn() } - const client = { documents, knowledge, retrieval: { query: vi.fn() } } + const client = { retrieval: { query: vi.fn() } } const readySource = makeSource({ status: "ready", knowhereDocumentId: "doc_legacy", - knowhereJobId: null, + knowhereJobId: "job_1", + }) + const durableUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_legacy/job_1/assets/pages/page-1.png" + mocks.readChunks.mockResolvedValue({ + document: { localDocumentId: "doc_legacy" }, + chunks: [ + { + position: 1, + chunkId: "c1", + chunkType: "page", + content: "Page", + readableContent: "Page", + sectionPath: "Page 1", + sourceChunkPath: "Page 1", + filePath: "pages/page-1.png", + assetUrl: durableUrl, + metadata: {}, + }, + ], + page: 1, + pageSize: 200, + totalChunks: 1, + totalPages: 1, }) - const snapshot = { - resultBlobUrl: "https://blob.example/manifest/current.json", - snapshotManifestUrl: "https://blob.example/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - assetUrlsByFilePath: { - "pages/page-1.png": "https://blob.example/pages/page-1.png", - }, - } - const ensureParsedSnapshotForRead = vi - .spyOn(sourceService, "ensureParsedSnapshotForRead") - .mockResolvedValue(snapshot) mocks.getAuthenticatedWithClient.mockResolvedValue({ user: { id: "user_1" }, workspace, @@ -177,7 +204,7 @@ describe("chat route services", () => { ) => Promise>> }) => { const assetUrls = await input.loadSourceAssetUrls?.(readySource) - expect(assetUrls).toEqual(snapshot.assetUrlsByFilePath) + expect(assetUrls).toEqual({ "pages/page-1.png": durableUrl }) return Either.right({ threadId: "thread_1", messages: [ @@ -193,13 +220,16 @@ describe("chat route services", () => { }) expect(result.status).toBe(200) - expect(ensureParsedSnapshotForRead).toHaveBeenCalledWith({ - workspaceId: workspace.id, - source: readySource, - client: { - documents, - knowledge, - }, + expect(mocks.makeKnowhereClientWithParsedStorage).toHaveBeenCalledWith( + "jwt_123", + { workspaceId: workspace.id }, + ) + expect(mocks.readChunks).toHaveBeenCalledWith({ + documentId: "doc_legacy", + revisionKey: "job_1", + page: 1, + pageSize: 200, + assetUrlPolicy: "durable", }) }) @@ -444,6 +474,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_123", knowhereDocumentId: "doc_1", stagedBlobPathname: null, diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index e931f04..7c0ce92 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -270,6 +270,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_123", knowhereDocumentId: "doc_included", stagedBlobPathname: null, diff --git a/src/domains/chunks/index.test.ts b/src/domains/chunks/index.test.ts index 0dfc22f..316bfd6 100644 --- a/src/domains/chunks/index.test.ts +++ b/src/domains/chunks/index.test.ts @@ -578,6 +578,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_123", knowhereDocumentId: "doc_123", stagedBlobPathname: null, diff --git a/src/domains/chunks/index.ts b/src/domains/chunks/index.ts index 8a285d7..5e1b25a 100644 --- a/src/domains/chunks/index.ts +++ b/src/domains/chunks/index.ts @@ -1,5 +1,5 @@ import { Effect } from "effect" -import type { DocumentChunk } from "@ontos-ai/knowhere-sdk" +import type { DocumentChunk, KnowledgeReadChunk } from "@ontos-ai/knowhere-sdk" import { parsedChunkNormalization } from "./normalization" import type { Source } from "@/infrastructure/db/schema" @@ -179,6 +179,35 @@ export function toParsedChunkView( }) } +/** + * Map an SDK `KnowledgeReadChunk` (from `knowledge.readChunks`) to the view + * model. Asset URLs on durable reads are already hardened SDK-side, so no + * `assetUrlsByFilePath` remap is needed here. + */ +export function toParsedChunkViewFromReadChunk( + chunk: KnowledgeReadChunk, + sourceTitle: string, + documentId?: string, +): ParsedChunkView { + return parsedChunkNormalization.createParsedChunkView({ + chunkId: chunk.chunkId, + documentId, + parserChunkId: chunk.chunkId, + sectionPath: chunk.sectionPath, + chunkType: chunk.chunkType, + contentSource: chunk.contentSource, + content: chunk.content, + metadata: chunk.metadata, + filePathCandidates: [ + chunk.filePath, + chunk.metadata["filePath"], + chunk.metadata["file_path"], + ], + assetUrl: chunk.assetUrl, + sourceTitle, + }) +} + export function resolveCitationChunk( citation: ChatCitationView, chunks: readonly ParsedChunkView[], diff --git a/src/domains/chunks/read.test.ts b/src/domains/chunks/read.test.ts new file mode 100644 index 0000000..1d0786d --- /dev/null +++ b/src/domains/chunks/read.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it, vi } from "vitest" +import type { Knowledge } from "@ontos-ai/knowhere-sdk" + +import { readAllSourceChunks, readSourceChunkPage } from "./read" + +function makeReadChunk(overrides: Record = {}) { + return { + position: 1, + chunkId: "parser_1", + chunkType: "text", + content: "Body", + readableContent: "Body", + sectionPath: "Summary", + sourceChunkPath: "Summary", + filePath: undefined, + assetUrl: undefined, + metadata: {}, + ...overrides, + } +} + +describe("readSourceChunkPage", () => { + it("reads a durable page and maps chunks to the view model", async () => { + const readChunks = vi.fn(async () => ({ + document: { localDocumentId: "doc_1" }, + chunks: [ + makeReadChunk({ + chunkType: "image", + filePath: "images/a.png", + assetUrl: "https://blob.example/images/a.png", + }), + ], + page: 2, + pageSize: 50, + totalChunks: 120, + totalPages: 3, + })) + const knowledge = { readChunks } as unknown as Knowledge + + const result = await readSourceChunkPage({ + knowledge, + source: { documentId: "doc_1", title: "notes.pdf", revisionKey: "rev_1" }, + params: { page: 2, pageSize: 50 }, + }) + + expect(readChunks).toHaveBeenCalledWith({ + documentId: "doc_1", + revisionKey: "rev_1", + page: 2, + pageSize: 50, + assetUrlPolicy: "durable", + }) + expect(result.pagination).toEqual({ + page: 2, + pageSize: 50, + total: 120, + totalPages: 3, + }) + expect(result.chunks[0]).toMatchObject({ + parserChunkId: "parser_1", + documentId: "doc_1", + assetUrl: "https://blob.example/images/a.png", + sourceTitle: "notes.pdf", + }) + }) + + it("omits revisionKey when the source has none", async () => { + const readChunks = vi.fn(async () => ({ + document: { localDocumentId: "doc_1" }, + chunks: [], + page: 1, + pageSize: 50, + totalChunks: 0, + totalPages: 1, + })) + const knowledge = { readChunks } as unknown as Knowledge + + await readSourceChunkPage({ + knowledge, + source: { documentId: "doc_1", title: "notes.pdf", revisionKey: null }, + params: { page: 1, pageSize: 50 }, + }) + + expect(readChunks).toHaveBeenCalledWith({ + documentId: "doc_1", + page: 1, + pageSize: 50, + assetUrlPolicy: "durable", + }) + }) +}) + +describe("readAllSourceChunks", () => { + it("pages the SDK to exhaustion", async () => { + const readChunks = vi + .fn() + .mockResolvedValueOnce({ + document: { localDocumentId: "doc_1" }, + chunks: [makeReadChunk({ chunkId: "c1" })], + page: 1, + pageSize: 200, + totalChunks: 2, + totalPages: 2, + }) + .mockResolvedValueOnce({ + document: { localDocumentId: "doc_1" }, + chunks: [makeReadChunk({ chunkId: "c2" })], + page: 2, + pageSize: 200, + totalChunks: 2, + totalPages: 2, + }) + const knowledge = { readChunks } as unknown as Knowledge + + const chunks = await readAllSourceChunks({ + knowledge, + source: { documentId: "doc_1", title: "notes.pdf", revisionKey: "rev_1" }, + }) + + expect(readChunks).toHaveBeenCalledTimes(2) + expect(chunks.map((chunk) => chunk.parserChunkId)).toEqual(["c1", "c2"]) + }) +}) diff --git a/src/domains/chunks/read.ts b/src/domains/chunks/read.ts new file mode 100644 index 0000000..f375d48 --- /dev/null +++ b/src/domains/chunks/read.ts @@ -0,0 +1,142 @@ +import "server-only" + +import type { Knowledge } from "@ontos-ai/knowhere-sdk" + +import { toParsedChunkViewFromReadChunk, type ChunkPage, type ChunkPageParams } from "@/domains/chunks" +import type { ParsedChunkView } from "@/domains/chunks/types" + +const loadAllPageSize = 200 + +type ReadableSource = { + readonly documentId: string + readonly title: string + readonly revisionKey?: string | null +} + +/** + * Read a single display page of parsed chunks through the SDK. The SDK serves + * from configured Blob storage when fresh and falls back to Knowhere remote + * otherwise, hardening visible asset URLs into durable Blob URLs + * (`assetUrlPolicy: "durable"`) and scheduling a background sync on a miss. + */ +export async function readSourceChunkPage(input: { + readonly knowledge: Knowledge + readonly source: ReadableSource + readonly params: ChunkPageParams +}): Promise { + const response = await input.knowledge.readChunks({ + documentId: input.source.documentId, + ...(input.source.revisionKey ? { revisionKey: input.source.revisionKey } : {}), + page: input.params.page, + pageSize: input.params.pageSize, + assetUrlPolicy: "durable", + }) + + const chunks = response.chunks.map((chunk) => + toParsedChunkViewFromReadChunk(chunk, input.source.title, input.source.documentId), + ) + + return { + chunks, + pagination: { + page: response.page ?? input.params.page, + pageSize: response.pageSize ?? input.params.pageSize, + total: response.totalChunks ?? chunks.length, + totalPages: + response.totalPages ?? + Math.max(1, Math.ceil(chunks.length / input.params.pageSize)), + }, + } +} + +/** + * Read every parsed chunk for a source by paging the SDK to exhaustion. Used by + * the tree view and load-all display mode. + */ +export async function readAllSourceChunks(input: { + readonly knowledge: Knowledge + readonly source: ReadableSource +}): Promise { + const chunks: ParsedChunkView[] = [] + let page = 1 + let totalPages = 1 + + do { + const response = await input.knowledge.readChunks({ + documentId: input.source.documentId, + ...(input.source.revisionKey + ? { revisionKey: input.source.revisionKey } + : {}), + page, + pageSize: loadAllPageSize, + assetUrlPolicy: "durable", + }) + for (const chunk of response.chunks) { + chunks.push( + toParsedChunkViewFromReadChunk( + chunk, + input.source.title, + input.source.documentId, + ), + ) + } + totalPages = Math.max(1, response.totalPages ?? 1) + page += 1 + } while (page <= totalPages) + + return chunks +} + +/** + * Build a `filePath -> durable Blob URL` map for a source by paging durable + * reads to exhaustion. This is the single asset-hardening path for chat: the + * SDK writes any missing asset into Blob during the durable read and returns + * the durable URL, which we index by both the chunk file path and any + * `metadata.pageAssets[].artifactRef`. + */ +export async function readSourceAssetUrls(input: { + readonly knowledge: Knowledge + readonly documentId: string + readonly revisionKey?: string | null +}): Promise>> { + const assetUrlsByFilePath: Record = {} + let page = 1 + let totalPages = 1 + + do { + const response = await input.knowledge.readChunks({ + documentId: input.documentId, + ...(input.revisionKey ? { revisionKey: input.revisionKey } : {}), + page, + pageSize: loadAllPageSize, + assetUrlPolicy: "durable", + }) + for (const chunk of response.chunks) { + if (chunk.filePath && chunk.assetUrl) { + assetUrlsByFilePath[chunk.filePath] = chunk.assetUrl + } + collectPageAssetUrls(chunk.metadata, assetUrlsByFilePath) + } + totalPages = Math.max(1, response.totalPages ?? 1) + page += 1 + } while (page <= totalPages) + + return assetUrlsByFilePath +} + +function collectPageAssetUrls( + metadata: Record, + target: Record, +): void { + const pageAssets = metadata["pageAssets"] + if (!Array.isArray(pageAssets)) return + for (const pageAsset of pageAssets) { + if (typeof pageAsset !== "object" || pageAsset === null) continue + const record = pageAsset as Record + const artifactRef = record["artifactRef"] + const assetUrl = record["assetUrl"] + if (typeof artifactRef === "string" && typeof assetUrl === "string") { + target[artifactRef] = assetUrl + } + } +} diff --git a/src/domains/chunks/server.test.ts b/src/domains/chunks/server.test.ts deleted file mode 100644 index 2442b53..0000000 --- a/src/domains/chunks/server.test.ts +++ /dev/null @@ -1,563 +0,0 @@ -import { describe, expect, it, vi, type Mock } from "vitest" -import { Effect } from "effect" -import type { DocumentChunk } from "@ontos-ai/knowhere-sdk" - -import type { Source } from "@/infrastructure/db/schema" -import { - loadChunkPageForSource, - loadChunksForSource, -} from "./server" - -describe("server chunk cache", () => { - it("loads ready source chunks from the parsed Blob snapshot without calling Knowhere listChunks", async () => { - const listChunks = vi.fn(async () => ({ - documentId: "doc_1", - chunks: [], - pagination: { - page: 1, - pageSize: 1, - total: 0, - totalPages: 0, - }, - })) - const snapshotStore = createSnapshotStore({ - "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json": - JSON.stringify({ - version: 1, - kind: "knowhere-parsed-result-snapshot", - jobId: "job_1", - documentId: "doc_1", - sourceFileName: "notes.pdf", - totalChunks: 1, - chunkPageSize: 1, - chunkPages: [ - { - page: 1, - pageSize: 1, - chunkCount: 1, - key: "workspaces/workspace_1/sources/source_1/parsed-result/chunks/page-1.json", - }, - ], - assetUrlsByFilePath: { - "images/chart.png": "https://blob.example/images/chart.png", - }, - createdAt: "2026-07-03T00:00:00.000Z", - }), - "workspaces/workspace_1/sources/source_1/parsed-result/chunks/page-1.json": - JSON.stringify({ - version: 1, - jobId: "job_1", - documentId: "doc_1", - sourceFileName: "notes.pdf", - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - chunks: [ - { - id: "chunk_image_1", - chunkId: "parser_image_1", - chunkType: "image", - content: "Chart summary", - sectionPath: "Revenue", - sourceChunkPath: "notes.pdf/Revenue", - filePath: "images/chart.png", - sortOrder: 1, - metadata: { summary: "Chart summary" }, - }, - ], - }), - }) - - const page = await Effect.runPromise( - loadChunkPageForSource( - makeSource({ status: "ready", knowhereJobId: "job_1" }), - { documents: { listChunks } }, - { page: 1, pageSize: 1 }, - { - snapshot: { - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - assetUrlsByFilePath: {}, - }, - snapshotStore, - workspaceId: "workspace_1", - }, - ), - ) - - expect(listChunks).not.toHaveBeenCalled() - expect(page).toEqual({ - chunks: [ - expect.objectContaining({ - chunkId: "chunk_image_1", - parserChunkId: "parser_image_1", - type: "image", - assetUrl: "https://blob.example/images/chart.png", - sourceTitle: "notes.pdf", - }), - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - }) - }) - - it("returns upstream chunks on a visible cache miss and warms mirrored assets in the background", async () => { - const warmTasks: Array<() => Promise> = [] - const cacheStore = createCacheStore() - const listChunks = vi.fn(async () => ({ - documentId: "doc_1", - jobResultId: "revision_1", - chunks: [ - makeDocumentChunk({ - id: "image_1", - chunkId: "parser_image_1", - chunkType: "image", - filePath: "images/image-1.png", - assetUrl: "https://knowhere.example/assets/image-1.png", - }), - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })) - const fetchAsset = vi.fn(async () => - new Response("image-body", { - headers: { "content-type": "image/png" }, - }), - ) - - const page = await Effect.runPromise( - loadChunkPageForSource( - makeSource({ knowhereJobId: "revision_1" }), - { documents: { listChunks } }, - { page: 1, pageSize: 1 }, - { - cacheStore, - fetchAsset, - scheduleWarm: (task) => warmTasks.push(task), - workspaceId: "workspace_1", - }, - ), - ) - - expect(page.chunks[0]?.assetUrl).toBe( - "https://knowhere.example/assets/image-1.png", - ) - expect(cacheStore.putMock).not.toHaveBeenCalled() - expect(warmTasks).toHaveLength(1) - - await warmTasks[0]?.() - - expect(fetchAsset).toHaveBeenCalledWith( - "https://knowhere.example/assets/image-1.png", - ) - expect(cacheStore.putMock).toHaveBeenCalledWith( - expect.stringContaining("/chunk-assets/revision_1/"), - Buffer.from("image-body"), - expect.objectContaining({ contentType: "image/png" }), - ) - const cachedPagePut = cacheStore.putMock.mock.calls.find( - ([pathname]) => - typeof pathname === "string" && pathname.endsWith(".json"), - ) - expect(cachedPagePut).toBeDefined() - const cachedPage = JSON.parse(String(cachedPagePut?.[1])) as { - readonly chunks: readonly { readonly assetUrl?: string }[] - } - expect(cachedPage.chunks[0]?.assetUrl).toContain( - "https://blob.example/workspaces/workspace_1/sources/source_1/chunk-assets/revision_1/", - ) - }) - - it("returns a cached visible page after verifying the current Knowhere revision", async () => { - const cachedPage = { - chunks: [ - { - chunkId: "image_1", - documentId: "doc_1", - type: "image", - content: "", - assetUrl: "https://blob.example/image-1.png", - sourceTitle: "notes.pdf", - }, - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - } - const listChunks = vi.fn(async () => ({ - documentId: "doc_1", - jobResultId: "revision_1", - chunks: [], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })) - const cacheStore = createCacheStore({ - get: vi.fn(async () => ({ - statusCode: 200, - stream: createTextStream(JSON.stringify(cachedPage)), - })), - }) - - const page = await Effect.runPromise( - loadChunkPageForSource( - makeSource({ knowhereJobId: "revision_1" }), - { documents: { listChunks } }, - { page: 1, pageSize: 1 }, - { - cacheStore, - workspaceId: "workspace_1", - }, - ), - ) - - expect(page).toEqual(cachedPage) - expect(listChunks).toHaveBeenCalledWith("doc_1", { - page: 1, - pageSize: 1, - includeAssetUrls: false, - }) - expect(cacheStore.getMock).toHaveBeenCalledWith( - expect.stringContaining("/revision_1/visible/page-1-size-1.json"), - { access: "public" }, - ) - }) - - it("ignores old cached pages when Knowhere reports a new job id", async () => { - const warmTasks: Array<() => Promise> = [] - const staleCachedPage = { - chunks: [ - { - chunkId: "stale_1", - documentId: "doc_1", - type: "text", - content: "Old chunk", - sourceTitle: "notes.pdf", - }, - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - } - const cacheStore = createCacheStore({ - get: vi.fn(async (pathname: string) => - pathname.includes("/job_old/") - ? { - statusCode: 200, - stream: createTextStream(JSON.stringify(staleCachedPage)), - } - : null, - ), - }) - const listChunks = vi.fn(async ( - _documentId: string, - params: { readonly includeAssetUrls: boolean }, - ) => ({ - documentId: "doc_1", - jobId: "job_new", - chunks: params.includeAssetUrls - ? [ - makeDocumentChunk({ - id: "text_new", - chunkId: "parser_text_new", - content: "New chunk", - }), - ] - : [], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })) - const onRevisionKey = vi.fn(async () => undefined) - - const page = await Effect.runPromise( - loadChunkPageForSource( - makeSource({ knowhereJobId: "job_old" }), - { documents: { listChunks } }, - { page: 1, pageSize: 1 }, - { - cacheStore, - onRevisionKey, - scheduleWarm: (task) => warmTasks.push(task), - workspaceId: "workspace_1", - }, - ), - ) - - expect(page.chunks[0]?.content).toBe("New chunk") - expect(cacheStore.getMock).toHaveBeenCalledWith( - expect.stringContaining("/job_new/visible/page-1-size-1.json"), - { access: "public" }, - ) - expect(cacheStore.getMock).not.toHaveBeenCalledWith( - expect.stringContaining("/job_old/visible/page-1-size-1.json"), - expect.anything(), - ) - expect(onRevisionKey).toHaveBeenCalledWith("job_new") - expect(warmTasks).toHaveLength(1) - }) - - it("uses structure-only chunk loading for full-tree requests", async () => { - const warmTasks: Array<() => Promise> = [] - const cacheStore = createCacheStore() - const listChunks = vi.fn(async () => ({ - documentId: "doc_1", - jobResultId: "revision_1", - chunks: [ - makeDocumentChunk({ - id: "text_1", - chunkId: "parser_text_1", - content: "A text chunk", - }), - ], - pagination: { - page: 1, - pageSize: 200, - total: 1, - totalPages: 1, - }, - })) - const fetchAsset = vi.fn() - - const chunks = await Effect.runPromise( - loadChunksForSource( - makeSource({ knowhereJobId: "revision_1" }), - { documents: { listChunks } }, - { - cacheStore, - fetchAsset, - scheduleWarm: (task) => warmTasks.push(task), - workspaceId: "workspace_1", - }, - ), - ) - - expect(chunks).toMatchObject([{ chunkId: "text_1" }]) - expect(listChunks).toHaveBeenCalledWith("doc_1", { - page: 1, - pageSize: 200, - includeAssetUrls: false, - }) - expect(fetchAsset).not.toHaveBeenCalled() - expect(warmTasks).toHaveLength(1) - await warmTasks[0]?.() - expect(cacheStore.putMock).toHaveBeenCalledWith( - expect.stringContaining("/structure/page-1-size-200.json"), - expect.any(String), - expect.objectContaining({ contentType: "application/json; charset=utf-8" }), - ) - }) - - it("caches media chunks without previews when Knowhere has no upstream asset URL", async () => { - const warmTasks: Array<() => Promise> = [] - const cacheStore = createCacheStore() - const listChunks = vi.fn(async () => ({ - documentId: "doc_1", - jobResultId: "revision_1", - chunks: [ - makeDocumentChunk({ - id: "image_1", - chunkId: "parser_image_1", - chunkType: "image", - filePath: "images/image-1.png", - assetUrl: null, - }), - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })) - const fetchAsset = vi.fn() - - const page = await Effect.runPromise( - loadChunkPageForSource( - makeSource({ knowhereJobId: "revision_1" }), - { documents: { listChunks } }, - { page: 1, pageSize: 1 }, - { - cacheStore, - fetchAsset, - scheduleWarm: (task) => warmTasks.push(task), - workspaceId: "workspace_1", - }, - ), - ) - - expect(page.chunks[0]?.assetUrl).toBeUndefined() - await warmTasks[0]?.() - expect(fetchAsset).not.toHaveBeenCalled() - const cachedPagePut = cacheStore.putMock.mock.calls.find( - ([pathname]) => - typeof pathname === "string" && pathname.endsWith(".json"), - ) - const cachedPage = JSON.parse(String(cachedPagePut?.[1])) as { - readonly chunks: readonly { readonly assetUrl?: string }[] - } - expect(cachedPage.chunks[0]?.assetUrl).toBeUndefined() - }) -}) - -type TestCacheGetResult = - | { - readonly statusCode: 200 - readonly stream: ReadableStream - } - | { - readonly statusCode: 304 - readonly stream: null - } - -type TestCachePutOptions = { - readonly access: "public" - readonly allowOverwrite: boolean - readonly contentType: string - readonly multipart?: boolean -} - -type TestCacheStore = { - readonly get: ( - pathname: string, - options: { readonly access: "public" }, - ) => Promise - readonly put: ( - pathname: string, - body: string | Buffer, - options: TestCachePutOptions, - ) => Promise<{ readonly url: string }> - readonly getMock: TestCacheGetMock - readonly putMock: TestCachePutMock -} - -type TestSnapshotStore = { - readonly get: ( - pathname: string, - options: { readonly access: "public" }, - ) => Promise -} - -type TestCacheGetMock = Mock< - ( - pathname: string, - options: { readonly access: "public" }, - ) => Promise -> - -type TestCachePutMock = Mock< - ( - pathname: string, - body: string | Buffer, - options: TestCachePutOptions, - ) => Promise<{ readonly url: string }> -> - -function createCacheStore(overrides: Partial<{ - readonly get: TestCacheGetMock - readonly put: TestCachePutMock -}> = {}): TestCacheStore { - const getMock = - overrides.get ?? - vi.fn(async () => null) - const putMock = - overrides.put ?? - vi.fn(async (pathname: string) => ({ - url: `https://blob.example/${pathname}`, - })) - - return { - get: (pathname, options) => getMock(pathname, options), - put: (pathname, body, options) => putMock(pathname, body, options), - getMock, - putMock, - } -} - -function createSnapshotStore( - entriesByPathname: Readonly>, -): TestSnapshotStore { - return { - get: vi.fn(async (pathname: string) => { - const entry = entriesByPathname[pathname] - if (!entry) return null - return { - statusCode: 200 as const, - stream: createTextStream(entry), - } - }), - } -} - -function makeDocumentChunk( - overrides: Partial = {}, -): DocumentChunk { - return { - id: "document_chunk_1", - chunkId: "parser_chunk_1", - chunkType: "text", - content: "Chunk content", - sectionId: null, - sectionPath: null, - sourceChunkPath: null, - filePath: null, - sortOrder: 1, - metadata: {}, - assetUrl: null, - ...overrides, - } -} - -function makeSource(overrides: Partial = {}): Source { - return { - id: "source_1", - workspaceId: "workspace_1", - title: "notes.pdf", - mimeType: "application/pdf", - sizeBytes: 100, - status: "ready", - failureReason: null, - knowhereJobId: "revision_1", - knowhereDocumentId: "doc_1", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-06T00:00:00Z"), - updatedAt: new Date("2026-05-06T00:00:00Z"), - deletedAt: null, - ...overrides, - } -} - -function createTextStream(text: string): ReadableStream { - const stream = new Response(text).body - if (!stream) throw new Error("Response body stream was not created.") - return stream -} diff --git a/src/domains/chunks/server.ts b/src/domains/chunks/server.ts deleted file mode 100644 index 64b7fe4..0000000 --- a/src/domains/chunks/server.ts +++ /dev/null @@ -1,911 +0,0 @@ -import "server-only" - -import path from "node:path" -import { createHash } from "node:crypto" -import { get as getBlob, put } from "@vercel/blob" -import { after } from "next/server" -import { Effect } from "effect" -import type { - DocumentChunk, -} from "@ontos-ai/knowhere-sdk" - -import { - resolveChunkConnectionTargets, - toParsedChunkView, - type ChunkKnowhereClient, - type ChunkPage, - type ChunkPageParams, - type LoadChunksOptions, -} from "@/domains/chunks" -import { - readParsedResultSnapshotChunkPage, - readParsedResultSnapshotManifest, - type ParsedResultSnapshotBlobStore, - type ParsedResultSnapshotChunk, - type ParsedResultSnapshotManifest, -} from "@/domains/sources/parse-result-storage-adapter" -import type { ParsedChunkView } from "@/domains/chunks/types" -import type { Source } from "@/infrastructure/db/schema" -import { logger } from "@/lib/logger" - -type ChunkPageMode = "visible" | "structure" - -type ChunkPageBlobGetResult = - | { - readonly statusCode: 200 - readonly stream: ReadableStream - } - | { - readonly statusCode: 304 - readonly stream: null - } - -type ChunkPageBlobPutOptions = { - readonly access: "public" - readonly allowOverwrite: boolean - readonly contentType: string - readonly multipart?: boolean -} - -type ChunkPageBlobStore = { - readonly get: ( - pathname: string, - options: { readonly access: "public" }, - ) => Promise - readonly put: ( - pathname: string, - body: string | Buffer, - options: ChunkPageBlobPutOptions, - ) => Promise<{ readonly url: string }> -} - -type FetchChunkAsset = (assetUrl: string) => Promise - -type ChunkPageWarmScheduler = (task: () => Promise) => void - -type ServerLoadChunksOptions = LoadChunksOptions & { - readonly workspaceId?: string - readonly snapshot?: SourceParsedSnapshotReadModel | null - readonly snapshotStore?: ParsedResultSnapshotBlobStore - readonly cacheStore?: ChunkPageBlobStore - readonly fetchAsset?: FetchChunkAsset - readonly mode?: ChunkPageMode - readonly onRevisionKey?: (revisionKey: string) => Promise - readonly scheduleWarm?: ChunkPageWarmScheduler -} - -type WarmChunkPageCacheInput = { - readonly source: Source - readonly client: ChunkKnowhereClient - readonly params: ChunkPageParams - readonly revisionKey: string - readonly workspaceId: string - readonly cacheStore: ChunkPageBlobStore - readonly fetchAsset: FetchChunkAsset - readonly scheduleWarm: ChunkPageWarmScheduler - readonly startAssetIndex?: number - readonly mirroredAssetUrlsByOriginalUrl?: Readonly> -} - -type MirrorableChunkAsset = { - readonly chunkId: string - readonly chunkType: "image" | "table" - readonly assetUrl: string - readonly filePath?: string -} - -export type SourceParsedSnapshotReadModel = { - readonly resultBlobUrl: string - readonly snapshotManifestUrl?: string | null - readonly snapshotManifestKey?: string | null - readonly assetUrlsByFilePath: Readonly> -} - -const documentChunkPageSize = 200 -const visibleChunkPageMode: ChunkPageMode = "visible" -const structureChunkPageMode: ChunkPageMode = "structure" -const maximumMirroredAssetsPerWarmStep = 50 -const maximumWarmStepDurationMs = 45_000 -const assetMirrorConcurrency = 10 - -const defaultBlobStore: ChunkPageBlobStore = { - get: (pathname, options) => getBlob(pathname, options), - put: (pathname, body, options) => - put(pathname, body, { - access: options.access, - allowOverwrite: options.allowOverwrite, - contentType: options.contentType, - multipart: options.multipart, - }), -} - -const defaultFetchAsset: FetchChunkAsset = (assetUrl: string) => fetch(assetUrl) - -const defaultScheduleWarm: ChunkPageWarmScheduler = ( - task: () => Promise, -) => { - try { - after(task) - } catch { - void task() - } -} - -export const loadChunksForSource = ( - source: Source, - client: ChunkKnowhereClient, - options: ServerLoadChunksOptions = {}, -) => - Effect.gen(function* () { - if (source.status !== "ready" || !source.knowhereDocumentId) return [] - - if (options.snapshot) { - const snapshotChunks = yield* Effect.promise(() => - loadAllSnapshotChunks({ - source, - snapshot: options.snapshot!, - blobStore: options.snapshotStore, - }), - ) - return resolveChunkConnectionTargets(snapshotChunks) - } - - const chunks: ParsedChunkView[] = [] - let page = 1 - let totalPages = 1 - - do { - const chunkPage = yield* loadChunkPageForSource(source, client, { - page, - pageSize: documentChunkPageSize, - }, { - ...options, - mode: structureChunkPageMode, - }) - chunks.push(...chunkPage.chunks) - totalPages = chunkPage.pagination.totalPages - page += 1 - } while (page <= totalPages) - - return resolveChunkConnectionTargets(chunks) - }) - -export const loadChunkPageForSource = ( - source: Source, - client: ChunkKnowhereClient, - params: ChunkPageParams, - options: ServerLoadChunksOptions = {}, -) => - Effect.gen(function* () { - const emptyPage = createEmptyChunkPage(params) - if (source.status !== "ready" || !source.knowhereDocumentId) { - return emptyPage - } - - if (options.snapshot) { - return yield* Effect.promise(() => - loadSnapshotChunkPage({ - source, - params, - snapshot: options.snapshot!, - blobStore: options.snapshotStore, - }), - ) - } - - const mode = options.mode ?? visibleChunkPageMode - const workspaceId = options.workspaceId ?? source.workspaceId - const cacheStore = options.cacheStore ?? defaultBlobStore - const includeAssetUrls = mode === visibleChunkPageMode - const revisionProbeResponse = yield* Effect.promise(() => - client.documents.listChunks(source.knowhereDocumentId!, { - page: params.page, - pageSize: params.pageSize, - includeAssetUrls: false, - }), - ) - const probeRevisionKey = getRevisionKey(revisionProbeResponse, source) - if (probeRevisionKey) { - scheduleRevisionKeyUpdate(source, probeRevisionKey, options.onRevisionKey) - const cachedPage = yield* Effect.promise(() => - readCachedChunkPage({ - cacheStore, - documentId: source.knowhereDocumentId!, - mode, - params, - revisionKey: probeRevisionKey, - workspaceId, - }), - ) - if (cachedPage) return cachedPage - } - - const response = includeAssetUrls - ? yield* Effect.promise(() => - client.documents.listChunks(source.knowhereDocumentId!, { - page: params.page, - pageSize: params.pageSize, - includeAssetUrls, - }), - ) - : revisionProbeResponse - const revisionKey = getRevisionKey(response, source) ?? probeRevisionKey - if (revisionKey && revisionKey !== probeRevisionKey) { - scheduleRevisionKeyUpdate(source, revisionKey, options.onRevisionKey) - } - - const chunkPage = createChunkPageFromResponse({ - response, - source, - params, - options: - mode === visibleChunkPageMode - ? { assetUrlsByFilePath: options.assetUrlsByFilePath } - : {}, - }) - - if (revisionKey) { - if (mode === visibleChunkPageMode) { - scheduleChunkPageWarm({ - source, - client, - params, - revisionKey, - workspaceId, - cacheStore, - fetchAsset: options.fetchAsset ?? defaultFetchAsset, - scheduleWarm: options.scheduleWarm ?? defaultScheduleWarm, - }) - } else { - scheduleStructurePageCacheWrite({ - cacheStore, - chunkPage, - documentId: source.knowhereDocumentId, - mode, - params, - revisionKey, - scheduleWarm: options.scheduleWarm ?? defaultScheduleWarm, - workspaceId, - }) - } - } - - return chunkPage - }) - -async function loadAllSnapshotChunks(input: { - readonly source: Source - readonly snapshot: SourceParsedSnapshotReadModel - readonly blobStore?: ParsedResultSnapshotBlobStore -}): Promise { - const manifest = await loadSnapshotManifest(input) - if (!manifest) return [] - - const pages = await Promise.all( - manifest.chunkPages.map((page) => - readParsedResultSnapshotChunkPage({ - pageKey: page.key, - blobStore: input.blobStore, - }), - ), - ) - const chunks = pages.flatMap((page) => page?.chunks ?? []) - return chunks.map((chunk) => - toParsedChunkView( - toDocumentChunkFromSnapshot(chunk), - input.source.title, - input.source.knowhereDocumentId ?? undefined, - { - assetUrlsByFilePath: { - ...manifest.assetUrlsByFilePath, - ...input.snapshot.assetUrlsByFilePath, - }, - }, - ), - ) -} - -async function loadSnapshotChunkPage(input: { - readonly source: Source - readonly params: ChunkPageParams - readonly snapshot: SourceParsedSnapshotReadModel - readonly blobStore?: ParsedResultSnapshotBlobStore -}): Promise { - const manifest = await loadSnapshotManifest(input) - if (!manifest) return createEmptyChunkPage(input.params) - - const pageReference = manifest.chunkPages.find( - (page) => page.page === input.params.page && page.pageSize === input.params.pageSize, - ) - if (!pageReference) { - return createEmptyChunkPage({ - page: input.params.page, - pageSize: input.params.pageSize, - }) - } - - const page = await readParsedResultSnapshotChunkPage({ - pageKey: pageReference.key, - blobStore: input.blobStore, - }) - if (!page) return createEmptyChunkPage(input.params) - - const assetUrlsByFilePath = { - ...manifest.assetUrlsByFilePath, - ...input.snapshot.assetUrlsByFilePath, - } - const chunks = page.chunks.map((chunk) => - toParsedChunkView( - toDocumentChunkFromSnapshot(chunk), - input.source.title, - input.source.knowhereDocumentId ?? undefined, - { assetUrlsByFilePath }, - ), - ) - - return { - chunks, - pagination: { - page: page.page, - pageSize: page.pageSize, - total: page.total, - totalPages: page.totalPages, - }, - } -} - -async function loadSnapshotManifest(input: { - readonly source: Source - readonly snapshot: SourceParsedSnapshotReadModel - readonly blobStore?: ParsedResultSnapshotBlobStore -}): Promise { - const manifestKey = input.snapshot.snapshotManifestKey - if (!manifestKey) { - logger.warn("chunks: ready source missing parsed snapshot manifest", { - sourceId: input.source.id, - documentId: input.source.knowhereDocumentId, - }) - return null - } - - const manifest = await readParsedResultSnapshotManifest({ - workspaceId: input.source.workspaceId, - sourceId: input.source.id, - manifestKey, - blobStore: input.blobStore, - }) - if (!manifest) { - logger.warn("chunks: parsed snapshot manifest could not be read", { - sourceId: input.source.id, - documentId: input.source.knowhereDocumentId, - manifestKey, - }) - } - return manifest -} - -function toDocumentChunkFromSnapshot(chunk: ParsedResultSnapshotChunk): DocumentChunk { - return { - id: chunk.id, - chunkId: chunk.chunkId, - chunkType: normalizeDocumentChunkType(chunk.chunkType), - content: chunk.content, - sectionId: null, - sectionPath: chunk.sectionPath ?? null, - sourceChunkPath: chunk.sourceChunkPath, - filePath: chunk.filePath ?? null, - sortOrder: chunk.sortOrder, - metadata: chunk.metadata, - assetUrl: chunk.assetUrl ?? null, - } -} - -function normalizeDocumentChunkType(value: string): DocumentChunk["chunkType"] { - if (value === "image" || value === "table" || value === "page") return value - return "text" -} - -export async function warmChunkPageCache( - input: WarmChunkPageCacheInput, -): Promise { - const response = await input.client.documents.listChunks( - input.source.knowhereDocumentId!, - { - page: input.params.page, - pageSize: input.params.pageSize, - includeAssetUrls: true, - }, - ) - const assets = collectMirrorableChunkAssets(response.chunks) - const startAssetIndex = input.startAssetIndex ?? 0 - const mirroredAssetUrls = new Map( - Object.entries(input.mirroredAssetUrlsByOriginalUrl ?? {}), - ) - - const stepStartedAt = Date.now() - const assetBatch = assets.slice( - startAssetIndex, - startAssetIndex + maximumMirroredAssetsPerWarmStep, - ) - const mirroredBatch = await mirrorChunkAssets({ - assets: assetBatch, - cacheStore: input.cacheStore, - fetchAsset: input.fetchAsset, - revisionKey: input.revisionKey, - source: input.source, - }) - for (const mirroredAsset of mirroredBatch) { - mirroredAssetUrls.set(mirroredAsset.assetUrl, mirroredAsset.blobUrl) - } - - const nextAssetIndex = startAssetIndex + assetBatch.length - if ( - nextAssetIndex < assets.length && - Date.now() - stepStartedAt < maximumWarmStepDurationMs - ) { - input.scheduleWarm(() => - warmChunkPageCache({ - ...input, - startAssetIndex: nextAssetIndex, - mirroredAssetUrlsByOriginalUrl: Object.fromEntries(mirroredAssetUrls), - }), - ) - return - } - - if (nextAssetIndex < assets.length) { - input.scheduleWarm(() => - warmChunkPageCache({ - ...input, - startAssetIndex: nextAssetIndex, - mirroredAssetUrlsByOriginalUrl: Object.fromEntries(mirroredAssetUrls), - }), - ) - return - } - - const rewrittenChunks = response.chunks.map((chunk) => - rewriteChunkAssetUrl(chunk, mirroredAssetUrls), - ) - const chunkPage = createChunkPageFromResponse({ - response: { - ...response, - chunks: rewrittenChunks, - }, - source: input.source, - params: input.params, - options: {}, - }) - - await writeCachedChunkPage({ - cacheStore: input.cacheStore, - chunkPage, - documentId: input.source.knowhereDocumentId!, - mode: visibleChunkPageMode, - params: input.params, - revisionKey: input.revisionKey, - workspaceId: input.workspaceId, - }) -} - -function scheduleChunkPageWarm(input: WarmChunkPageCacheInput): void { - input.scheduleWarm(async () => { - try { - await warmChunkPageCache(input) - } catch (error) { - logger.warn("chunks: chunk page cache warm failed", { - sourceId: input.source.id, - documentId: input.source.knowhereDocumentId, - page: input.params.page, - pageSize: input.params.pageSize, - revisionKey: input.revisionKey, - error: getErrorMessage(error), - }) - } - }) -} - -function scheduleStructurePageCacheWrite(input: { - readonly cacheStore: ChunkPageBlobStore - readonly chunkPage: ChunkPage - readonly documentId: string - readonly mode: ChunkPageMode - readonly params: ChunkPageParams - readonly revisionKey: string - readonly scheduleWarm: ChunkPageWarmScheduler - readonly workspaceId: string -}): void { - input.scheduleWarm(async () => { - try { - await writeCachedChunkPage(input) - } catch (error) { - logger.warn("chunks: structure chunk page cache write failed", { - documentId: input.documentId, - page: input.params.page, - pageSize: input.params.pageSize, - revisionKey: input.revisionKey, - error: getErrorMessage(error), - }) - } - }) -} - -async function readCachedChunkPage(input: { - readonly cacheStore: ChunkPageBlobStore - readonly documentId: string - readonly mode: ChunkPageMode - readonly params: ChunkPageParams - readonly revisionKey: string - readonly workspaceId: string -}): Promise { - const pathname = getChunkPageCachePathname(input) - const result = await input.cacheStore.get(pathname, { access: "public" }) - if (!result || result.statusCode !== 200) return null - - const text = await new Response(result.stream).text() - return parseCachedChunkPage(text) -} - -async function writeCachedChunkPage(input: { - readonly cacheStore: ChunkPageBlobStore - readonly chunkPage: ChunkPage - readonly documentId: string - readonly mode: ChunkPageMode - readonly params: ChunkPageParams - readonly revisionKey: string - readonly workspaceId: string -}): Promise { - await input.cacheStore.put( - getChunkPageCachePathname(input), - JSON.stringify(input.chunkPage), - { - access: "public", - allowOverwrite: true, - contentType: "application/json; charset=utf-8", - }, - ) -} - -function createChunkPageFromResponse(input: { - readonly response: { - readonly chunks: readonly DocumentChunk[] - readonly pagination?: { - readonly page?: number - readonly pageSize?: number - readonly total?: number - readonly totalPages?: number - } - } - readonly source: Source - readonly params: ChunkPageParams - readonly options: LoadChunksOptions -}): ChunkPage { - const chunks = input.response.chunks.map((chunk) => - toParsedChunkView( - chunk, - input.source.title, - input.source.knowhereDocumentId ?? undefined, - input.options, - ), - ) - - return { - chunks, - pagination: { - page: getFiniteNonNegativeNumber( - input.response.pagination?.page, - input.params.page, - ), - pageSize: getFiniteNonNegativeNumber( - input.response.pagination?.pageSize, - input.params.pageSize, - ), - total: getFiniteNonNegativeNumber( - input.response.pagination?.total, - chunks.length, - ), - totalPages: getFiniteNonNegativeNumber( - input.response.pagination?.totalPages, - Math.ceil(chunks.length / input.params.pageSize), - ), - }, - } -} - -function createEmptyChunkPage(params: ChunkPageParams): ChunkPage { - return { - chunks: [], - pagination: { - page: params.page, - pageSize: params.pageSize, - total: 0, - totalPages: 0, - }, - } -} - -function parseCachedChunkPage(text: string): ChunkPage | null { - try { - const value: unknown = JSON.parse(text) - if (!isRecord(value)) return null - const chunks = value["chunks"] - const pagination = value["pagination"] - if (!Array.isArray(chunks) || !isRecord(pagination)) return null - const page = getNumber(pagination["page"]) - const pageSize = getNumber(pagination["pageSize"]) - const total = getNumber(pagination["total"]) - const totalPages = getNumber(pagination["totalPages"]) - if ( - page === undefined || - pageSize === undefined || - total === undefined || - totalPages === undefined - ) { - return null - } - - return { - chunks: chunks.filter(isParsedChunkView), - pagination: { - page, - pageSize, - total, - totalPages, - }, - } - } catch { - return null - } -} - -async function mirrorChunkAssets(input: { - readonly assets: readonly MirrorableChunkAsset[] - readonly cacheStore: ChunkPageBlobStore - readonly fetchAsset: FetchChunkAsset - readonly revisionKey: string - readonly source: Source -}): Promise< - readonly { - readonly assetUrl: string - readonly blobUrl: string - }[] -> { - return mapWithConcurrency( - input.assets, - assetMirrorConcurrency, - async (asset) => { - const response = await input.fetchAsset(asset.assetUrl) - if (!response.ok) { - throw new Error( - `Chunk asset fetch failed with status ${response.status}.`, - ) - } - - const body = Buffer.from(await response.arrayBuffer()) - const blob = await input.cacheStore.put( - getMirroredAssetPathname({ - asset, - revisionKey: input.revisionKey, - source: input.source, - }), - body, - { - access: "public", - allowOverwrite: true, - contentType: getMirroredAssetContentType(asset, response), - multipart: true, - }, - ) - return { - assetUrl: asset.assetUrl, - blobUrl: blob.url, - } - }, - ) -} - -async function mapWithConcurrency( - inputs: readonly Input[], - concurrency: number, - mapInput: (input: Input) => Promise, -): Promise { - const results: Array = [] - let nextIndex = 0 - const workerCount = Math.min(concurrency, inputs.length) - - async function runWorker(): Promise { - while (nextIndex < inputs.length) { - const index = nextIndex - nextIndex += 1 - const input = inputs[index] - if (input === undefined) return - results[index] = await mapInput(input) - } - } - - await Promise.all( - Array.from({ length: workerCount }, () => runWorker()), - ) - - return results.filter( - (result): result is Output => result !== undefined, - ) -} - -function collectMirrorableChunkAssets( - chunks: readonly DocumentChunk[], -): readonly MirrorableChunkAsset[] { - return chunks.flatMap((chunk): readonly MirrorableChunkAsset[] => { - if (chunk.chunkType !== "image" && chunk.chunkType !== "table") return [] - if (typeof chunk.assetUrl !== "string" || chunk.assetUrl.length === 0) { - return [] - } - - return [ - { - chunkId: chunk.id, - chunkType: chunk.chunkType, - assetUrl: chunk.assetUrl, - ...(chunk.filePath ? { filePath: chunk.filePath } : {}), - }, - ] - }) -} - -function rewriteChunkAssetUrl( - chunk: DocumentChunk, - mirroredAssetUrls: ReadonlyMap, -): DocumentChunk { - if (typeof chunk.assetUrl !== "string") return chunk - const mirroredAssetUrl = mirroredAssetUrls.get(chunk.assetUrl) - return mirroredAssetUrl ? { ...chunk, assetUrl: mirroredAssetUrl } : chunk -} - -function getChunkPageCachePathname(input: { - readonly documentId: string - readonly mode: ChunkPageMode - readonly params: ChunkPageParams - readonly revisionKey: string - readonly workspaceId: string -}): string { - return [ - "workspaces", - encodePathSegment(input.workspaceId), - "chunk-pages", - encodePathSegment(input.documentId), - encodePathSegment(input.revisionKey), - input.mode, - `page-${input.params.page}-size-${input.params.pageSize}.json`, - ].join("/") -} - -function getMirroredAssetPathname(input: { - readonly asset: MirrorableChunkAsset - readonly revisionKey: string - readonly source: Source -}): string { - const basename = getMirroredAssetBasename(input.asset) - return [ - "workspaces", - encodePathSegment(input.source.workspaceId), - "sources", - encodePathSegment(input.source.id), - "chunk-assets", - encodePathSegment(input.revisionKey), - `${hashValue(input.asset.assetUrl)}-${basename}`, - ].join("/") -} - -function getMirroredAssetBasename(asset: MirrorableChunkAsset): string { - const candidate = - asset.filePath ?? - getUrlPathname(asset.assetUrl).split("/").filter(Boolean).at(-1) ?? - `${asset.chunkId}.bin` - return candidate.replace(/[^a-zA-Z0-9._-]+/g, "-") -} - -function getMirroredAssetContentType( - asset: MirrorableChunkAsset, - response: Response, -): string { - const headerContentType = response.headers.get("content-type") - if (headerContentType) return headerContentType - if (asset.chunkType === "table") return "text/html; charset=utf-8" - - const extension = path.extname(asset.filePath ?? getUrlPathname(asset.assetUrl)) - .toLowerCase() - if (extension === ".jpg" || extension === ".jpeg") return "image/jpeg" - if (extension === ".png") return "image/png" - if (extension === ".webp") return "image/webp" - if (extension === ".gif") return "image/gif" - return "application/octet-stream" -} - -function getRevisionKey( - response: { - readonly jobId?: string | null - readonly jobResultId?: string | null - }, - source: Source, -): string | null { - return ( - getNonEmptyString(response.jobId) ?? - getNonEmptyString(response.jobResultId) ?? - getFallbackRevisionKey(source) - ) -} - -function getFallbackRevisionKey(source: Source): string | null { - return ( - getNonEmptyString(source.knowhereJobId) ?? - getNonEmptyString(source.knowhereDocumentId) - ) -} - -function scheduleRevisionKeyUpdate( - source: Source, - revisionKey: string, - onRevisionKey: ((revisionKey: string) => Promise) | undefined, -): void { - if (!onRevisionKey || source.knowhereJobId === revisionKey) return - - void onRevisionKey(revisionKey).catch((error: unknown) => { - logger.warn("chunks: source revision key update failed", { - sourceId: source.id, - documentId: source.knowhereDocumentId, - revisionKey, - error: getErrorMessage(error), - }) - }) -} - -function getFiniteNonNegativeNumber( - value: number | undefined, - fallback: number, -): number { - return typeof value === "number" && Number.isFinite(value) && value >= 0 - ? value - : fallback -} - -function isParsedChunkView(value: unknown): value is ParsedChunkView { - if (!isRecord(value)) return false - return ( - typeof value["chunkId"] === "string" && - typeof value["type"] === "string" && - typeof value["content"] === "string" && - typeof value["sourceTitle"] === "string" - ) -} - -function getNumber(value: unknown): number | undefined { - return typeof value === "number" && Number.isFinite(value) ? value : undefined -} - -function getNonEmptyString(value: unknown): string | null { - return typeof value === "string" && value.length > 0 ? value : null -} - -function isRecord(value: unknown): value is Readonly> { - return typeof value === "object" && value !== null && !Array.isArray(value) -} - -function getUrlPathname(value: string): string { - try { - return new URL(value).pathname - } catch { - return value.split("?")[0] ?? value - } -} - -function encodePathSegment(value: string): string { - return encodeURIComponent(value) -} - -function hashValue(value: string): string { - return createHash("sha256").update(value).digest("hex").slice(0, 16) -} - -function getErrorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error) -} diff --git a/src/domains/sources/counts.test.ts b/src/domains/sources/counts.test.ts index e520eff..22f0b8c 100644 --- a/src/domains/sources/counts.test.ts +++ b/src/domains/sources/counts.test.ts @@ -14,6 +14,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 1, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: "doc_1", stagedBlobPathname: null, @@ -29,36 +30,11 @@ function makeSource(overrides: Partial = {}): Source { } describe("countChunksBySourceId", () => { - it("counts ready source chunks from the parsed snapshot manifest", async () => { - const listChunks = vi.fn() + it("counts ready source chunks from the document total", async () => { + const listChunks = vi.fn(async () => ({ pagination: { total: 12 } })) const mockClient = { documents: { listChunks }, } as unknown as Knowhere - const repository = { - getParseSnapshotMetadata: vi.fn(async (workspaceId: string, sourceId: string) => - sourceId === "ready" - ? { - resultBlobUrl: "https://blob.example/manifest/current.json", - snapshotManifestUrl: "https://blob.example/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/ready/parsed-result/manifest/current.json", - assetUrlsByFilePath: {}, - } - : null, - ), - } - const readSnapshotManifest = vi.fn(async () => ({ - version: 1 as const, - kind: "knowhere-parsed-result-snapshot" as const, - jobId: "job_1", - documentId: "doc_ready", - sourceFileName: "notes.pdf", - totalChunks: 12, - chunkPageSize: 50, - chunkPages: [], - assetUrlsByFilePath: {}, - createdAt: "2026-07-04T00:00:00.000Z", - })) const { countChunksBySourceId } = await import("./counts") @@ -66,48 +42,32 @@ describe("countChunksBySourceId", () => { countChunksBySourceId( [ makeSource({ id: "ready", knowhereDocumentId: "doc_ready" }), - makeSource({ id: "parsing", status: "parsing", knowhereDocumentId: null }), + makeSource({ + id: "parsing", + status: "parsing", + knowhereDocumentId: null, + }), makeSource({ id: "missing-doc", knowhereDocumentId: null }), ], mockClient, - { - repository, - readSnapshotManifest, - }, ), ) - expect(listChunks).not.toHaveBeenCalled() - expect(repository.getParseSnapshotMetadata).toHaveBeenCalledWith( - "workspace_1", - "ready", - ) - expect(readSnapshotManifest).toHaveBeenCalledWith({ - workspaceId: "workspace_1", - sourceId: "ready", - manifestKey: - "workspaces/workspace_1/sources/ready/parsed-result/manifest/current.json", + expect(listChunks).toHaveBeenCalledTimes(1) + expect(listChunks).toHaveBeenCalledWith("doc_ready", { + page: 1, + pageSize: 1, }) expect(counts).toEqual(new Map([["ready", 12]])) }) - it("skips a source count when snapshot manifest lookup fails", async () => { - const listChunks = vi.fn() + it("skips a source count when the document total lookup fails", async () => { + const listChunks = vi.fn(async () => { + throw new Error("temporary outage") + }) const mockClient = { documents: { listChunks }, } as unknown as Knowhere - const repository = { - getParseSnapshotMetadata: vi.fn(async () => ({ - resultBlobUrl: "https://blob.example/manifest/current.json", - snapshotManifestUrl: "https://blob.example/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/ready/parsed-result/manifest/current.json", - assetUrlsByFilePath: {}, - })), - } - const readSnapshotManifest = vi.fn(async () => { - throw new Error("temporary outage") - }) const { countChunksBySourceId } = await import("./counts") @@ -115,15 +75,11 @@ describe("countChunksBySourceId", () => { countChunksBySourceId( [makeSource({ id: "ready", knowhereDocumentId: "doc_ready" })], mockClient, - { - repository, - readSnapshotManifest, - }, ), ) expect(counts.size).toBe(0) - expect(listChunks).not.toHaveBeenCalled() + expect(listChunks).toHaveBeenCalledTimes(1) }) it("does not count materialized demo sources through their copied document id", async () => { diff --git a/src/domains/sources/counts.ts b/src/domains/sources/counts.ts index 8f934bb..e6e0c9b 100644 --- a/src/domains/sources/counts.ts +++ b/src/domains/sources/counts.ts @@ -4,29 +4,24 @@ import { Effect } from "effect" import type Knowhere from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" -import { - readParsedResultSnapshotManifest, - type ParsedResultSnapshotManifest, -} from "./parse-result-storage-adapter" -import { sourceWorkflowRuntime } from "./workflow-runtime" -type ChunkCountRepository = Pick< - typeof sourceWorkflowRuntime, - "getParseSnapshotMetadata" -> - -type ChunkCountOptions = { - readonly readSnapshotManifest?: typeof readParsedResultSnapshotManifest - readonly repository?: ChunkCountRepository +type CountChunksClient = { + readonly documents: { + listChunks( + documentId: string, + params: { readonly page: number; readonly pageSize: number }, + ): Promise<{ + readonly pagination?: { readonly total?: number } + }> + } } export const countChunksBySourceId = ( sources: readonly Source[], client: Knowhere, - options: ChunkCountOptions = {}, ) => Effect.gen(function* () { - void client + const countClient = client as unknown as CountChunksClient const readySources = sources.filter( (source) => !source.demoKey && @@ -38,19 +33,10 @@ export const countChunksBySourceId = ( const entries = yield* Effect.all( readySources.map((source) => Effect.gen(function* () { - const manifest = yield* Effect.tryPromise(() => - loadSourceSnapshotManifest(source, options), - ).pipe( - Effect.catchAll(() => - Effect.succeed(null), - ), - ) - if (!manifest) return [source.id, undefined] as const - - return [ - source.id, - manifest.totalChunks, - ] as const + const total = yield* Effect.tryPromise(() => + loadSourceChunkCount(countClient, source.knowhereDocumentId!), + ).pipe(Effect.catchAll(() => Effect.succeed(undefined))) + return [source.id, total] as const }), ), { concurrency: "unbounded" }, @@ -67,10 +53,9 @@ export const countChunksBySourceId = ( export const sourceViewOptionsBySourceId = ( sources: readonly Source[], client: Knowhere, - options: ChunkCountOptions = {}, ) => Effect.gen(function* () { - const counts = yield* countChunksBySourceId(sources, client, options) + const counts = yield* countChunksBySourceId(sources, client) return new Map( sources.map((source) => [ source.id, @@ -79,41 +64,14 @@ export const sourceViewOptionsBySourceId = ( ) }) -async function loadSourceSnapshotManifest( - source: Source, - options: ChunkCountOptions, -): Promise { - const repository = options.repository ?? sourceWorkflowRuntime - const readSnapshotManifest = - options.readSnapshotManifest ?? readParsedResultSnapshotManifest - const snapshot = await repository.getParseSnapshotMetadata( - source.workspaceId, - source.id, - ) - if (!isCompleteSnapshot(snapshot)) return null - - return readSnapshotManifest({ - workspaceId: source.workspaceId, - sourceId: source.id, - manifestKey: snapshot.snapshotManifestKey, +async function loadSourceChunkCount( + client: CountChunksClient, + documentId: string, +): Promise { + const response = await client.documents.listChunks(documentId, { + page: 1, + pageSize: 1, }) -} - -function isCompleteSnapshot( - snapshot: - | { - readonly snapshotManifestKey?: string | null - readonly snapshotManifestUrl?: string | null - } - | null, -): snapshot is { - readonly snapshotManifestKey: string - readonly snapshotManifestUrl: string -} { - return ( - typeof snapshot?.snapshotManifestKey === "string" && - snapshot.snapshotManifestKey.length > 0 && - typeof snapshot.snapshotManifestUrl === "string" && - snapshot.snapshotManifestUrl.length > 0 - ) + const total = response.pagination?.total + return typeof total === "number" && Number.isFinite(total) ? total : undefined } diff --git a/src/domains/sources/parse-result-storage-adapter.ts b/src/domains/sources/parse-result-storage-adapter.ts deleted file mode 100644 index d195b8f..0000000 --- a/src/domains/sources/parse-result-storage-adapter.ts +++ /dev/null @@ -1,269 +0,0 @@ -import "server-only" - -import { get, head, put, BlobNotFoundError } from "@vercel/blob" -import type { - KnowhereAssetStorageObject, - KnowhereAssetStorageOptions, -} from "@ontos-ai/knowhere-sdk" - -export type ParsedResultAssetStorageInput = { - readonly workspaceId: string - readonly sourceId: string -} - -export type ParsedResultAssetIndex = { - readonly workspaceId: string - readonly sourceId: string - readonly assetUrlsByFilePath: Readonly> - readonly updatedAt: string -} - -export type ParsedResultSnapshotChunk = { - readonly id: string - readonly chunkId: string - readonly chunkType: string - readonly contentSource?: string - readonly content: string - readonly sectionPath?: string - readonly sourceChunkPath: string - readonly filePath?: string - readonly sortOrder: number - readonly metadata: Record - readonly assetUrl?: string -} - -export type ParsedResultSnapshotChunkPage = { - readonly version: 1 - readonly jobId: string - readonly documentId?: string - readonly namespace?: string - readonly sourceFileName: string - readonly page: number - readonly pageSize: number - readonly total: number - readonly totalPages: number - readonly chunks: readonly ParsedResultSnapshotChunk[] -} - -export type ParsedResultSnapshotManifest = { - readonly version: 1 - readonly kind: "knowhere-parsed-result-snapshot" - readonly jobId: string - readonly documentId?: string - readonly namespace?: string - readonly sourceFileName: string - readonly totalChunks: number - readonly chunkPageSize: number - readonly chunkPages: readonly { - readonly page: number - readonly pageSize: number - readonly chunkCount: number - readonly key: string - readonly url?: string - }[] - readonly assetUrlsByFilePath: Readonly> - readonly createdAt: string -} - -const parsedResultDirectoryName = "parsed-result" -const parsedResultAssetIndexFileName = "asset-index.json" -const parsedResultSnapshotManifestPath = "manifest/current.json" - -export function createParsedResultStorageAdapter({ - workspaceId, - sourceId, -}: ParsedResultAssetStorageInput): KnowhereAssetStorageOptions { - return { - adapter: { - async headObject(key) { - try { - const blob = await head(key) - return { - key, - contentType: blob.contentType, - contentLength: blob.size, - url: blob.url, - } - } catch (error) { - if (error instanceof BlobNotFoundError) return null - throw error - } - }, - async getObjectUrl(key) { - try { - return (await head(key)).url - } catch (error) { - if (error instanceof BlobNotFoundError) return null - throw error - } - }, - async writeObject(input: KnowhereAssetStorageObject) { - const blob = await put(input.key, Buffer.from(input.body), { - access: "public", - addRandomSuffix: false, - allowOverwrite: true, - contentType: input.contentType, - multipart: true, - }) - - return { - key: blob.pathname, - url: blob.url, - } - }, - }, - keyPrefix: getParsedResultBlobPrefix(workspaceId, sourceId), - skipExisting: true, - } -} - -export function getParsedResultSnapshotManifestKey({ - workspaceId, - sourceId, -}: ParsedResultAssetStorageInput): string { - return `${getParsedResultBlobPrefix( - workspaceId, - sourceId, - )}/${parsedResultSnapshotManifestPath}` -} - -export async function readParsedResultSnapshotManifest(input: { - readonly workspaceId: string - readonly sourceId: string - readonly manifestKey?: string | null - readonly blobStore?: ParsedResultSnapshotBlobStore -}): Promise { - const key = - input.manifestKey ?? - getParsedResultSnapshotManifestKey({ - workspaceId: input.workspaceId, - sourceId: input.sourceId, - }) - const text = await readBlobText(key, input.blobStore ?? vercelSnapshotBlobStore) - if (!text) return null - return parseSnapshotManifest(text) -} - -export async function readParsedResultSnapshotChunkPage(input: { - readonly pageKey: string - readonly blobStore?: ParsedResultSnapshotBlobStore -}): Promise { - const text = await readBlobText(input.pageKey, input.blobStore ?? vercelSnapshotBlobStore) - if (!text) return null - return parseSnapshotChunkPage(text) -} - -export async function writeParsedResultAssetIndex(input: { - readonly workspaceId: string - readonly sourceId: string - readonly assetUrlsByFilePath: Readonly> -}): Promise { - const index: ParsedResultAssetIndex = { - workspaceId: input.workspaceId, - sourceId: input.sourceId, - assetUrlsByFilePath: input.assetUrlsByFilePath, - updatedAt: new Date().toISOString(), - } - const pathname = `${getParsedResultBlobPrefix( - input.workspaceId, - input.sourceId, - )}/${parsedResultAssetIndexFileName}` - const blob = await put(pathname, JSON.stringify(index, null, 2), { - access: "public", - addRandomSuffix: false, - allowOverwrite: true, - contentType: "application/json; charset=utf-8", - }) - - return blob.url -} - -function getParsedResultBlobPrefix(workspaceId: string, sourceId: string): string { - return `workspaces/${workspaceId}/sources/${sourceId}/${parsedResultDirectoryName}` -} - -type ParsedResultSnapshotBlobGetResult = - | { - readonly statusCode: 200 - readonly stream: ReadableStream - } - | { - readonly statusCode: 304 - readonly stream: null - } - -export type ParsedResultSnapshotBlobStore = { - readonly get: ( - pathname: string, - options: { readonly access: "public" }, - ) => Promise -} - -const vercelSnapshotBlobStore: ParsedResultSnapshotBlobStore = { - get: (pathname, options) => get(pathname, options), -} - -async function readBlobText( - key: string, - blobStore: ParsedResultSnapshotBlobStore, -): Promise { - try { - const result = await blobStore.get(key, { access: "public" }) - if (!result || result.statusCode !== 200) return null - return new Response(result.stream).text() - } catch (error) { - if (error instanceof BlobNotFoundError) return null - throw error - } -} - -function parseSnapshotManifest(text: string): ParsedResultSnapshotManifest | null { - try { - const value: unknown = JSON.parse(text) - return isSnapshotManifest(value) ? value : null - } catch { - return null - } -} - -function parseSnapshotChunkPage(text: string): ParsedResultSnapshotChunkPage | null { - try { - const value: unknown = JSON.parse(text) - return isSnapshotChunkPage(value) ? value : null - } catch { - return null - } -} - -function isSnapshotManifest(value: unknown): value is ParsedResultSnapshotManifest { - if (!isRecord(value)) return false - return ( - value["kind"] === "knowhere-parsed-result-snapshot" && - value["version"] === 1 && - typeof value["jobId"] === "string" && - typeof value["sourceFileName"] === "string" && - typeof value["totalChunks"] === "number" && - typeof value["chunkPageSize"] === "number" && - Array.isArray(value["chunkPages"]) && - isRecord(value["assetUrlsByFilePath"]) && - typeof value["createdAt"] === "string" - ) -} - -function isSnapshotChunkPage(value: unknown): value is ParsedResultSnapshotChunkPage { - if (!isRecord(value)) return false - return ( - value["version"] === 1 && - typeof value["jobId"] === "string" && - typeof value["sourceFileName"] === "string" && - typeof value["page"] === "number" && - typeof value["pageSize"] === "number" && - typeof value["total"] === "number" && - typeof value["totalPages"] === "number" && - Array.isArray(value["chunks"]) - ) -} - -function isRecord(value: unknown): value is Readonly> { - return typeof value === "object" && value !== null && !Array.isArray(value) -} diff --git a/src/domains/sources/parsed-document-blob-storage.test.ts b/src/domains/sources/parsed-document-blob-storage.test.ts new file mode 100644 index 0000000..8a3c427 --- /dev/null +++ b/src/domains/sources/parsed-document-blob-storage.test.ts @@ -0,0 +1,223 @@ +import { describe, expect, it } from "vitest" +import type { + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, + ParsedDocumentSyncProgress, +} from "@ontos-ai/knowhere-sdk" + +import { + BlobParsedDocumentStorage, + type ParsedDocumentBlobStore, +} from "./parsed-document-blob-storage" + +type StoredBlob = { + readonly body: string | Buffer + readonly contentType: string +} + +function createFakeBlobStore(): { + readonly store: ParsedDocumentBlobStore + readonly objects: Map +} { + const objects = new Map() + const toUrl = (pathname: string) => + `https://fake.public.blob.vercel-storage.com/${pathname}` + + const store: ParsedDocumentBlobStore = { + get: async (pathname) => { + const object = objects.get(pathname) + if (!object) return null + const body = + typeof object.body === "string" + ? object.body + : new Uint8Array(object.body) + return { + statusCode: 200, + stream: new Response(body).body as ReadableStream, + } + }, + put: async (pathname, body, options) => { + objects.set(pathname, { body, contentType: options.contentType }) + return { url: toUrl(pathname), pathname } + }, + head: async (pathname) => { + const object = objects.get(pathname) + return object ? { url: toUrl(pathname) } : null + }, + del: async (pathname) => { + objects.delete(pathname) + }, + } + + return { store, objects } +} + +const documentId = "doc_123" +const revisionKey = "job_result_1" + +const manifest: KnowhereParsedSnapshotManifest = { + version: 1, + kind: "knowhere-parsed-result-snapshot", + jobId: "job_1", + revisionKey, + documentId, + sourceFileName: "example.pdf", + totalChunks: 2, + chunkPageSize: 200, + chunkPages: [{ page: 1, pageSize: 200, chunkCount: 2, key: "chunks/page-1.json" }], + assetUrlsByFilePath: {}, + createdAt: "2026-07-04T00:00:00.000Z", +} + +const chunkPage: KnowhereParsedSnapshotChunkPage = { + version: 1, + jobId: "job_1", + revisionKey, + documentId, + sourceFileName: "example.pdf", + page: 1, + pageSize: 200, + total: 2, + totalPages: 1, + chunks: [ + { + id: "c1", + chunkId: "c1", + chunkType: "text", + content: "hello", + sourceChunkPath: "example.pdf", + sortOrder: 0, + metadata: {}, + }, + ], +} + +describe("BlobParsedDocumentStorage", () => { + it("round-trips a manifest keyed by workspace/document/revision", async () => { + const { store, objects } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + await storage.writeManifest({ documentId, revisionKey, manifest }) + + expect([...objects.keys()]).toContain( + "workspaces/ws_1/parsed-documents/doc_123/job_result_1/manifest/current.json", + ) + const read = await storage.readManifest({ documentId, revisionKey }) + expect(read).toEqual(manifest) + }) + + it("round-trips a chunk page", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + await storage.writeChunkPage({ documentId, revisionKey, page: chunkPage }) + const read = await storage.readChunkPage({ documentId, revisionKey, page: 1 }) + expect(read).toEqual(chunkPage) + }) + + it("round-trips sync progress", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + const progress: ParsedDocumentSyncProgress = { + documentId, + revisionKey, + nextChunkPage: 3, + nextAssetIndex: 0, + status: "running", + updatedAt: "2026-07-04T00:00:00.000Z", + } + + await storage.writeSyncProgress(progress) + const read = await storage.readSyncProgress({ documentId, revisionKey }) + expect(read).toEqual(progress) + }) + + it("writes an asset and resolves a durable URL", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + const written = await storage.writeAsset({ + documentId, + revisionKey, + sourcePath: "images/fig-1.png", + body: new Uint8Array([1, 2, 3]), + contentType: "image/png", + }) + expect(written.url).toContain( + "workspaces/ws_1/parsed-documents/doc_123/job_result_1/assets/images/fig-1.png", + ) + + const url = await storage.getAssetUrl({ + documentId, + revisionKey, + sourcePath: "images/fig-1.png", + }) + expect(url).toBe(written.url) + }) + + it("returns null for a missing manifest, chunk page, progress, and asset", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + expect(await storage.readManifest({ documentId, revisionKey })).toBeNull() + expect( + await storage.readChunkPage({ documentId, revisionKey, page: 9 }), + ).toBeNull() + expect(await storage.readSyncProgress({ documentId, revisionKey })).toBeNull() + expect( + await storage.getAssetUrl({ documentId, revisionKey, sourcePath: "x.png" }), + ).toBeNull() + }) + + it("isolates artifacts by revision key so a stale revision does not read", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + await storage.writeManifest({ documentId, revisionKey, manifest }) + const staleRead = await storage.readManifest({ + documentId, + revisionKey: "job_result_2", + }) + expect(staleRead).toBeNull() + }) + + it("rejects traversal in the document id, revision key, and asset path", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + await expect( + storage.readManifest({ documentId: "../escape", revisionKey }), + ).rejects.toThrow(/Invalid parsed storage segment/) + await expect( + storage.readManifest({ documentId, revisionKey: "a/b" }), + ).rejects.toThrow(/Invalid parsed storage segment/) + await expect( + storage.getAssetUrl({ + documentId, + revisionKey, + sourcePath: "../../etc/passwd", + }), + ).rejects.toThrow(/Invalid parsed storage path/) + }) +}) diff --git a/src/domains/sources/parsed-document-blob-storage.ts b/src/domains/sources/parsed-document-blob-storage.ts new file mode 100644 index 0000000..23a93a0 --- /dev/null +++ b/src/domains/sources/parsed-document-blob-storage.ts @@ -0,0 +1,291 @@ +import "server-only" + +import { del, get, head, put, BlobNotFoundError } from "@vercel/blob" +import type { + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, + ParsedDocumentStorage, + ParsedDocumentStorageAsset, + ParsedDocumentStorageAssetParams, + ParsedDocumentStorageChunkPageParams, + ParsedDocumentStorageDocument, + ParsedDocumentStorageManifestParams, + ParsedDocumentSyncProgress, +} from "@ontos-ai/knowhere-sdk" + +/** + * Vercel Blob backed implementation of the SDK `ParsedDocumentStorage` + * interface. The SDK only ever calls this with `{ documentId, revisionKey }`, + * so the adapter is constructed with the enclosing workspace and derives all + * blob keys from documentId + revisionKey. Keying every artifact under the + * revision key gives the SDK freshness contract for free: a manifest read at a + * revision path always carries a matching `revisionKey`. + * + * The `blobStore` seam mirrors the pattern used elsewhere in the sources + * domain so tests can inject a fake store instead of hitting Vercel Blob. + */ + +const parsedDocumentsDirectoryName = "parsed-documents" +const manifestStoragePath = "manifest/current.json" +const syncProgressStoragePath = "sync-progress.json" +const jsonContentType = "application/json; charset=utf-8" + +type BlobGetResult = + | { + readonly statusCode: 200 + readonly stream: ReadableStream + } + | { + readonly statusCode: 304 + readonly stream: null + } + +type BlobPutResult = { + readonly url: string + readonly pathname: string +} + +type BlobHeadResult = { + readonly url: string +} + +/** + * Minimal blob operations the adapter depends on. The default binds directly to + * `@vercel/blob`; tests provide an in-memory stand-in. + */ +export type ParsedDocumentBlobStore = { + readonly get: ( + pathname: string, + options: { readonly access: "public" }, + ) => Promise + readonly put: ( + pathname: string, + body: string | Buffer, + options: { + readonly access: "public" + readonly allowOverwrite: boolean + readonly contentType: string + readonly multipart?: boolean + }, + ) => Promise + readonly head: (pathname: string) => Promise + readonly del: (pathname: string) => Promise +} + +export type BlobParsedDocumentStorageInput = { + readonly workspaceId: string + readonly blobStore?: ParsedDocumentBlobStore +} + +const vercelBlobStore: ParsedDocumentBlobStore = { + get: (pathname, options) => get(pathname, options), + put: async (pathname, body, options) => { + const blob = await put(pathname, body, { + access: options.access, + addRandomSuffix: false, + allowOverwrite: options.allowOverwrite, + contentType: options.contentType, + multipart: options.multipart, + }) + return { url: blob.url, pathname: blob.pathname } + }, + head: async (pathname) => { + try { + const blob = await head(pathname) + return { url: blob.url } + } catch (error) { + if (error instanceof BlobNotFoundError) return null + throw error + } + }, + del: (pathname) => del(pathname), +} + +export class BlobParsedDocumentStorage implements ParsedDocumentStorage { + private readonly workspaceId: string + private readonly blobStore: ParsedDocumentBlobStore + + constructor(input: BlobParsedDocumentStorageInput) { + this.workspaceId = input.workspaceId + this.blobStore = input.blobStore ?? vercelBlobStore + } + + async readManifest( + params: ParsedDocumentStorageManifestParams, + ): Promise { + return this.readJson( + this.getManifestKey(params.documentId, params.revisionKey), + ) + } + + async writeManifest(params: { + readonly documentId: string + readonly revisionKey: string + readonly manifest: KnowhereParsedSnapshotManifest + }): Promise { + await this.writeJson( + this.getManifestKey(params.documentId, params.revisionKey), + params.manifest, + ) + } + + async readChunkPage( + params: ParsedDocumentStorageChunkPageParams, + ): Promise { + // chunkType filtering happens SDK-side after read; storage returns the full page. + return this.readJson( + this.getChunkPageKey(params.documentId, params.revisionKey, params.page), + ) + } + + async writeChunkPage(params: { + readonly documentId: string + readonly revisionKey: string + readonly page: KnowhereParsedSnapshotChunkPage + }): Promise { + await this.writeJson( + this.getChunkPageKey( + params.documentId, + params.revisionKey, + params.page.page, + ), + params.page, + ) + } + + async writeAsset( + params: ParsedDocumentStorageDocument & ParsedDocumentStorageAsset, + ): Promise<{ readonly sourcePath: string; readonly url?: string }> { + const blob = await this.blobStore.put( + this.getAssetKey(params.documentId, params.revisionKey, params.sourcePath), + Buffer.from(params.body), + { + access: "public", + allowOverwrite: true, + contentType: params.contentType, + multipart: true, + }, + ) + return { sourcePath: params.sourcePath, url: blob.url } + } + + async getAssetUrl( + params: ParsedDocumentStorageAssetParams, + ): Promise { + const result = await this.blobStore.head( + this.getAssetKey(params.documentId, params.revisionKey, params.sourcePath), + ) + return result?.url ?? null + } + + async readSyncProgress( + params: ParsedDocumentStorageDocument, + ): Promise { + return this.readJson( + this.getSyncProgressKey(params.documentId, params.revisionKey), + ) + } + + async writeSyncProgress(params: ParsedDocumentSyncProgress): Promise { + await this.writeJson( + this.getSyncProgressKey(params.documentId, params.revisionKey), + params, + ) + } + + private getRevisionPrefix(documentId: string, revisionKey: string): string { + return [ + "workspaces", + normalizePathSegment(this.workspaceId), + parsedDocumentsDirectoryName, + normalizePathSegment(documentId), + normalizePathSegment(revisionKey), + ].join("/") + } + + private getManifestKey(documentId: string, revisionKey: string): string { + return `${this.getRevisionPrefix(documentId, revisionKey)}/${manifestStoragePath}` + } + + private getChunkPageKey( + documentId: string, + revisionKey: string, + page: number, + ): string { + return `${this.getRevisionPrefix(documentId, revisionKey)}/chunks/page-${page}.json` + } + + private getSyncProgressKey(documentId: string, revisionKey: string): string { + return `${this.getRevisionPrefix(documentId, revisionKey)}/${syncProgressStoragePath}` + } + + private getAssetKey( + documentId: string, + revisionKey: string, + sourcePath: string, + ): string { + return `${this.getRevisionPrefix(documentId, revisionKey)}/assets/${normalizeRelativeStoragePath(sourcePath)}` + } + + private async readJson(key: string): Promise { + const text = await this.readBlobText(key) + if (text === null) return null + try { + return JSON.parse(text) as T + } catch { + return null + } + } + + private async writeJson(key: string, value: unknown): Promise { + await this.blobStore.put(key, JSON.stringify(value), { + access: "public", + allowOverwrite: true, + contentType: jsonContentType, + }) + } + + private async readBlobText(key: string): Promise { + try { + const result = await this.blobStore.get(key, { access: "public" }) + if (!result || result.statusCode !== 200) return null + return await new Response(result.stream).text() + } catch (error) { + if (error instanceof BlobNotFoundError) return null + throw error + } + } +} + +/** + * Reject path segments that could traverse outside the intended prefix. Mirrors + * the SDK `DiskParsedDocumentStorage` guard so blob keys stay well-formed. + */ +function normalizePathSegment(value: string): string { + if ( + value.length === 0 || + value.includes("\0") || + value.includes("/") || + value.includes("\\") || + value === "." || + value === ".." + ) { + throw new Error(`Invalid parsed storage segment: ${value}`) + } + return value +} + +function normalizeRelativeStoragePath(value: string): string { + const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "") + if ( + normalized.length === 0 || + normalized.includes("\0") || + normalized.startsWith("/") || + normalized + .split("/") + .some((part) => part.length === 0 || part === "." || part === "..") + ) { + throw new Error(`Invalid parsed storage path: ${value}`) + } + return normalized +} diff --git a/src/domains/sources/parsed-document-sync-scheduler.ts b/src/domains/sources/parsed-document-sync-scheduler.ts new file mode 100644 index 0000000..5d2957d --- /dev/null +++ b/src/domains/sources/parsed-document-sync-scheduler.ts @@ -0,0 +1,154 @@ +import "server-only" + +import { Client } from "@upstash/workflow" +import type { ParsedDocumentSyncScheduler } from "@ontos-ai/knowhere-sdk" + +import { logger } from "@/lib/logger" + +/** + * A `ParsedDocumentSyncScheduler` whose `schedule` enqueues a durable QStash + * continuation instead of running the (non-serializable) task closure inline. + * The SDK schedules a background sync on every storage read-miss; the closure + * captures a per-request client we cannot serialize across serverless + * invocations, so we discard it and trigger `/api/sources/parsed-sync`, which + * rebuilds a parsed-storage client and loops `syncParsedDocument` to completion. + * + * The scheduler is pre-bound to a single `{ workspaceId, sourceId, documentId }` + * because the SDK invokes `schedule(task)` with no arguments. `revisionKey` is + * intentionally NOT part of the bound identity — the sync route resolves the + * current revision itself so a scheduler bound before the revision is known + * still enqueues correctly. + */ + +export type ParsedSyncTrigger = (input: { + readonly url: string + readonly body: ParsedSyncPayload + readonly workflowRunId: string +}) => Promise + +export type ParsedSyncPayload = { + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly apiKey: string + readonly revisionKey?: string + readonly segmentIndex?: number +} + +export type CreateParsedDocumentSyncSchedulerInput = { + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly apiKey: string + readonly revisionKey?: string + readonly trigger?: ParsedSyncTrigger +} + +// Bounded per-document guard: the SDK may schedule on every read-miss within a +// single request. The cooldown collapses duplicate same-process enqueues +// without permanently blocking future syncs. +const triggerCooldownMs: number = 60_000 +const lastTriggeredAtByKey: Map = new Map() + +function resolveBaseURL(): string { + return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" +} + +export function getParsedSyncWorkflowRunId(input: { + readonly documentId: string + readonly revisionKey: string + readonly segmentIndex: number +}): string { + return `${input.documentId}-sync-${input.revisionKey}-${input.segmentIndex}` +} + +const defaultTrigger: ParsedSyncTrigger = async (input) => { + const token = process.env.QSTASH_TOKEN + if (!token) { + throw new Error("QSTASH_TOKEN is required to schedule parsed document sync.") + } + await new Client({ token }).trigger({ + url: input.url, + body: input.body, + workflowRunId: input.workflowRunId, + retries: 3, + }) +} + +export function getParsedSyncUrl(): string { + return `${resolveBaseURL()}/api/sources/parsed-sync` +} + +/** + * Enqueue a durable parsed-document sync from outside the SDK read path — used + * by the reconcile workflow to hand off to the resumable parsed-sync route, and + * by retry to resume a failed storage sync. Starts at segment 0. + */ +export async function enqueueParsedDocumentSync( + input: { + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly apiKey: string + readonly revisionKey?: string + }, + trigger: ParsedSyncTrigger = defaultTrigger, +): Promise { + await trigger({ + url: getParsedSyncUrl(), + body: { ...input, segmentIndex: 0 }, + workflowRunId: getParsedSyncWorkflowRunId({ + documentId: input.documentId, + revisionKey: input.revisionKey ?? "initial", + segmentIndex: 0, + }), + }) +} + +export function createParsedDocumentSyncScheduler( + input: CreateParsedDocumentSyncSchedulerInput, +): ParsedDocumentSyncScheduler { + const trigger = input.trigger ?? defaultTrigger + const cooldownKey = `${input.documentId}|${input.revisionKey ?? "unknown"}` + + return { + schedule: () => { + const now = Date.now() + const lastTriggeredAt = lastTriggeredAtByKey.get(cooldownKey) + if ( + lastTriggeredAt !== undefined && + now - lastTriggeredAt < triggerCooldownMs + ) { + return + } + lastTriggeredAtByKey.set(cooldownKey, now) + + const url = `${resolveBaseURL()}/api/sources/parsed-sync` + const workflowRunId = getParsedSyncWorkflowRunId({ + documentId: input.documentId, + revisionKey: input.revisionKey ?? "initial", + segmentIndex: 0, + }) + void trigger({ + url, + body: { + workspaceId: input.workspaceId, + sourceId: input.sourceId, + documentId: input.documentId, + apiKey: input.apiKey, + revisionKey: input.revisionKey, + segmentIndex: 0, + }, + workflowRunId, + }).catch((error: unknown) => { + lastTriggeredAtByKey.delete(cooldownKey) + logger.error("parsed-sync-scheduler: failed to enqueue sync", { + workspaceId: input.workspaceId, + sourceId: input.sourceId, + documentId: input.documentId, + error: error instanceof Error ? error.message : String(error), + }) + }) + }, + } +} diff --git a/src/domains/sources/parsed-sync-route-workflow.test.ts b/src/domains/sources/parsed-sync-route-workflow.test.ts new file mode 100644 index 0000000..555c186 --- /dev/null +++ b/src/domains/sources/parsed-sync-route-workflow.test.ts @@ -0,0 +1,201 @@ +import { afterEach, describe, expect, it, vi } from "vitest" + +const mocks = vi.hoisted(() => ({ + makeKnowhereClientWithParsedStorage: vi.fn(), + updateSyncStatus: vi.fn(), + findInWorkspace: vi.fn(), + markFailed: vi.fn(), + markReady: vi.fn(), + markSourceReadyAfterReconciliation: vi.fn(), + loggerInfo: vi.fn(), + loggerError: vi.fn(), +})) + +vi.mock("@/integrations/knowhere", () => ({ + makeKnowhereClientWithParsedStorage: mocks.makeKnowhereClientWithParsedStorage, +})) + +vi.mock("./workflow-runtime", () => ({ + sourceWorkflowRuntime: { + updateSyncStatus: mocks.updateSyncStatus, + findInWorkspace: mocks.findInWorkspace, + markFailed: mocks.markFailed, + markReady: mocks.markReady, + }, +})) + +vi.mock("./source-reconcile-workflow", () => ({ + markSourceReadyAfterReconciliation: mocks.markSourceReadyAfterReconciliation, +})) + +vi.mock("@/lib/logger", () => ({ + logger: { info: mocks.loggerInfo, error: mocks.loggerError }, +})) + +import { parsedSyncRouteWorkflow } from "./parsed-sync-route-workflow" + +type RunStep = (id: string, task: () => Promise | T) => Promise + +function createContext(overrides: { url?: string } = {}) { + const run: RunStep = async (_id, task) => task() + return { + run, + url: overrides.url ?? "https://notebook.example/api/sources/parsed-sync", + } +} + +const basePayload = { + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + apiKey: "key_1", + segmentIndex: 0, +} + +describe("parsedSyncRouteWorkflow.runParsedSyncWorkflow", () => { + afterEach(() => { + vi.clearAllMocks() + }) + + it("marks the source ready when sync completes in one segment", async () => { + const syncParsedDocument = vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_1", + completed: true, + })) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: {}, + knowledge: { syncParsedDocument }, + }) + mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ + status: "ready", + }) + + await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ + context: createContext(), + payload: basePayload, + }) + + expect(syncParsedDocument).toHaveBeenCalledTimes(1) + expect(syncParsedDocument).toHaveBeenCalledWith({ documentId: "doc_1" }) + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { revisionKey: "rev_1", syncStatus: "completed" }, + ) + expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + }) + }) + + it("triggers a continuation and does not mark ready when sync is incomplete", async () => { + const syncParsedDocument = vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_1", + completed: false, + })) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: {}, + knowledge: { syncParsedDocument }, + }) + const triggered: Array<{ workflowRunId: string; segmentIndex?: number }> = [] + const restore = parsedSyncRouteWorkflow.setContinuationTriggerForTesting( + async (input) => { + triggered.push({ + workflowRunId: input.workflowRunId, + segmentIndex: input.payload.segmentIndex, + }) + }, + ) + + try { + await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ + context: createContext(), + payload: basePayload, + }) + } finally { + restore() + } + + expect(mocks.markSourceReadyAfterReconciliation).not.toHaveBeenCalled() + expect(triggered).toHaveLength(1) + expect(triggered[0]?.segmentIndex).toBe(1) + expect(triggered[0]?.workflowRunId).toBe("doc_1-sync-rev_1-1") + expect(mocks.updateSyncStatus).toHaveBeenLastCalledWith( + "workspace_1", + "source_1", + { revisionKey: "rev_1", syncStatus: "running" }, + ) + }) + + it("passes an explicit revisionKey into syncParsedDocument when provided", async () => { + const syncParsedDocument = vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_9", + completed: true, + })) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: {}, + knowledge: { syncParsedDocument }, + }) + mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ + status: "ready", + }) + + await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ + context: createContext(), + payload: { ...basePayload, revisionKey: "rev_9" }, + }) + + expect(syncParsedDocument).toHaveBeenCalledWith({ + documentId: "doc_1", + revisionKey: "rev_9", + }) + }) +}) + +describe("parsedSyncRouteWorkflow.markSyncFailedAfterWorkflowFailure", () => { + afterEach(() => { + vi.clearAllMocks() + }) + + it("fails a parsing source with failure_stage storage_sync", async () => { + mocks.findInWorkspace.mockResolvedValue({ id: "source_1", status: "parsing" }) + + await parsedSyncRouteWorkflow.markSyncFailedAfterWorkflowFailure( + { ...basePayload, revisionKey: "rev_1" }, + "boom", + ) + + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { revisionKey: "rev_1", syncStatus: "failed", syncError: "boom" }, + ) + expect(mocks.markFailed).toHaveBeenCalledWith( + "workspace_1", + "source_1", + expect.stringContaining("storage sync failed"), + "parsing", + "storage_sync", + ) + }) + + it("does not fail an already-ready source, only records sync_status", async () => { + mocks.findInWorkspace.mockResolvedValue({ id: "source_1", status: "ready" }) + + await parsedSyncRouteWorkflow.markSyncFailedAfterWorkflowFailure( + basePayload, + "boom", + ) + + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { revisionKey: undefined, syncStatus: "failed", syncError: "boom" }, + ) + expect(mocks.markFailed).not.toHaveBeenCalled() + }) +}) diff --git a/src/domains/sources/parsed-sync-route-workflow.ts b/src/domains/sources/parsed-sync-route-workflow.ts new file mode 100644 index 0000000..c33ace8 --- /dev/null +++ b/src/domains/sources/parsed-sync-route-workflow.ts @@ -0,0 +1,226 @@ +import "server-only" + +import { Client, type WorkflowContext } from "@upstash/workflow" +import type { KnowledgeSyncParsedDocumentResponse } from "@ontos-ai/knowhere-sdk" + +import { makeKnowhereClientWithParsedStorage } from "@/integrations/knowhere" +import { logger } from "@/lib/logger" +import { + getParsedSyncWorkflowRunId, + type ParsedSyncPayload, +} from "./parsed-document-sync-scheduler" +import { markSourceReadyAfterReconciliation } from "./source-reconcile-workflow" +import { sourceWorkflowRuntime } from "./workflow-runtime" + +type ParsedSyncWorkflowContext = Pick< + WorkflowContext, + "run" | "url" +> + +type NormalizedParsedSyncPayload = { + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly apiKey: string + readonly revisionKey?: string + readonly segmentIndex: number +} + +type ContinuationTriggerInput = { + readonly url: string + readonly payload: ParsedSyncPayload + readonly workflowRunId: string +} + +// Sync steps per workflow segment. Each `syncParsedDocument` call is bounded by +// the SDK limits (pages + deadline); this caps how many bounded steps we run in +// one serverless invocation before handing off to a fresh continuation. +const maxSyncStepsPerSegment = 4 + +let triggerContinuation: typeof triggerParsedSyncContinuation = + triggerParsedSyncContinuation + +function normalizeParsedSyncPayload( + payload: ParsedSyncPayload, +): NormalizedParsedSyncPayload { + return { + workspaceId: payload.workspaceId, + sourceId: payload.sourceId, + documentId: payload.documentId, + apiKey: payload.apiKey, + revisionKey: payload.revisionKey, + segmentIndex: + typeof payload.segmentIndex === "number" && + Number.isInteger(payload.segmentIndex) && + payload.segmentIndex >= 0 + ? payload.segmentIndex + : 0, + } +} + +async function runParsedSyncWorkflow(input: { + readonly context: ParsedSyncWorkflowContext + readonly payload: NormalizedParsedSyncPayload +}): Promise { + const { context, payload } = input + const { workspaceId, sourceId, documentId, apiKey } = payload + const { knowledge } = makeKnowhereClientWithParsedStorage(apiKey, { + workspaceId, + }) + + let revisionKey = payload.revisionKey + let completed = false + + for (let step = 0; step < maxSyncStepsPerSegment; step++) { + const result: KnowledgeSyncParsedDocumentResponse = await context.run( + `sync-${payload.segmentIndex}-${step}`, + async () => + knowledge.syncParsedDocument({ + documentId, + ...(revisionKey ? { revisionKey } : {}), + }), + ) + revisionKey = result.revisionKey + + await context.run(`record-progress-${payload.segmentIndex}-${step}`, () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: result.completed ? "completed" : "running", + }), + ) + + if (result.completed) { + completed = true + break + } + } + + if (!completed) { + const nextSegmentIndex = payload.segmentIndex + 1 + await context.run( + `trigger-sync-continuation-${nextSegmentIndex}`, + async () => + triggerContinuation({ + url: context.url, + payload: { + workspaceId, + sourceId, + documentId, + apiKey, + revisionKey, + segmentIndex: nextSegmentIndex, + }, + workflowRunId: getParsedSyncWorkflowRunId({ + documentId, + revisionKey: revisionKey ?? "initial", + segmentIndex: nextSegmentIndex, + }), + }), + ) + logger.info("parsed-sync: continuation triggered", { + sourceId, + documentId, + segmentIndex: nextSegmentIndex, + }) + return + } + + // A parsing source is only readied once its parsed snapshot is fully synced. + // For an already-ready source (read-miss backfill) markReady is a no-op guard. + const ready = await context.run("source-ready", async () => + markSourceReadyAfterReconciliation({ + workspaceId, + sourceId, + documentId, + }), + ) + logger.info("parsed-sync: parsed document sync finished", { + sourceId, + documentId, + revisionKey, + status: ready.status, + }) +} + +async function triggerParsedSyncContinuation( + input: ContinuationTriggerInput, +): Promise { + const token = process.env.QSTASH_TOKEN + if (!token) { + throw new Error("QSTASH_TOKEN is required to continue parsed document sync.") + } + + await new Client({ token }).trigger({ + url: input.url, + body: input.payload, + workflowRunId: input.workflowRunId, + retries: 3, + }) +} + +async function markSyncFailedAfterWorkflowFailure( + payload: ParsedSyncPayload, + failResponse: string, +): Promise { + const normalized = normalizeParsedSyncPayload(payload) + const reason = getSafeFailureReason(failResponse) + + // Record the storage-sync failure. A source still `parsing` is failed with + // failure_stage=storage_sync so a retry resumes sync without reparsing; an + // already-ready source is left ready (it still serves via remote fallback), + // only its sync_status is marked failed for observability. + await sourceWorkflowRuntime.updateSyncStatus( + normalized.workspaceId, + normalized.sourceId, + { + revisionKey: normalized.revisionKey, + syncStatus: "failed", + syncError: reason, + }, + ) + + const source = await sourceWorkflowRuntime.findInWorkspace( + normalized.workspaceId, + normalized.sourceId, + ) + if (source?.status === "parsing") { + await sourceWorkflowRuntime.markFailed( + normalized.workspaceId, + normalized.sourceId, + `Parsed document storage sync failed: ${reason}`, + "parsing", + "storage_sync", + ) + } + + logger.error("parsed-sync: marked sync failed after workflow failure", { + workspaceId: normalized.workspaceId, + sourceId: normalized.sourceId, + documentId: normalized.documentId, + segmentIndex: normalized.segmentIndex, + sourceStatus: source?.status, + }) +} + +function getSafeFailureReason(value: string): string { + const normalized = value.replace(/\s+/g, " ").trim() + if (normalized.length === 0) return "retry attempts were exhausted." + return normalized.slice(0, 500) +} + +function setContinuationTriggerForTesting( + trigger: typeof triggerParsedSyncContinuation, +): () => void { + const previous = triggerContinuation + triggerContinuation = trigger + return () => { + triggerContinuation = previous + } +} + +export const parsedSyncRouteWorkflow = { + markSyncFailedAfterWorkflowFailure, + normalizeParsedSyncPayload, + runParsedSyncWorkflow, + setContinuationTriggerForTesting, +} diff --git a/src/domains/sources/reconcile.test.ts b/src/domains/sources/reconcile.test.ts index 4c90172..5be0cea 100644 --- a/src/domains/sources/reconcile.test.ts +++ b/src/domains/sources/reconcile.test.ts @@ -20,6 +20,7 @@ function makeSource(overrides: Partial): Source { sizeBytes: 1, status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: null, stagedBlobPathname: null, diff --git a/src/domains/sources/remote-parsed-snapshot-sync.test.ts b/src/domains/sources/remote-parsed-snapshot-sync.test.ts deleted file mode 100644 index 6a91582..0000000 --- a/src/domains/sources/remote-parsed-snapshot-sync.test.ts +++ /dev/null @@ -1,316 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest" - -const mocks = vi.hoisted(() => ({ - createParsedResultStorageAdapter: vi.fn(), - loggerInfo: vi.fn(), -})) - -vi.mock("./parse-result-storage-adapter", () => ({ - createParsedResultStorageAdapter: mocks.createParsedResultStorageAdapter, -})) - -vi.mock("@/lib/logger", () => ({ - logger: { - info: mocks.loggerInfo, - }, -})) - -import type { Source, SourceParseResult } from "@/infrastructure/db/schema" -import { syncRemoteParsedSnapshot } from "./remote-parsed-snapshot-sync" - -describe("syncRemoteParsedSnapshot", () => { - afterEach(() => { - vi.clearAllMocks() - }) - - it("loads an already-parsed remote document into the Notebook Blob snapshot", async () => { - const storageAdapter = { - adapter: { - writeObject: vi.fn(), - }, - keyPrefix: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result", - } - const listChunks = vi.fn(async () => ({ - jobResultId: "job_result_1", - })) - const loadJobResult = vi.fn(async () => ({ - assetUrlsByFilePath: { - "pages/page-1.png": "https://blob.example/pages/page-1.png", - }, - parsedSnapshot: { - manifestKey: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - manifestUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - }, - })) - const repository = createRepository() - mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) - - const snapshot = await syncRemoteParsedSnapshot({ - workspaceId: "workspace_1", - source: makeSource({ - knowhereJobId: null, - }), - client: { - documents: { - listChunks, - }, - knowledge: { - loadJobResult, - }, - }, - repository, - }) - - expect(listChunks).toHaveBeenCalledWith("doc_remote", { - page: 1, - pageSize: 1, - includeAssetUrls: false, - }) - expect(mocks.createParsedResultStorageAdapter).toHaveBeenCalledWith({ - workspaceId: "workspace_1", - sourceId: "00000000-0000-0000-0000-000000000009", - }) - expect(loadJobResult).toHaveBeenCalledWith({ - jobId: "job_result_1", - storageAdapter, - }) - expect(repository.markParsing).toHaveBeenCalledWith( - "workspace_1", - "00000000-0000-0000-0000-000000000009", - "job_result_1", - "doc_remote", - ) - expect(repository.saveParseResult).toHaveBeenCalledWith( - "workspace_1", - "00000000-0000-0000-0000-000000000009", - { - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - assetUrlsByFilePath: { - "pages/page-1.png": "https://blob.example/pages/page-1.png", - }, - }, - ) - expect(repository.markReady).toHaveBeenCalledWith( - "workspace_1", - "00000000-0000-0000-0000-000000000009", - "doc_remote", - ) - expect(snapshot).toEqual({ - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - assetUrlsByFilePath: { - "pages/page-1.png": "https://blob.example/pages/page-1.png", - }, - }) - }) - - it("returns the existing complete snapshot without calling Knowhere", async () => { - const existingSnapshot = { - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - assetUrlsByFilePath: {}, - } - const repository = createRepository({ - getParseSnapshotMetadata: vi.fn(async () => existingSnapshot), - }) - const listChunks = vi.fn() - const loadJobResult = vi.fn() - - const snapshot = await syncRemoteParsedSnapshot({ - workspaceId: "workspace_1", - source: makeSource(), - client: { - documents: { - listChunks, - }, - knowledge: { - loadJobResult, - }, - }, - repository, - }) - - expect(snapshot).toEqual(existingSnapshot) - expect(listChunks).not.toHaveBeenCalled() - expect(loadJobResult).not.toHaveBeenCalled() - expect(repository.markParsing).not.toHaveBeenCalled() - expect(repository.markReady).not.toHaveBeenCalled() - }) - - it("marks a parsing source ready when a complete snapshot already exists", async () => { - const existingSnapshot = { - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - assetUrlsByFilePath: {}, - } - const repository = createRepository({ - getParseSnapshotMetadata: vi.fn(async () => existingSnapshot), - }) - const listChunks = vi.fn() - const loadJobResult = vi.fn() - - const snapshot = await syncRemoteParsedSnapshot({ - workspaceId: "workspace_1", - source: makeSource({ - status: "parsing", - knowhereDocumentId: "doc_remote", - }), - client: { - documents: { - listChunks, - }, - knowledge: { - loadJobResult, - }, - }, - repository, - }) - - expect(snapshot).toEqual(existingSnapshot) - expect(listChunks).not.toHaveBeenCalled() - expect(loadJobResult).not.toHaveBeenCalled() - expect(repository.markReady).toHaveBeenCalledWith( - "workspace_1", - "00000000-0000-0000-0000-000000000009", - "doc_remote", - ) - }) - - it("retries an interrupted remote snapshot sync from a parsing source row", async () => { - const storageAdapter = { - adapter: { - writeObject: vi.fn(), - }, - keyPrefix: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result", - } - const listChunks = vi.fn() - const loadJobResult = vi.fn(async () => ({ - assetUrlsByFilePath: {}, - parsedSnapshot: { - manifestKey: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - manifestUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - }, - })) - const repository = createRepository() - mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) - - await syncRemoteParsedSnapshot({ - workspaceId: "workspace_1", - source: makeSource({ - status: "parsing", - knowhereJobId: "job_result_1", - }), - client: { - documents: { - listChunks, - }, - knowledge: { - loadJobResult, - }, - }, - repository, - }) - - expect(listChunks).not.toHaveBeenCalled() - expect(loadJobResult).toHaveBeenCalledWith({ - jobId: "job_result_1", - storageAdapter, - }) - expect(repository.markReady).toHaveBeenCalledWith( - "workspace_1", - "00000000-0000-0000-0000-000000000009", - "doc_remote", - ) - }) -}) - -type SyncRemoteParsedSnapshotInput = Parameters< - typeof syncRemoteParsedSnapshot ->[0] - -type TestRemoteParsedSnapshotRepository = NonNullable< - SyncRemoteParsedSnapshotInput["repository"] -> - -function createRepository( - overrides: Partial = {}, -): TestRemoteParsedSnapshotRepository { - return { - ...createRepositoryShape(), - ...overrides, - } -} - -function createRepositoryShape(): TestRemoteParsedSnapshotRepository { - return { - getParseSnapshotMetadata: vi.fn(async () => null), - markParsing: vi.fn(async () => makeSource({ status: "parsing" })), - markReady: vi.fn(async () => makeSource()), - saveParseResult: vi.fn(async () => makeSourceParseResult()), - } -} - -function makeSource(overrides: Partial = {}): Source { - return { - id: "00000000-0000-0000-0000-000000000009", - workspaceId: "workspace_1", - title: "remote.pdf", - mimeType: "application/pdf", - sizeBytes: 0, - status: "ready", - failureReason: null, - knowhereJobId: "job_result_1", - knowhereDocumentId: "doc_remote", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-10T00:00:00.000Z"), - updatedAt: new Date("2026-05-10T00:00:00.000Z"), - deletedAt: null, - ...overrides, - } -} - -function makeSourceParseResult( - overrides: Partial = {}, -): SourceParseResult { - return { - id: "parse_result_1", - sourceId: "00000000-0000-0000-0000-000000000009", - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/00000000-0000-0000-0000-000000000009/parsed-result/manifest/current.json", - assetUrls: {}, - createdAt: new Date("2026-05-10T00:00:00.000Z"), - updatedAt: new Date("2026-05-10T00:00:00.000Z"), - ...overrides, - } -} diff --git a/src/domains/sources/remote-parsed-snapshot-sync.ts b/src/domains/sources/remote-parsed-snapshot-sync.ts deleted file mode 100644 index 3e939c4..0000000 --- a/src/domains/sources/remote-parsed-snapshot-sync.ts +++ /dev/null @@ -1,210 +0,0 @@ -import "server-only" - -import { Effect } from "effect" - -import type { Source } from "@/infrastructure/db/schema" -import { logger } from "@/lib/logger" -import { createParsedResultStorageAdapter } from "./parse-result-storage-adapter" -import { sourceWorkflowRuntime } from "./workflow-runtime" - -type RemoteParsedSnapshotSource = Pick< - Source, - "id" | "workspaceId" | "knowhereDocumentId" | "knowhereJobId" | "status" -> - -type RemoteParsedSnapshotClient = { - readonly documents: { - listChunks( - documentId: string, - params: { - readonly page: number - readonly pageSize: number - readonly includeAssetUrls: boolean - }, - ): Promise<{ - readonly jobId?: string | null - readonly jobResultId?: string | null - }> - } - readonly knowledge: { - loadJobResult(params: { - readonly jobId: string - readonly storageAdapter: ReturnType - }): Promise - } -} - -type RemoteParsedSnapshotResponse = { - readonly assetUrlsByFilePath?: Readonly> - readonly parsedSnapshot?: { - readonly manifestKey: string - readonly manifestUrl?: string - } -} - -type RemoteParsedSnapshotRepository = { - readonly getParseSnapshotMetadata: typeof sourceWorkflowRuntime.getParseSnapshotMetadata - readonly markParsing: typeof sourceWorkflowRuntime.markParsing - readonly markReady: typeof sourceWorkflowRuntime.markReady - readonly saveParseResult: typeof sourceWorkflowRuntime.saveParseResult -} - -type SyncRemoteParsedSnapshotInput = { - readonly workspaceId: string - readonly source: RemoteParsedSnapshotSource - readonly client: RemoteParsedSnapshotClient - readonly repository?: RemoteParsedSnapshotRepository -} - -type RemoteParsedSnapshotReadModel = { - readonly resultBlobUrl: string - readonly snapshotManifestUrl?: string | null - readonly snapshotManifestKey?: string | null - readonly assetUrlsByFilePath: Readonly> -} - -const jobProbePage = 1 -const jobProbePageSize = 1 - -const syncRemoteParsedSnapshotEffect = Effect.fn( - "syncRemoteParsedSnapshot", -)(function* ({ - workspaceId, - source, - client, - repository = sourceWorkflowRuntime, -}: SyncRemoteParsedSnapshotInput) { - const existingSnapshot = yield* Effect.tryPromise(() => - repository.getParseSnapshotMetadata(workspaceId, source.id), - ) - if (isCompleteSnapshot(existingSnapshot)) { - if (source.status === "parsing" && source.knowhereDocumentId) { - yield* Effect.tryPromise(() => - repository.markReady(workspaceId, source.id, source.knowhereDocumentId!), - ) - } - return existingSnapshot - } - - if (source.status !== "ready" && source.status !== "parsing") { - return existingSnapshot - } - - const documentId = source.knowhereDocumentId - if (!documentId) { - return yield* Effect.die( - new Error("Remote source is missing a Knowhere document id."), - ) - } - - const jobId = yield* resolveRemoteDocumentJobId({ - client, - documentId, - source, - }) - yield* Effect.tryPromise(() => - repository.markParsing(workspaceId, source.id, jobId, documentId), - ) - - const snapshot = yield* Effect.tryPromise(async () => { - const loadedResult = await client.knowledge.loadJobResult({ - jobId, - storageAdapter: createParsedResultStorageAdapter({ - workspaceId, - sourceId: source.id, - }), - }) - const manifest = loadedResult.parsedSnapshot - if (!manifest?.manifestUrl || !manifest.manifestKey) { - throw new Error( - "Remote parsed snapshot was not written; refusing to mark source ready.", - ) - } - return { - assetUrlsByFilePath: loadedResult.assetUrlsByFilePath ?? {}, - snapshotManifestUrl: manifest.manifestUrl, - snapshotManifestKey: manifest.manifestKey, - } - }) - - const savedSnapshot = yield* Effect.tryPromise(() => - repository.saveParseResult(workspaceId, source.id, { - resultBlobUrl: snapshot.snapshotManifestUrl, - snapshotManifestUrl: snapshot.snapshotManifestUrl, - snapshotManifestKey: snapshot.snapshotManifestKey, - assetUrlsByFilePath: snapshot.assetUrlsByFilePath, - }), - ) - if (!savedSnapshot) { - return yield* Effect.die( - new Error("Remote parsed snapshot could not be saved for the source."), - ) - } - - yield* Effect.tryPromise(() => - repository.markReady(workspaceId, source.id, documentId), - ) - - logger.info("sources: remote parsed snapshot synced", { - sourceId: source.id, - documentId, - jobId, - snapshotManifestKey: snapshot.snapshotManifestKey, - }) - - return { - resultBlobUrl: snapshot.snapshotManifestUrl, - snapshotManifestUrl: snapshot.snapshotManifestUrl, - snapshotManifestKey: snapshot.snapshotManifestKey, - assetUrlsByFilePath: snapshot.assetUrlsByFilePath, - } -}) - -export async function syncRemoteParsedSnapshot( - input: SyncRemoteParsedSnapshotInput, -): Promise { - return Effect.runPromise(syncRemoteParsedSnapshotEffect(input)) -} - -const resolveRemoteDocumentJobId = Effect.fn("resolveRemoteDocumentJobId")( - function* ({ - client, - documentId, - source, - }: { - readonly client: RemoteParsedSnapshotClient - readonly documentId: string - readonly source: RemoteParsedSnapshotSource - }) { - if (source.knowhereJobId) return source.knowhereJobId - - const response = yield* Effect.tryPromise(() => - client.documents.listChunks(documentId, { - page: jobProbePage, - pageSize: jobProbePageSize, - includeAssetUrls: false, - }), - ) - const jobId = response.jobId ?? response.jobResultId - if (!jobId) { - return yield* Effect.die( - new Error("Remote document chunk metadata did not include a job id."), - ) - } - return jobId - }, -) - -function isCompleteSnapshot( - snapshot: RemoteParsedSnapshotReadModel | null, -): snapshot is RemoteParsedSnapshotReadModel & { - readonly snapshotManifestKey: string - readonly snapshotManifestUrl: string -} { - return ( - typeof snapshot?.snapshotManifestKey === "string" && - snapshot.snapshotManifestKey.length > 0 && - typeof snapshot.snapshotManifestUrl === "string" && - snapshot.snapshotManifestUrl.length > 0 - ) -} diff --git a/src/domains/sources/repository.ts b/src/domains/sources/repository.ts index 4d9c061..3c6d01c 100644 --- a/src/domains/sources/repository.ts +++ b/src/domains/sources/repository.ts @@ -23,6 +23,7 @@ type SourceRepository = { readonly getParseResultProgressEffect: typeof sourceParseResultRepository.getParseResultProgressEffect readonly getParseSnapshotMetadataEffect: typeof sourceParseResultRepository.getParseSnapshotMetadataEffect readonly getParseAssetUrlsEffect: typeof sourceParseResultRepository.getParseAssetUrlsEffect + readonly updateSyncStatusEffect: typeof sourceParseResultRepository.updateSyncStatusEffect } export const sourceRepository: SourceRepository = { @@ -49,4 +50,5 @@ export const sourceRepository: SourceRepository = { getParseSnapshotMetadataEffect: sourceParseResultRepository.getParseSnapshotMetadataEffect, getParseAssetUrlsEffect: sourceParseResultRepository.getParseAssetUrlsEffect, + updateSyncStatusEffect: sourceParseResultRepository.updateSyncStatusEffect, } diff --git a/src/domains/sources/retry.test.ts b/src/domains/sources/retry.test.ts index 86c03a1..19eb565 100644 --- a/src/domains/sources/retry.test.ts +++ b/src/domains/sources/retry.test.ts @@ -18,6 +18,7 @@ describe("retrySourceToKnowhereEffect", () => { const parsingSource = makeSource({ status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_retry", knowhereDocumentId: "doc_retry", }) @@ -123,6 +124,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 5, status: "failed", failureReason: "Knowhere upload failed.", + failureStage: null, knowhereJobId: null, knowhereDocumentId: null, stagedBlobPathname: null, diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index b8fbca2..7d97920 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -1,6 +1,8 @@ import { Effect } from "effect" import { demoView } from "@/domains/demo/view" +import { readAllSourceChunks, readSourceChunkPage } from "@/domains/chunks/read" +import { resolveChunkConnectionTargets } from "@/domains/chunks" import type { DemoChunkPage } from "@/integrations/knowhere-demo" import { logger } from "@/lib/logger" import { routeResult } from "@/lib/route-result" @@ -8,7 +10,10 @@ import { decodeRemoteSourceId, findRemoteLibraryDocumentBySourceId, } from "./remote-library" -import { getClientForWorkspace } from "./route-dependencies" +import { + getClientForWorkspace, + getKnowledgeForSource, +} from "./route-dependencies" import { sourceRowRepository } from "./source-row-repository" import type { JsonRouteResult, @@ -23,8 +28,6 @@ type RouteChunksDependencies = Pick< | "ensureApiKeyForWorkspace" | "ensureWorkspace" | "getCurrentUser" - | "loadChunkPageForSource" - | "loadChunksForSource" | "makeKnowhereClient" | "sourceService" > @@ -85,67 +88,42 @@ const loadSourceChunksEffect = ( return demoResult ?? sourceNotFound() } - const client = yield* Effect.tryPromise(() => - getClientForWorkspace(workspace.id, input.cookieHeader, deps), - ) - const snapshot = yield* Effect.tryPromise(() => - deps.sourceService.ensureParsedSnapshotForRead({ - workspaceId: workspace.id, - source, - client: client.knowledge - ? { - documents: client.documents, - knowledge: client.knowledge, - } - : null, - }), - ).pipe( - Effect.catchAll((error) => - Effect.sync(() => { - logger.warn("sources: parsed snapshot sync for read failed", { - workspaceId: workspace.id, - sourceId: source.id, - documentId: source.knowhereDocumentId, - error: getErrorMessage(error), - }) - return null - }), - ), - ) - if (!isCompleteSnapshot(snapshot)) { + // Reads never return zero chunks for a ready remote document; the SDK falls + // back to Knowhere when Blob storage is missing or stale. A source that is + // not yet ready has no published document to read. + if (source.status !== "ready" || !source.knowhereDocumentId) { return sourceSnapshotProcessing(input) } + + const apiKey = yield* Effect.tryPromise(() => + deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), + ) + const knowledge = getKnowledgeForSource({ + apiKey, + workspaceId: workspace.id, + sourceId: source.id, + documentId: source.knowhereDocumentId, + revisionKey: source.knowhereJobId, + }) + const readableSource = { + documentId: source.knowhereDocumentId, + title: source.title, + revisionKey: source.knowhereJobId, + } + if (input.shouldLoadAll) { - const chunks = yield* deps.loadChunksForSource(source, client, { - snapshot, - workspaceId: workspace.id, - onRevisionKey: async (revisionKey) => { - await deps.sourceService.updateSourceRevisionKey( - workspace.id, - source.id, - revisionKey, - ) - }, - }) - return routeResult.ok({ chunks }) + const chunks = yield* Effect.tryPromise(() => + readAllSourceChunks({ knowledge, source: readableSource }), + ) + return routeResult.ok({ chunks: resolveChunkConnectionTargets(chunks) }) } - const chunkPage = yield* deps.loadChunkPageForSource( - source, - client, - input.pageParams, - { - assetUrlsByFilePath: snapshot.assetUrlsByFilePath, - snapshot, - workspaceId: workspace.id, - onRevisionKey: async (revisionKey) => { - await deps.sourceService.updateSourceRevisionKey( - workspace.id, - source.id, - revisionKey, - ) - }, - }, + const chunkPage = yield* Effect.tryPromise(() => + readSourceChunkPage({ + knowledge, + source: readableSource, + params: input.pageParams, + }), ) return routeResult.ok(chunkPage) }) @@ -163,6 +141,9 @@ const loadRemoteChunkPageEffect = ( const workspace = yield* Effect.tryPromise(() => deps.ensureWorkspace(user.id), ) + const apiKey = yield* Effect.tryPromise(() => + deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), + ) const client = yield* Effect.tryPromise(() => getClientForWorkspace(workspace.id, input.cookieHeader, deps), ) @@ -185,60 +166,34 @@ const loadRemoteChunkPageEffect = ( revisionKey: remoteDocument.revisionKey ?? null, }), ) - const knowledge = client.knowledge - if (!knowledge) { - throw new Error("Knowhere client does not support parsed snapshot caching.") - } - const snapshot = yield* Effect.tryPromise(() => - deps.sourceService.syncRemoteParsedSnapshot({ - workspaceId: workspace.id, - source, - client: { - documents: client.documents, - knowledge, - }, - }), - ) - if (!isCompleteSnapshot(snapshot)) { - return sourceSnapshotProcessing(input) - } - const readySource = { - ...source, - status: "ready" as const, - knowhereDocumentId: source.knowhereDocumentId ?? remoteDocument.documentId, + const documentId = source.knowhereDocumentId ?? remoteDocument.documentId + + const knowledge = getKnowledgeForSource({ + apiKey, + workspaceId: workspace.id, + sourceId: source.id, + documentId, + revisionKey: source.knowhereJobId ?? remoteDocument.revisionKey ?? null, + }) + const readableSource = { + documentId, + title: source.title, + revisionKey: source.knowhereJobId ?? remoteDocument.revisionKey ?? null, } if (input.shouldLoadAll) { - const chunks = yield* deps.loadChunksForSource(readySource, client, { - snapshot, - workspaceId: workspace.id, - onRevisionKey: async (revisionKey) => { - await deps.sourceService.updateSourceRevisionKey( - workspace.id, - source.id, - revisionKey, - ) - }, - }) - return routeResult.ok({ chunks }) + const chunks = yield* Effect.tryPromise(() => + readAllSourceChunks({ knowledge, source: readableSource }), + ) + return routeResult.ok({ chunks: resolveChunkConnectionTargets(chunks) }) } - const chunkPage = yield* deps.loadChunkPageForSource( - readySource, - client, - input.pageParams, - { - assetUrlsByFilePath: snapshot.assetUrlsByFilePath, - snapshot, - workspaceId: workspace.id, - onRevisionKey: async (revisionKey) => { - await deps.sourceService.updateSourceRevisionKey( - workspace.id, - source.id, - revisionKey, - ) - }, - }, + const chunkPage = yield* Effect.tryPromise(() => + readSourceChunkPage({ + knowledge, + source: readableSource, + params: input.pageParams, + }), ) return routeResult.ok(chunkPage) }) @@ -361,7 +316,7 @@ function sourceSnapshotProcessing( return routeResult.ok( { chunks: [], - message: "Source parsed snapshot is still being prepared.", + message: "Source is still being prepared.", }, 202, ) @@ -376,29 +331,10 @@ function sourceSnapshotProcessing( total: 0, totalPages: 0, }, - message: "Source parsed snapshot is still being prepared.", + message: "Source is still being prepared.", }, 202, ) } -function isCompleteSnapshot( - snapshot: - | { - readonly snapshotManifestKey?: string | null - readonly snapshotManifestUrl?: string | null - } - | null, -): snapshot is { - readonly snapshotManifestKey: string - readonly snapshotManifestUrl: string -} { - return ( - typeof snapshot?.snapshotManifestKey === "string" && - snapshot.snapshotManifestKey.length > 0 && - typeof snapshot.snapshotManifestUrl === "string" && - snapshot.snapshotManifestUrl.length > 0 - ) -} - export { createRouteChunks } diff --git a/src/domains/sources/route-dependencies.ts b/src/domains/sources/route-dependencies.ts index af81a3c..a39635f 100644 --- a/src/domains/sources/route-dependencies.ts +++ b/src/domains/sources/route-dependencies.ts @@ -1,17 +1,18 @@ import "server-only" import { del } from "@vercel/blob" +import type { Knowledge } from "@ontos-ai/knowhere-sdk" -import { - loadChunkPageForSource, - loadChunksForSource, -} from "@/domains/chunks/server" import { ensureApiKeyForWorkspace } from "@/integrations/dashboard/api-key-service" -import { makeKnowhereClient as makeDefaultKnowhereClient } from "@/integrations/knowhere" +import { + makeKnowhereClient as makeDefaultKnowhereClient, + makeKnowhereClientWithParsedStorage, +} from "@/integrations/knowhere" import { knowhereDemoApi } from "@/integrations/knowhere-demo" import { getCurrentUser, requireUser } from "@/infrastructure/auth" import { workspaceService } from "@/domains/workspace/service" import { sourceViewOptionsBySourceId as getDefaultSourceViewOptionsBySourceId } from "./counts" +import { createParsedDocumentSyncScheduler } from "./parsed-document-sync-scheduler" import { reconcileSourcesForWorkspace as reconcileDefaultSourcesForWorkspace } from "./reconcile" import { sourceWorkflowRuntime } from "./workflow-runtime" import { sourceService as defaultSourceService } from "./service" @@ -32,8 +33,6 @@ const defaultDependencies: SourceRouteServiceDependencies = { sources, client as ReturnType, ), - loadChunkPageForSource, - loadChunksForSource, makeKnowhereClient: (apiKey: string) => makeDefaultKnowhereClient(apiKey) as SourceRouteKnowhereClient, listSourcesForWorkspace: sourceWorkflowRuntime.listForWorkspace, @@ -44,20 +43,14 @@ const defaultDependencies: SourceRouteServiceDependencies = { ), requireUser, sourceService: { - ensureParsedSnapshotForRead: - defaultSourceService.ensureParsedSnapshotForRead, findInWorkspace: defaultSourceService.findInWorkspace, - getParseAssetUrls: defaultSourceService.getParseAssetUrls, - getParseSnapshotMetadata: defaultSourceService.getParseSnapshotMetadata, hideDemoSource: defaultSourceService.hideDemoSource, listHiddenDemoSourceIds: defaultSourceService.listHiddenDemoSourceIds, localizeRemoteDocument: defaultSourceService.localizeRemoteDocument, - updateSourceRevisionKey: defaultSourceService.updateSourceRevisionKey, softDelete: defaultSourceService.softDelete, upsertMaterializedDemoSource: defaultSourceService.upsertMaterializedDemoSource, retrySourceToKnowhere: defaultSourceService.retrySourceToKnowhere, - syncRemoteParsedSnapshot: defaultSourceService.syncRemoteParsedSnapshot, uploadSourceBlobToKnowhere: defaultSourceService.uploadSourceBlobToKnowhere, uploadSourceToKnowhere: defaultSourceService.uploadSourceToKnowhere, }, @@ -92,4 +85,35 @@ async function getClientForWorkspace( return deps.makeKnowhereClient(apiKey) } -export { createSourceRouteDependencies, getClientForWorkspace } +/** + * Build a `Knowledge` configured with Vercel-Blob parsed storage plus a + * QStash-backed background-sync scheduler bound to this source's document. + * Chunk reads go through the returned `knowledge`; on a storage miss the SDK + * serves from Knowhere remote and the scheduler enqueues a durable backfill. + */ +function getKnowledgeForSource(input: { + readonly apiKey: string + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly revisionKey?: string | null +}): Knowledge { + const scheduler = createParsedDocumentSyncScheduler({ + workspaceId: input.workspaceId, + sourceId: input.sourceId, + documentId: input.documentId, + apiKey: input.apiKey, + revisionKey: input.revisionKey ?? undefined, + }) + const { knowledge } = makeKnowhereClientWithParsedStorage(input.apiKey, { + workspaceId: input.workspaceId, + scheduler, + }) + return knowledge +} + +export { + createSourceRouteDependencies, + getClientForWorkspace, + getKnowledgeForSource, +} diff --git a/src/domains/sources/route-retry.test.ts b/src/domains/sources/route-retry.test.ts new file mode 100644 index 0000000..fc1316c --- /dev/null +++ b/src/domains/sources/route-retry.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it, vi } from "vitest" + +import { createRouteRetry } from "./route-retry" +import type { Source, Workspace } from "@/infrastructure/db/schema" + +const workspace: Workspace = { + id: "workspace_1", + userId: "user_1", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), +} + +function makeSource(overrides: Partial = {}): Source { + return { + id: "source_1", + workspaceId: "workspace_1", + title: "notes.pdf", + mimeType: "application/pdf", + sizeBytes: 5, + status: "failed", + failureReason: "boom", + failureStage: null, + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + ...overrides, + } +} + +function baseDeps() { + return { + ensureApiKeyForWorkspace: vi.fn(async () => "jwt_123"), + ensureWorkspace: vi.fn(async () => workspace), + makeKnowhereClient: vi.fn(), + requireUser: vi.fn(async () => ({ id: "user_1", email: null, name: null })), + sourceService: { + retrySourceToKnowhere: vi.fn(), + }, + } +} + +describe("createRouteRetry", () => { + it("resumes parsed sync for a storage_sync failure without reparsing", async () => { + const storageSyncFailed = makeSource({ failureStage: "storage_sync" }) + const resumedSource = makeSource({ status: "parsing", failureStage: null }) + const resumeParsedSync = vi.fn(async () => resumedSource) + const deps = baseDeps() + const retry = createRouteRetry({ + ...deps, + resumeParsedSync, + sourceService: { + findInWorkspace: vi.fn(async () => storageSyncFailed), + retrySourceToKnowhere: deps.sourceService.retrySourceToKnowhere, + }, + } as unknown as Parameters[0]) + + const result = await retry.retrySource({ + cookieHeader: "session=abc", + sourceId: "source_1", + }) + + expect(result.status).toBe(200) + expect(resumeParsedSync).toHaveBeenCalledWith({ + workspace, + source: storageSyncFailed, + apiKey: "jwt_123", + }) + expect(deps.sourceService.retrySourceToKnowhere).not.toHaveBeenCalled() + }) + + it("reparses a plain parse failure with a saved original blob", async () => { + const parseFailed = makeSource({ + failureStage: "parse", + originalBlobUrl: "https://blob.example/source-uploads/u/document.pdf", + originalBlobPathname: "source-uploads/u/document.pdf", + }) + const parsingSource = makeSource({ status: "parsing", failureStage: null }) + const resumeParsedSync = vi.fn() + const retrySourceToKnowhere = vi.fn(async () => parsingSource) + const deps = baseDeps() + const retry = createRouteRetry({ + ...deps, + resumeParsedSync, + makeKnowhereClient: vi.fn(() => ({})), + sourceService: { + findInWorkspace: vi.fn(async () => parseFailed), + retrySourceToKnowhere, + }, + } as unknown as Parameters[0]) + + const result = await retry.retrySource({ + cookieHeader: "session=abc", + sourceId: "source_1", + }) + + expect(result.status).toBe(200) + expect(resumeParsedSync).not.toHaveBeenCalled() + expect(retrySourceToKnowhere).toHaveBeenCalled() + }) +}) diff --git a/src/domains/sources/route-retry.ts b/src/domains/sources/route-retry.ts index 68464bb..094c91c 100644 --- a/src/domains/sources/route-retry.ts +++ b/src/domains/sources/route-retry.ts @@ -1,8 +1,12 @@ import { Effect } from "effect" +import { logger } from "@/lib/logger" import { routeResult } from "@/lib/route-result" import { startBackgroundReconciliation } from "./background-reconcile" +import { enqueueParsedDocumentSync } from "./parsed-document-sync-scheduler" +import { sourceWorkflowRuntime } from "./workflow-runtime" import { toSourceView } from "./view" +import type { Source, Workspace } from "@/infrastructure/db/schema" import type { JsonRouteResult, RetrySourceBody, @@ -17,7 +21,9 @@ type RouteRetryDependencies = Pick< | "makeKnowhereClient" | "requireUser" | "sourceService" -> +> & { + readonly resumeParsedSync?: typeof resumeParsedSync +} type RouteRetry = { readonly retrySource: ( @@ -51,6 +57,23 @@ const retrySourceEffect = ( if (source.status !== "failed") { return routeResult.error(409, "Only failed sources can be retried.") } + + // A storage-sync failure keeps the parsed Knowhere document intact — resume + // the parsed-document sync from the stored revision instead of reparsing. + if (source.failureStage === "storage_sync" && source.knowhereDocumentId) { + const apiKey = yield* Effect.tryPromise(() => + deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), + ) + const resumedSource = yield* Effect.tryPromise(() => + (deps.resumeParsedSync ?? resumeParsedSync)({ + workspace, + source, + apiKey, + }), + ) + return routeResult.ok({ source: toSourceView(resumedSource) }) + } + if (!source.originalBlobUrl || !source.originalBlobPathname) { return routeResult.error( 409, @@ -75,4 +98,50 @@ const retrySourceEffect = ( return routeResult.ok({ source: toSourceView(retriedSource) }) }) -export { createRouteRetry } +/** + * Resume a parsed-document storage sync that failed after a successful parse. + * Moves the source back to `parsing` (clearing the failure stage), resets the + * sync status, and re-enqueues the resumable parsed-sync workflow from the + * stored document + revision — no reparse. + */ +async function resumeParsedSync(input: { + readonly workspace: Workspace + readonly source: Source + readonly apiKey: string +}): Promise { + const { workspace, source, apiKey } = input + const documentId = source.knowhereDocumentId + if (!documentId) return source + + const revisionKey = source.knowhereJobId ?? undefined + const parsingSource = await sourceWorkflowRuntime.markParsing( + workspace.id, + source.id, + revisionKey ?? documentId, + documentId, + "failed", + ) + if (!parsingSource) return source + + await sourceWorkflowRuntime.updateSyncStatus(workspace.id, source.id, { + revisionKey, + syncStatus: "running", + syncError: null, + }) + await enqueueParsedDocumentSync({ + workspaceId: workspace.id, + sourceId: source.id, + documentId, + apiKey, + revisionKey, + }) + logger.info("sources: resumed parsed document storage sync on retry", { + workspaceId: workspace.id, + sourceId: source.id, + documentId, + revisionKey, + }) + return parsingSource +} + +export { createRouteRetry, resumeParsedSync } diff --git a/src/domains/sources/route-service.test.ts b/src/domains/sources/route-service.test.ts index ca37390..26e60cb 100644 --- a/src/domains/sources/route-service.test.ts +++ b/src/domains/sources/route-service.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it, vi } from "vitest"; import { Effect } from "effect"; import type { Job } from "@ontos-ai/knowhere-sdk"; -import type { ChunkPage } from "@/domains/chunks"; import type { Source, Workspace } from "@/infrastructure/db/schema"; import { createRouteListing } from "./route-listing"; import { createSourceRouteService } from "./route-service"; @@ -23,6 +22,7 @@ const source: Source = { sizeBytes: 5, status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: null, stagedBlobPathname: null, @@ -120,147 +120,15 @@ describe("source route service", () => { expect(listHiddenDemoSourceIds).toHaveBeenCalledWith(workspace.id); }); - it("syncs a ready legacy source snapshot before loading displayed chunks", async () => { - const readySource: Source = { - ...source, - id: "00000000-0000-4000-8000-000000000001", - knowhereJobId: null, - knowhereDocumentId: "doc_legacy", - status: "ready", - }; - const snapshot = { - resultBlobUrl: "https://blob.example/manifest/current.json", - snapshotManifestUrl: "https://blob.example/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/00000000-0000-4000-8000-000000000001/parsed-result/manifest/current.json", - assetUrlsByFilePath: { - "images/chart.png": "https://blob.example/images/chart.png", - }, - }; - const listChunks = vi.fn(async () => { - throw new Error("display path must not call Knowhere listChunks"); - }); - const knowledge = { - loadJobResult: vi.fn(), - }; - const knowhereClient = { - documents: { - archive: vi.fn(async () => undefined), - listChunks, - }, - jobs: { - create: vi.fn(), - get: vi.fn(), - upload: vi.fn(), - }, - knowledge, - }; - const loadedPage: ChunkPage = { - chunks: [ - { - chunkId: "chunk_1", - type: "text", - content: "Chunk body", - sourceTitle: "notes.pdf", - }, - ], - pagination: { - page: 1, - pageSize: 50, - total: 1, - totalPages: 1, - }, - }; - const loadChunkPageForSource = vi.fn(() => Effect.succeed(loadedPage)); - const syncRemoteParsedSnapshot = vi.fn(async () => snapshot); - const service = createSourceRouteService({ - ensureApiKeyForWorkspace: vi.fn(async () => "jwt_123"), - ensureWorkspace: vi.fn(async () => workspace), - getCurrentUser: vi.fn(async () => ({ - id: "user_1", - email: null, - name: null, - })), - loadChunkPageForSource, - makeKnowhereClient: vi.fn(() => knowhereClient), - sourceService: { - findInWorkspace: vi.fn(async () => readySource), - ensureParsedSnapshotForRead: syncRemoteParsedSnapshot, - updateSourceRevisionKey: vi.fn(async () => readySource), - }, - }); - - const result = await service.loadSourceChunks({ - cookieHeader: "session=abc", - sourceId: readySource.id, - shouldLoadAll: false, - pageParams: { page: 1, pageSize: 50 }, - }); - - expect(result).toEqual({ - status: 200, - body: { - chunks: [ - { - chunkId: "chunk_1", - type: "text", - content: "Chunk body", - sourceTitle: "notes.pdf", - }, - ], - pagination: { - page: 1, - pageSize: 50, - total: 1, - totalPages: 1, - }, - }, - }); - expect(syncRemoteParsedSnapshot).toHaveBeenCalledWith({ - workspaceId: workspace.id, - source: readySource, - client: { - documents: knowhereClient.documents, - knowledge, - }, - }); - expect(loadChunkPageForSource).toHaveBeenCalledWith( - readySource, - knowhereClient, - { page: 1, pageSize: 50 }, - expect.objectContaining({ - assetUrlsByFilePath: snapshot.assetUrlsByFilePath, - snapshot, - workspaceId: workspace.id, - }), - ); - expect(listChunks).not.toHaveBeenCalled(); - }); - - it("returns processing when a ready legacy source snapshot is not synced yet", async () => { - const readySource: Source = { + it("returns processing without building a client when a source is not ready", async () => { + const parsingSource: Source = { ...source, id: "00000000-0000-4000-8000-000000000002", knowhereDocumentId: "doc_legacy", knowhereJobId: null, - status: "ready", - }; - const knowledge = { - loadJobResult: vi.fn(), - }; - const knowhereClient = { - documents: { - archive: vi.fn(async () => undefined), - listChunks: vi.fn(), - }, - jobs: { - create: vi.fn(), - get: vi.fn(), - upload: vi.fn(), - }, - knowledge, + status: "parsing", }; - const loadChunkPageForSource = vi.fn(); + const makeKnowhereClient = vi.fn(); const service = createSourceRouteService({ ensureApiKeyForWorkspace: vi.fn(async () => "jwt_123"), ensureWorkspace: vi.fn(async () => workspace), @@ -269,19 +137,15 @@ describe("source route service", () => { email: null, name: null, })), - loadChunkPageForSource, - makeKnowhereClient: vi.fn(() => knowhereClient), + makeKnowhereClient, sourceService: { - findInWorkspace: vi.fn(async () => readySource), - ensureParsedSnapshotForRead: vi.fn(async () => { - throw new Error("snapshot import still running"); - }), + findInWorkspace: vi.fn(async () => parsingSource), }, }); const result = await service.loadSourceChunks({ cookieHeader: "session=abc", - sourceId: readySource.id, + sourceId: parsingSource.id, shouldLoadAll: false, pageParams: { page: 1, pageSize: 50 }, }); @@ -296,11 +160,9 @@ describe("source route service", () => { total: 0, totalPages: 0, }, - message: "Source parsed snapshot is still being prepared.", + message: "Source is still being prepared.", }, }); - expect(loadChunkPageForSource).not.toHaveBeenCalled(); - expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled(); }); it("lists shared default and legacy namespace documents as lightweight remote sources", async () => { @@ -985,6 +847,7 @@ describe("source route service", () => { ...failedSource, status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_retry", }; const knowhereClient = { diff --git a/src/domains/sources/route-types.ts b/src/domains/sources/route-types.ts index a32cad2..96bd80c 100644 --- a/src/domains/sources/route-types.ts +++ b/src/domains/sources/route-types.ts @@ -3,10 +3,6 @@ import type { ChunkPage, ChunkPageParams, } from "@/domains/chunks" -import type { - loadChunkPageForSource, - loadChunksForSource, -} from "@/domains/chunks/server" import type { ParsedChunkView } from "@/domains/chunks/types" import type { SourceStatus, SourceView } from "@/domains/sources/types" import type { AuthUser } from "@/infrastructure/auth" @@ -18,13 +14,8 @@ import type { import type { RouteResult } from "@/lib/route-result" import type { SourceBlobUploadInput } from "./blob-upload" import type { sourceViewOptionsBySourceId } from "./counts" -import type { syncRemoteParsedSnapshot } from "./remote-parsed-snapshot-sync" import type { UploadKnowhereClient } from "./upload" -type SyncRemoteParsedSnapshotInput = Parameters< - typeof syncRemoteParsedSnapshot ->[0] - type SourceRouteKnowhereClient = UploadKnowhereClient & ChunkKnowhereClient & { readonly documents: ChunkKnowhereClient["documents"] & { @@ -59,7 +50,6 @@ type SourceRouteKnowhereClient = UploadKnowhereClient & }> archive(documentId: string): Promise } - readonly knowledge?: SyncRemoteParsedSnapshotInput["client"]["knowledge"] } type SourceUploadRequest = @@ -162,16 +152,6 @@ type SourceRouteService = { } type SourceWorkflowService = { - readonly ensureParsedSnapshotForRead: (input: { - readonly workspaceId: string - readonly source: Source - readonly client?: SyncRemoteParsedSnapshotInput["client"] | null - }) => Promise<{ - readonly resultBlobUrl: string - readonly snapshotManifestUrl?: string | null - readonly snapshotManifestKey?: string | null - readonly assetUrlsByFilePath: Readonly> - } | null> readonly uploadSourceToKnowhere: ( workspace: Workspace, file: File, @@ -187,7 +167,6 @@ type SourceWorkflowService = { source: Source, knowhere: UploadKnowhereClient, ) => Promise - readonly syncRemoteParsedSnapshot: typeof syncRemoteParsedSnapshot readonly findInWorkspace: ( workspaceId: string, sourceId: string, @@ -196,19 +175,6 @@ type SourceWorkflowService = { workspaceId: string, sourceId: string, ) => Promise - readonly getParseAssetUrls: ( - workspaceId: string, - sourceId: string, - ) => Promise>> - readonly getParseSnapshotMetadata: ( - workspaceId: string, - sourceId: string, - ) => Promise<{ - readonly resultBlobUrl: string - readonly snapshotManifestUrl?: string | null - readonly snapshotManifestKey?: string | null - readonly assetUrlsByFilePath: Readonly> - } | null> readonly hideDemoSource: ( workspaceId: string, demoSourceId: string, @@ -226,11 +192,6 @@ type SourceWorkflowService = { readonly revisionKey?: string | null }, ) => Promise - readonly updateSourceRevisionKey: ( - workspaceId: string, - sourceId: string, - revisionKey: string, - ) => Promise readonly upsertMaterializedDemoSource: ( workspaceId: string, input: { @@ -266,8 +227,6 @@ type SourceRouteServiceDependencies = { sources: readonly Source[], client: SourceRouteKnowhereClient, ) => ReturnType - readonly loadChunkPageForSource: typeof loadChunkPageForSource - readonly loadChunksForSource: typeof loadChunksForSource readonly makeKnowhereClient: (apiKey: string) => SourceRouteKnowhereClient readonly listSourcesForWorkspace: (workspaceId: string) => Promise readonly reconcileSourcesForWorkspace: ( diff --git a/src/domains/sources/service.ts b/src/domains/sources/service.ts index 11e9199..4fcbfd5 100644 --- a/src/domains/sources/service.ts +++ b/src/domains/sources/service.ts @@ -9,36 +9,19 @@ import { uploadSourceBlobToKnowhereEffect, uploadSourceToKnowhereEffect, } from "./upload" -import { syncRemoteParsedSnapshot } from "./remote-parsed-snapshot-sync" import { retrySourceToKnowhereEffect } from "./retry" import { sourceWorkflowRuntime } from "./workflow-runtime" type SourceService = { - readonly ensureParsedSnapshotForRead: ( - input: EnsureParsedSnapshotForReadInput, - ) => Promise readonly findInWorkspace: ( workspaceId: string, sourceId: string, ) => Promise - readonly getParseAssetUrls: ( - workspaceId: string, - sourceId: string, - ) => Promise>> - readonly getParseSnapshotMetadata: ( - workspaceId: string, - sourceId: string, - ) => ReturnType readonly listForWorkspace: (workspaceId: string) => Promise readonly localizeRemoteDocument: ( workspaceId: string, input: Parameters[1], ) => Promise - readonly updateSourceRevisionKey: ( - workspaceId: string, - sourceId: string, - revisionKey: string, - ) => Promise readonly listHiddenDemoSourceIds: (workspaceId: string) => Promise readonly hideDemoSource: ( workspaceId: string, @@ -69,20 +52,6 @@ type SourceService = { source: Source, knowhere: UploadKnowhereClient, ) => Promise - readonly syncRemoteParsedSnapshot: typeof syncRemoteParsedSnapshot -} - -type EnsureParsedSnapshotForReadInput = { - readonly workspaceId: string - readonly source: Source - readonly client?: Parameters[0]["client"] | null -} - -type EnsureParsedSnapshotForReadResult = { - readonly resultBlobUrl: string - readonly snapshotManifestUrl?: string | null - readonly snapshotManifestKey?: string | null - readonly assetUrlsByFilePath: Readonly> } const uploadSourceToKnowhere: SourceService["uploadSourceToKnowhere"] = ( @@ -125,64 +94,16 @@ const retrySourceToKnowhere: SourceService["retrySourceToKnowhere"] = ( }), ) -const ensureParsedSnapshotForRead: SourceService["ensureParsedSnapshotForRead"] = - async ({ workspaceId, source, client }: EnsureParsedSnapshotForReadInput) => { - const existingSnapshot = - await sourceWorkflowRuntime.getParseSnapshotMetadata(workspaceId, source.id) - if (isCompleteParsedSnapshot(existingSnapshot)) return existingSnapshot - - if (!canSyncParsedSnapshotForRead(source, client)) { - return existingSnapshot - } - - return syncRemoteParsedSnapshot({ - workspaceId, - source, - client, - }) - } - export const sourceService: SourceService = { - ensureParsedSnapshotForRead, findInWorkspace: sourceWorkflowRuntime.findInWorkspace, - getParseAssetUrls: sourceWorkflowRuntime.getParseAssetUrls, - getParseSnapshotMetadata: sourceWorkflowRuntime.getParseSnapshotMetadata, hideDemoSource: sourceWorkflowRuntime.hideDemoSource, listHiddenDemoSourceIds: sourceWorkflowRuntime.listHiddenDemoSourceIds, listForWorkspace: sourceWorkflowRuntime.listForWorkspace, localizeRemoteDocument: sourceWorkflowRuntime.localizeRemoteDocument, - updateSourceRevisionKey: sourceWorkflowRuntime.updateRevisionKey, softDelete: sourceWorkflowRuntime.softDelete, upsertMaterializedDemoSource: sourceWorkflowRuntime.upsertMaterializedDemoSource, uploadSourceToKnowhere, uploadSourceBlobToKnowhere, retrySourceToKnowhere, - syncRemoteParsedSnapshot, -} - -function canSyncParsedSnapshotForRead( - source: Source, - client: EnsureParsedSnapshotForReadInput["client"], -): client is Parameters[0]["client"] { - return ( - Boolean(client) && - (source.status === "ready" || source.status === "parsing") && - typeof source.knowhereDocumentId === "string" && - source.knowhereDocumentId.length > 0 - ) -} - -function isCompleteParsedSnapshot( - snapshot: EnsureParsedSnapshotForReadResult | null, -): snapshot is EnsureParsedSnapshotForReadResult & { - readonly snapshotManifestKey: string - readonly snapshotManifestUrl: string -} { - return ( - typeof snapshot?.snapshotManifestKey === "string" && - snapshot.snapshotManifestKey.length > 0 && - typeof snapshot.snapshotManifestUrl === "string" && - snapshot.snapshotManifestUrl.length > 0 - ) } diff --git a/src/domains/sources/source-parse-result-repository.ts b/src/domains/sources/source-parse-result-repository.ts index c3e9859..6ddc1e2 100644 --- a/src/domains/sources/source-parse-result-repository.ts +++ b/src/domains/sources/source-parse-result-repository.ts @@ -14,13 +14,25 @@ type SaveSourceParseResultInput = { readonly resultBlobUrl: string readonly snapshotManifestUrl?: string readonly snapshotManifestKey?: string + readonly revisionKey?: string + readonly syncStatus?: SourceParseSyncStatus + readonly syncError?: string | null readonly assetUrlsByFilePath: Readonly> } +export type SourceParseSyncStatus = + | "pending" + | "running" + | "completed" + | "failed" + type SourceParseResultProgress = { readonly resultBlobUrl: string readonly snapshotManifestUrl?: string | null readonly snapshotManifestKey?: string | null + readonly revisionKey?: string | null + readonly syncStatus?: string | null + readonly syncError?: string | null readonly assetUrlsByFilePath: Readonly> } @@ -28,9 +40,18 @@ export type SourceParseSnapshotMetadata = { readonly resultBlobUrl: string readonly snapshotManifestUrl?: string | null readonly snapshotManifestKey?: string | null + readonly revisionKey?: string | null + readonly syncStatus?: string | null + readonly syncError?: string | null readonly assetUrlsByFilePath: Readonly> } +type UpdateSyncStatusInput = { + readonly revisionKey?: string + readonly syncStatus: SourceParseSyncStatus + readonly syncError?: string | null +} + type SourceParseResultRepository = { readonly saveParseResultEffect: ( workspaceId: string, @@ -54,6 +75,11 @@ type SourceParseResultRepository = { workspaceId: string, sourceId: string, ) => Effect.Effect>, never, DbClient> + readonly updateSyncStatusEffect: ( + workspaceId: string, + sourceId: string, + input: UpdateSyncStatusInput, + ) => Effect.Effect } export function buildAtomicAssetUrlsMergeSql( @@ -79,6 +105,9 @@ const saveParseResultEffect: SourceParseResultRepository["saveParseResultEffect" resultBlobUrl: input.resultBlobUrl, snapshotManifestUrl: input.snapshotManifestUrl, snapshotManifestKey: input.snapshotManifestKey, + revisionKey: input.revisionKey, + syncStatus: input.syncStatus, + syncError: input.syncError, assetUrls: input.assetUrlsByFilePath, }) .onConflictDoUpdate({ @@ -87,6 +116,9 @@ const saveParseResultEffect: SourceParseResultRepository["saveParseResultEffect" resultBlobUrl: input.resultBlobUrl, snapshotManifestUrl: input.snapshotManifestUrl, snapshotManifestKey: input.snapshotManifestKey, + revisionKey: input.revisionKey, + syncStatus: input.syncStatus, + syncError: input.syncError, assetUrls: input.assetUrlsByFilePath, updatedAt: sql`now()`, }, @@ -114,6 +146,9 @@ const mergeParseAssetUrlsEffect: SourceParseResultRepository["mergeParseAssetUrl resultBlobUrl: input.resultBlobUrl, snapshotManifestUrl: input.snapshotManifestUrl, snapshotManifestKey: input.snapshotManifestKey, + revisionKey: input.revisionKey, + syncStatus: input.syncStatus, + syncError: input.syncError, assetUrls: input.assetUrlsByFilePath, }) .onConflictDoUpdate({ @@ -122,6 +157,9 @@ const mergeParseAssetUrlsEffect: SourceParseResultRepository["mergeParseAssetUrl resultBlobUrl: input.resultBlobUrl, snapshotManifestUrl: input.snapshotManifestUrl, snapshotManifestKey: input.snapshotManifestKey, + revisionKey: input.revisionKey, + syncStatus: input.syncStatus, + syncError: input.syncError, assetUrls: buildAtomicAssetUrlsMergeSql( input.assetUrlsByFilePath, ), @@ -149,6 +187,9 @@ const getParseResultProgressEffect: SourceParseResultRepository["getParseResultP resultBlobUrl: sourceParseResults.resultBlobUrl, snapshotManifestUrl: sourceParseResults.snapshotManifestUrl, snapshotManifestKey: sourceParseResults.snapshotManifestKey, + revisionKey: sourceParseResults.revisionKey, + syncStatus: sourceParseResults.syncStatus, + syncError: sourceParseResults.syncError, assetUrls: sourceParseResults.assetUrls, }) .from(sourceParseResults) @@ -159,13 +200,54 @@ const getParseResultProgressEffect: SourceParseResultRepository["getParseResultP if (!progress) return null return { - resultBlobUrl: progress.resultBlobUrl, + resultBlobUrl: progress.resultBlobUrl ?? "", snapshotManifestUrl: progress.snapshotManifestUrl, snapshotManifestKey: progress.snapshotManifestKey, + revisionKey: progress.revisionKey, + syncStatus: progress.syncStatus, + syncError: progress.syncError, assetUrlsByFilePath: progress.assetUrls, } }) +const updateSyncStatusEffect: SourceParseResultRepository["updateSyncStatusEffect"] = + (workspaceId: string, sourceId: string, input: UpdateSyncStatusInput) => + Effect.gen(function* () { + const db = yield* DbClient + const source = yield* Effect.promise(() => + sourceRowRepository.findInWorkspaceWithDb(db, workspaceId, sourceId), + ) + if (!source) return null + + // Upsert: a read-miss backfill may set sync status before any parsed + // snapshot row exists, so insert a bare row when one is not present yet. + const [result] = yield* Effect.promise(() => + db + .insert(sourceParseResults) + .values({ + sourceId, + revisionKey: input.revisionKey, + syncStatus: input.syncStatus, + syncError: input.syncError ?? null, + assetUrls: {}, + }) + .onConflictDoUpdate({ + target: sourceParseResults.sourceId, + set: { + ...(input.revisionKey !== undefined + ? { revisionKey: input.revisionKey } + : {}), + syncStatus: input.syncStatus, + syncError: input.syncError ?? null, + updatedAt: sql`now()`, + }, + }) + .returning(), + ) + + return result ?? null + }) + const getParseAssetUrlsEffect: SourceParseResultRepository["getParseAssetUrlsEffect"] = (workspaceId: string, sourceId: string) => Effect.gen(function* () { @@ -199,4 +281,5 @@ export const sourceParseResultRepository: SourceParseResultRepository = { getParseResultProgressEffect, getParseSnapshotMetadataEffect, getParseAssetUrlsEffect, + updateSyncStatusEffect, } diff --git a/src/domains/sources/source-reconcile-route-workflow.test.ts b/src/domains/sources/source-reconcile-route-workflow.test.ts index ea68cbe..52f9c6b 100644 --- a/src/domains/sources/source-reconcile-route-workflow.test.ts +++ b/src/domains/sources/source-reconcile-route-workflow.test.ts @@ -1,13 +1,13 @@ import { afterEach, describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ - createParsedResultStorageAdapter: vi.fn(), - saveParseResult: vi.fn(), + enqueueParsedDocumentSync: vi.fn(), + updateSyncStatus: vi.fn(), + markFailed: vi.fn(), loggerError: vi.fn(), loggerInfo: vi.fn(), loggerWarn: vi.fn(), - makeKnowhereClient: vi.fn(), - markFailed: vi.fn(), + makeKnowhereClientWithParsedStorage: vi.fn(), markSourceReadyAfterReconciliation: vi.fn(), pollSourceReconciliation: vi.fn(), })) @@ -20,16 +20,17 @@ vi.mock("@/domains/sources/source-reconcile-workflow", () => ({ vi.mock("@/domains/sources/workflow-runtime", () => ({ sourceWorkflowRuntime: { markFailed: mocks.markFailed, - saveParseResult: mocks.saveParseResult, + updateSyncStatus: mocks.updateSyncStatus, }, })) -vi.mock("./parse-result-storage-adapter", () => ({ - createParsedResultStorageAdapter: mocks.createParsedResultStorageAdapter, +vi.mock("./parsed-document-sync-scheduler", () => ({ + enqueueParsedDocumentSync: mocks.enqueueParsedDocumentSync, })) vi.mock("@/integrations/knowhere", () => ({ - makeKnowhereClient: mocks.makeKnowhereClient, + makeKnowhereClientWithParsedStorage: + mocks.makeKnowhereClientWithParsedStorage, })) vi.mock("@/lib/logger", () => ({ @@ -42,6 +43,30 @@ vi.mock("@/lib/logger", () => ({ import { sourceReconcileRouteWorkflow } from "./source-reconcile-route-workflow" +function createClient(overrides: { + syncParsedDocument?: ReturnType + jobResultId?: string + jobId?: string +}) { + const listChunks = vi.fn(async () => ({ + jobResultId: overrides.jobResultId ?? "rev_1", + jobId: overrides.jobId ?? "job_1", + })) + return { + client: { jobs: {}, documents: { listChunks } }, + knowledge: { + syncParsedDocument: + overrides.syncParsedDocument ?? + vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_1", + completed: true, + })), + }, + listChunks, + } +} + describe("sourceReconcileRouteWorkflow", () => { afterEach(() => { vi.clearAllMocks() @@ -64,7 +89,7 @@ describe("sourceReconcileRouteWorkflow", () => { }) }) - it("writes a parsed snapshot before marking the source ready", async () => { + it("syncs the parsed document then marks the source ready", async () => { const context = createWorkflowContext() const continuations: ContinuationTriggerInput[] = [] const restore = @@ -73,36 +98,19 @@ describe("sourceReconcileRouteWorkflow", () => { continuations.push(input) }, ) - const storageAdapter = { - adapter: { - writeObject: vi.fn(), - }, - keyPrefix: "workspaces/workspace_1/sources/source_1/parsed-result", - } - const loadJobResult = vi.fn().mockResolvedValue({ - assetUrlsByFilePath: { - "page_citation_assets/page-1.png": - "https://blob.example/page_citation_assets/page-1.png", - }, - parsedSnapshot: { - manifestKey: - "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - manifestUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - }, + const wired = createClient({}) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: wired.client, + knowledge: wired.knowledge, + }) + mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ + status: "ready", }) - const client = { jobs: {}, knowledge: { loadJobResult } } - mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) - mocks.makeKnowhereClient.mockReturnValue(client) - mocks.saveParseResult.mockResolvedValue({ id: "parse_result_1" }) mocks.pollSourceReconciliation.mockResolvedValue({ kind: "ready-to-prepare", jobId: "job_1", documentId: "doc_1", }) - mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ - status: "ready", - }) try { await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ @@ -117,54 +125,70 @@ describe("sourceReconcileRouteWorkflow", () => { restore() } - expect(mocks.createParsedResultStorageAdapter).toHaveBeenCalledWith({ - workspaceId: "workspace_1", - sourceId: "source_1", - }) - expect(loadJobResult).toHaveBeenCalledWith({ - jobId: "job_1", - storageAdapter, + expect(wired.knowledge.syncParsedDocument).toHaveBeenCalledWith({ + documentId: "doc_1", + revisionKey: "rev_1", }) - expect(mocks.saveParseResult).toHaveBeenCalledWith( + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( "workspace_1", "source_1", - { - resultBlobUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestUrl: - "https://blob.example/workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - snapshotManifestKey: - "workspaces/workspace_1/sources/source_1/parsed-result/manifest/current.json", - assetUrlsByFilePath: { - "page_citation_assets/page-1.png": - "https://blob.example/page_citation_assets/page-1.png", - }, - }, + { revisionKey: "rev_1", syncStatus: "completed" }, ) expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ workspaceId: "workspace_1", sourceId: "source_1", documentId: "doc_1", }) - expect(mocks.loggerWarn).not.toHaveBeenCalled() + expect(mocks.enqueueParsedDocumentSync).not.toHaveBeenCalled() expect(continuations).toEqual([]) }) - it("does not mark ready when snapshot manifest storage is missing", async () => { + it("hands off to parsed-sync and stays parsing when sync is incomplete", async () => { const context = createWorkflowContext() - const storageAdapter = { - adapter: { - writeObject: vi.fn(), - }, - keyPrefix: "workspaces/workspace_1/sources/source_1/parsed-result", - } - const loadJobResult = vi.fn().mockResolvedValue({ - assetUrlsByFilePath: {}, + const syncParsedDocument = vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_1", + completed: false, + })) + const wired = createClient({ syncParsedDocument }) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: wired.client, + knowledge: wired.knowledge, }) - mocks.createParsedResultStorageAdapter.mockReturnValue(storageAdapter) - mocks.makeKnowhereClient.mockReturnValue({ - jobs: {}, - knowledge: { loadJobResult }, + mocks.pollSourceReconciliation.mockResolvedValue({ + kind: "ready-to-prepare", + jobId: "job_1", + documentId: "doc_1", + }) + + await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ + context, + payload: sourceReconcileRouteWorkflow.normalizeReconcilePayload({ + workspaceId: "workspace_1", + sourceId: "source_1", + apiKey: "jwt_1", + }), + }) + + expect(mocks.markSourceReadyAfterReconciliation).not.toHaveBeenCalled() + expect(mocks.enqueueParsedDocumentSync).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + apiKey: "jwt_1", + revisionKey: "rev_1", + }) + }) + + it("fails the source with storage_sync stage when sync throws", async () => { + const context = createWorkflowContext() + const syncParsedDocument = vi.fn(async () => { + throw new Error("blob write failed") + }) + const wired = createClient({ syncParsedDocument }) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: wired.client, + knowledge: wired.knowledge, }) mocks.pollSourceReconciliation.mockResolvedValue({ kind: "ready-to-prepare", @@ -181,9 +205,15 @@ describe("sourceReconcileRouteWorkflow", () => { apiKey: "jwt_1", }), }), - ).rejects.toThrow("Parsed result snapshot was not written") + ).rejects.toThrow("blob write failed") - expect(mocks.saveParseResult).not.toHaveBeenCalled() + expect(mocks.markFailed).toHaveBeenCalledWith( + "workspace_1", + "source_1", + expect.stringContaining("storage sync failed"), + "parsing", + "storage_sync", + ) expect(mocks.markSourceReadyAfterReconciliation).not.toHaveBeenCalled() }) @@ -196,7 +226,10 @@ describe("sourceReconcileRouteWorkflow", () => { continuations.push(input) }, ) - mocks.makeKnowhereClient.mockReturnValue({ jobs: {} }) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: { jobs: {}, documents: { listChunks: vi.fn() } }, + knowledge: { syncParsedDocument: vi.fn() }, + }) mocks.pollSourceReconciliation.mockResolvedValue({ kind: "waiting", jobId: "job_1", diff --git a/src/domains/sources/source-reconcile-route-workflow.ts b/src/domains/sources/source-reconcile-route-workflow.ts index 16f9054..cfa90b3 100644 --- a/src/domains/sources/source-reconcile-route-workflow.ts +++ b/src/domains/sources/source-reconcile-route-workflow.ts @@ -6,11 +6,9 @@ import { markSourceReadyAfterReconciliation, pollSourceReconciliation, } from "@/domains/sources/source-reconcile-workflow" -import { makeKnowhereClient } from "@/integrations/knowhere" +import { makeKnowhereClientWithParsedStorage } from "@/integrations/knowhere" import { logger } from "@/lib/logger" -import { - createParsedResultStorageAdapter, -} from "./parse-result-storage-adapter" +import { enqueueParsedDocumentSync } from "./parsed-document-sync-scheduler" import { sourceWorkflowRuntime } from "./workflow-runtime" type ReconcilePayload = { @@ -23,14 +21,6 @@ type ReconcilePayload = { type ReconcilePhase = "poll-and-ready" | "poll-and-mirror" | "asset-batches" -type ParsedSnapshotResponse = { - readonly assetUrlsByFilePath?: Readonly> - readonly parsedSnapshot?: { - readonly manifestKey: string - readonly manifestUrl?: string - } -} - type NormalizedReconcilePayload = { readonly workspaceId: string readonly sourceId: string @@ -53,6 +43,7 @@ type ContinuationTriggerInput = { const maxPollAttempts = 25 const initialDelaySeconds = 3 const maxDelaySeconds = 30 +const maxSyncStepsPerReconcile = 4 let triggerContinuation: typeof triggerReconcileContinuation = triggerReconcileContinuation @@ -63,7 +54,9 @@ async function runPollAndMirrorWorkflow(input: { }): Promise { const { context, payload } = input const { workspaceId, sourceId, apiKey } = payload - const client = makeKnowhereClient(apiKey) + const { client, knowledge } = makeKnowhereClientWithParsedStorage(apiKey, { + workspaceId, + }) let delay = initialDelaySeconds let completedJob: { readonly jobId: string @@ -129,35 +122,88 @@ async function runPollAndMirrorWorkflow(input: { return } - const snapshot = await context.run("load-job-result-snapshot", async () => { - const loadedResult = (await client.knowledge.loadJobResult({ - jobId: jobToPrepare.jobId, - storageAdapter: createParsedResultStorageAdapter({ + // Resolve the parsed revision and record that storage sync is in progress. + const revisionKey = await context.run("resolve-revision-key", async () => { + const firstPage = await client.documents.listChunks( + jobToPrepare.documentId, + { page: 1, pageSize: 1, includeAssetUrls: false }, + ) + return firstPage.jobResultId ?? firstPage.jobId ?? jobToPrepare.jobId + }) + await context.run("record-sync-running", async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "running", + }), + ) + + // Sync the parsed snapshot into Blob before the source is considered ready. + // Each step is bounded by SDK limits; if it does not finish in this segment, + // hand off to the resumable parsed-sync workflow and stay `parsing`. A sync + // error stages the failure as `storage_sync` so a retry resumes sync without + // reparsing, then rethrows to fail the workflow run. + let syncCompleted = false + for (let step = 0; step < maxSyncStepsPerReconcile; step++) { + const result = await context.run(`parsed-sync-${step}`, async () => { + try { + return await knowledge.syncParsedDocument({ + documentId: jobToPrepare.documentId, + revisionKey, + }) + } catch (error) { + await sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "failed", + syncError: error instanceof Error ? error.message : String(error), + }) + await sourceWorkflowRuntime.markFailed( + workspaceId, + sourceId, + `Parsed document storage sync failed: ${ + error instanceof Error ? error.message : String(error) + }`, + "parsing", + "storage_sync", + ) + throw error + } + }) + if (result.completed) { + syncCompleted = true + break + } + } + + if (!syncCompleted) { + await context.run("record-sync-progress", async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "running", + }), + ) + await context.run("enqueue-parsed-sync-continuation", async () => + enqueueParsedDocumentSync({ workspaceId, sourceId, + documentId: jobToPrepare.documentId, + apiKey, + revisionKey, }), - })) as ParsedSnapshotResponse - const manifest = loadedResult.parsedSnapshot - if (!manifest?.manifestUrl || !manifest.manifestKey) { - throw new Error( - "Parsed result snapshot was not written; refusing to mark source ready.", - ) - } - return { - assetUrlsByFilePath: loadedResult.assetUrlsByFilePath ?? {}, - snapshotManifestUrl: manifest.manifestUrl, - snapshotManifestKey: manifest.manifestKey, - } - }) - await context.run("save-parse-result-assets", async () => - sourceWorkflowRuntime.saveParseResult(workspaceId, sourceId, { - resultBlobUrl: snapshot.snapshotManifestUrl, - snapshotManifestUrl: snapshot.snapshotManifestUrl, - snapshotManifestKey: snapshot.snapshotManifestKey, - assetUrlsByFilePath: snapshot.assetUrlsByFilePath, + ) + logger.info("workflow: parsed storage sync handed off to parsed-sync", { + sourceId, + documentId: jobToPrepare.documentId, + revisionKey, + }) + return + } + + await context.run("record-sync-completed", async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "completed", }), ) - const ready = await context.run("source-ready", async () => markSourceReadyAfterReconciliation({ workspaceId, @@ -168,6 +214,7 @@ async function runPollAndMirrorWorkflow(input: { logger.info("workflow: source parse reconciliation finished", { sourceId, jobId: jobToPrepare.jobId, + revisionKey, status: ready.status, }) } diff --git a/src/domains/sources/source-reconcile-workflow.test.ts b/src/domains/sources/source-reconcile-workflow.test.ts index f1907ef..5757da8 100644 --- a/src/domains/sources/source-reconcile-workflow.test.ts +++ b/src/domains/sources/source-reconcile-workflow.test.ts @@ -24,6 +24,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 1, status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: null, stagedBlobPathname: null, diff --git a/src/domains/sources/source-row-repository.ts b/src/domains/sources/source-row-repository.ts index 77c56dd..2c06765 100644 --- a/src/domains/sources/source-row-repository.ts +++ b/src/domains/sources/source-row-repository.ts @@ -26,6 +26,7 @@ type SourceUpdate = Partial< | "sizeBytes" | "status" | "failureReason" + | "failureStage" | "knowhereJobId" | "knowhereDocumentId" | "stagedBlobPathname" @@ -83,6 +84,7 @@ type SourceRowRepository = { sourceId: string, reason: string, requiredStatus?: string, + failureStage?: string, ) => Effect.Effect readonly clearStagedBlobEffect: ( workspaceId: string, @@ -193,6 +195,7 @@ const markParsingEffect: SourceRowRepository["markParsingEffect"] = ( knowhereJobId: jobId, knowhereDocumentId: documentId, failureReason: null, + failureStage: null, }, requiredStatus) const markReadyEffect: SourceRowRepository["markReadyEffect"] = ( @@ -204,6 +207,7 @@ const markReadyEffect: SourceRowRepository["markReadyEffect"] = ( status: "ready", knowhereDocumentId: documentId, failureReason: null, + failureStage: null, }, "parsing") const updateRevisionKeyEffect: SourceRowRepository["updateRevisionKeyEffect"] = ( @@ -220,10 +224,12 @@ const markFailedEffect: SourceRowRepository["markFailedEffect"] = ( sourceId: string, reason: string, requiredStatus?: string, + failureStage?: string, ) => updateInWorkspaceEffect(workspaceId, sourceId, { status: "failed", failureReason: reason, + failureStage: failureStage ?? null, }, requiredStatus) const clearStagedBlobEffect: SourceRowRepository["clearStagedBlobEffect"] = ( diff --git a/src/domains/sources/upload.test.ts b/src/domains/sources/upload.test.ts index 912ab9d..77c9022 100644 --- a/src/domains/sources/upload.test.ts +++ b/src/domains/sources/upload.test.ts @@ -23,6 +23,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 12, status: "uploading", failureReason: null, + failureStage: null, knowhereJobId: null, knowhereDocumentId: null, stagedBlobPathname: null, diff --git a/src/domains/sources/view.test.ts b/src/domains/sources/view.test.ts index e854c29..cc5b223 100644 --- a/src/domains/sources/view.test.ts +++ b/src/domains/sources/view.test.ts @@ -12,6 +12,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 1, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: "doc_1", stagedBlobPathname: null, diff --git a/src/domains/sources/workflow-runtime.test.ts b/src/domains/sources/workflow-runtime.test.ts index 3cc714a..08e9853 100644 --- a/src/domains/sources/workflow-runtime.test.ts +++ b/src/domains/sources/workflow-runtime.test.ts @@ -31,6 +31,7 @@ function makeSource(status: Source["status"]): Source { sizeBytes: 1024, status, failureReason: status === "failed" ? "failed" : null, + failureStage: null, knowhereJobId: status === "parsing" ? "job_1" : null, knowhereDocumentId: status === "ready" ? "document_1" : null, stagedBlobPathname: null, diff --git a/src/domains/sources/workflow-runtime.ts b/src/domains/sources/workflow-runtime.ts index 8527fab..24464e0 100644 --- a/src/domains/sources/workflow-runtime.ts +++ b/src/domains/sources/workflow-runtime.ts @@ -2,6 +2,7 @@ import "server-only" import { databaseRuntime } from "@/domains/workspace/database-runtime" import { sourceRepository } from "./repository" +import type { SourceParseSnapshotMetadata } from "./source-parse-result-repository" import type { Source, SourceParseResult } from "@/infrastructure/db/schema" import type { UploadSourceRepository } from "./upload" @@ -17,6 +18,10 @@ type SaveSourceParseResultInput = Parameters< typeof sourceRepository.saveParseResultEffect >[2] +type UpdateSyncStatusInput = Parameters< + typeof sourceRepository.updateSyncStatusEffect +>[2] + type UpsertMaterializedDemoSourceInput = Parameters< typeof sourceRepository.upsertMaterializedDemoSourceEffect >[1] @@ -38,6 +43,7 @@ type UploadRepositoryRuntime = { sourceId: string, reason: string, requiredStatus?: string, + failureStage?: string, ) => Promise } @@ -60,21 +66,11 @@ type SourceWorkflowRuntime = UploadRepositoryRuntime & { readonly getParseResultProgress: ( workspaceId: string, sourceId: string, - ) => Promise<{ - readonly resultBlobUrl: string - readonly snapshotManifestUrl?: string | null - readonly snapshotManifestKey?: string | null - readonly assetUrlsByFilePath: Readonly> - } | null> + ) => Promise readonly getParseSnapshotMetadata: ( workspaceId: string, sourceId: string, - ) => Promise<{ - readonly resultBlobUrl: string - readonly snapshotManifestUrl?: string | null - readonly snapshotManifestKey?: string | null - readonly assetUrlsByFilePath: Readonly> - } | null> + ) => Promise readonly listForWorkspace: (workspaceId: string) => Promise readonly localizeRemoteDocument: ( workspaceId: string, @@ -105,6 +101,11 @@ type SourceWorkflowRuntime = UploadRepositoryRuntime & { sourceId: string, input: SaveSourceParseResultInput, ) => Promise + readonly updateSyncStatus: ( + workspaceId: string, + sourceId: string, + input: UpdateSyncStatusInput, + ) => Promise readonly softDelete: ( workspaceId: string, sourceId: string, @@ -200,9 +201,16 @@ const markFailed: SourceWorkflowRuntime["markFailed"] = ( sourceId: string, reason: string, requiredStatus?: string, + failureStage?: string, ) => databaseRuntime.runPromise( - sourceRepository.markFailedEffect(workspaceId, sourceId, reason, requiredStatus), + sourceRepository.markFailedEffect( + workspaceId, + sourceId, + reason, + requiredStatus, + failureStage, + ), ) const clearStagedBlob: SourceWorkflowRuntime["clearStagedBlob"] = ( @@ -245,6 +253,15 @@ const mergeParseAssetUrls: SourceWorkflowRuntime["mergeParseAssetUrls"] = ( sourceRepository.mergeParseAssetUrlsEffect(workspaceId, sourceId, input), ) +const updateSyncStatus: SourceWorkflowRuntime["updateSyncStatus"] = ( + workspaceId: string, + sourceId: string, + input: UpdateSyncStatusInput, +) => + databaseRuntime.runPromise( + sourceRepository.updateSyncStatusEffect(workspaceId, sourceId, input), + ) + const getParseResultProgress: SourceWorkflowRuntime["getParseResultProgress"] = (workspaceId: string, sourceId: string) => databaseRuntime.runPromise( @@ -316,5 +333,6 @@ export const sourceWorkflowRuntime: SourceWorkflowRuntime = { mergeParseAssetUrls, saveParseResult, softDelete, + updateSyncStatus, upsertMaterializedDemoSource, } diff --git a/src/domains/workspace/initial-state.test.ts b/src/domains/workspace/initial-state.test.ts index 9ea24e3..23bd50f 100644 --- a/src/domains/workspace/initial-state.test.ts +++ b/src/domains/workspace/initial-state.test.ts @@ -647,6 +647,7 @@ function makeSource( sizeBytes: 1024, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: "document_1", stagedBlobPathname: null, diff --git a/src/domains/workspace/integration.test.ts b/src/domains/workspace/integration.test.ts index 9d27083..8b6d80b 100644 --- a/src/domains/workspace/integration.test.ts +++ b/src/domains/workspace/integration.test.ts @@ -145,7 +145,7 @@ describeIfDb("workspace helpers — integration", () => { markSourceReady: sourceWorkflowRuntime.markReady, markSourceFailed: sourceWorkflowRuntime.markFailed, saveSourceParseResult: sourceWorkflowRuntime.saveParseResult, - getParseAssetUrls: sourceService.getParseAssetUrls, + getParseAssetUrls: sourceWorkflowRuntime.getParseAssetUrls, hideDemoSource: sourceService.hideDemoSource, listHiddenDemoSourceIds: sourceService.listHiddenDemoSourceIds, upsertMaterializedDemoSource: diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index 8c909a9..c24b27c 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -66,6 +66,8 @@ export type NewWorkspace = typeof workspaces.$inferInsert; * - `size_bytes` — original upload size (for display + quota) * - `status` — lifecycle: uploading | parsing | ready | failed * - `failure_reason` — human-readable error text when status=failed + * - `failure_stage` — which stage failed: parse | storage_sync; drives + * whether a retry reparses or only resumes storage sync * - `knowhere_job_id` — set once the parse job is created * - `knowhere_document_id` — set when parsing completes; used to import * parsed snapshots and to exclude a source from a @@ -95,6 +97,7 @@ export const sources = pgTable( sizeBytes: bigint("size_bytes", { mode: "number" }).notNull(), status: text("status").notNull(), failureReason: text("failure_reason"), + failureStage: text("failure_stage"), knowhereJobId: text("knowhere_job_id"), knowhereDocumentId: text("knowhere_document_id"), stagedBlobPathname: text("staged_blob_pathname"), @@ -162,8 +165,18 @@ export type NewDemoSourceVisibility = typeof demoSourceVisibilities.$inferInsert * Notebook-owned parse-result artifact index for one source. * * Blob is the Notebook-owned read model for parsed chunks after source - * reconciliation completes. This row stores the current parsed snapshot - * manifest and the file-path-to-public-URL map for parsed media artifacts. + * reconciliation completes. The parsed snapshot itself (manifest, chunk pages, + * assets, and resumable sync progress) lives in Vercel Blob under + * `workspaces/{ws}/parsed-documents/{documentId}/{revisionKey}/...`, managed by + * the SDK `ParsedDocumentStorage`. This row records: + * - `revision_key` — current parsed revision (jobResultId ?? jobId); the + * storage fast-path key passed into SDK reads + * - `sync_status` — pending | running | completed | failed for the + * background/parse-time storage sync + * - `sync_error` — last storage-sync error detail when sync_status=failed + * - `result_blob_url` / `snapshot_manifest_*` — legacy columns retained for + * rows written by the pre-migration manifest format + * - `asset_urls` — legacy file-path-to-public-URL map for older rows */ export const sourceParseResults = pgTable( "source_parse_results", @@ -173,9 +186,12 @@ export const sourceParseResults = pgTable( .notNull() .references(() => sources.id, { onDelete: "cascade" }) .unique(), - resultBlobUrl: text("result_blob_url").notNull(), + resultBlobUrl: text("result_blob_url"), snapshotManifestUrl: text("snapshot_manifest_url"), snapshotManifestKey: text("snapshot_manifest_key"), + revisionKey: text("revision_key"), + syncStatus: text("sync_status"), + syncError: text("sync_error"), assetUrls: jsonb("asset_urls") .$type>>() .notNull(), diff --git a/src/integrations/knowhere.ts b/src/integrations/knowhere.ts index 81f6726..c0d074d 100644 --- a/src/integrations/knowhere.ts +++ b/src/integrations/knowhere.ts @@ -1,5 +1,35 @@ import Knowhere from "@ontos-ai/knowhere-sdk" +import type { + Knowledge, + ParsedDocumentStorageLimits, + ParsedDocumentSyncScheduler, +} from "@ontos-ai/knowhere-sdk" import { logger } from "@/lib/logger" +import { BlobParsedDocumentStorage } from "@/domains/sources/parsed-document-blob-storage" + +/** + * Vercel-safe defaults for parsed-document reads and background sync. Each read + * miss and each background sync step is bounded by page count and a deadline so + * a single serverless invocation stays well under the platform ceiling; the + * SDK returns `completed:false` and the caller re-enqueues to continue. + */ +const defaultParsedStorageLimits: ParsedDocumentStorageLimits = { + chunkPageSize: 200, + remotePageSize: 100, + maxPagesPerSync: 10, + maxAssetsPerSync: 20, + syncDeadlineMs: 8000, + grepMaxPages: 50, + grepDeadlineMs: 8000, + outlineMaxPages: 50, + outlineDeadlineMs: 8000, +} + +type ParsedStorageOptions = { + readonly workspaceId: string + readonly scheduler?: ParsedDocumentSyncScheduler + readonly limits?: ParsedDocumentStorageLimits +} /** * Create a Knowhere client with the given API key. @@ -14,6 +44,29 @@ export function makeKnowhereClient(apiKey: string): Knowhere { return wrapKnowhereClient(client) } +/** + * Create a Knowhere client plus a `Knowledge` configured with a Vercel-Blob + * `ParsedDocumentStorage`. Reads through `knowledge` serve from Blob first and + * fall back to Knowhere remote transparently; the `scheduler` (when provided) + * backfills Blob in the background. Use `client` for retrieval/documents/jobs. + * + * `withParsedStorage` is invoked through the logging Proxy so `this` binds to + * the real Knowledge; the returned Knowledge wraps the unwrapped inner client + * (its internal `documents.listChunks` calls are not logged, which is fine). + */ +export function makeKnowhereClientWithParsedStorage( + apiKey: string, + options: ParsedStorageOptions, +): { readonly client: Knowhere; readonly knowledge: Knowledge } { + const client = makeKnowhereClient(apiKey) + const knowledge = client.knowledge.withParsedStorage({ + storage: new BlobParsedDocumentStorage({ workspaceId: options.workspaceId }), + scheduler: options.scheduler, + limits: options.limits ?? defaultParsedStorageLimits, + }) + return { client, knowledge } +} + function wrapKnowhereClient(client: Knowhere): Knowhere { return new Proxy(client, { get(target, prop, receiver) { From 012d729526abeca8e92bf421f7fc639d3c0cab90 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sun, 5 Jul 2026 14:09:21 +0800 Subject: [PATCH 11/12] Add parsed sync capacity guard --- drizzle/0012_lazy_wendell_rand.sql | 21 + drizzle/meta/0012_snapshot.json | 932 ++++++++++++++++++ drizzle/meta/_journal.json | 7 + .../parsed-document-sync-capacity.test.ts | 53 + .../sources/parsed-document-sync-capacity.ts | 304 ++++++ .../sources/parsed-document-sync-scheduler.ts | 4 + .../parsed-sync-route-workflow.test.ts | 113 ++- .../sources/parsed-sync-route-workflow.ts | 166 +++- .../source-reconcile-route-workflow.test.ts | 130 ++- .../source-reconcile-route-workflow.ts | 180 ++-- src/infrastructure/db/schema.ts | 55 ++ 11 files changed, 1853 insertions(+), 112 deletions(-) create mode 100644 drizzle/0012_lazy_wendell_rand.sql create mode 100644 drizzle/meta/0012_snapshot.json create mode 100644 src/domains/sources/parsed-document-sync-capacity.test.ts create mode 100644 src/domains/sources/parsed-document-sync-capacity.ts diff --git a/drizzle/0012_lazy_wendell_rand.sql b/drizzle/0012_lazy_wendell_rand.sql new file mode 100644 index 0000000..f50ab54 --- /dev/null +++ b/drizzle/0012_lazy_wendell_rand.sql @@ -0,0 +1,21 @@ +CREATE TABLE "parsed_document_sync_leases" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "source_id" uuid NOT NULL, + "document_id" text NOT NULL, + "revision_key" text, + "lease_token" text NOT NULL, + "acquired_at" timestamp with time zone DEFAULT now() NOT NULL, + "expires_at" timestamp with time zone NOT NULL, + "released_at" timestamp with time zone, + "release_reason" text, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "parsed_document_sync_leases" ADD CONSTRAINT "parsed_document_sync_leases_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "parsed_document_sync_leases" ADD CONSTRAINT "parsed_document_sync_leases_source_id_sources_id_fk" FOREIGN KEY ("source_id") REFERENCES "public"."sources"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "parsed_document_sync_leases_token_idx" ON "parsed_document_sync_leases" USING btree ("lease_token");--> statement-breakpoint +CREATE INDEX "parsed_document_sync_leases_active_idx" ON "parsed_document_sync_leases" USING btree ("expires_at") WHERE released_at IS NULL;--> statement-breakpoint +CREATE INDEX "parsed_document_sync_leases_workspace_active_idx" ON "parsed_document_sync_leases" USING btree ("workspace_id") WHERE released_at IS NULL;--> statement-breakpoint +CREATE INDEX "parsed_document_sync_leases_document_active_idx" ON "parsed_document_sync_leases" USING btree ("document_id") WHERE released_at IS NULL; \ No newline at end of file diff --git a/drizzle/meta/0012_snapshot.json b/drizzle/meta/0012_snapshot.json new file mode 100644 index 0000000..4011068 --- /dev/null +++ b/drizzle/meta/0012_snapshot.json @@ -0,0 +1,932 @@ +{ + "id": "a2a2f9a9-d567-4413-89d8-fa714b994351", + "prevId": "bab84445-8eb2-4d15-856e-b2f59edcaf47", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index fc814c6..93f0ced 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -85,6 +85,13 @@ "when": 1783175614191, "tag": "0011_rare_deadpool", "breakpoints": true + }, + { + "idx": 12, + "version": "7", + "when": 1783231408481, + "tag": "0012_lazy_wendell_rand", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/domains/sources/parsed-document-sync-capacity.test.ts b/src/domains/sources/parsed-document-sync-capacity.test.ts new file mode 100644 index 0000000..5800e09 --- /dev/null +++ b/src/domains/sources/parsed-document-sync-capacity.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest" + +import { parsedDocumentSyncCapacityGuard } from "./parsed-document-sync-capacity" + +describe("parsedDocumentSyncCapacityGuard", () => { + it("uses the configured defaults when environment variables are absent", () => { + expect(parsedDocumentSyncCapacityGuard.readPolicy({})).toEqual({ + globalActiveLimit: 100, + workspaceActiveLimit: 5, + documentActiveLimit: 1, + waitSeconds: 60, + }) + }) + + it("reads positive integer capacity settings from the environment", () => { + expect( + parsedDocumentSyncCapacityGuard.readPolicy({ + SYNC_GLOBAL_ACTIVE_LIMIT: "12", + SYNC_WORKSPACE_ACTIVE_LIMIT: "3", + SYNC_DOCUMENT_ACTIVE_LIMIT: "2", + SYNC_WAIT_SECONDS: "90", + }), + ).toEqual({ + globalActiveLimit: 12, + workspaceActiveLimit: 3, + documentActiveLimit: 2, + waitSeconds: 90, + }) + }) + + it("denies capacity by document, workspace, then global cap", () => { + const policy = parsedDocumentSyncCapacityGuard.readPolicy({}) + + expect( + parsedDocumentSyncCapacityGuard.selectDenialReason( + { globalActive: 0, workspaceActive: 0, documentActive: 1 }, + policy, + ), + ).toBe("document") + expect( + parsedDocumentSyncCapacityGuard.selectDenialReason( + { globalActive: 0, workspaceActive: 5, documentActive: 0 }, + policy, + ), + ).toBe("workspace") + expect( + parsedDocumentSyncCapacityGuard.selectDenialReason( + { globalActive: 100, workspaceActive: 0, documentActive: 0 }, + policy, + ), + ).toBe("global") + }) +}) diff --git a/src/domains/sources/parsed-document-sync-capacity.ts b/src/domains/sources/parsed-document-sync-capacity.ts new file mode 100644 index 0000000..cff5584 --- /dev/null +++ b/src/domains/sources/parsed-document-sync-capacity.ts @@ -0,0 +1,304 @@ +import "server-only" + +import { randomUUID } from "node:crypto" + +import { and, eq, isNull, sql } from "drizzle-orm" +import { Effect } from "effect" + +import { databaseRuntime } from "@/domains/workspace/database-runtime" +import { DbClient } from "@/infrastructure/db" +import { parsedDocumentSyncLeases } from "@/infrastructure/db/schema" + +type SyncCapacityPolicy = { + readonly globalActiveLimit: number + readonly workspaceActiveLimit: number + readonly documentActiveLimit: number + readonly waitSeconds: number +} + +type ActiveSyncCounts = { + readonly globalActive: number + readonly workspaceActive: number + readonly documentActive: number +} + +type SyncCapacityDenialReason = "global" | "workspace" | "document" + +type AcquireSyncLeaseInput = { + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly revisionKey?: string + readonly policy?: SyncCapacityPolicy +} + +type AcquireSyncLeaseResult = + | { + readonly kind: "acquired" + readonly leaseToken: string + readonly activeCounts: ActiveSyncCounts + } + | { + readonly kind: "capacity-full" + readonly reason: SyncCapacityDenialReason + readonly waitSeconds: number + readonly activeCounts: ActiveSyncCounts + } + | { + readonly kind: "source-missing" + } + +type ReleaseSyncLeaseInput = { + readonly leaseToken: string + readonly releaseReason: SyncLeaseReleaseReason +} + +type SyncLeaseReleaseReason = "completed" | "incomplete" | "failed" + +type AcquireSyncLeaseRow = { + readonly leaseId: string | null + readonly leaseToken: string | null + readonly hasSource: boolean + readonly globalActive: number | string + readonly workspaceActive: number | string + readonly documentActive: number | string +} + +type RawRowsResult = + | readonly Row[] + | { + readonly rows: readonly Row[] + } + +const defaultGlobalActiveLimit: number = 100 +const defaultWorkspaceActiveLimit: number = 5 +const defaultDocumentActiveLimit: number = 1 +const defaultWaitSeconds: number = 60 +const leaseDurationSeconds: number = 10 * 60 + +const capacityLockClassId: number = 24_071_105 +const capacityLockObjectId: number = 2_607_198 + +function readPolicy( + env: Readonly> = process.env, +): SyncCapacityPolicy { + return { + globalActiveLimit: readPositiveIntegerEnv( + "SYNC_GLOBAL_ACTIVE_LIMIT", + defaultGlobalActiveLimit, + env, + ), + workspaceActiveLimit: readPositiveIntegerEnv( + "SYNC_WORKSPACE_ACTIVE_LIMIT", + defaultWorkspaceActiveLimit, + env, + ), + documentActiveLimit: readPositiveIntegerEnv( + "SYNC_DOCUMENT_ACTIVE_LIMIT", + defaultDocumentActiveLimit, + env, + ), + waitSeconds: readPositiveIntegerEnv( + "SYNC_WAIT_SECONDS", + defaultWaitSeconds, + env, + ), + } +} + +function readPositiveIntegerEnv( + name: string, + defaultValue: number, + env: Readonly>, +): number { + const rawValue = env[name] + if (!rawValue) return defaultValue + + const value = Number(rawValue) + if (!Number.isInteger(value) || value < 1) return defaultValue + + return value +} + +function selectDenialReason( + counts: ActiveSyncCounts, + policy: SyncCapacityPolicy, +): SyncCapacityDenialReason | null { + if (counts.documentActive >= policy.documentActiveLimit) return "document" + if (counts.workspaceActive >= policy.workspaceActiveLimit) return "workspace" + if (counts.globalActive >= policy.globalActiveLimit) return "global" + + return null +} + +const acquireEffect = ( + input: AcquireSyncLeaseInput, +): Effect.Effect => + Effect.gen(function* () { + const db = yield* DbClient + const policy = input.policy ?? readPolicy() + const leaseToken = randomUUID() + const result = yield* Effect.promise(() => + db.execute(sql` + WITH lock AS ( + SELECT pg_advisory_xact_lock(${capacityLockClassId}, ${capacityLockObjectId}) + ), + expired AS ( + UPDATE parsed_document_sync_leases + SET released_at = now(), + release_reason = 'expired', + updated_at = now() + WHERE released_at IS NULL + AND expires_at <= now() + RETURNING id + ), + counts AS ( + SELECT + EXISTS ( + SELECT 1 + FROM sources + WHERE id = ${input.sourceId}::uuid + AND workspace_id = ${input.workspaceId}::uuid + AND deleted_at IS NULL + ) AS "hasSource", + ( + SELECT count(*)::int + FROM parsed_document_sync_leases + WHERE released_at IS NULL + AND expires_at > now() + ) AS "globalActive", + ( + SELECT count(*)::int + FROM parsed_document_sync_leases + WHERE workspace_id = ${input.workspaceId}::uuid + AND released_at IS NULL + AND expires_at > now() + ) AS "workspaceActive", + ( + SELECT count(*)::int + FROM parsed_document_sync_leases + WHERE document_id = ${input.documentId} + AND released_at IS NULL + AND expires_at > now() + ) AS "documentActive" + FROM lock + ), + inserted AS ( + INSERT INTO parsed_document_sync_leases ( + workspace_id, + source_id, + document_id, + revision_key, + lease_token, + expires_at + ) + SELECT + ${input.workspaceId}::uuid, + ${input.sourceId}::uuid, + ${input.documentId}, + ${input.revisionKey ?? null}, + ${leaseToken}, + now() + (${leaseDurationSeconds} * interval '1 second') + FROM counts + WHERE "hasSource" + AND "globalActive" < ${policy.globalActiveLimit} + AND "workspaceActive" < ${policy.workspaceActiveLimit} + AND "documentActive" < ${policy.documentActiveLimit} + RETURNING id AS "leaseId", lease_token AS "leaseToken" + ) + SELECT + (SELECT "leaseId" FROM inserted) AS "leaseId", + (SELECT "leaseToken" FROM inserted) AS "leaseToken", + counts."hasSource", + counts."globalActive", + counts."workspaceActive", + counts."documentActive" + FROM counts + `), + ) + const row = getFirstRow(getRawRows(result)) + if (!row) return { kind: "source-missing" } + if (!row.hasSource) return { kind: "source-missing" } + + const activeCounts = normalizeActiveCounts(row) + if (row.leaseToken) { + return { + kind: "acquired", + leaseToken: row.leaseToken, + activeCounts, + } + } + + return { + kind: "capacity-full", + reason: selectDenialReason(activeCounts, policy) ?? "global", + waitSeconds: policy.waitSeconds, + activeCounts, + } + }) + +const releaseEffect = ( + input: ReleaseSyncLeaseInput, +): Effect.Effect => + Effect.gen(function* () { + const db = yield* DbClient + yield* Effect.promise(() => + db + .update(parsedDocumentSyncLeases) + .set({ + releasedAt: sql`now()`, + releaseReason: input.releaseReason, + updatedAt: sql`now()`, + }) + .where( + and( + eq(parsedDocumentSyncLeases.leaseToken, input.leaseToken), + isNull(parsedDocumentSyncLeases.releasedAt), + ), + ), + ) + }) + +function normalizeActiveCounts(row: AcquireSyncLeaseRow): ActiveSyncCounts { + return { + globalActive: normalizeCount(row.globalActive), + workspaceActive: normalizeCount(row.workspaceActive), + documentActive: normalizeCount(row.documentActive), + } +} + +function normalizeCount(value: number | string): number { + return typeof value === "number" ? value : Number(value) +} + +function getRawRows(value: RawRowsResult): readonly Row[] { + if (isReadonlyArray(value)) return value + return value.rows +} + +function getFirstRow(rows: readonly Row[]): Row | undefined { + return rows[0] +} + +function isReadonlyArray( + value: RawRowsResult, +): value is readonly Row[] { + return Array.isArray(value) +} + +async function acquire( + input: AcquireSyncLeaseInput, +): Promise { + return databaseRuntime.runPromise(acquireEffect(input)) +} + +async function release(input: ReleaseSyncLeaseInput): Promise { + await databaseRuntime.runPromise(releaseEffect(input)) +} + +export const parsedDocumentSyncCapacityGuard = { + acquire, + readPolicy, + release, + selectDenialReason, +} diff --git a/src/domains/sources/parsed-document-sync-scheduler.ts b/src/domains/sources/parsed-document-sync-scheduler.ts index 5d2957d..c0ee4ce 100644 --- a/src/domains/sources/parsed-document-sync-scheduler.ts +++ b/src/domains/sources/parsed-document-sync-scheduler.ts @@ -24,6 +24,7 @@ export type ParsedSyncTrigger = (input: { readonly url: string readonly body: ParsedSyncPayload readonly workflowRunId: string + readonly delaySeconds?: number }) => Promise export type ParsedSyncPayload = { @@ -72,6 +73,7 @@ const defaultTrigger: ParsedSyncTrigger = async (input) => { body: input.body, workflowRunId: input.workflowRunId, retries: 3, + delay: input.delaySeconds, }) } @@ -91,6 +93,7 @@ export async function enqueueParsedDocumentSync( readonly documentId: string readonly apiKey: string readonly revisionKey?: string + readonly delaySeconds?: number }, trigger: ParsedSyncTrigger = defaultTrigger, ): Promise { @@ -102,6 +105,7 @@ export async function enqueueParsedDocumentSync( revisionKey: input.revisionKey ?? "initial", segmentIndex: 0, }), + delaySeconds: input.delaySeconds, }) } diff --git a/src/domains/sources/parsed-sync-route-workflow.test.ts b/src/domains/sources/parsed-sync-route-workflow.test.ts index 555c186..91095f5 100644 --- a/src/domains/sources/parsed-sync-route-workflow.test.ts +++ b/src/domains/sources/parsed-sync-route-workflow.test.ts @@ -1,7 +1,9 @@ -import { afterEach, describe, expect, it, vi } from "vitest" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ + acquireSyncCapacity: vi.fn(), makeKnowhereClientWithParsedStorage: vi.fn(), + releaseSyncCapacity: vi.fn(), updateSyncStatus: vi.fn(), findInWorkspace: vi.fn(), markFailed: vi.fn(), @@ -28,6 +30,13 @@ vi.mock("./source-reconcile-workflow", () => ({ markSourceReadyAfterReconciliation: mocks.markSourceReadyAfterReconciliation, })) +vi.mock("./parsed-document-sync-capacity", () => ({ + parsedDocumentSyncCapacityGuard: { + acquire: mocks.acquireSyncCapacity, + release: mocks.releaseSyncCapacity, + }, +})) + vi.mock("@/lib/logger", () => ({ logger: { info: mocks.loggerInfo, error: mocks.loggerError }, })) @@ -53,6 +62,22 @@ const basePayload = { } describe("parsedSyncRouteWorkflow.runParsedSyncWorkflow", () => { + beforeEach(() => { + mocks.acquireSyncCapacity.mockResolvedValue({ + kind: "acquired", + leaseToken: "lease_1", + activeCounts: { + globalActive: 0, + workspaceActive: 0, + documentActive: 0, + }, + }) + mocks.releaseSyncCapacity.mockResolvedValue(undefined) + mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ + status: "ready", + }) + }) + afterEach(() => { vi.clearAllMocks() }) @@ -67,9 +92,6 @@ describe("parsedSyncRouteWorkflow.runParsedSyncWorkflow", () => { client: {}, knowledge: { syncParsedDocument }, }) - mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ - status: "ready", - }) await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ context: createContext(), @@ -88,9 +110,13 @@ describe("parsedSyncRouteWorkflow.runParsedSyncWorkflow", () => { sourceId: "source_1", documentId: "doc_1", }) + expect(mocks.releaseSyncCapacity).toHaveBeenCalledWith({ + leaseToken: "lease_1", + releaseReason: "completed", + }) }) - it("triggers a continuation and does not mark ready when sync is incomplete", async () => { + it("marks ready and triggers a continuation when sync is incomplete", async () => { const syncParsedDocument = vi.fn(async () => ({ documentId: "doc_1", revisionKey: "rev_1", @@ -119,7 +145,11 @@ describe("parsedSyncRouteWorkflow.runParsedSyncWorkflow", () => { restore() } - expect(mocks.markSourceReadyAfterReconciliation).not.toHaveBeenCalled() + expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + }) expect(triggered).toHaveLength(1) expect(triggered[0]?.segmentIndex).toBe(1) expect(triggered[0]?.workflowRunId).toBe("doc_1-sync-rev_1-1") @@ -128,6 +158,10 @@ describe("parsedSyncRouteWorkflow.runParsedSyncWorkflow", () => { "source_1", { revisionKey: "rev_1", syncStatus: "running" }, ) + expect(mocks.releaseSyncCapacity).toHaveBeenCalledWith({ + leaseToken: "lease_1", + releaseReason: "incomplete", + }) }) it("passes an explicit revisionKey into syncParsedDocument when provided", async () => { @@ -140,9 +174,6 @@ describe("parsedSyncRouteWorkflow.runParsedSyncWorkflow", () => { client: {}, knowledge: { syncParsedDocument }, }) - mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ - status: "ready", - }) await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ context: createContext(), @@ -154,6 +185,70 @@ describe("parsedSyncRouteWorkflow.runParsedSyncWorkflow", () => { revisionKey: "rev_9", }) }) + + it("schedules a delayed retry when capacity is full", async () => { + const syncParsedDocument = vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_1", + completed: true, + })) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: {}, + knowledge: { syncParsedDocument }, + }) + mocks.acquireSyncCapacity.mockResolvedValue({ + kind: "capacity-full", + reason: "workspace", + waitSeconds: 60, + activeCounts: { + globalActive: 10, + workspaceActive: 5, + documentActive: 0, + }, + }) + const triggered: Array<{ + readonly workflowRunId: string + readonly segmentIndex?: number + readonly delaySeconds?: number + }> = [] + const restore = parsedSyncRouteWorkflow.setContinuationTriggerForTesting( + async (input) => { + triggered.push({ + workflowRunId: input.workflowRunId, + segmentIndex: input.payload.segmentIndex, + delaySeconds: input.delaySeconds, + }) + }, + ) + + try { + await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ + context: createContext(), + payload: { ...basePayload, revisionKey: "rev_1" }, + }) + } finally { + restore() + } + + expect(syncParsedDocument).not.toHaveBeenCalled() + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { + revisionKey: "rev_1", + syncStatus: "pending", + syncError: null, + }, + ) + expect(triggered).toEqual([ + { + workflowRunId: "doc_1-sync-rev_1-1", + segmentIndex: 1, + delaySeconds: 60, + }, + ]) + expect(mocks.releaseSyncCapacity).not.toHaveBeenCalled() + }) }) describe("parsedSyncRouteWorkflow.markSyncFailedAfterWorkflowFailure", () => { diff --git a/src/domains/sources/parsed-sync-route-workflow.ts b/src/domains/sources/parsed-sync-route-workflow.ts index c33ace8..6be3b23 100644 --- a/src/domains/sources/parsed-sync-route-workflow.ts +++ b/src/domains/sources/parsed-sync-route-workflow.ts @@ -9,6 +9,7 @@ import { getParsedSyncWorkflowRunId, type ParsedSyncPayload, } from "./parsed-document-sync-scheduler" +import { parsedDocumentSyncCapacityGuard } from "./parsed-document-sync-capacity" import { markSourceReadyAfterReconciliation } from "./source-reconcile-workflow" import { sourceWorkflowRuntime } from "./workflow-runtime" @@ -30,8 +31,11 @@ type ContinuationTriggerInput = { readonly url: string readonly payload: ParsedSyncPayload readonly workflowRunId: string + readonly delaySeconds?: number } +type SyncLeaseReleaseReason = "completed" | "incomplete" | "failed" + // Sync steps per workflow segment. Each `syncParsedDocument` call is bounded by // the SDK limits (pages + deadline); this caps how many bounded steps we run in // one serverless invocation before handing off to a fresh continuation. @@ -68,37 +72,45 @@ async function runParsedSyncWorkflow(input: { workspaceId, }) + const preSyncReady = await context.run( + `source-ready-before-sync-${payload.segmentIndex}`, + async () => + markSourceReadyAfterReconciliation({ + workspaceId, + sourceId, + documentId, + }), + ) + if (preSyncReady.status === "gone") return + let revisionKey = payload.revisionKey let completed = false + let releaseReason: SyncLeaseReleaseReason = "incomplete" - for (let step = 0; step < maxSyncStepsPerSegment; step++) { - const result: KnowledgeSyncParsedDocumentResponse = await context.run( - `sync-${payload.segmentIndex}-${step}`, - async () => - knowledge.syncParsedDocument({ - documentId, - ...(revisionKey ? { revisionKey } : {}), - }), - ) - revisionKey = result.revisionKey - - await context.run(`record-progress-${payload.segmentIndex}-${step}`, () => - sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + const capacity = await context.run( + `acquire-sync-capacity-${payload.segmentIndex}`, + async () => + parsedDocumentSyncCapacityGuard.acquire({ + workspaceId, + sourceId, + documentId, revisionKey, - syncStatus: result.completed ? "completed" : "running", }), + ) + if (capacity.kind === "source-missing") return + if (capacity.kind === "capacity-full") { + await context.run( + `record-capacity-wait-${payload.segmentIndex}`, + async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "pending", + syncError: null, + }), ) - - if (result.completed) { - completed = true - break - } - } - - if (!completed) { const nextSegmentIndex = payload.segmentIndex + 1 await context.run( - `trigger-sync-continuation-${nextSegmentIndex}`, + `trigger-sync-capacity-retry-${nextSegmentIndex}`, async () => triggerContinuation({ url: context.url, @@ -115,33 +127,116 @@ async function runParsedSyncWorkflow(input: { revisionKey: revisionKey ?? "initial", segmentIndex: nextSegmentIndex, }), + delaySeconds: capacity.waitSeconds, }), ) - logger.info("parsed-sync: continuation triggered", { + logger.info("parsed-sync: capacity full; retry scheduled", { sourceId, documentId, - segmentIndex: nextSegmentIndex, + reason: capacity.reason, + waitSeconds: capacity.waitSeconds, + activeCounts: capacity.activeCounts, }) return } - // A parsing source is only readied once its parsed snapshot is fully synced. - // For an already-ready source (read-miss backfill) markReady is a no-op guard. - const ready = await context.run("source-ready", async () => - markSourceReadyAfterReconciliation({ - workspaceId, - sourceId, - documentId, - }), - ) + try { + for (let step = 0; step < maxSyncStepsPerSegment; step++) { + const result: KnowledgeSyncParsedDocumentResponse = await context.run( + `sync-${payload.segmentIndex}-${step}`, + async () => + knowledge.syncParsedDocument({ + documentId, + ...(revisionKey ? { revisionKey } : {}), + }), + ) + revisionKey = result.revisionKey + + await context.run(`record-progress-${payload.segmentIndex}-${step}`, () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: result.completed ? "completed" : "running", + }), + ) + + if (result.completed) { + completed = true + releaseReason = "completed" + break + } + } + + if (!completed) { + const nextSegmentIndex = payload.segmentIndex + 1 + await context.run( + `trigger-sync-continuation-${nextSegmentIndex}`, + async () => + triggerContinuation({ + url: context.url, + payload: { + workspaceId, + sourceId, + documentId, + apiKey, + revisionKey, + segmentIndex: nextSegmentIndex, + }, + workflowRunId: getParsedSyncWorkflowRunId({ + documentId, + revisionKey: revisionKey ?? "initial", + segmentIndex: nextSegmentIndex, + }), + }), + ) + logger.info("parsed-sync: continuation triggered", { + sourceId, + documentId, + segmentIndex: nextSegmentIndex, + }) + return + } + } catch (error) { + releaseReason = "failed" + throw error + } finally { + await context.run(`release-sync-capacity-${payload.segmentIndex}`, async () => + releaseCapacityLease({ + leaseToken: capacity.leaseToken, + releaseReason, + sourceId, + documentId, + }), + ) + } + logger.info("parsed-sync: parsed document sync finished", { sourceId, documentId, revisionKey, - status: ready.status, + status: preSyncReady.status, }) } +async function releaseCapacityLease(input: { + readonly leaseToken: string + readonly releaseReason: SyncLeaseReleaseReason + readonly sourceId: string + readonly documentId: string +}): Promise { + try { + await parsedDocumentSyncCapacityGuard.release({ + leaseToken: input.leaseToken, + releaseReason: input.releaseReason, + }) + } catch (error) { + logger.error("parsed-sync: failed to release capacity lease", { + sourceId: input.sourceId, + documentId: input.documentId, + error: error instanceof Error ? error.message : String(error), + }) + } +} + async function triggerParsedSyncContinuation( input: ContinuationTriggerInput, ): Promise { @@ -155,6 +250,7 @@ async function triggerParsedSyncContinuation( body: input.payload, workflowRunId: input.workflowRunId, retries: 3, + delay: input.delaySeconds, }) } diff --git a/src/domains/sources/source-reconcile-route-workflow.test.ts b/src/domains/sources/source-reconcile-route-workflow.test.ts index 52f9c6b..ee415ff 100644 --- a/src/domains/sources/source-reconcile-route-workflow.test.ts +++ b/src/domains/sources/source-reconcile-route-workflow.test.ts @@ -1,7 +1,9 @@ -import { afterEach, describe, expect, it, vi } from "vitest" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ + acquireSyncCapacity: vi.fn(), enqueueParsedDocumentSync: vi.fn(), + releaseSyncCapacity: vi.fn(), updateSyncStatus: vi.fn(), markFailed: vi.fn(), loggerError: vi.fn(), @@ -28,6 +30,13 @@ vi.mock("./parsed-document-sync-scheduler", () => ({ enqueueParsedDocumentSync: mocks.enqueueParsedDocumentSync, })) +vi.mock("./parsed-document-sync-capacity", () => ({ + parsedDocumentSyncCapacityGuard: { + acquire: mocks.acquireSyncCapacity, + release: mocks.releaseSyncCapacity, + }, +})) + vi.mock("@/integrations/knowhere", () => ({ makeKnowhereClientWithParsedStorage: mocks.makeKnowhereClientWithParsedStorage, @@ -43,6 +52,12 @@ vi.mock("@/lib/logger", () => ({ import { sourceReconcileRouteWorkflow } from "./source-reconcile-route-workflow" +const activeCounts = { + globalActive: 0, + workspaceActive: 0, + documentActive: 0, +} + function createClient(overrides: { syncParsedDocument?: ReturnType jobResultId?: string @@ -68,6 +83,18 @@ function createClient(overrides: { } describe("sourceReconcileRouteWorkflow", () => { + beforeEach(() => { + mocks.acquireSyncCapacity.mockResolvedValue({ + kind: "acquired", + leaseToken: "lease_1", + activeCounts, + }) + mocks.releaseSyncCapacity.mockResolvedValue(undefined) + mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ + status: "ready", + }) + }) + afterEach(() => { vi.clearAllMocks() }) @@ -89,7 +116,7 @@ describe("sourceReconcileRouteWorkflow", () => { }) }) - it("syncs the parsed document then marks the source ready", async () => { + it("marks the source ready then syncs the parsed document", async () => { const context = createWorkflowContext() const continuations: ContinuationTriggerInput[] = [] const restore = @@ -103,9 +130,6 @@ describe("sourceReconcileRouteWorkflow", () => { client: wired.client, knowledge: wired.knowledge, }) - mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ - status: "ready", - }) mocks.pollSourceReconciliation.mockResolvedValue({ kind: "ready-to-prepare", jobId: "job_1", @@ -139,11 +163,21 @@ describe("sourceReconcileRouteWorkflow", () => { sourceId: "source_1", documentId: "doc_1", }) + expect(mocks.acquireSyncCapacity).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + revisionKey: "rev_1", + }) + expect(mocks.releaseSyncCapacity).toHaveBeenCalledWith({ + leaseToken: "lease_1", + releaseReason: "completed", + }) expect(mocks.enqueueParsedDocumentSync).not.toHaveBeenCalled() expect(continuations).toEqual([]) }) - it("hands off to parsed-sync and stays parsing when sync is incomplete", async () => { + it("hands off to parsed-sync when sync is incomplete", async () => { const context = createWorkflowContext() const syncParsedDocument = vi.fn(async () => ({ documentId: "doc_1", @@ -170,7 +204,11 @@ describe("sourceReconcileRouteWorkflow", () => { }), }) - expect(mocks.markSourceReadyAfterReconciliation).not.toHaveBeenCalled() + expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + }) expect(mocks.enqueueParsedDocumentSync).toHaveBeenCalledWith({ workspaceId: "workspace_1", sourceId: "source_1", @@ -178,9 +216,13 @@ describe("sourceReconcileRouteWorkflow", () => { apiKey: "jwt_1", revisionKey: "rev_1", }) + expect(mocks.releaseSyncCapacity).toHaveBeenCalledWith({ + leaseToken: "lease_1", + releaseReason: "incomplete", + }) }) - it("fails the source with storage_sync stage when sync throws", async () => { + it("records sync failure when Blob sync throws", async () => { const context = createWorkflowContext() const syncParsedDocument = vi.fn(async () => { throw new Error("blob write failed") @@ -207,14 +249,76 @@ describe("sourceReconcileRouteWorkflow", () => { }), ).rejects.toThrow("blob write failed") - expect(mocks.markFailed).toHaveBeenCalledWith( + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( "workspace_1", "source_1", - expect.stringContaining("storage sync failed"), - "parsing", - "storage_sync", + { + revisionKey: "rev_1", + syncStatus: "failed", + syncError: "blob write failed", + }, + ) + expect(mocks.markFailed).not.toHaveBeenCalled() + expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + }) + expect(mocks.releaseSyncCapacity).toHaveBeenCalledWith( + { + leaseToken: "lease_1", + releaseReason: "failed", + }, ) - expect(mocks.markSourceReadyAfterReconciliation).not.toHaveBeenCalled() + }) + + it("marks the source ready and schedules delayed sync when capacity is full", async () => { + const context = createWorkflowContext() + const wired = createClient({}) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: wired.client, + knowledge: wired.knowledge, + }) + mocks.acquireSyncCapacity.mockResolvedValue({ + kind: "capacity-full", + reason: "document", + waitSeconds: 60, + activeCounts: { + globalActive: 10, + workspaceActive: 1, + documentActive: 1, + }, + }) + mocks.pollSourceReconciliation.mockResolvedValue({ + kind: "ready-to-prepare", + jobId: "job_1", + documentId: "doc_1", + }) + + await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ + context, + payload: sourceReconcileRouteWorkflow.normalizeReconcilePayload({ + workspaceId: "workspace_1", + sourceId: "source_1", + apiKey: "jwt_1", + }), + }) + + expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + }) + expect(wired.knowledge.syncParsedDocument).not.toHaveBeenCalled() + expect(mocks.enqueueParsedDocumentSync).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + apiKey: "jwt_1", + revisionKey: "rev_1", + delaySeconds: 60, + }) + expect(mocks.releaseSyncCapacity).not.toHaveBeenCalled() }) it("triggers a fresh poll run when Knowhere is still running after the segment budget", async () => { diff --git a/src/domains/sources/source-reconcile-route-workflow.ts b/src/domains/sources/source-reconcile-route-workflow.ts index cfa90b3..1e45127 100644 --- a/src/domains/sources/source-reconcile-route-workflow.ts +++ b/src/domains/sources/source-reconcile-route-workflow.ts @@ -9,6 +9,7 @@ import { import { makeKnowhereClientWithParsedStorage } from "@/integrations/knowhere" import { logger } from "@/lib/logger" import { enqueueParsedDocumentSync } from "./parsed-document-sync-scheduler" +import { parsedDocumentSyncCapacityGuard } from "./parsed-document-sync-capacity" import { sourceWorkflowRuntime } from "./workflow-runtime" type ReconcilePayload = { @@ -40,6 +41,8 @@ type ContinuationTriggerInput = { readonly workflowRunId: string } +type SyncLeaseReleaseReason = "completed" | "incomplete" | "failed" + const maxPollAttempts = 25 const initialDelaySeconds = 3 const maxDelaySeconds = 30 @@ -122,7 +125,6 @@ async function runPollAndMirrorWorkflow(input: { return } - // Resolve the parsed revision and record that storage sync is in progress. const revisionKey = await context.run("resolve-revision-key", async () => { const firstPage = await client.documents.listChunks( jobToPrepare.documentId, @@ -130,87 +132,131 @@ async function runPollAndMirrorWorkflow(input: { ) return firstPage.jobResultId ?? firstPage.jobId ?? jobToPrepare.jobId }) - await context.run("record-sync-running", async () => + await context.run("record-sync-pending", async () => sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { revisionKey, - syncStatus: "running", + syncStatus: "pending", + syncError: null, }), ) - // Sync the parsed snapshot into Blob before the source is considered ready. - // Each step is bounded by SDK limits; if it does not finish in this segment, - // hand off to the resumable parsed-sync workflow and stay `parsing`. A sync - // error stages the failure as `storage_sync` so a retry resumes sync without - // reparsing, then rethrows to fail the workflow run. - let syncCompleted = false - for (let step = 0; step < maxSyncStepsPerReconcile; step++) { - const result = await context.run(`parsed-sync-${step}`, async () => { - try { - return await knowledge.syncParsedDocument({ - documentId: jobToPrepare.documentId, - revisionKey, - }) - } catch (error) { - await sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { - revisionKey, - syncStatus: "failed", - syncError: error instanceof Error ? error.message : String(error), - }) - await sourceWorkflowRuntime.markFailed( - workspaceId, - sourceId, - `Parsed document storage sync failed: ${ - error instanceof Error ? error.message : String(error) - }`, - "parsing", - "storage_sync", - ) - throw error - } - }) - if (result.completed) { - syncCompleted = true - break - } - } + const ready = await context.run("source-ready", async () => + markSourceReadyAfterReconciliation({ + workspaceId, + sourceId, + documentId: jobToPrepare.documentId, + }), + ) + if (ready.status === "gone") return - if (!syncCompleted) { - await context.run("record-sync-progress", async () => - sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { - revisionKey, - syncStatus: "running", - }), - ) - await context.run("enqueue-parsed-sync-continuation", async () => + const capacity = await context.run("acquire-sync-capacity", async () => + parsedDocumentSyncCapacityGuard.acquire({ + workspaceId, + sourceId, + documentId: jobToPrepare.documentId, + revisionKey, + }), + ) + if (capacity.kind === "source-missing") return + if (capacity.kind === "capacity-full") { + await context.run("enqueue-capacity-retry", async () => enqueueParsedDocumentSync({ workspaceId, sourceId, documentId: jobToPrepare.documentId, apiKey, revisionKey, + delaySeconds: capacity.waitSeconds, }), ) - logger.info("workflow: parsed storage sync handed off to parsed-sync", { + logger.info("workflow: parsed storage sync delayed by capacity guard", { sourceId, documentId: jobToPrepare.documentId, revisionKey, + reason: capacity.reason, + waitSeconds: capacity.waitSeconds, + activeCounts: capacity.activeCounts, }) return } + let syncCompleted = false + let releaseReason: SyncLeaseReleaseReason = "incomplete" + try { + await context.run("record-sync-running", async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "running", + syncError: null, + }), + ) + + for (let step = 0; step < maxSyncStepsPerReconcile; step++) { + const result = await context.run(`parsed-sync-${step}`, async () => { + try { + return await knowledge.syncParsedDocument({ + documentId: jobToPrepare.documentId, + revisionKey, + }) + } catch (error) { + await sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "failed", + syncError: getErrorMessage(error), + }) + throw error + } + }) + if (result.completed) { + syncCompleted = true + releaseReason = "completed" + break + } + } + + if (!syncCompleted) { + await context.run("record-sync-progress", async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "running", + }), + ) + await context.run("enqueue-parsed-sync-continuation", async () => + enqueueParsedDocumentSync({ + workspaceId, + sourceId, + documentId: jobToPrepare.documentId, + apiKey, + revisionKey, + }), + ) + logger.info("workflow: parsed storage sync handed off to parsed-sync", { + sourceId, + documentId: jobToPrepare.documentId, + revisionKey, + }) + return + } + } catch (error) { + releaseReason = "failed" + throw error + } finally { + await context.run("release-sync-capacity", async () => + releaseCapacityLease({ + leaseToken: capacity.leaseToken, + releaseReason, + sourceId, + documentId: jobToPrepare.documentId, + }), + ) + } + await context.run("record-sync-completed", async () => sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { revisionKey, syncStatus: "completed", }), ) - const ready = await context.run("source-ready", async () => - markSourceReadyAfterReconciliation({ - workspaceId, - sourceId, - documentId: jobToPrepare.documentId, - }), - ) logger.info("workflow: source parse reconciliation finished", { sourceId, jobId: jobToPrepare.jobId, @@ -219,6 +265,26 @@ async function runPollAndMirrorWorkflow(input: { }) } +async function releaseCapacityLease(input: { + readonly leaseToken: string + readonly releaseReason: SyncLeaseReleaseReason + readonly sourceId: string + readonly documentId: string +}): Promise { + try { + await parsedDocumentSyncCapacityGuard.release({ + leaseToken: input.leaseToken, + releaseReason: input.releaseReason, + }) + } catch (error) { + logger.error("workflow: failed to release sync capacity lease", { + sourceId: input.sourceId, + documentId: input.documentId, + error: getErrorMessage(error), + }) + } +} + function normalizeReconcilePayload( payload: ReconcilePayload, ): NormalizedReconcilePayload { @@ -295,6 +361,10 @@ function getSafeFailureReason(value: string): string { return normalized.slice(0, 500) } +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + export const sourceReconcileRouteWorkflow = { getPollWorkflowRunId, markSourceFailedAfterWorkflowFailure, diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index c24b27c..4e9470f 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -208,6 +208,61 @@ export const sourceParseResults = pgTable( export type SourceParseResult = typeof sourceParseResults.$inferSelect; export type NewSourceParseResult = typeof sourceParseResults.$inferInsert; +/** + * Durable active-work leases for Notebook-owned parsed-document Blob sync. + * + * The sync workers acquire one row before calling the Knowhere SDK's + * Vercel-Blob mirror. Active rows (`released_at IS NULL`) are counted globally, + * per workspace, and per document so Vercel can scale route invocations without + * allowing one user or one document to consume all sync capacity. Expired rows + * are released during the next acquire attempt; normal workers release in a + * `finally` block after their bounded sync segment exits. + */ +export const parsedDocumentSyncLeases = pgTable( + "parsed_document_sync_leases", + { + id: uuid("id").primaryKey().defaultRandom(), + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + sourceId: uuid("source_id") + .notNull() + .references(() => sources.id, { onDelete: "cascade" }), + documentId: text("document_id").notNull(), + revisionKey: text("revision_key"), + leaseToken: text("lease_token").notNull(), + acquiredAt: timestamp("acquired_at", { withTimezone: true }) + .notNull() + .defaultNow(), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + releasedAt: timestamp("released_at", { withTimezone: true }), + releaseReason: text("release_reason"), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + uniqueIndex("parsed_document_sync_leases_token_idx").on(t.leaseToken), + index("parsed_document_sync_leases_active_idx") + .on(t.expiresAt) + .where(sql`released_at IS NULL`), + index("parsed_document_sync_leases_workspace_active_idx") + .on(t.workspaceId) + .where(sql`released_at IS NULL`), + index("parsed_document_sync_leases_document_active_idx") + .on(t.documentId) + .where(sql`released_at IS NULL`), + ], +); + +export type ParsedDocumentSyncLease = + typeof parsedDocumentSyncLeases.$inferSelect; +export type NewParsedDocumentSyncLease = + typeof parsedDocumentSyncLeases.$inferInsert; + /** * A chat thread is a conversation within a workspace. `demo_key` is retained * for legacy seeded demo conversations. From 1d0d6a9ec684ac41a4f967a4df45c470a4217e96 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Sun, 5 Jul 2026 15:46:58 +0800 Subject: [PATCH 12/12] Update Knowhere SDK to 2.1.1 --- package-lock.json | 10 +++++----- package.json | 2 +- pnpm-lock.yaml | 13 +++++-------- pnpm-workspace.yaml | 5 +---- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index b8d1d91..e465210 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@antv/chart-visualization-skills": "0.1.3", "@effect/platform": "^0.96.1", "@neondatabase/serverless": "^1.1.0", - "@ontos-ai/knowhere-sdk": "^2.0.0", + "@ontos-ai/knowhere-sdk": "^2.1.1", "@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-dialog": "^1.1.15", @@ -2275,16 +2275,16 @@ } }, "node_modules/@ontos-ai/knowhere-sdk": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@ontos-ai/knowhere-sdk/-/knowhere-sdk-2.0.0.tgz", - "integrity": "sha512-Zhx8mn/8pYyhBaO9BOdQYsVZybGXyq15YctjHSU8TwgNjJEzX5lEiBEkp0kIGFPh3S+/7X3rZGakefEPf5G2tQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@ontos-ai/knowhere-sdk/-/knowhere-sdk-2.1.1.tgz", + "integrity": "sha512-K33ylB/QjVYlLgow+/Hw+uNhQbWWrMTIqZ3IAlx5buHSI2EHPVP9+kDsfslwpfffYSn85j9Ab0/E9qTG0QotTw==", "license": "MIT", "dependencies": { "axios": "^1.15.2", "jszip": "^3.10.0" }, "engines": { - "node": ">=20.19.0", + "node": ">=22.13.0", "npm": ">=10.0.0", "pnpm": ">=9.0.0" } diff --git a/package.json b/package.json index c998de4..94d1c3a 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "@effect/platform": "^0.96.1", "@napi-rs/canvas": "^1.0.2", "@neondatabase/serverless": "^1.1.0", - "@ontos-ai/knowhere-sdk": "^2.0.0", + "@ontos-ai/knowhere-sdk": "^2.1.1", "@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-dialog": "^1.1.15", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 94e3f05..48aabe5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,9 +4,6 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false -overrides: - '@ontos-ai/knowhere-sdk': file:../knowhere-node-sdk - importers: .: @@ -27,8 +24,8 @@ importers: specifier: ^1.1.0 version: 1.1.0 '@ontos-ai/knowhere-sdk': - specifier: file:../knowhere-node-sdk - version: file:../knowhere-node-sdk + specifier: ^2.1.1 + version: 2.1.1 '@radix-ui/react-alert-dialog': specifier: ^1.1.15 version: 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -1606,8 +1603,8 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} - '@ontos-ai/knowhere-sdk@file:../knowhere-node-sdk': - resolution: {directory: ../knowhere-node-sdk, type: directory} + '@ontos-ai/knowhere-sdk@2.1.1': + resolution: {integrity: sha512-K33ylB/QjVYlLgow+/Hw+uNhQbWWrMTIqZ3IAlx5buHSI2EHPVP9+kDsfslwpfffYSn85j9Ab0/E9qTG0QotTw==} engines: {node: '>=22.13.0', npm: '>=10.0.0', pnpm: '>=9.0.0'} '@open-draft/deferred-promise@2.2.0': @@ -6734,7 +6731,7 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} - '@ontos-ai/knowhere-sdk@file:../knowhere-node-sdk': + '@ontos-ai/knowhere-sdk@2.1.1': dependencies: axios: 1.18.1 jszip: 3.10.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a49c9d9..80ee5bb 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -3,7 +3,4 @@ ignoredBuiltDependencies: - unrs-resolver minimumReleaseAgeExclude: - - '@ontos-ai/knowhere-sdk@2.0.0' - -overrides: - '@ontos-ai/knowhere-sdk': file:../knowhere-node-sdk + - '@ontos-ai/knowhere-sdk@2.1.1'