diff --git a/.gitignore b/.gitignore index 74889db..2a2a9f1 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ # misc .DS_Store *.pem +.repos/effect # debug npm-debug.log* diff --git a/drizzle/0010_minor_piledriver.sql b/drizzle/0010_minor_piledriver.sql new file mode 100644 index 0000000..973029b --- /dev/null +++ b/drizzle/0010_minor_piledriver.sql @@ -0,0 +1,2 @@ +ALTER TABLE "source_parse_results" ADD COLUMN "snapshot_manifest_url" text;--> statement-breakpoint +ALTER TABLE "source_parse_results" ADD COLUMN "snapshot_manifest_key" text; \ No newline at end of file diff --git a/drizzle/0011_rare_deadpool.sql b/drizzle/0011_rare_deadpool.sql new file mode 100644 index 0000000..0cf5000 --- /dev/null +++ b/drizzle/0011_rare_deadpool.sql @@ -0,0 +1,5 @@ +ALTER TABLE "source_parse_results" ALTER COLUMN "result_blob_url" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "source_parse_results" ADD COLUMN "revision_key" text;--> statement-breakpoint +ALTER TABLE "source_parse_results" ADD COLUMN "sync_status" text;--> statement-breakpoint +ALTER TABLE "source_parse_results" ADD COLUMN "sync_error" text;--> statement-breakpoint +ALTER TABLE "sources" ADD COLUMN "failure_stage" text; \ No newline at end of file diff --git a/drizzle/0012_lazy_wendell_rand.sql b/drizzle/0012_lazy_wendell_rand.sql new file mode 100644 index 0000000..f50ab54 --- /dev/null +++ b/drizzle/0012_lazy_wendell_rand.sql @@ -0,0 +1,21 @@ +CREATE TABLE "parsed_document_sync_leases" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "source_id" uuid NOT NULL, + "document_id" text NOT NULL, + "revision_key" text, + "lease_token" text NOT NULL, + "acquired_at" timestamp with time zone DEFAULT now() NOT NULL, + "expires_at" timestamp with time zone NOT NULL, + "released_at" timestamp with time zone, + "release_reason" text, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "parsed_document_sync_leases" ADD CONSTRAINT "parsed_document_sync_leases_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "parsed_document_sync_leases" ADD CONSTRAINT "parsed_document_sync_leases_source_id_sources_id_fk" FOREIGN KEY ("source_id") REFERENCES "public"."sources"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "parsed_document_sync_leases_token_idx" ON "parsed_document_sync_leases" USING btree ("lease_token");--> statement-breakpoint +CREATE INDEX "parsed_document_sync_leases_active_idx" ON "parsed_document_sync_leases" USING btree ("expires_at") WHERE released_at IS NULL;--> statement-breakpoint +CREATE INDEX "parsed_document_sync_leases_workspace_active_idx" ON "parsed_document_sync_leases" USING btree ("workspace_id") WHERE released_at IS NULL;--> statement-breakpoint +CREATE INDEX "parsed_document_sync_leases_document_active_idx" ON "parsed_document_sync_leases" USING btree ("document_id") WHERE released_at IS NULL; \ No newline at end of file diff --git a/drizzle/meta/0010_snapshot.json b/drizzle/meta/0010_snapshot.json new file mode 100644 index 0000000..b9b5a4f --- /dev/null +++ b/drizzle/meta/0010_snapshot.json @@ -0,0 +1,728 @@ +{ + "id": "cc41c815-4e52-4d01-ac2e-abbc791331b7", + "prevId": "c59ee2f8-6b59-4a2c-98a3-d71a67388d47", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/0011_snapshot.json b/drizzle/meta/0011_snapshot.json new file mode 100644 index 0000000..0f8448c --- /dev/null +++ b/drizzle/meta/0011_snapshot.json @@ -0,0 +1,752 @@ +{ + "id": "bab84445-8eb2-4d15-856e-b2f59edcaf47", + "prevId": "cc41c815-4e52-4d01-ac2e-abbc791331b7", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/0012_snapshot.json b/drizzle/meta/0012_snapshot.json new file mode 100644 index 0000000..4011068 --- /dev/null +++ b/drizzle/meta/0012_snapshot.json @@ -0,0 +1,932 @@ +{ + "id": "a2a2f9a9-d567-4413-89d8-fa714b994351", + "prevId": "bab84445-8eb2-4d15-856e-b2f59edcaf47", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 568b77f..93f0ced 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -71,6 +71,27 @@ "when": 1782472980935, "tag": "0009_true_lila_cheney", "breakpoints": true + }, + { + "idx": 10, + "version": "7", + "when": 1783098230819, + "tag": "0010_minor_piledriver", + "breakpoints": true + }, + { + "idx": 11, + "version": "7", + "when": 1783175614191, + "tag": "0011_rare_deadpool", + "breakpoints": true + }, + { + "idx": 12, + "version": "7", + "when": 1783231408481, + "tag": "0012_lazy_wendell_rand", + "breakpoints": true } ] } \ No newline at end of file diff --git a/eslint.config.mjs b/eslint.config.mjs index ac9d10f..6ed5798 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -13,6 +13,7 @@ const eslintConfig = defineConfig([ "build/**", "playwright-report/**", "test-results/**", + ".repos/effect/**", "next-env.d.ts", ]), ]); diff --git a/next.config.ts b/next.config.ts index e294c70..b73f520 100644 --- a/next.config.ts +++ b/next.config.ts @@ -7,6 +7,9 @@ const nextConfig: NextConfig = { "pg", "@neondatabase/serverless", "postgres", + "@ontos-ai/knowhere-sdk", + "@napi-rs/canvas", + "piscina", ], allowedDevOrigins: [ "127.0.0.1", diff --git a/package-lock.json b/package-lock.json index b8d1d91..e465210 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@antv/chart-visualization-skills": "0.1.3", "@effect/platform": "^0.96.1", "@neondatabase/serverless": "^1.1.0", - "@ontos-ai/knowhere-sdk": "^2.0.0", + "@ontos-ai/knowhere-sdk": "^2.1.1", "@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-dialog": "^1.1.15", @@ -2275,16 +2275,16 @@ } }, "node_modules/@ontos-ai/knowhere-sdk": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@ontos-ai/knowhere-sdk/-/knowhere-sdk-2.0.0.tgz", - "integrity": "sha512-Zhx8mn/8pYyhBaO9BOdQYsVZybGXyq15YctjHSU8TwgNjJEzX5lEiBEkp0kIGFPh3S+/7X3rZGakefEPf5G2tQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@ontos-ai/knowhere-sdk/-/knowhere-sdk-2.1.1.tgz", + "integrity": "sha512-K33ylB/QjVYlLgow+/Hw+uNhQbWWrMTIqZ3IAlx5buHSI2EHPVP9+kDsfslwpfffYSn85j9Ab0/E9qTG0QotTw==", "license": "MIT", "dependencies": { "axios": "^1.15.2", "jszip": "^3.10.0" }, "engines": { - "node": ">=20.19.0", + "node": ">=22.13.0", "npm": ">=10.0.0", "pnpm": ">=9.0.0" } diff --git a/package.json b/package.json index 73b1e26..94d1c3a 100644 --- a/package.json +++ b/package.json @@ -2,11 +2,14 @@ "name": "knowhere-notebook", "version": "0.1.0", "private": true, + "engines": { + "node": ">=22.13.0" + }, "scripts": { "dev": "pnpm run upstash:dev & next dev", "build": "next build", "typecheck": "tsc --noEmit", - "prepare": "effect-language-service patch", + "prepare": "./scripts/prepare-effect.sh && effect-language-service patch", "start": "next start", "lint": "eslint", "test": "vitest run", @@ -23,8 +26,9 @@ "@ai-sdk/react": "^3.0.177", "@antv/chart-visualization-skills": "0.1.3", "@effect/platform": "^0.96.1", + "@napi-rs/canvas": "^1.0.2", "@neondatabase/serverless": "^1.1.0", - "@ontos-ai/knowhere-sdk": "^2.0.0", + "@ontos-ai/knowhere-sdk": "^2.1.1", "@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-dialog": "^1.1.15", @@ -51,6 +55,7 @@ "next": "16.2.4", "next-themes": "^0.4.6", "pdfjs-dist": "5.4.296", + "piscina": "^5.2.0", "postgres": "^3.4.9", "posthog-js": "^1.386.8", "react": "19.2.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1c4223f..48aabe5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,12 +17,15 @@ importers: '@effect/platform': specifier: ^0.96.1 version: 0.96.1(effect@3.21.2) + '@napi-rs/canvas': + specifier: ^1.0.2 + version: 1.0.2 '@neondatabase/serverless': specifier: ^1.1.0 version: 1.1.0 '@ontos-ai/knowhere-sdk': - specifier: ^2.0.0 - version: 2.0.0 + specifier: ^2.1.1 + version: 2.1.1 '@radix-ui/react-alert-dialog': specifier: ^1.1.15 version: 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -101,6 +104,9 @@ importers: pdfjs-dist: specifier: 5.4.296 version: 5.4.296 + piscina: + specifier: ^5.2.0 + version: 5.2.0 postgres: specifier: ^3.4.9 version: 3.4.9 @@ -167,7 +173,7 @@ importers: version: 19.2.3(@types/react@19.2.14) '@vitejs/plugin-react': specifier: ^6.0.1 - version: 6.0.1(babel-plugin-react-compiler@1.0.0)(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)) + version: 6.0.1(babel-plugin-react-compiler@1.0.0)(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)) babel-plugin-react-compiler: specifier: ^1.0.0 version: 1.0.0 @@ -194,7 +200,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.1.5 - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@20.19.39)(jsdom@29.1.1(@noble/hashes@1.8.0))(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@20.19.39)(jsdom@29.1.1(@noble/hashes@1.8.0))(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)) packages: @@ -1241,24 +1247,48 @@ packages: cpu: [arm64] os: [android] + '@napi-rs/canvas-android-arm64@1.0.2': + resolution: {integrity: sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + '@napi-rs/canvas-darwin-arm64@0.1.100': resolution: {integrity: sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] + '@napi-rs/canvas-darwin-arm64@1.0.2': + resolution: {integrity: sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + '@napi-rs/canvas-darwin-x64@0.1.100': resolution: {integrity: sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] + '@napi-rs/canvas-darwin-x64@1.0.2': + resolution: {integrity: sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.100': resolution: {integrity: sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==} engines: {node: '>= 10'} cpu: [arm] os: [linux] + '@napi-rs/canvas-linux-arm-gnueabihf@1.0.2': + resolution: {integrity: sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + '@napi-rs/canvas-linux-arm64-gnu@0.1.100': resolution: {integrity: sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==} engines: {node: '>= 10'} @@ -1266,6 +1296,13 @@ packages: os: [linux] libc: [glibc] + '@napi-rs/canvas-linux-arm64-gnu@1.0.2': + resolution: {integrity: sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + '@napi-rs/canvas-linux-arm64-musl@0.1.100': resolution: {integrity: sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==} engines: {node: '>= 10'} @@ -1273,6 +1310,13 @@ packages: os: [linux] libc: [musl] + '@napi-rs/canvas-linux-arm64-musl@1.0.2': + resolution: {integrity: sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + '@napi-rs/canvas-linux-riscv64-gnu@0.1.100': resolution: {integrity: sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==} engines: {node: '>= 10'} @@ -1280,6 +1324,13 @@ packages: os: [linux] libc: [glibc] + '@napi-rs/canvas-linux-riscv64-gnu@1.0.2': + resolution: {integrity: sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + '@napi-rs/canvas-linux-x64-gnu@0.1.100': resolution: {integrity: sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==} engines: {node: '>= 10'} @@ -1287,6 +1338,13 @@ packages: os: [linux] libc: [glibc] + '@napi-rs/canvas-linux-x64-gnu@1.0.2': + resolution: {integrity: sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + '@napi-rs/canvas-linux-x64-musl@0.1.100': resolution: {integrity: sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==} engines: {node: '>= 10'} @@ -1294,22 +1352,158 @@ packages: os: [linux] libc: [musl] + '@napi-rs/canvas-linux-x64-musl@1.0.2': + resolution: {integrity: sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + '@napi-rs/canvas-win32-arm64-msvc@0.1.100': resolution: {integrity: sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] + '@napi-rs/canvas-win32-arm64-msvc@1.0.2': + resolution: {integrity: sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + '@napi-rs/canvas-win32-x64-msvc@0.1.100': resolution: {integrity: sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==} engines: {node: '>= 10'} cpu: [x64] os: [win32] + '@napi-rs/canvas-win32-x64-msvc@1.0.2': + resolution: {integrity: sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + '@napi-rs/canvas@0.1.100': resolution: {integrity: sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==} engines: {node: '>= 10'} + '@napi-rs/canvas@1.0.2': + resolution: {integrity: sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ==} + engines: {node: '>= 10'} + + '@napi-rs/nice-android-arm-eabi@1.1.1': + resolution: {integrity: sha512-kjirL3N6TnRPv5iuHw36wnucNqXAO46dzK9oPb0wj076R5Xm8PfUVA9nAFB5ZNMmfJQJVKACAPd/Z2KYMppthw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@napi-rs/nice-android-arm64@1.1.1': + resolution: {integrity: sha512-blG0i7dXgbInN5urONoUCNf+DUEAavRffrO7fZSeoRMJc5qD+BJeNcpr54msPF6qfDD6kzs9AQJogZvT2KD5nw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/nice-darwin-arm64@1.1.1': + resolution: {integrity: sha512-s/E7w45NaLqTGuOjC2p96pct4jRfo61xb9bU1unM/MJ/RFkKlJyJDx7OJI/O0ll/hrfpqKopuAFDV8yo0hfT7A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/nice-darwin-x64@1.1.1': + resolution: {integrity: sha512-dGoEBnVpsdcC+oHHmW1LRK5eiyzLwdgNQq3BmZIav+9/5WTZwBYX7r5ZkQC07Nxd3KHOCkgbHSh4wPkH1N1LiQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/nice-freebsd-x64@1.1.1': + resolution: {integrity: sha512-kHv4kEHAylMYmlNwcQcDtXjklYp4FCf0b05E+0h6nDHsZ+F0bDe04U/tXNOqrx5CmIAth4vwfkjjUmp4c4JktQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/nice-linux-arm-gnueabihf@1.1.1': + resolution: {integrity: sha512-E1t7K0efyKXZDoZg1LzCOLxgolxV58HCkaEkEvIYQx12ht2pa8hoBo+4OB3qh7e+QiBlp1SRf+voWUZFxyhyqg==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/nice-linux-arm64-gnu@1.1.1': + resolution: {integrity: sha512-CIKLA12DTIZlmTaaKhQP88R3Xao+gyJxNWEn04wZwC2wmRapNnxCUZkVwggInMJvtVElA+D4ZzOU5sX4jV+SmQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@napi-rs/nice-linux-arm64-musl@1.1.1': + resolution: {integrity: sha512-+2Rzdb3nTIYZ0YJF43qf2twhqOCkiSrHx2Pg6DJaCPYhhaxbLcdlV8hCRMHghQ+EtZQWGNcS2xF4KxBhSGeutg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@napi-rs/nice-linux-ppc64-gnu@1.1.1': + resolution: {integrity: sha512-4FS8oc0GeHpwvv4tKciKkw3Y4jKsL7FRhaOeiPei0X9T4Jd619wHNe4xCLmN2EMgZoeGg+Q7GY7BsvwKpL22Tg==} + engines: {node: '>= 10'} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@napi-rs/nice-linux-riscv64-gnu@1.1.1': + resolution: {integrity: sha512-HU0nw9uD4FO/oGCCk409tCi5IzIZpH2agE6nN4fqpwVlCn5BOq0MS1dXGjXaG17JaAvrlpV5ZeyZwSon10XOXw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@napi-rs/nice-linux-s390x-gnu@1.1.1': + resolution: {integrity: sha512-2YqKJWWl24EwrX0DzCQgPLKQBxYDdBxOHot1KWEq7aY2uYeX+Uvtv4I8xFVVygJDgf6/92h9N3Y43WPx8+PAgQ==} + engines: {node: '>= 10'} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@napi-rs/nice-linux-x64-gnu@1.1.1': + resolution: {integrity: sha512-/gaNz3R92t+dcrfCw/96pDopcmec7oCcAQ3l/M+Zxr82KT4DljD37CpgrnXV+pJC263JkW572pdbP3hP+KjcIg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@napi-rs/nice-linux-x64-musl@1.1.1': + resolution: {integrity: sha512-xScCGnyj/oppsNPMnevsBe3pvNaoK7FGvMjT35riz9YdhB2WtTG47ZlbxtOLpjeO9SqqQ2J2igCmz6IJOD5JYw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@napi-rs/nice-openharmony-arm64@1.1.1': + resolution: {integrity: sha512-6uJPRVwVCLDeoOaNyeiW0gp2kFIM4r7PL2MczdZQHkFi9gVlgm+Vn+V6nTWRcu856mJ2WjYJiumEajfSm7arPQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [openharmony] + + '@napi-rs/nice-win32-arm64-msvc@1.1.1': + resolution: {integrity: sha512-uoTb4eAvM5B2aj/z8j+Nv8OttPf2m+HVx3UjA5jcFxASvNhQriyCQF1OB1lHL43ZhW+VwZlgvjmP5qF3+59atA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/nice-win32-ia32-msvc@1.1.1': + resolution: {integrity: sha512-CNQqlQT9MwuCsg1Vd/oKXiuH+TcsSPJmlAFc5frFyX/KkOh0UpBLEj7aoY656d5UKZQMQFP7vJNa1DNUNORvug==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/nice-win32-x64-msvc@1.1.1': + resolution: {integrity: sha512-vB+4G/jBQCAh0jelMTY3+kgFy00Hlx2f2/1zjMoH821IbplbWZOkLiTYXQkygNTzQJTq5cvwBDgn2ppHD+bglQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/nice@1.1.1': + resolution: {integrity: sha512-xJIPs+bYuc9ASBl+cvGsKbGrJmS6fAKaSZCnT0lhahT5rhA2VVy9/EcIgd2JhtEuFOJNx7UHNn/qiTPTY4nrQw==} + engines: {node: '>= 10'} + '@napi-rs/wasm-runtime@0.2.12': resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} @@ -1409,9 +1603,9 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} - '@ontos-ai/knowhere-sdk@2.0.0': - resolution: {integrity: sha512-Zhx8mn/8pYyhBaO9BOdQYsVZybGXyq15YctjHSU8TwgNjJEzX5lEiBEkp0kIGFPh3S+/7X3rZGakefEPf5G2tQ==} - engines: {node: '>=20.19.0', npm: '>=10.0.0', pnpm: '>=9.0.0'} + '@ontos-ai/knowhere-sdk@2.1.1': + resolution: {integrity: sha512-K33ylB/QjVYlLgow+/Hw+uNhQbWWrMTIqZ3IAlx5buHSI2EHPVP9+kDsfslwpfffYSn85j9Ab0/E9qTG0QotTw==} + engines: {node: '>=22.13.0', npm: '>=10.0.0', pnpm: '>=9.0.0'} '@open-draft/deferred-promise@2.2.0': resolution: {integrity: sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==} @@ -2415,6 +2609,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + agent-base@7.1.4: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} @@ -2533,8 +2731,8 @@ packages: resolution: {integrity: sha512-KunSNx+TVpkAw/6ULfhnx+HWRecjqZGTOyquAoWHYLRSdK1tB5Ihce1ZW+UY3fj33bYAFWPu7W/GRSmmrCGuxA==} engines: {node: '>=4'} - axios@1.16.0: - resolution: {integrity: sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==} + axios@1.18.1: + resolution: {integrity: sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==} axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} @@ -3370,8 +3568,8 @@ packages: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} - form-data@4.0.5: - resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} engines: {node: '>= 6'} formdata-polyfill@4.0.10: @@ -3522,6 +3720,10 @@ packages: resolution: {integrity: sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==} engines: {node: '>= 0.4'} + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} @@ -3552,6 +3754,10 @@ packages: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + https-proxy-agent@7.0.6: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} @@ -4501,6 +4707,10 @@ packages: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} + piscina@5.2.0: + resolution: {integrity: sha512-DszUCKeVN/5G5QKo6jAVHL8fmKnkJvQ0ACiVgY7YGCq3TUB2oznAOayvZPIAdEThvhczkXR+qm3IHsNXpFCYfA==} + engines: {node: '>=20.x'} + pkce-challenge@5.0.1: resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} engines: {node: '>=16.20.0'} @@ -5413,6 +5623,11 @@ packages: yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + yargs-parser@21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} @@ -6284,36 +6499,69 @@ snapshots: '@napi-rs/canvas-android-arm64@0.1.100': optional: true + '@napi-rs/canvas-android-arm64@1.0.2': + optional: true + '@napi-rs/canvas-darwin-arm64@0.1.100': optional: true + '@napi-rs/canvas-darwin-arm64@1.0.2': + optional: true + '@napi-rs/canvas-darwin-x64@0.1.100': optional: true + '@napi-rs/canvas-darwin-x64@1.0.2': + optional: true + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.100': optional: true + '@napi-rs/canvas-linux-arm-gnueabihf@1.0.2': + optional: true + '@napi-rs/canvas-linux-arm64-gnu@0.1.100': optional: true + '@napi-rs/canvas-linux-arm64-gnu@1.0.2': + optional: true + '@napi-rs/canvas-linux-arm64-musl@0.1.100': optional: true + '@napi-rs/canvas-linux-arm64-musl@1.0.2': + optional: true + '@napi-rs/canvas-linux-riscv64-gnu@0.1.100': optional: true + '@napi-rs/canvas-linux-riscv64-gnu@1.0.2': + optional: true + '@napi-rs/canvas-linux-x64-gnu@0.1.100': optional: true + '@napi-rs/canvas-linux-x64-gnu@1.0.2': + optional: true + '@napi-rs/canvas-linux-x64-musl@0.1.100': optional: true + '@napi-rs/canvas-linux-x64-musl@1.0.2': + optional: true + '@napi-rs/canvas-win32-arm64-msvc@0.1.100': optional: true + '@napi-rs/canvas-win32-arm64-msvc@1.0.2': + optional: true + '@napi-rs/canvas-win32-x64-msvc@0.1.100': optional: true + '@napi-rs/canvas-win32-x64-msvc@1.0.2': + optional: true + '@napi-rs/canvas@0.1.100': optionalDependencies: '@napi-rs/canvas-android-arm64': 0.1.100 @@ -6329,6 +6577,92 @@ snapshots: '@napi-rs/canvas-win32-x64-msvc': 0.1.100 optional: true + '@napi-rs/canvas@1.0.2': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 1.0.2 + '@napi-rs/canvas-darwin-arm64': 1.0.2 + '@napi-rs/canvas-darwin-x64': 1.0.2 + '@napi-rs/canvas-linux-arm-gnueabihf': 1.0.2 + '@napi-rs/canvas-linux-arm64-gnu': 1.0.2 + '@napi-rs/canvas-linux-arm64-musl': 1.0.2 + '@napi-rs/canvas-linux-riscv64-gnu': 1.0.2 + '@napi-rs/canvas-linux-x64-gnu': 1.0.2 + '@napi-rs/canvas-linux-x64-musl': 1.0.2 + '@napi-rs/canvas-win32-arm64-msvc': 1.0.2 + '@napi-rs/canvas-win32-x64-msvc': 1.0.2 + + '@napi-rs/nice-android-arm-eabi@1.1.1': + optional: true + + '@napi-rs/nice-android-arm64@1.1.1': + optional: true + + '@napi-rs/nice-darwin-arm64@1.1.1': + optional: true + + '@napi-rs/nice-darwin-x64@1.1.1': + optional: true + + '@napi-rs/nice-freebsd-x64@1.1.1': + optional: true + + '@napi-rs/nice-linux-arm-gnueabihf@1.1.1': + optional: true + + '@napi-rs/nice-linux-arm64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-arm64-musl@1.1.1': + optional: true + + '@napi-rs/nice-linux-ppc64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-riscv64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-s390x-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-x64-gnu@1.1.1': + optional: true + + '@napi-rs/nice-linux-x64-musl@1.1.1': + optional: true + + '@napi-rs/nice-openharmony-arm64@1.1.1': + optional: true + + '@napi-rs/nice-win32-arm64-msvc@1.1.1': + optional: true + + '@napi-rs/nice-win32-ia32-msvc@1.1.1': + optional: true + + '@napi-rs/nice-win32-x64-msvc@1.1.1': + optional: true + + '@napi-rs/nice@1.1.1': + optionalDependencies: + '@napi-rs/nice-android-arm-eabi': 1.1.1 + '@napi-rs/nice-android-arm64': 1.1.1 + '@napi-rs/nice-darwin-arm64': 1.1.1 + '@napi-rs/nice-darwin-x64': 1.1.1 + '@napi-rs/nice-freebsd-x64': 1.1.1 + '@napi-rs/nice-linux-arm-gnueabihf': 1.1.1 + '@napi-rs/nice-linux-arm64-gnu': 1.1.1 + '@napi-rs/nice-linux-arm64-musl': 1.1.1 + '@napi-rs/nice-linux-ppc64-gnu': 1.1.1 + '@napi-rs/nice-linux-riscv64-gnu': 1.1.1 + '@napi-rs/nice-linux-s390x-gnu': 1.1.1 + '@napi-rs/nice-linux-x64-gnu': 1.1.1 + '@napi-rs/nice-linux-x64-musl': 1.1.1 + '@napi-rs/nice-openharmony-arm64': 1.1.1 + '@napi-rs/nice-win32-arm64-msvc': 1.1.1 + '@napi-rs/nice-win32-ia32-msvc': 1.1.1 + '@napi-rs/nice-win32-x64-msvc': 1.1.1 + optional: true + '@napi-rs/wasm-runtime@0.2.12': dependencies: '@emnapi/core': 1.10.0 @@ -6397,12 +6731,13 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} - '@ontos-ai/knowhere-sdk@2.0.0': + '@ontos-ai/knowhere-sdk@2.1.1': dependencies: - axios: 1.16.0 + axios: 1.18.1 jszip: 3.10.1 transitivePeerDependencies: - debug + - supports-color '@open-draft/deferred-promise@2.2.0': {} @@ -7238,10 +7573,10 @@ snapshots: '@vercel/oidc@3.2.0': {} - '@vitejs/plugin-react@6.0.1(babel-plugin-react-compiler@1.0.0)(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0))': + '@vitejs/plugin-react@6.0.1(babel-plugin-react-compiler@1.0.0)(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0))': dependencies: '@rolldown/pluginutils': 1.0.0-rc.7 - vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0) + vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0) optionalDependencies: babel-plugin-react-compiler: 1.0.0 @@ -7254,14 +7589,14 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.5(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0))': + '@vitest/mocker@4.1.5(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: msw: 2.14.3(@types/node@20.19.39)(typescript@6.0.3) - vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0) + vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0) '@vitest/pretty-format@4.1.5': dependencies: @@ -7300,6 +7635,12 @@ snapshots: acorn@8.16.0: {} + agent-base@6.0.2: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + agent-base@7.1.4: {} ai@6.0.175(zod@4.4.3): @@ -7443,13 +7784,15 @@ snapshots: axe-core@4.11.4: {} - axios@1.16.0: + axios@1.18.1: dependencies: follow-redirects: 1.16.0 - form-data: 4.0.5 + form-data: 4.0.6 + https-proxy-agent: 5.0.1 proxy-from-env: 2.1.0 transitivePeerDependencies: - debug + - supports-color axobject-query@4.1.0: {} @@ -8397,12 +8740,12 @@ snapshots: dependencies: is-callable: 1.2.7 - form-data@4.0.5: + form-data@4.0.6: dependencies: asynckit: 0.4.0 combined-stream: 1.0.8 es-set-tostringtag: 2.1.0 - hasown: 2.0.3 + hasown: 2.0.4 mime-types: 2.1.35 formdata-polyfill@4.0.10: @@ -8539,6 +8882,10 @@ snapshots: dependencies: function-bind: 1.1.2 + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + hast-util-to-jsx-runtime@2.3.6: dependencies: '@types/estree': 1.0.8 @@ -8592,6 +8939,13 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 + https-proxy-agent@5.0.1: + dependencies: + agent-base: 6.0.2 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.4 @@ -9715,6 +10069,10 @@ snapshots: picomatch@4.0.4: {} + piscina@5.2.0: + optionalDependencies: + '@napi-rs/nice': 1.1.1 + pkce-challenge@5.0.1: {} playwright-core@1.58.2: {} @@ -10653,7 +11011,7 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0): + vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -10666,11 +11024,12 @@ snapshots: fsevents: 2.3.3 jiti: 2.7.0 tsx: 4.21.0 + yaml: 2.9.0 - vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@20.19.39)(jsdom@29.1.1(@noble/hashes@1.8.0))(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)): + vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@20.19.39)(jsdom@29.1.1(@noble/hashes@1.8.0))(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.5 - '@vitest/mocker': 4.1.5(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)) + '@vitest/mocker': 4.1.5(msw@2.14.3(@types/node@20.19.39)(typescript@6.0.3))(vite@8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.5 '@vitest/runner': 4.1.5 '@vitest/snapshot': 4.1.5 @@ -10687,7 +11046,7 @@ snapshots: tinyexec: 1.1.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0) + vite: 8.0.10(@types/node@20.19.39)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.0 @@ -10799,6 +11158,9 @@ snapshots: yallist@3.1.1: {} + yaml@2.9.0: + optional: true + yargs-parser@21.1.1: {} yargs@17.7.2: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index d989c76..80ee5bb 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,5 +1,6 @@ ignoredBuiltDependencies: - sharp - unrs-resolver + minimumReleaseAgeExclude: - - '@ontos-ai/knowhere-sdk@2.0.0' + - '@ontos-ai/knowhere-sdk@2.1.1' diff --git a/scripts/prepare-effect.sh b/scripts/prepare-effect.sh new file mode 100755 index 0000000..24a9d31 --- /dev/null +++ b/scripts/prepare-effect.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +set -eu + +repo_dir=".repos/effect" +repo_url="https://github.com/Effect-TS/effect-smol" + +if [ -d "$repo_dir/.git" ]; then + exit 0 +fi + +mkdir -p ".repos" +git clone "$repo_url" "$repo_dir" diff --git a/src/agent-harness/ledger.ts b/src/agent-harness/ledger.ts index da197e8..103f9c0 100644 --- a/src/agent-harness/ledger.ts +++ b/src/agent-harness/ledger.ts @@ -68,10 +68,14 @@ export function createEvidenceLedger() { chunk: { ref: `r${retrievalIndex}:referenced:${index + 1}`, kind: "referenced_chunk", + chunkId: chunk.chunkId, content, contentPreview: content, chunkType: chunk.chunkType, score: null, + sourceChunkPath: chunk.sourceChunkPath, + filePath: chunk.filePath, + metadata: chunk.metadata, source: { documentId: chunk.documentId, sourceFileName: null, @@ -146,6 +150,10 @@ function addChunkFromResult(input: { contentPreview: buildContentPreview(input.result.content), chunkType: input.result.chunkType, score: input.result.score, + chunkId: input.result.chunkId, + sourceChunkPath: input.result.sourceChunkPath, + filePath: input.result.filePath, + metadata: input.result.metadata, source: { documentId: input.result.source.documentId, sourceFileName: input.result.source.sourceFileName, diff --git a/src/agent-harness/types.ts b/src/agent-harness/types.ts index 1a1cce9..c559623 100644 --- a/src/agent-harness/types.ts +++ b/src/agent-harness/types.ts @@ -94,10 +94,14 @@ export type RetrievalCapability = { export type EvidenceChunk = { readonly ref: string readonly kind: "result" | "referenced_chunk" + readonly chunkId?: string readonly content: string readonly contentPreview: string readonly chunkType: string readonly score: number | null + readonly sourceChunkPath?: string | null + readonly filePath?: string | null + readonly metadata?: Readonly> readonly source: { readonly documentId?: string | null readonly sourceFileName?: string | null diff --git a/src/app/api/demo-sources/materialize/route.test.ts b/src/app/api/demo-sources/materialize/route.test.ts index 9b41725..0818dca 100644 --- a/src/app/api/demo-sources/materialize/route.test.ts +++ b/src/app/api/demo-sources/materialize/route.test.ts @@ -232,6 +232,7 @@ function makeSource( sizeBytes: 5648867, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: null, knowhereDocumentId: "doc_user_copy", stagedBlobPathname: null, diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index 524a657..85fb1bd 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -2,19 +2,17 @@ import { NextRequest } from "next/server" import { beforeEach, describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ - blobGet: vi.fn(), - blobPut: vi.fn(), deleteBlob: vi.fn(), ensureApiKeyForWorkspace: vi.fn(), ensureWorkspace: vi.fn(), fetchDemoChunkPage: vi.fn(), findSourceInWorkspace: vi.fn(), getCurrentUser: vi.fn(), - getSourceParseAssetUrls: vi.fn(), localizeRemoteDocument: vi.fn(), makeKnowhereClient: vi.fn(), + makeKnowhereClientWithParsedStorage: vi.fn(), + readChunks: vi.fn(), requireUser: vi.fn(), - updateSourceRevisionKey: vi.fn(), })) vi.mock("next/headers", () => ({ @@ -39,20 +37,18 @@ vi.mock("@/infrastructure/auth", () => ({ vi.mock("@/integrations/knowhere", () => ({ makeKnowhereClient: mocks.makeKnowhereClient, + makeKnowhereClientWithParsedStorage: + mocks.makeKnowhereClientWithParsedStorage, })) vi.mock("@vercel/blob", () => ({ del: mocks.deleteBlob, - get: mocks.blobGet, - put: mocks.blobPut, })) vi.mock("@/domains/sources/service", () => ({ sourceService: { findInWorkspace: mocks.findSourceInWorkspace, - getParseAssetUrls: mocks.getSourceParseAssetUrls, localizeRemoteDocument: mocks.localizeRemoteDocument, - updateSourceRevisionKey: mocks.updateSourceRevisionKey, }, })) @@ -67,11 +63,10 @@ import { GET } from "./route" describe("GET /api/sources/[sourceId]/chunks", () => { beforeEach(() => { vi.clearAllMocks() - mocks.blobGet.mockResolvedValue(null) - mocks.blobPut.mockImplementation(async (pathname: string) => ({ - url: `https://blob.example/${pathname}`, - })) - mocks.updateSourceRevisionKey.mockResolvedValue(null) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: { documents: { listChunks: vi.fn() } }, + knowledge: { readChunks: mocks.readChunks }, + }) }) it("serves API-owned demo chunks for anonymous canonical demo sources", async () => { @@ -132,7 +127,6 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }) expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() - expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() }) it("loads every API-owned demo chunk page for full anonymous chunk requests", async () => { @@ -287,7 +281,6 @@ describe("GET /api/sources/[sourceId]/chunks", () => { expect(mocks.findSourceInWorkspace).not.toHaveBeenCalled() expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() - expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() }) it("serves demo chunks for authenticated materialized demo sources", async () => { @@ -377,7 +370,6 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }) expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() - expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() }) it("logs the demo chunk load failure before returning 404", async () => { @@ -429,32 +421,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { } }) - it("loads authenticated workspace chunks without probing the demo endpoint first", async () => { - const knowhereClient = { - documents: { - listChunks: vi.fn(async () => ({ - chunks: [ - { - id: "dchk_1", - chunkId: "parser_1", - chunkType: "text", - content: "Workspace chunk", - sectionPath: "Summary", - sourceChunkPath: "Default_Root/notes.pdf/Summary", - filePath: null, - metadata: {}, - sortOrder: 0, - }, - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })), - }, - } + it("loads authenticated workspace chunks through the SDK durable read", async () => { mocks.getCurrentUser.mockResolvedValue({ id: "user_1", email: null, @@ -466,42 +433,55 @@ describe("GET /api/sources/[sourceId]/chunks", () => { namespace: "notebook-workspace_1", createdAt: new Date("2026-05-10T00:00:00.000Z"), }) - mocks.findSourceInWorkspace.mockResolvedValue({ - id: "00000000-0000-0000-0000-000000000002", - workspaceId: "workspace_1", - title: "notes.pdf", - mimeType: "application/pdf", - sizeBytes: 1024, - status: "ready", - failureReason: null, - knowhereJobId: "job_1", - knowhereDocumentId: "doc_1", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-10T00:00:00.000Z"), - updatedAt: new Date("2026-05-10T00:00:00.000Z"), - deletedAt: null, - }) + mocks.findSourceInWorkspace.mockResolvedValue( + makeReadySource({ + id: "00000000-0000-0000-0000-000000000002", + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + }), + ) mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") - mocks.makeKnowhereClient.mockReturnValue(knowhereClient) - mocks.getSourceParseAssetUrls.mockResolvedValue({}) + mocks.readChunks.mockResolvedValue({ + document: { localDocumentId: "doc_1" }, + chunks: [ + { + position: 1, + chunkId: "parser_1", + chunkType: "image", + content: "Workspace chunk", + readableContent: "Workspace chunk", + sectionPath: "Summary", + sourceChunkPath: "Summary", + filePath: "images/chart.png", + assetUrl: + "https://fake.public.blob.vercel-storage.com/images/chart.png", + metadata: {}, + }, + ], + page: 1, + pageSize: 1, + totalChunks: 1, + totalPages: 1, + }) const response = await GET( new NextRequest( "http://localhost:3001/api/sources/00000000-0000-0000-0000-000000000002/chunks?page=1&pageSize=1", ), - { params: Promise.resolve({ sourceId: "00000000-0000-0000-0000-000000000002" }) }, + { + params: Promise.resolve({ + sourceId: "00000000-0000-0000-0000-000000000002", + }), + }, ) await expect(response.json()).resolves.toMatchObject({ chunks: [ { - chunkId: "dchk_1", parserChunkId: "parser_1", documentId: "doc_1", + assetUrl: + "https://fake.public.blob.vercel-storage.com/images/chart.png", sourceTitle: "notes.pdf", }, ], @@ -513,14 +493,62 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }) expect(response.status).toBe(200) expect(mocks.fetchDemoChunkPage).not.toHaveBeenCalled() - expect(knowhereClient.documents.listChunks).toHaveBeenCalledWith("doc_1", { + expect(mocks.readChunks).toHaveBeenCalledWith({ + documentId: "doc_1", + revisionKey: "job_1", page: 1, pageSize: 1, - includeAssetUrls: true, + assetUrlPolicy: "durable", + }) + }) + + it("returns processing when a workspace source is not ready", async () => { + mocks.getCurrentUser.mockResolvedValue({ + id: "user_1", + email: null, + name: null, + }) + mocks.ensureWorkspace.mockResolvedValue({ + id: "workspace_1", + userId: "user_1", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + }) + mocks.findSourceInWorkspace.mockResolvedValue( + makeReadySource({ + id: "00000000-0000-0000-0000-000000000002", + status: "parsing", + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + }), + ) + + const response = await GET( + new NextRequest( + "http://localhost:3001/api/sources/00000000-0000-0000-0000-000000000002/chunks?page=1&pageSize=1", + ), + { + params: Promise.resolve({ + sourceId: "00000000-0000-0000-0000-000000000002", + }), + }, + ) + + await expect(response.json()).resolves.toEqual({ + chunks: [], + pagination: { + page: 1, + pageSize: 1, + total: 0, + totalPages: 0, + }, + message: "Source is still being prepared.", }) + expect(response.status).toBe(202) + expect(mocks.readChunks).not.toHaveBeenCalled() }) - it("materializes a remote source id on open before loading chunks", async () => { + it("materializes a remote source id and reads chunks through the SDK", async () => { const knowhereClient = { documents: { list: vi.fn(async () => ({ @@ -537,29 +565,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }, ], })), - listChunks: vi.fn(async () => ({ - documentId: "doc_remote", - jobResultId: "job_result_1", - chunks: [ - { - id: "dchk_remote", - chunkId: "parser_remote", - chunkType: "text", - content: "Remote chunk", - sectionPath: "Summary", - sourceChunkPath: "Default_Root/remote.pdf/Summary", - filePath: null, - metadata: {}, - sortOrder: 0, - }, - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })), + listChunks: vi.fn(), }, } mocks.getCurrentUser.mockResolvedValue({ @@ -576,24 +582,35 @@ describe("GET /api/sources/[sourceId]/chunks", () => { mocks.fetchDemoChunkPage.mockRejectedValue(new Error("not a demo")) mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123") mocks.makeKnowhereClient.mockReturnValue(knowhereClient) - mocks.localizeRemoteDocument.mockResolvedValue({ - id: "00000000-0000-0000-0000-000000000009", - workspaceId: "workspace_1", - title: "remote.pdf", - mimeType: "application/pdf", - sizeBytes: 0, - status: "ready", - failureReason: null, - knowhereJobId: "job_result_1", - knowhereDocumentId: "doc_remote", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-10T00:00:00.000Z"), - updatedAt: new Date("2026-05-10T00:00:00.000Z"), - deletedAt: null, + mocks.localizeRemoteDocument.mockResolvedValue( + makeReadySource({ + id: "00000000-0000-0000-0000-000000000009", + title: "remote.pdf", + knowhereJobId: "job_result_1", + knowhereDocumentId: "doc_remote", + }), + ) + mocks.readChunks.mockResolvedValue({ + document: { localDocumentId: "doc_remote" }, + chunks: [ + { + position: 1, + chunkId: "parser_remote", + chunkType: "page", + content: "Remote chunk", + readableContent: "Remote chunk", + sectionPath: "Summary", + sourceChunkPath: "Summary", + filePath: "pages/page-1.png", + assetUrl: + "https://fake.public.blob.vercel-storage.com/pages/page-1.png", + metadata: {}, + }, + ], + page: 1, + pageSize: 1, + totalChunks: 1, + totalPages: 1, }) const response = await GET( @@ -610,9 +627,10 @@ describe("GET /api/sources/[sourceId]/chunks", () => { await expect(response.json()).resolves.toMatchObject({ chunks: [ { - chunkId: "dchk_remote", parserChunkId: "parser_remote", documentId: "doc_remote", + assetUrl: + "https://fake.public.blob.vercel-storage.com/pages/page-1.png", sourceTitle: "remote.pdf", }, ], @@ -628,25 +646,45 @@ describe("GET /api/sources/[sourceId]/chunks", () => { "workspace_1", "session=abc", ) - expect(mocks.localizeRemoteDocument).toHaveBeenCalledWith( - "workspace_1", - { - documentId: "doc_remote", - namespace: "default", - status: "ready", - title: "remote.pdf", - mimeType: "application/pdf", - sizeBytes: undefined, - revisionKey: "job_result_1", - }, - ) - expect(knowhereClient.documents.listChunks).toHaveBeenCalledWith( - "doc_remote", - { - page: 1, - pageSize: 1, - includeAssetUrls: true, - }, - ) + expect(mocks.localizeRemoteDocument).toHaveBeenCalledWith("workspace_1", { + documentId: "doc_remote", + namespace: "default", + status: "ready", + title: "remote.pdf", + mimeType: "application/pdf", + sizeBytes: undefined, + revisionKey: "job_result_1", + }) + expect(mocks.readChunks).toHaveBeenCalledWith({ + documentId: "doc_remote", + revisionKey: "job_result_1", + page: 1, + pageSize: 1, + assetUrlPolicy: "durable", + }) }) }) + +function makeReadySource(overrides: Record) { + return { + id: "00000000-0000-0000-0000-000000000002", + workspaceId: "workspace_1", + title: "notes.pdf", + mimeType: "application/pdf", + sizeBytes: 1024, + status: "ready", + failureReason: null, + failureStage: null, + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + ...overrides, + } +} diff --git a/src/app/api/sources/parsed-sync/route.ts b/src/app/api/sources/parsed-sync/route.ts new file mode 100644 index 0000000..5272176 --- /dev/null +++ b/src/app/api/sources/parsed-sync/route.ts @@ -0,0 +1,27 @@ +import { serve } from "@upstash/workflow/nextjs" + +import { parsedSyncRouteWorkflow } from "@/domains/sources/parsed-sync-route-workflow" + +type ParsedSyncPayload = Parameters< + typeof parsedSyncRouteWorkflow.normalizeParsedSyncPayload +>[0] + +export const { POST } = serve( + async (context) => { + const payload = parsedSyncRouteWorkflow.normalizeParsedSyncPayload( + context.requestPayload, + ) + await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ + context, + payload, + }) + }, + { + failureFunction: async ({ context, failResponse }) => { + await parsedSyncRouteWorkflow.markSyncFailedAfterWorkflowFailure( + context.requestPayload, + failResponse, + ) + }, + }, +) diff --git a/src/app/api/sources/route.test.ts b/src/app/api/sources/route.test.ts index a78639b..a7b10c6 100644 --- a/src/app/api/sources/route.test.ts +++ b/src/app/api/sources/route.test.ts @@ -67,6 +67,7 @@ const source: Source = { sizeBytes: 5, status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: null, stagedBlobPathname: null, diff --git a/src/components/chat-message-list.test.ts b/src/components/chat-message-list.test.ts index 8d58630..f94478e 100644 --- a/src/components/chat-message-list.test.ts +++ b/src/components/chat-message-list.test.ts @@ -129,6 +129,56 @@ describe("ChatMessageList", () => { ); }); + it("renders a separate page image link without replacing source focus", async () => { + const user = userEvent.setup(); + const onCitationClick = vi.fn(); + + render( + React.createElement(ChatMessageList, { + messages: [ + { + id: "assistant_1", + role: "assistant", + content: "The referenced page discusses revenue.", + citations: [ + { + chunkType: "page", + score: 0.9, + pageCitationAssetUrl: + "https://blob.example/pages/page-000004.png", + source: { + documentId: "doc_1", + sourceFileName: "report.pdf", + sectionPath: "Page 4", + }, + }, + ], + }, + ], + onCitationClick, + }), + ); + + const citationButton = screen.getByRole("button", { + name: "Open source report.pdf", + }); + const pageImageLink = screen.getByRole("link", { + name: "Open page image for report.pdf", + }); + + expect(pageImageLink.getAttribute("href")).toBe( + "https://blob.example/pages/page-000004.png", + ); + + await user.click(citationButton); + expect(onCitationClick).toHaveBeenCalledWith( + expect.objectContaining({ + pageCitationAssetUrl: "https://blob.example/pages/page-000004.png", + }), + "assistant_1:0", + ); + }); + it("removes description-only source labels without changing other markdown whitespace", () => { render( React.createElement(ChatMessageList, { diff --git a/src/components/chat-message-list.tsx b/src/components/chat-message-list.tsx index e5e6460..531c71f 100644 --- a/src/components/chat-message-list.tsx +++ b/src/components/chat-message-list.tsx @@ -2,7 +2,7 @@ import { type CSSProperties, type ReactElement } from "react"; import { type VirtualItem } from "@tanstack/react-virtual"; -import { ImageIcon, MessageCircle } from "lucide-react"; +import { ExternalLink, ImageIcon, MessageCircle } from "lucide-react"; import ReactMarkdown, { defaultUrlTransform, type Components, @@ -633,28 +633,60 @@ function CitationChip({ citationId: string, ) => void; }): ReactElement { + const pageCitationAssetUrl = getTrimmedCitationField( + citation.pageCitationAssetUrl, + ); + return ( - - - + + - {label} - - - - {tooltipLabel} - - + {tooltipLabel} + + + {pageCitationAssetUrl ? ( + + + + + + + + Open page image + + + ) : null} + ); } diff --git a/src/components/chunks-panel.tsx b/src/components/chunks-panel.tsx index 898ca39..78cd170 100644 --- a/src/components/chunks-panel.tsx +++ b/src/components/chunks-panel.tsx @@ -57,6 +57,7 @@ export type ChunksPanelProps = { isLoadingMore?: boolean; isLoadingAllChunks?: boolean; hasMoreChunks?: boolean; + processingMessage?: string | null; onLoadMore?: () => void; onLoadAllChunks?: () => void; onLoginClick?: () => void; @@ -84,6 +85,7 @@ export function ChunksPanel({ isLoadingMore = false, isLoadingAllChunks = false, hasMoreChunks = false, + processingMessage = null, onLoadMore, onLoadAllChunks, onLoginClick, @@ -360,7 +362,7 @@ export function ChunksPanel({ className="mx-auto flex w-[90%] min-w-0 max-w-[1600px] flex-col items-center p-3 sm:p-6" > {isLoading ? ( - + ) : chunks.length === 0 ? ( selectedSource ? ( @@ -1138,13 +1140,19 @@ function EmptyChunks(): ReactNode { ); } -function LoadingChunks(): ReactNode { +function LoadingChunks({ + message, +}: { + readonly message?: string | null; +}): ReactNode { return (
-

Loading parsed chunks...

+

+ {message ?? "Loading parsed chunks..."} +

); } diff --git a/src/components/source-row.tsx b/src/components/source-row.tsx index abfe8cb..e964c93 100644 --- a/src/components/source-row.tsx +++ b/src/components/source-row.tsx @@ -102,7 +102,7 @@ export function SourceRow({ }`} > {isReady - ? `${getReadySourceLabel(source)} · ${source.chunkCount ?? 0} chunks` + ? getReadySourceStatusText(source) : source.status === "parsing" ? "Preparing" : source.status === "uploading" @@ -197,6 +197,11 @@ function getReadySourceLabel(source: SourceView): string { return "Processed"; } +function getReadySourceStatusText(source: SourceView): string { + if (typeof source.chunkCount !== "number") return getReadySourceLabel(source); + return `${getReadySourceLabel(source)} · ${source.chunkCount} chunks`; +} + function fileIconTint(title: string): { bg: string; fg: string } { const ext = title.split(".").pop()?.toLowerCase(); switch (ext) { diff --git a/src/components/workspace-citation-focus.ts b/src/components/workspace-citation-focus.ts index b0d0f66..3485dd2 100644 --- a/src/components/workspace-citation-focus.ts +++ b/src/components/workspace-citation-focus.ts @@ -43,6 +43,7 @@ type WorkspaceCitationFocus = { readonly requestChunkFocus: (chunkId: string | null) => void readonly isSelectedChunksLoading: boolean readonly isSelectedChunksLoadingMore: boolean + readonly selectedChunksMessage: string | null readonly selectedChunks: ParsedChunkView[] readonly selectedSource: SourceView | undefined } @@ -80,6 +81,7 @@ export function useWorkspaceCitationFocus({ handleLoadMoreChunks, isSelectedChunksLoading, isSelectedChunksLoadingMore, + selectedChunksMessage, selectedChunks, selectedSource, } = useWorkspaceSelectedChunks({ @@ -277,6 +279,7 @@ export function useWorkspaceCitationFocus({ pendingCitationId, prefetchedChunksBySourceId, requestChunkFocus, + selectedChunksMessage, selectedChunks, selectedSource, } diff --git a/src/components/workspace-selected-chunks.test.ts b/src/components/workspace-selected-chunks.test.ts index 2772c85..fa08df5 100644 --- a/src/components/workspace-selected-chunks.test.ts +++ b/src/components/workspace-selected-chunks.test.ts @@ -129,6 +129,38 @@ describe("useWorkspaceSelectedChunks", () => { }); }); + it("treats a processing chunk page as a loading state", async () => { + fetchChunkPageMock.mockResolvedValue({ + chunks: [], + isProcessing: true, + message: "Source parsed snapshot is still being prepared.", + pagination: { + page: 1, + pageSize: 50, + total: 0, + totalPages: 0, + }, + }); + + const { result } = renderHook( + () => + useWorkspaceSelectedChunks({ + selectedSourceId: "source_1", + sources: [readySource], + prefetchedChunksBySourceId: {}, + }), + { wrapper: createSWRWrapper }, + ); + + await waitFor(() => + expect(result.current.selectedChunksMessage).toBe( + "Source parsed snapshot is still being prepared.", + ), + ); + expect(result.current.isSelectedChunksLoading).toBe(true); + expect(result.current.selectedChunks).toEqual([]); + }); + it("returns an empty chunk list when no source is selected", () => { const { result } = renderHook( () => diff --git a/src/components/workspace-selected-chunks.ts b/src/components/workspace-selected-chunks.ts index 4ec9f97..6d79175 100644 --- a/src/components/workspace-selected-chunks.ts +++ b/src/components/workspace-selected-chunks.ts @@ -24,6 +24,7 @@ type WorkspaceSelectedChunks = { readonly handleLoadMoreChunks: () => void readonly isSelectedChunksLoading: boolean readonly isSelectedChunksLoadingMore: boolean + readonly selectedChunksMessage: string | null readonly selectedChunks: ParsedChunkView[] readonly selectedSource: SourceView | undefined } @@ -57,8 +58,11 @@ export function useWorkspaceSelectedChunks({ { revalidateIfStale: false, keepPreviousData: false, + refreshInterval: (pages: readonly SourceChunksResponse[] | undefined) => + hasProcessingChunkPage(pages) ? 2_000 : 0, }, ) + const selectedChunksMessage = getSelectedChunksMessage(selectedChunkPages) const pagedSelectedChunks = useMemo( () => resolveChunkConnectionTargets( @@ -90,10 +94,11 @@ export function useWorkspaceSelectedChunks({ typeof selectedChunkPages[selectedChunkPageCount - 1] === "undefined", ) const isSelectedChunksLoading = - selectedChunkSourceId !== null && - !prefetchedSelectedChunks && - !selectedChunkPages && - isChunksLoading + Boolean(selectedChunksMessage) || + (selectedChunkSourceId !== null && + !prefetchedSelectedChunks && + !selectedChunkPages && + isChunksLoading) function handleLoadMoreChunks(): void { if (!hasMoreSelectedChunks || isSelectedChunksLoadingMore) return @@ -105,6 +110,7 @@ export function useWorkspaceSelectedChunks({ handleLoadMoreChunks, isSelectedChunksLoading, isSelectedChunksLoadingMore, + selectedChunksMessage, selectedChunks, selectedSource, } @@ -118,6 +124,22 @@ function fetchChunksByKey([ return workspaceClient.fetchChunkPage(sourceId, page) } +function hasProcessingChunkPage( + pages: readonly SourceChunksResponse[] | undefined, +): boolean { + return pages?.some((page) => page.isProcessing) ?? false +} + +function getSelectedChunksMessage( + pages: readonly SourceChunksResponse[] | undefined, +): string | null { + const page = pages?.find( + (candidate) => + candidate.isProcessing && typeof candidate.message === "string", + ) + return page?.message ?? null +} + function mergeVisibleChunkAssetUrls( chunks: readonly ParsedChunkView[], visibleChunks: readonly ParsedChunkView[], diff --git a/src/components/workspace-shell-layout.tsx b/src/components/workspace-shell-layout.tsx index 235eb07..3b9e997 100644 --- a/src/components/workspace-shell-layout.tsx +++ b/src/components/workspace-shell-layout.tsx @@ -80,6 +80,7 @@ export type WorkspaceShellLayoutProps = { readonly pendingCitationId: string | null readonly readySourceCount: number readonly selectedChunks: readonly ParsedChunkView[] + readonly selectedChunksMessage: string | null readonly selectedSourceFile: SourceOriginalFileView | null readonly selectedSourceId: string | null readonly selectedSourceTitle: string | null @@ -271,6 +272,7 @@ export function WorkspaceShellLayout( isLoading={props.isSelectedChunksLoading} isLoadingAllChunks={props.isSelectedAllChunksLoading} isLoadingMore={props.isSelectedChunksLoadingMore} + processingMessage={props.selectedChunksMessage} hasMoreChunks={props.hasMoreSelectedChunks} onLoadAllChunks={props.onLoadAllChunks} onLoadMore={props.onLoadMoreChunks} @@ -412,6 +414,7 @@ export function WorkspaceShellLayout( isLoading={props.isSelectedChunksLoading} isLoadingAllChunks={props.isSelectedAllChunksLoading} isLoadingMore={props.isSelectedChunksLoadingMore} + processingMessage={props.selectedChunksMessage} hasMoreChunks={props.hasMoreSelectedChunks} onLoadAllChunks={props.onLoadAllChunks} onLoadMore={props.onLoadMoreChunks} diff --git a/src/components/workspace-shell.tsx b/src/components/workspace-shell.tsx index 254c91e..4dd8b9c 100644 --- a/src/components/workspace-shell.tsx +++ b/src/components/workspace-shell.tsx @@ -236,6 +236,7 @@ function WorkspaceShellContent({ pendingCitationId={citationFocus.pendingCitationId} readySourceCount={sourceWorkflow.readySourceCount} selectedChunks={citationFocus.selectedChunks} + selectedChunksMessage={citationFocus.selectedChunksMessage} selectedSourceFile={citationFocus.selectedSource?.originalFile ?? null} selectedSourceId={sourceWorkflow.selectedSourceId} selectedSourceTitle={selectedSourceTitle} diff --git a/src/domains/chat/chat-citation-persistence.ts b/src/domains/chat/chat-citation-persistence.ts index bf2a1a1..54e89de 100644 --- a/src/domains/chat/chat-citation-persistence.ts +++ b/src/domains/chat/chat-citation-persistence.ts @@ -86,6 +86,7 @@ function toCitationView( chunkType: citation.chunkType, score: citation.score, assetUrl: citation.assetUrl, + pageCitationAssetUrl: citation.pageCitationAssetUrl, description: "description" in citation ? citation.description : undefined, source: { documentId: citation.source.documentId, diff --git a/src/domains/chat/citations.test.ts b/src/domains/chat/citations.test.ts index ce681ec..b6daa0a 100644 --- a/src/domains/chat/citations.test.ts +++ b/src/domains/chat/citations.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest" -import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import { toChatCitationViews } from "./citations" +import type { PageCitationAssetRetrievalResult } from "./page-citation-assets" describe("toChatCitationViews", () => { it("extracts first citation description for each generated source label", () => { @@ -31,11 +31,24 @@ describe("toChatCitationViews", () => { { ...secondResult, description: "margin expansion" }, ]) }) + + it("preserves page citation asset URLs", () => { + const result = makeRetrievalResult({ + chunkType: "page", + pageCitationAssetUrl: "https://blob.example/pages/page-000004.png", + }) + + const citations = toChatCitationViews([result], "Grounded answer.") + + expect(citations[0]?.pageCitationAssetUrl).toBe( + "https://blob.example/pages/page-000004.png", + ) + }) }) function makeRetrievalResult( - overrides: Partial = {}, -): RetrievalResult { + overrides: Partial = {}, +): PageCitationAssetRetrievalResult { return { content: "Grounding content", chunkType: "text", diff --git a/src/domains/chat/citations.ts b/src/domains/chat/citations.ts index e010bdc..b6b4038 100644 --- a/src/domains/chat/citations.ts +++ b/src/domains/chat/citations.ts @@ -2,9 +2,10 @@ import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" import type { ChatCitationView } from "@/domains/chat/types" +import type { PageCitationAssetRetrievalResult } from "./page-citation-assets" export function toChatCitationViews( - results: readonly RetrievalResult[], + results: readonly PageCitationAssetRetrievalResult[], answer: string, ): ChatCitationView[] { const descriptionsBySourceNumber = getCitationDescriptions(answer) @@ -16,6 +17,9 @@ export function toChatCitationViews( chunkType: result.chunkType, score: result.score, ...(result.assetUrl ? { assetUrl: result.assetUrl } : {}), + ...(result.pageCitationAssetUrl + ? { pageCitationAssetUrl: result.pageCitationAssetUrl } + : {}), ...(description ? { description } : {}), source: { documentId: result.source.documentId, diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 0c8cfd6..1b51180 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -9,7 +9,10 @@ import { generateAgenticOutputManifest, parseChatRequestBody, } from "." -import type { HardenMediaAssetUrlsInput } from "./media-asset-hardening" +import type { + HardenableRetrievalResult, + HardenMediaAssetUrlsInput, +} from "./media-asset-hardening" import type { Source } from "@/infrastructure/db/schema" import type { ChatArtifactView } from "@/domains/chat/types" @@ -665,6 +668,223 @@ describe("answerQuestionWithRetrieval", () => { expect(answer.artifacts?.[0]?.citation?.assetUrl).toBe(hardenedAssetUrl); }); + it("hardens page citation asset URLs before returning citations", async () => { + const rawPageAssetUrl = + "https://knowhere-storage.example/results/job_1/page_citation_assets/page-4.png?AWSAccessKeyId=test"; + const storedPageAssetUrl = + "https://blob.example/workspaces/workspace_1/sources/source_pages/parsed-result/page_citation_assets/page-4.png"; + const hardenedPageAssetUrl = + "https://blob.example/workspaces/workspace_1/chat-assets/source-source_pages/page-4.png"; + const result = makeRetrievalResult({ + chunkType: "page", + metadata: { + pageNums: [4], + pageAssets: [ + { + pageNum: 4, + artifactRef: "page_citation_assets/page-4.png", + assetUrl: rawPageAssetUrl, + contentType: "image/png", + }, + ], + }, + source: { + documentId: "doc_pages", + sourceFileName: "document-generated.pdf", + sectionPath: "Page 4", + }, + }); + const retrieval = { + query: vi.fn().mockResolvedValue({ + results: [result], + evidenceText: "Page four evidence.", + referencedChunks: [], + namespace: "notebook-workspace", + query: "page four evidence", + routerUsed: "workflow_single_step", + answerText: null, + }), + }; + const generateAnswer = vi.fn(async ({ searchSources }) => { + await searchSources({ query: "page four evidence" }); + return makeHarnessRunResult(`This page has the answer. ${storedPageAssetUrl}`); + }); + const hardenMediaAssetUrls = vi.fn( + async ({ + results, + artifacts, + }: HardenMediaAssetUrlsInput): Promise<{ + results: HardenableRetrievalResult[] + artifacts?: ChatArtifactView[] + }> => ({ + results: results.map( + (candidate): HardenableRetrievalResult => ({ + ...candidate, + pageCitationAssetUrl: + candidate.pageCitationAssetUrl === rawPageAssetUrl + ? hardenedPageAssetUrl + : candidate.pageCitationAssetUrl === storedPageAssetUrl + ? hardenedPageAssetUrl + : candidate.pageCitationAssetUrl, + }), + ), + ...(artifacts ? { artifacts: [...artifacts] } : {}), + }), + ); + + const answer = await Effect.runPromise( + answerQuestionWithRetrieval({ + question: "What is on page four?", + namespace: "notebook-workspace", + sources: [ + makeSource({ + id: "source_pages", + title: "deck.pdf", + knowhereDocumentId: "doc_pages", + }), + ], + excludedSourceIds: [], + retrieval, + generateAnswer, + hardenMediaAssetUrls, + loadSourceAssetUrls: vi.fn(async () => ({ + "page_citation_assets/page-4.png": storedPageAssetUrl, + })), + messages: [], + }), + ); + + expect(hardenMediaAssetUrls).toHaveBeenCalledWith({ + results: [ + expect.objectContaining({ + pageCitationAssetUrl: storedPageAssetUrl, + source: expect.objectContaining({ + sourceFileName: "deck.pdf", + }), + }), + ], + artifacts: undefined, + }); + expect(answer.answer).toBe("This page has the answer."); + expect(answer.answer).not.toContain("knowhere-storage.example"); + expect(answer.citations).toEqual([ + expect.objectContaining({ + chunkType: "page", + pageCitationAssetUrl: hardenedPageAssetUrl, + source: expect.objectContaining({ + sourceFileName: "deck.pdf", + }), + }), + ]); + expect(answer.citations[0]?.pageCitationAssetUrl).not.toBe(rawPageAssetUrl); + }); + + it("hardens page citation asset URLs from referenced chunk metadata", async () => { + const rawPageAssetUrl = + "https://knowhere-storage.example/results/job_1/page_citation_assets/page-6.png?AWSAccessKeyId=test"; + const storedPageAssetUrl = + "https://blob.example/workspaces/workspace_1/sources/source_pages/parsed-result/page_citation_assets/page-6.png"; + const hardenedPageAssetUrl = + "https://blob.example/workspaces/workspace_1/chat-assets/source-source_pages/page-6.png"; + const retrieval = { + query: vi.fn().mockResolvedValue({ + results: [], + evidenceText: "Page six evidence.", + referencedChunks: [ + { + chunkId: "chunk_page_6", + documentId: "doc_pages", + chunkType: "page", + sectionPath: "Page 6", + filePath: null, + jobId: "job_1", + assetUrl: rawPageAssetUrl, + metadata: { + pageNums: [6], + pageAssets: [ + { + pageNum: 6, + artifactRef: "page_citation_assets/page-6.png", + assetUrl: rawPageAssetUrl, + contentType: "image/png", + }, + ], + }, + }, + ], + namespace: "notebook-workspace", + query: "page six evidence", + routerUsed: "workflow_single_step", + answerText: null, + }), + }; + const generateAnswer = vi.fn(async ({ searchSources }) => { + await searchSources({ query: "page six evidence" }); + return makeHarnessRunResult("This page has referenced evidence."); + }); + const hardenMediaAssetUrls = vi.fn( + async ({ + results, + artifacts, + }: HardenMediaAssetUrlsInput): Promise<{ + results: HardenableRetrievalResult[] + artifacts?: ChatArtifactView[] + }> => ({ + results: results.map( + (candidate): HardenableRetrievalResult => ({ + ...candidate, + pageCitationAssetUrl: + candidate.pageCitationAssetUrl === rawPageAssetUrl + ? hardenedPageAssetUrl + : candidate.pageCitationAssetUrl === storedPageAssetUrl + ? hardenedPageAssetUrl + : candidate.pageCitationAssetUrl, + }), + ), + ...(artifacts ? { artifacts: [...artifacts] } : {}), + }), + ); + + const answer = await Effect.runPromise( + answerQuestionWithRetrieval({ + question: "What is on page six?", + namespace: "notebook-workspace", + sources: [ + makeSource({ + id: "source_pages", + title: "deck.pdf", + knowhereDocumentId: "doc_pages", + }), + ], + excludedSourceIds: [], + retrieval, + generateAnswer, + hardenMediaAssetUrls, + loadSourceAssetUrls: vi.fn(async () => ({ + "page_citation_assets/page-6.png": storedPageAssetUrl, + })), + messages: [], + }), + ); + + expect(hardenMediaAssetUrls).toHaveBeenCalledWith({ + results: [ + expect.objectContaining({ + metadata: expect.objectContaining({ + pageAssets: [ + expect.objectContaining({ + assetUrl: rawPageAssetUrl, + }), + ], + }), + pageCitationAssetUrl: storedPageAssetUrl, + }), + ], + artifacts: undefined, + }); + expect(answer.citations[0]?.pageCitationAssetUrl).toBe(hardenedPageAssetUrl); + }); + it("returns only harness-selected artifacts when retrieval has extra media candidates", async () => { const frontAssetUrl = "https://blob.example/images/id-front.jpg"; const backAssetUrl = "https://blob.example/images/id-back.jpg"; @@ -1773,6 +1993,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_123", knowhereDocumentId: "doc_included", stagedBlobPathname: null, diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index 2d7245d..bbf95b9 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -38,6 +38,8 @@ import { enrichRetrievalResultsWithAssetUrls, removeRetrievedMediaAssetUrls, } from "./media-assets" +import { enrichRetrievalResultsWithPageCitationAssetUrls } from "./page-citation-assets" +import type { HardenableRetrievalResult } from "./media-asset-hardening" const DEFAULT_TOP_K = 8 const MAX_AGENTIC_TOP_K = 12 @@ -244,11 +246,18 @@ export const answerQuestionWithRetrieval = ( evidenceText: formatRetrievalEvidenceText(retrievalResponses), }), ) + const pageCitationResults = yield* Effect.tryPromise(() => + enrichRetrievalResultsWithPageCitationAssetUrls({ + results: enrichedResults, + sources: input.sources, + loadSourceAssetUrls: input.loadSourceAssetUrls, + }), + ) const artifacts = toChatArtifactViewsFromHarness(generatedAnswer, input.sources) const hardenedMedia = yield* Effect.tryPromise(() => hardenAnswerMediaAssetUrls({ input, - results: enrichedResults, + results: pageCitationResults, artifacts, }), ) @@ -257,21 +266,21 @@ export const answerQuestionWithRetrieval = ( results: getGeneratedAnswerSanitizerResults({ rawResults, enrichedResults, + pageCitationResults, hardenedResults: hardenedMedia.results, artifacts, hardenedArtifacts: hardenedMedia.artifacts, }), }) - const citationResults = hardenedMedia.results const displayArtifacts = hardenedMedia.artifacts ?? [] logger.info("chat-agent: answer complete", { answerLength: answer.length, - citationCount: citationResults.length, + citationCount: hardenedMedia.results.length, artifactCount: displayArtifacts.length, }) return { answer, - citations: toChatCitationViews(citationResults, answer), + citations: toChatCitationViews(hardenedMedia.results, answer), artifacts: displayArtifacts, } }) @@ -416,7 +425,7 @@ function normalizeHarnessSource( type AnswerMediaAssetHardeningInput = { readonly input: AnswerQuestionInput - readonly results: readonly RetrievalResult[] + readonly results: readonly HardenableRetrievalResult[] readonly artifacts?: readonly ChatArtifactView[] } @@ -425,7 +434,7 @@ async function hardenAnswerMediaAssetUrls({ results, artifacts, }: AnswerMediaAssetHardeningInput): Promise<{ - readonly results: RetrievalResult[] + readonly results: HardenableRetrievalResult[] readonly artifacts?: ChatArtifactView[] }> { if (!input.hardenMediaAssetUrls) { @@ -456,7 +465,8 @@ async function hardenAnswerMediaAssetUrls({ type GeneratedAnswerSanitizerResultsInput = { readonly rawResults: readonly RetrievalResult[] readonly enrichedResults: readonly RetrievalResult[] - readonly hardenedResults: readonly RetrievalResult[] + readonly pageCitationResults: readonly HardenableRetrievalResult[] + readonly hardenedResults: readonly HardenableRetrievalResult[] readonly artifacts?: readonly ChatArtifactView[] readonly hardenedArtifacts?: readonly ChatArtifactView[] } @@ -464,6 +474,7 @@ type GeneratedAnswerSanitizerResultsInput = { function getGeneratedAnswerSanitizerResults({ rawResults, enrichedResults, + pageCitationResults, hardenedResults, artifacts, hardenedArtifacts, @@ -471,12 +482,32 @@ function getGeneratedAnswerSanitizerResults({ return [ ...rawResults, ...enrichedResults, + ...toPageCitationSanitizerResults(pageCitationResults), ...hardenedResults, + ...toPageCitationSanitizerResults(hardenedResults), ...toArtifactSanitizerResults(artifacts), ...toArtifactSanitizerResults(hardenedArtifacts), ] } +function toPageCitationSanitizerResults( + results: readonly HardenableRetrievalResult[], +): RetrievalResult[] { + return results.flatMap((result): RetrievalResult[] => { + if (!result.pageCitationAssetUrl) return [] + + return [ + { + content: result.content, + chunkType: result.chunkType, + score: result.score, + assetUrl: result.pageCitationAssetUrl, + source: result.source, + }, + ] + }) +} + function toArtifactSanitizerResults( artifacts: readonly ChatArtifactView[] | undefined, ): RetrievalResult[] { @@ -500,6 +531,15 @@ function toArtifactSanitizerResults( }), ) } + if (artifact.citation?.pageCitationAssetUrl) { + results.push( + toArtifactSanitizerResult({ + assetUrl: artifact.citation.pageCitationAssetUrl, + artifact, + citation: artifact.citation, + }), + ) + } return results }) } @@ -792,17 +832,7 @@ function mapManifestCitationsToResults( resolveChunkForAssetRef(citation.ref, assetsByRef, chunksByRef) if (!chunk) continue - const retrievalResult: RetrievalResult = { - content: chunk.content, - chunkType: chunk.chunkType, - score: chunk.score, - ...(chunk.assetUrl ? { assetUrl: chunk.assetUrl } : {}), - source: { - documentId: chunk.source.documentId ?? undefined, - sourceFileName: chunk.source.sourceFileName ?? undefined, - sectionPath: chunk.source.sectionPath ?? undefined, - }, - } + const retrievalResult = toRetrievalResultFromEvidenceChunk(chunk) const key = getRetrievalResultKey(retrievalResult) if (seenKeys.has(key)) continue @@ -889,10 +919,14 @@ function toRetrievalResultFromEvidenceChunk( chunk: EvidenceChunk, ): RetrievalResult { return { + ...(chunk.chunkId ? { chunkId: chunk.chunkId } : {}), content: chunk.content, chunkType: chunk.chunkType, score: chunk.score, ...(chunk.assetUrl ? { assetUrl: chunk.assetUrl } : {}), + ...(chunk.sourceChunkPath ? { sourceChunkPath: chunk.sourceChunkPath } : {}), + ...(chunk.filePath ? { filePath: chunk.filePath } : {}), + ...(chunk.metadata ? { metadata: chunk.metadata } : {}), source: { documentId: chunk.source.documentId ?? undefined, sourceFileName: chunk.source.sourceFileName ?? undefined, @@ -921,10 +955,14 @@ function collectRetrievalResults( for (const result of [ ...response.results, ...response.referencedChunks.map((chunk): RetrievalResult => ({ + chunkId: chunk.chunkId, content: "", chunkType: chunk.chunkType, score: null, ...(chunk.assetUrl ? { assetUrl: chunk.assetUrl } : {}), + ...(chunk.sourceChunkPath ? { sourceChunkPath: chunk.sourceChunkPath } : {}), + ...(chunk.filePath ? { filePath: chunk.filePath } : {}), + ...(chunk.metadata ? { metadata: chunk.metadata } : {}), source: { documentId: chunk.documentId, sourceFileName: sourceTitlesByDocumentId.get(chunk.documentId), diff --git a/src/domains/chat/media-asset-hardening.test.ts b/src/domains/chat/media-asset-hardening.test.ts index 70cf562..d3c87b2 100644 --- a/src/domains/chat/media-asset-hardening.test.ts +++ b/src/domains/chat/media-asset-hardening.test.ts @@ -1,11 +1,10 @@ import { afterEach, describe, expect, it, vi } from "vitest" -import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" import { hardenChatMediaAssetUrls, - type ChatMediaAssetBlobStore, - type FetchChatMediaAsset, + isNotebookOwnedAssetUrl, + type HardenableRetrievalResult, } from "./media-asset-hardening" const loggerMock = vi.hoisted(() => ({ @@ -20,68 +19,42 @@ vi.mock("@/lib/logger", () => ({ afterEach(() => { vi.clearAllMocks() - delete process.env.KNOWHERE_BASE_URL }) describe("hardenChatMediaAssetUrls", () => { - it("copies upstream absolute asset URLs into Notebook chat assets", async () => { - const rawAssetUrl = - "https://knowhere-storage.example/results/job_1/images/image-6-%E6%83%85%E6%84%9F%E5%88%86%E7%B1%BB%E6%A8%A1%E5%9E%8B.jpg?AWSAccessKeyId=test&Signature=secret" - const blobStore = makeBlobStore( - "https://blob.example/workspaces/workspace_1/chat-assets/source-source_1/image-6.jpg", - ) - const fetchAsset = makeFetchAsset("image-bytes", "image/jpeg") + it("keeps an already Notebook-owned asset URL without loading the asset map", async () => { + const ownedUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_1/rev_1/assets/images/a.png" + const loadSourceAssetUrls = vi.fn(async () => ({})) const result = await hardenChatMediaAssetUrls({ workspaceId: "workspace_1", - sources: [ - makeSource({ - id: "source_1", - knowhereDocumentId: "doc_model", - }), - ], + sources: [makeSource({ id: "source_1", knowhereDocumentId: "doc_1" })], results: [ makeRetrievalResult({ chunkType: "image", - assetUrl: rawAssetUrl, + assetUrl: ownedUrl, source: { - documentId: "doc_model", + documentId: "doc_1", sourceFileName: "model.pdf", sectionPath: "Root", }, }), ], - blobStore, - fetchAsset, + loadSourceAssetUrls, }) - expect(fetchAsset).toHaveBeenCalledWith(rawAssetUrl) - expect(blobStore.put).toHaveBeenCalledWith( - expect.stringMatching( - /^workspaces\/workspace_1\/chat-assets\/source-source_1\/[a-f0-9]{24}-image-6\.jpg$/, - ), - expect.any(Buffer), - { - access: "public", - allowOverwrite: true, - contentType: "image/jpeg", - multipart: true, - }, - ) - expect(result.results[0]?.assetUrl).toBe( - "https://blob.example/workspaces/workspace_1/chat-assets/source-source_1/image-6.jpg", - ) + expect(result.results[0]?.assetUrl).toBe(ownedUrl) + expect(loadSourceAssetUrls).not.toHaveBeenCalled() }) - it("uses an existing parsed asset URL before fetching the upstream URL", async () => { + it("resolves a raw asset URL to the durable parsed asset URL", async () => { const rawAssetUrl = "https://knowhere-storage.example/results/job_1/images/id-front.jpg?AWSAccessKeyId=test" - const parsedAssetUrl = - "https://blob.example/workspaces/workspace_1/sources/source_identity/parsed-result/images/id-front.jpg" - const blobStore = makeBlobStore("https://blob.example/should-not-upload.jpg") - const fetchAsset = makeFetchAsset("should-not-fetch", "image/jpeg") + const durableUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_identity/rev_1/assets/images/id-front.jpg" const loadSourceAssetUrls = vi.fn().mockResolvedValue({ - "images/id-front.jpg": parsedAssetUrl, + "images/id-front.jpg": durableUrl, }) const result = await hardenChatMediaAssetUrls({ @@ -104,99 +77,49 @@ describe("hardenChatMediaAssetUrls", () => { }), ], loadSourceAssetUrls, - blobStore, - fetchAsset, }) expect(loadSourceAssetUrls).toHaveBeenCalledWith( expect.objectContaining({ id: "source_identity" }), ) - expect(fetchAsset).not.toHaveBeenCalled() - expect(blobStore.put).not.toHaveBeenCalled() - expect(result.results[0]?.assetUrl).toBe(parsedAssetUrl) - }) - - it("fetches demo asset routes from the upstream demo API", async () => { - process.env.KNOWHERE_BASE_URL = "https://knowhere.example" - const demoAssetUrl = - "/api/demo-sources/demo_source_1/assets/images/demo%20chart.png" - const blobStore = makeBlobStore( - "https://blob.example/workspaces/workspace_1/chat-assets/demo-demo_source_1/demo-chart.png", - ) - const fetchAsset = makeFetchAsset("demo-image", "image/png") - - const result = await hardenChatMediaAssetUrls({ - workspaceId: "workspace_1", - sources: [], - results: [ - makeRetrievalResult({ - chunkType: "image", - assetUrl: demoAssetUrl, - source: { - documentId: "demo_doc", - sourceFileName: "demo.pdf", - sectionPath: "images/demo chart.png", - }, - }), - ], - blobStore, - fetchAsset, - }) - - expect(fetchAsset).toHaveBeenCalledWith( - "https://knowhere.example/api/v1/demo/sources/demo_source_1/assets/images/demo%20chart.png", - ) - expect(fetchAsset).not.toHaveBeenCalledWith(demoAssetUrl) - expect(blobStore.put).toHaveBeenCalledWith( - expect.stringContaining("/chat-assets/demo-demo_source_1/"), - expect.any(Buffer), - expect.objectContaining({ contentType: "image/png" }), - ) - expect(result.results[0]?.assetUrl).toBe( - "https://blob.example/workspaces/workspace_1/chat-assets/demo-demo_source_1/demo-chart.png", - ) + expect(result.results[0]?.assetUrl).toBe(durableUrl) }) - it("falls back to the raw URL when hardening fails", async () => { + it("omits an asset URL that cannot be resolved to a durable URL", async () => { const rawAssetUrl = "https://knowhere-storage.example/results/job_1/tables/table-1.html?AWSAccessKeyId=test" - const blobStore = makeBlobStore("https://blob.example/should-not-exist.html") - const fetchAsset: FetchChatMediaAsset = vi - .fn() - .mockRejectedValue(new Error("expired URL")) + const loadSourceAssetUrls = vi.fn().mockResolvedValue({}) const result = await hardenChatMediaAssetUrls({ workspaceId: "workspace_1", - sources: [], + sources: [ + makeSource({ id: "source_1", knowhereDocumentId: "doc_1" }), + ], results: [ makeRetrievalResult({ chunkType: "table", assetUrl: rawAssetUrl, + source: { + documentId: "doc_1", + sourceFileName: "source.pdf", + sectionPath: "tables/table-1.html", + }, }), ], - blobStore, - fetchAsset, + loadSourceAssetUrls, }) - expect(result.results[0]?.assetUrl).toBe(rawAssetUrl) - expect(blobStore.put).not.toHaveBeenCalled() - expect(loggerMock.warn).toHaveBeenCalledWith( - "chat-agent: media asset hardening failed; keeping raw URL", - expect.objectContaining({ - assetUrl: - "https://knowhere-storage.example/results/job_1/tables/table-1.html", - error: "expired URL", - }), - ) + expect(result.results[0]?.assetUrl).toBeUndefined() }) - it("rewrites artifact asset URLs and nested citation asset URLs", async () => { + it("resolves an artifact asset URL and its nested citation URL", async () => { const rawAssetUrl = "https://knowhere-storage.example/results/job_1/images/front.jpg?AWSAccessKeyId=test" - const blobAssetUrl = - "https://blob.example/workspaces/workspace_1/chat-assets/source-source_identity/front.jpg" - const blobStore = makeBlobStore(blobAssetUrl) - const fetchAsset = makeFetchAsset("front-image", "image/jpeg") + const durableUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_identity/rev_1/assets/images/front.jpg" + const loadSourceAssetUrls = vi.fn().mockResolvedValue({ + "images/front.jpg": durableUrl, + }) const result = await hardenChatMediaAssetUrls({ workspaceId: "workspace_1", @@ -225,40 +148,38 @@ describe("hardenChatMediaAssetUrls", () => { }, }, ], - blobStore, - fetchAsset, + loadSourceAssetUrls, }) const [artifact] = result.artifacts ?? [] - expect(fetchAsset).toHaveBeenCalledTimes(1) - expect(artifact?.assetUrl).toBe(blobAssetUrl) - expect(artifact?.citation?.assetUrl).toBe(blobAssetUrl) + expect(artifact?.assetUrl).toBe(durableUrl) + expect(artifact?.citation?.assetUrl).toBe(durableUrl) }) }) -function makeFetchAsset( - body: string, - contentType: string, -): FetchChatMediaAsset { - return vi.fn().mockResolvedValue( - new Response(Buffer.from(body), { - status: 200, - headers: { - "content-type": contentType, - }, - }), - ) -} - -function makeBlobStore(url: string): ChatMediaAssetBlobStore { - return { - put: vi.fn().mockResolvedValue({ url }), - } -} +describe("isNotebookOwnedAssetUrl", () => { + it("treats Vercel Blob hosts and parsed-document paths as owned", () => { + expect( + isNotebookOwnedAssetUrl( + "https://fake.public.blob.vercel-storage.com/x/y.png", + ), + ).toBe(true) + expect( + isNotebookOwnedAssetUrl( + "https://cdn.example/workspaces/w/parsed-documents/d/r/assets/a.png", + ), + ).toBe(true) + expect( + isNotebookOwnedAssetUrl( + "https://knowhere-storage.example/results/job_1/images/a.png?sig=x", + ), + ).toBe(false) + }) +}) function makeRetrievalResult( - overrides: Partial = {}, -): RetrievalResult { + overrides: Partial = {}, +): HardenableRetrievalResult { return { content: "Asset evidence", chunkType: "text", @@ -281,6 +202,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_123", knowhereDocumentId: "doc_1", stagedBlobPathname: null, diff --git a/src/domains/chat/media-asset-hardening.ts b/src/domains/chat/media-asset-hardening.ts index e0a813e..f768515 100644 --- a/src/domains/chat/media-asset-hardening.ts +++ b/src/domains/chat/media-asset-hardening.ts @@ -1,6 +1,3 @@ -import path from "node:path" -import { createHash } from "node:crypto" -import { put } from "@vercel/blob" import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import type { @@ -8,18 +5,21 @@ import type { ChatCitationView, } from "@/domains/chat/types" import type { Source } from "@/infrastructure/db/schema" -import { knowhereDemoApi } from "@/integrations/knowhere-demo" import { logger } from "@/lib/logger" import type { LoadSourceAssetUrls } from "./media-assets" import { resolveAssetUrlFromReferenceText } from "./media-assets" +export type HardenableRetrievalResult = RetrievalResult & { + readonly pageCitationAssetUrl?: string +} + export type HardenMediaAssetUrlsInput = { - readonly results: readonly RetrievalResult[] + readonly results: readonly HardenableRetrievalResult[] readonly artifacts?: readonly ChatArtifactView[] } export type HardenMediaAssetUrlsResult = { - readonly results: RetrievalResult[] + readonly results: HardenableRetrievalResult[] readonly artifacts?: ChatArtifactView[] } @@ -27,30 +27,11 @@ export type HardenMediaAssetUrls = ( input: HardenMediaAssetUrlsInput, ) => Promise -export type ChatMediaAssetBlobStore = { - readonly put: ( - pathname: string, - body: Buffer, - options: ChatMediaAssetBlobPutOptions, - ) => Promise<{ readonly url: string }> -} - -export type ChatMediaAssetBlobPutOptions = { - readonly access?: "public" - readonly allowOverwrite?: boolean - readonly contentType: string - readonly multipart?: boolean -} - -export type FetchChatMediaAsset = (url: string) => Promise - export type HardenChatMediaAssetUrlsForWorkspaceInput = HardenMediaAssetUrlsInput & { readonly workspaceId: string readonly sources: readonly Source[] - readonly loadSourceAssetUrls?: LoadSourceAssetUrls - readonly blobStore?: ChatMediaAssetBlobStore - readonly fetchAsset?: FetchChatMediaAsset + readonly loadSourceAssetUrls: LoadSourceAssetUrls } type AssetReferenceSource = ChatCitationView["source"] @@ -61,67 +42,42 @@ type AssetUrlReference = { readonly content?: string } -type AssetFetchRequest = { - readonly fetchUrl: string - readonly canonicalKey: string - readonly sourceSegment: string - readonly suggestedFileName: string -} - type HardeningContext = { - readonly workspaceId: string readonly sourcesByDocumentId: ReadonlyMap - readonly loadSourceAssetUrls?: LoadSourceAssetUrls + readonly loadSourceAssetUrls: LoadSourceAssetUrls readonly assetUrlsBySourceId: Map< string, Promise>> > - readonly hardenedAssetUrlByKey: Map> - readonly blobStore: ChatMediaAssetBlobStore - readonly fetchAsset: FetchChatMediaAsset -} - -type DemoAssetRoute = { - readonly demoSourceId: string - readonly encodedAssetPath: string - readonly decodedAssetPath: string } -const chatAssetsDirectoryName = "chat-assets" const parsedResultDirectoryName = "parsed-result" -const fallbackContentType = "application/octet-stream" -const defaultFetchAsset: FetchChatMediaAsset = (url) => fetch(url) -const defaultBlobStore: ChatMediaAssetBlobStore = { - put: (pathname, body, options) => - put(pathname, body, { - access: options.access ?? "public", - allowOverwrite: options.allowOverwrite, - contentType: options.contentType, - multipart: options.multipart, - }), -} +const chatAssetsDirectoryName = "chat-assets" +/** + * Resolve chat citation/media asset URLs to durable Notebook Blob URLs. The + * single hardening path is the SDK's `assetUrlPolicy: "durable"` read that + * `loadSourceAssetUrls` performs; here we only map a retrieval result's + * reference text to the durable URL that read produced. + * + * An asset that is already Notebook-owned is kept as-is. An asset that cannot + * be resolved to a durable URL is omitted rather than exposing a presigned + * Knowhere URL to the client. + */ export async function hardenChatMediaAssetUrls({ results, artifacts, - workspaceId, sources, loadSourceAssetUrls, - blobStore = defaultBlobStore, - fetchAsset = defaultFetchAsset, }: HardenChatMediaAssetUrlsForWorkspaceInput): Promise { const context: HardeningContext = { - workspaceId, sourcesByDocumentId: createSourcesByDocumentId(sources), loadSourceAssetUrls, assetUrlsBySourceId: new Map(), - hardenedAssetUrlByKey: new Map(), - blobStore, - fetchAsset, } const hardenedResults = await Promise.all( - results.map((result): Promise => + results.map((result): Promise => hardenRetrievalResult(result, context), ), ) @@ -140,26 +96,36 @@ export async function hardenChatMediaAssetUrls({ } async function hardenRetrievalResult( - result: RetrievalResult, + result: HardenableRetrievalResult, context: HardeningContext, -): Promise { +): Promise { const assetUrl = getTrimmedString(result.assetUrl) - if (!assetUrl) return result - - const hardenedAssetUrl = await hardenAssetUrl( - { - assetUrl, - source: result.source, - content: result.content, - }, - context, - ) - if (hardenedAssetUrl === result.assetUrl) return result + const pageCitationAssetUrl = getTrimmedString(result.pageCitationAssetUrl) + if (!assetUrl && !pageCitationAssetUrl) return result - return { - ...result, + const hardenedAssetUrl = assetUrl + ? await resolveDurableAssetUrl( + { assetUrl, source: result.source, content: result.content }, + context, + ) + : undefined + const hardenedPageCitationAssetUrl = pageCitationAssetUrl + ? await resolveDurableAssetUrl( + { + assetUrl: pageCitationAssetUrl, + source: result.source, + content: result.content, + }, + context, + ) + : undefined + + return applyAssetUrls(result, { + hadAssetUrl: Boolean(assetUrl), + hadPageCitationAssetUrl: Boolean(pageCitationAssetUrl), assetUrl: hardenedAssetUrl, - } + pageCitationAssetUrl: hardenedPageCitationAssetUrl, + }) } async function hardenArtifact( @@ -170,27 +136,26 @@ async function hardenArtifact( ? await hardenCitation(artifact.citation, context) : undefined const assetUrl = getTrimmedString(artifact.assetUrl) - if (!assetUrl) { - return citation && citation !== artifact.citation - ? { ...artifact, citation } - : artifact - } + const hardenedAssetUrl = assetUrl + ? await resolveDurableAssetUrl( + { + assetUrl, + source: artifact.citation?.source, + content: artifact.label, + }, + context, + ) + : undefined - const hardenedAssetUrl = await hardenAssetUrl( - { - assetUrl, - source: artifact.citation?.source, - content: artifact.label, - }, - context, - ) - const hasAssetUrlChange = hardenedAssetUrl !== artifact.assetUrl - const hasCitationChange = citation && citation !== artifact.citation - if (!hasAssetUrlChange && !hasCitationChange) return artifact + const citationChanged = citation && citation !== artifact.citation + const assetUrlChanged = assetUrl + ? hardenedAssetUrl !== artifact.assetUrl + : false + if (!citationChanged && !assetUrlChanged) return artifact return { ...artifact, - assetUrl: hardenedAssetUrl, + ...(assetUrl ? { assetUrl: hardenedAssetUrl } : {}), ...(citation ? { citation } : {}), } } @@ -200,70 +165,53 @@ async function hardenCitation( context: HardeningContext, ): Promise { const assetUrl = getTrimmedString(citation.assetUrl) - if (!assetUrl) return citation - - const hardenedAssetUrl = await hardenAssetUrl( - { - assetUrl, - source: citation.source, - content: citation.content, - }, - context, - ) - if (hardenedAssetUrl === citation.assetUrl) return citation + const pageCitationAssetUrl = getTrimmedString(citation.pageCitationAssetUrl) + if (!assetUrl && !pageCitationAssetUrl) return citation - return { - ...citation, + const hardenedAssetUrl = assetUrl + ? await resolveDurableAssetUrl( + { assetUrl, source: citation.source, content: citation.content }, + context, + ) + : undefined + const hardenedPageCitationAssetUrl = pageCitationAssetUrl + ? await resolveDurableAssetUrl( + { + assetUrl: pageCitationAssetUrl, + source: citation.source, + content: citation.content, + }, + context, + ) + : undefined + + return applyAssetUrls(citation, { + hadAssetUrl: Boolean(assetUrl), + hadPageCitationAssetUrl: Boolean(pageCitationAssetUrl), assetUrl: hardenedAssetUrl, - } + pageCitationAssetUrl: hardenedPageCitationAssetUrl, + }) } -async function hardenAssetUrl( +/** + * Return a durable Notebook-owned URL for a reference: keep already-owned URLs, + * otherwise resolve against the source's durable parsed asset map. Returns + * `undefined` when no durable URL is available so callers omit the URL rather + * than leak a presigned Knowhere URL. + */ +async function resolveDurableAssetUrl( reference: AssetUrlReference, context: HardeningContext, -): Promise { +): Promise { if (isNotebookOwnedAssetUrl(reference.assetUrl)) { return reference.assetUrl } - const parsedAssetUrl = await resolveParsedAssetUrl(reference, context) - if (parsedAssetUrl) return parsedAssetUrl - - const fetchRequest = resolveAssetFetchRequest(reference.assetUrl) - if (!fetchRequest) return reference.assetUrl - const source = resolveSourceForReference(reference, context) - const sourceSegment = source - ? `source-${toSafePathSegment(source.id)}` - : fetchRequest.sourceSegment - const hardeningKey = [ - context.workspaceId, - source?.id ?? reference.source?.documentId ?? "", - fetchRequest.canonicalKey, - ].join("\0") - const cached = context.hardenedAssetUrlByKey.get(hardeningKey) - if (cached) return cached - - const hardenedAssetUrl = copyAssetToBlob({ - reference, - fetchRequest, - context, - sourceSegment, - hardeningKey, - }) - context.hardenedAssetUrlByKey.set(hardeningKey, hardenedAssetUrl) - return hardenedAssetUrl -} - -async function resolveParsedAssetUrl( - reference: AssetUrlReference, - context: HardeningContext, -): Promise { - const source = resolveSourceForReference(reference, context) - if (!source || !context.loadSourceAssetUrls) return null + if (!source) return undefined const assetUrlsByFilePath = await getCachedSourceAssetUrls(source, context) - return resolveAssetUrlFromReferenceText({ + const durableUrl = resolveAssetUrlFromReferenceText({ values: [ reference.source?.sectionPath, reference.content, @@ -271,6 +219,46 @@ async function resolveParsedAssetUrl( ], assetUrlsByFilePath, }) + return durableUrl ?? undefined +} + +function applyAssetUrls< + T extends { + readonly assetUrl?: string | null + readonly pageCitationAssetUrl?: string | null + }, +>( + value: T, + hardened: { + readonly hadAssetUrl: boolean + readonly hadPageCitationAssetUrl: boolean + readonly assetUrl: string | undefined + readonly pageCitationAssetUrl: string | undefined + }, +): T { + const assetUrlChanged = + hardened.hadAssetUrl && hardened.assetUrl !== value.assetUrl + const pageCitationChanged = + hardened.hadPageCitationAssetUrl && + hardened.pageCitationAssetUrl !== value.pageCitationAssetUrl + if (!assetUrlChanged && !pageCitationChanged) return value + + const next: Record = { ...value } + if (hardened.hadAssetUrl) { + if (hardened.assetUrl) { + next["assetUrl"] = hardened.assetUrl + } else { + delete next["assetUrl"] + } + } + if (hardened.hadPageCitationAssetUrl) { + if (hardened.pageCitationAssetUrl) { + next["pageCitationAssetUrl"] = hardened.pageCitationAssetUrl + } else { + delete next["pageCitationAssetUrl"] + } + } + return next as T } async function getCachedSourceAssetUrls( @@ -280,116 +268,25 @@ async function getCachedSourceAssetUrls( const cached = context.assetUrlsBySourceId.get(source.id) if (cached) return cached - const loaded = context.loadSourceAssetUrls - ? context.loadSourceAssetUrls(source).catch((error: unknown) => { - logger.warn("chat-agent: failed to load parsed asset map", { - sourceId: source.id, - error: formatUnknownError(error), - }) - return {} + const loaded = context + .loadSourceAssetUrls(source) + .catch((error: unknown) => { + logger.warn("chat: failed to load durable parsed asset map", { + sourceId: source.id, + error: formatUnknownError(error), }) - : Promise.resolve({}) + return {} + }) context.assetUrlsBySourceId.set(source.id, loaded) return loaded } -async function copyAssetToBlob(input: { - readonly reference: AssetUrlReference - readonly fetchRequest: AssetFetchRequest - readonly context: HardeningContext - readonly sourceSegment: string - readonly hardeningKey: string -}): Promise { - try { - const response = await input.context.fetchAsset(input.fetchRequest.fetchUrl) - if (!response.ok) { - logger.warn("chat-agent: media asset hardening fetch failed", { - assetUrl: redactAssetUrl(input.reference.assetUrl), - status: response.status, - }) - return input.reference.assetUrl - } - - const body = Buffer.from(await response.arrayBuffer()) - const contentType = normalizeContentType( - response.headers.get("content-type"), - input.fetchRequest.suggestedFileName, - ) - const blobPathname = getChatAssetBlobPathname({ - workspaceId: input.context.workspaceId, - sourceSegment: input.sourceSegment, - hardeningKey: input.hardeningKey, - suggestedFileName: input.fetchRequest.suggestedFileName, - contentType, - }) - const blob = await input.context.blobStore.put(blobPathname, body, { - access: "public", - allowOverwrite: true, - contentType, - multipart: true, - }) - return blob.url - } catch (error) { - logger.warn("chat-agent: media asset hardening failed; keeping raw URL", { - assetUrl: redactAssetUrl(input.reference.assetUrl), - error: formatUnknownError(error), - }) - return input.reference.assetUrl - } -} - -function resolveAssetFetchRequest(assetUrl: string): AssetFetchRequest | null { - const demoAsset = parseDemoAssetRoute(assetUrl) - if (demoAsset) { - return { - fetchUrl: knowhereDemoApi.resolveApiURL( - `/api/v1/demo/sources/${encodeURIComponent( - demoAsset.demoSourceId, - )}/assets/${demoAsset.encodedAssetPath}`, - ), - canonicalKey: `demo:${demoAsset.demoSourceId}:${demoAsset.decodedAssetPath}`, - sourceSegment: `demo-${toSafePathSegment(demoAsset.demoSourceId)}`, - suggestedFileName: getPathBasename(demoAsset.decodedAssetPath), - } - } - - const absoluteUrl = parseAbsoluteHttpUrl(assetUrl) - if (!absoluteUrl) return null - - return { - fetchUrl: assetUrl, - canonicalKey: `url:${absoluteUrl.origin}${absoluteUrl.pathname}`, - sourceSegment: `external-${hashText(absoluteUrl.origin).slice(0, 16)}`, - suggestedFileName: getPathBasename(absoluteUrl.pathname), - } -} - -function parseDemoAssetRoute(assetUrl: string): DemoAssetRoute | null { - const pathname = getAssetUrlPathname(assetUrl) - const match = /^\/api\/demo-sources\/([^/]+)\/assets\/(.+)$/.exec(pathname) - const encodedDemoSourceId = match?.[1] - const encodedAssetPath = match?.[2] - if (!encodedDemoSourceId || !encodedAssetPath) return null - - const demoSourceId = decodeUrlComponent(encodedDemoSourceId) - const assetPathSegments = encodedAssetPath - .split("/") - .map(decodeUrlComponent) - .filter((segment): boolean => segment.length > 0) - if (!demoSourceId || assetPathSegments.length === 0) return null - - return { - demoSourceId, - encodedAssetPath: assetPathSegments.map(encodeURIComponent).join("/"), - decodedAssetPath: assetPathSegments.join("/"), - } -} - -function isNotebookOwnedAssetUrl(assetUrl: string): boolean { +export function isNotebookOwnedAssetUrl(assetUrl: string): boolean { const pathname = getAssetUrlPathname(assetUrl).toLowerCase() if ( pathname.includes(`/${parsedResultDirectoryName}/`) || - pathname.includes(`/${chatAssetsDirectoryName}/`) + pathname.includes(`/${chatAssetsDirectoryName}/`) || + pathname.includes("/parsed-documents/") ) { return true } @@ -399,80 +296,6 @@ function isNotebookOwnedAssetUrl(assetUrl: string): boolean { return hostname?.endsWith(".blob.vercel-storage.com") === true } -function getChatAssetBlobPathname(input: { - readonly workspaceId: string - readonly sourceSegment: string - readonly hardeningKey: string - readonly suggestedFileName: string - readonly contentType: string -}): string { - const hash = hashText(input.hardeningKey).slice(0, 24) - const fileName = toSafeFileName(input.suggestedFileName, input.contentType) - return [ - "workspaces", - toSafePathSegment(input.workspaceId), - chatAssetsDirectoryName, - input.sourceSegment, - `${hash}-${fileName}`, - ].join("/") -} - -function normalizeContentType( - value: string | null, - fileName: string, -): string { - const normalized = value?.replace(/\s+/g, " ").trim() - if (normalized) return normalized - return getContentTypeForPath(fileName) -} - -function getContentTypeForPath(filePath: string): string { - const extension = path.extname(filePath).toLowerCase() - if (extension === ".jpg" || extension === ".jpeg") return "image/jpeg" - if (extension === ".png") return "image/png" - if (extension === ".gif") return "image/gif" - if (extension === ".webp") return "image/webp" - if (extension === ".svg") return "image/svg+xml" - if (extension === ".html" || extension === ".htm") { - return "text/html; charset=utf-8" - } - if (extension === ".csv") return "text/csv; charset=utf-8" - if (extension === ".pdf") return "application/pdf" - return fallbackContentType -} - -function getExtensionForContentType(contentType: string): string { - const normalized = contentType.split(";")[0]?.trim().toLowerCase() - if (normalized === "image/jpeg") return ".jpg" - if (normalized === "image/png") return ".png" - if (normalized === "image/gif") return ".gif" - if (normalized === "image/webp") return ".webp" - if (normalized === "image/svg+xml") return ".svg" - if (normalized === "text/html") return ".html" - if (normalized === "text/csv") return ".csv" - if (normalized === "application/pdf") return ".pdf" - return ".bin" -} - -function toSafeFileName(fileName: string, contentType: string): string { - const extension = getSafeFileExtension(fileName, contentType) - const stem = path.basename(fileName, path.extname(fileName)) - const safeStem = toSafePathSegment(stem) - return `${safeStem}${extension}` -} - -function getSafeFileExtension(fileName: string, contentType: string): string { - const extension = path.extname(fileName).toLowerCase() - if (/^\.[a-z0-9]{1,12}$/.test(extension)) return extension - return getExtensionForContentType(contentType) -} - -function getPathBasename(value: string): string { - const decodedPath = decodeUrlComponent(value) - const basename = decodedPath.replaceAll("\\", "/").split("/").pop() - return basename && basename.trim().length > 0 ? basename : "asset" -} - function getAssetUrlPathname(assetUrl: string): string { try { return new URL(assetUrl, "http://notebook.local").pathname @@ -508,33 +331,6 @@ function createSourcesByDocumentId( ) } -function toSafePathSegment(value: string): string { - const decoded = decodeUrlComponent(value) - const normalized = decoded - .replace(/[^A-Za-z0-9._-]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 80) - return normalized || hashText(value).slice(0, 16) -} - -function hashText(value: string): string { - return createHash("sha256").update(value).digest("hex") -} - -function decodeUrlComponent(value: string): string { - try { - return decodeURIComponent(value) - } catch { - return value - } -} - -function redactAssetUrl(assetUrl: string): string { - const absoluteUrl = parseAbsoluteHttpUrl(assetUrl) - if (absoluteUrl) return `${absoluteUrl.origin}${absoluteUrl.pathname}` - return getAssetUrlPathname(assetUrl) -} - function formatUnknownError(error: unknown): string { if (error instanceof Error) return error.message return String(error) diff --git a/src/domains/chat/media-assets.test.ts b/src/domains/chat/media-assets.test.ts index f79338b..2fc84cb 100644 --- a/src/domains/chat/media-assets.test.ts +++ b/src/domains/chat/media-assets.test.ts @@ -298,6 +298,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: "doc_1", stagedBlobPathname: null, diff --git a/src/domains/chat/page-citation-assets.test.ts b/src/domains/chat/page-citation-assets.test.ts new file mode 100644 index 0000000..d99dbcb --- /dev/null +++ b/src/domains/chat/page-citation-assets.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it, vi } from "vitest" +import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" + +import type { Source } from "@/infrastructure/db/schema" +import { enrichRetrievalResultsWithPageCitationAssetUrls } from "./page-citation-assets" + +describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { + it("does not use direct server-provided page asset URLs from result metadata", async () => { + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "page", + metadata: { + pageNums: [2], + pageAssets: [ + { + pageNum: 2, + artifactRef: "page_citation_assets/page-2.png", + assetUrl: "https://assets.example/pages/page-2.png", + }, + ], + }, + }), + ], + sources: [makeSource()], + }) + + expect(result?.pageCitationAssetUrl).toBeUndefined() + }) + + it("uses stored Blob URLs for page citation assets", async () => { + const loadSourceAssetUrls = vi.fn().mockResolvedValue({ + "page_citation_assets/page-2.png": + "https://blob.example/page_citation_assets/page-2.png", + }) + + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "page", + metadata: { + pageNums: [2], + pageAssets: [ + { + pageNum: 2, + artifactRef: "page_citation_assets/page-2.png", + assetUrl: "https://assets.example/pages/page-2.png", + }, + ], + }, + }), + ], + sources: [makeSource()], + loadSourceAssetUrls, + }) + + expect(loadSourceAssetUrls).toHaveBeenCalledWith( + expect.objectContaining({ id: "source_1" }), + ) + expect(result?.pageCitationAssetUrl).toBe( + "https://blob.example/page_citation_assets/page-2.png", + ) + }) + + it("chooses the stored asset matching the citation page metadata", async () => { + const loadSourceAssetUrls = vi.fn().mockResolvedValue({ + "page_citation_assets/page-4.png": + "https://blob.example/page_citation_assets/page-4.png", + }) + + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "page", + metadata: { + pageNums: [4], + pageAssets: [ + { + pageNum: 2, + artifactRef: "page_citation_assets/page-2.png", + assetUrl: "https://assets.example/pages/page-2.png", + }, + { + pageNum: 4, + artifactRef: "page_citation_assets/page-4.png", + assetUrl: "https://assets.example/pages/page-4.png", + }, + ], + }, + }), + ], + sources: [makeSource()], + loadSourceAssetUrls, + }) + + expect(result?.pageCitationAssetUrl).toBe( + "https://blob.example/page_citation_assets/page-4.png", + ) + }) + + it("does not invent a link when the server only provides an artifact ref", async () => { + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "page", + metadata: { + pageNums: [4], + pageAssets: [ + { + pageNum: 4, + artifactRef: "page_citation_assets/page-4.png", + }, + ], + }, + }), + ], + sources: [makeSource()], + }) + + expect(result?.pageCitationAssetUrl).toBeUndefined() + }) + + it("leaves non-page results unchanged even when they have page metadata", async () => { + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [ + makeRetrievalResult({ + chunkType: "text", + metadata: { + pageNums: [4], + pageAssets: [ + { + pageNum: 4, + artifactRef: "page_citation_assets/page-4.png", + assetUrl: "https://assets.example/pages/page-4.png", + }, + ], + }, + }), + ], + sources: [makeSource()], + }) + + expect(result?.pageCitationAssetUrl).toBeUndefined() + }) +}) + +function makeRetrievalResult( + overrides: Partial = {}, +): RetrievalResult { + return { + chunkId: "chunk_page_4", + content: "Page four summary", + chunkType: "page", + score: 0.8, + metadata: { pageNums: [4] }, + source: { + documentId: "doc_1", + sourceFileName: "source.pdf", + sectionPath: "Page 4", + }, + ...overrides, + } +} + +function makeSource(overrides: Partial = {}): Source { + return { + id: "source_1", + workspaceId: "workspace_1", + title: "source.pdf", + mimeType: "application/pdf", + sizeBytes: 1, + status: "ready", + failureReason: null, + failureStage: null, + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-07-03T00:00:00.000Z"), + updatedAt: new Date("2026-07-03T00:00:00.000Z"), + deletedAt: null, + ...overrides, + } +} diff --git a/src/domains/chat/page-citation-assets.ts b/src/domains/chat/page-citation-assets.ts new file mode 100644 index 0000000..77b8d53 --- /dev/null +++ b/src/domains/chat/page-citation-assets.ts @@ -0,0 +1,203 @@ +import "server-only" + +import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" + +import type { Source } from "@/infrastructure/db/schema" +import type { LoadSourceAssetUrls } from "./media-assets" + +export type PageCitationAssetRetrievalResult = RetrievalResult & { + readonly pageCitationAssetUrl?: string +} + +type EnrichRetrievalResultsWithPageCitationAssetUrlsInput = { + readonly results: readonly RetrievalResult[] + readonly sources: readonly Source[] + readonly loadSourceAssetUrls?: LoadSourceAssetUrls +} + +type PageCitationAssetCandidate = { + readonly pageNum: number + readonly artifactRef?: string + readonly assetUrl?: string +} + +export async function enrichRetrievalResultsWithPageCitationAssetUrls({ + results, + sources, + loadSourceAssetUrls, +}: EnrichRetrievalResultsWithPageCitationAssetUrlsInput): Promise< + PageCitationAssetRetrievalResult[] +> { + if (results.length === 0) return [] + + const sourcesByDocumentId = new Map( + sources.flatMap((source): readonly [string, Source][] => + source.knowhereDocumentId ? [[source.knowhereDocumentId, source]] : [], + ), + ) + const assetUrlsBySourceId = new Map< + string, + Promise>> + >() + + return Promise.all( + results.map((result) => + enrichRetrievalResultWithPageCitationAssetUrl({ + result, + sourcesByDocumentId, + loadSourceAssetUrls, + assetUrlsBySourceId, + }), + ), + ) +} + +async function enrichRetrievalResultWithPageCitationAssetUrl(input: { + readonly result: RetrievalResult + readonly sourcesByDocumentId: ReadonlyMap + readonly loadSourceAssetUrls?: LoadSourceAssetUrls + readonly assetUrlsBySourceId: Map< + string, + Promise>> + > +}): Promise { + if (!isPageResult(input.result)) return input.result + + const pageNumbers = getPageNumbers(input.result.metadata) + const directAsset = getDirectPageCitationAsset(input.result, pageNumbers) + const sourceAssetUrl = await getStoredPageCitationAssetUrl({ + result: input.result, + directAsset, + sourcesByDocumentId: input.sourcesByDocumentId, + loadSourceAssetUrls: input.loadSourceAssetUrls, + assetUrlsBySourceId: input.assetUrlsBySourceId, + }) + if (sourceAssetUrl) { + return { + ...input.result, + pageCitationAssetUrl: sourceAssetUrl, + } + } + + return input.result +} + +async function getStoredPageCitationAssetUrl(input: { + readonly result: RetrievalResult + readonly directAsset: PageCitationAssetCandidate | null + readonly sourcesByDocumentId: ReadonlyMap + readonly loadSourceAssetUrls?: LoadSourceAssetUrls + readonly assetUrlsBySourceId: Map< + string, + Promise>> + > +}): Promise { + const artifactRef = getTrimmedString(input.directAsset?.artifactRef) + const documentId = getTrimmedString(input.result.source.documentId) + if (!artifactRef || !documentId || !input.loadSourceAssetUrls) return null + + const source = input.sourcesByDocumentId.get(documentId) + if (!source) return null + + const assetUrls = await getCachedSourceAssetUrls( + source, + input.loadSourceAssetUrls, + input.assetUrlsBySourceId, + ) + return getTrimmedString(assetUrls[artifactRef]) +} + +function isPageResult(result: RetrievalResult): boolean { + return result.chunkType.toLowerCase() === "page" +} + +function getDirectPageCitationAsset( + result: RetrievalResult, + pageNumbers: readonly number[], +): PageCitationAssetCandidate | null { + const candidates = parsePageCitationAssetCandidates(result.metadata?.pageAssets) + + if (pageNumbers.length > 0) { + const matchingCandidates = candidates.filter((candidate) => + pageNumbers.includes(candidate.pageNum), + ) + if (matchingCandidates[0]) return matchingCandidates[0] + } + + return candidates[0] ?? null +} + +function parsePageCitationAssetCandidates( + value: unknown, +): readonly PageCitationAssetCandidate[] { + if (!Array.isArray(value)) return [] + + return value.flatMap((item): PageCitationAssetCandidate[] => { + if (!isRecord(item)) return [] + const pageNum = getPositiveInteger(item.pageNum) + if (!pageNum) return [] + + return [ + { + pageNum, + artifactRef: getTrimmedString(item.artifactRef) ?? undefined, + assetUrl: getTrimmedString(item.assetUrl) ?? undefined, + }, + ] + }) +} + +async function getCachedSourceAssetUrls( + source: Source, + loadSourceAssetUrls: LoadSourceAssetUrls, + cache: Map>>>, +): Promise>> { + let cached = cache.get(source.id) + if (!cached) { + cached = loadSourceAssetUrls(source).catch(() => ({})) + cache.set(source.id, cached) + } + return cached +} + +function getPageNumbers( + metadata: Readonly> | undefined, +): readonly number[] { + if (!metadata) return [] + + const values = [metadata.pageNums, metadata.page_nums, metadata.pageNum] + const pageNumbers = new Set() + + for (const value of values) { + if (Array.isArray(value)) { + for (const item of value) { + const pageNum = getPositiveInteger(item) + if (pageNum) pageNumbers.add(pageNum) + } + continue + } + + const pageNum = getPositiveInteger(value) + if (pageNum) pageNumbers.add(pageNum) + } + + return [...pageNumbers].sort((left, right) => left - right) +} + +function getTrimmedString(value: unknown): string | null { + if (typeof value !== "string") return null + const trimmed = value.trim() + return trimmed.length > 0 ? trimmed : null +} + +function getPositiveInteger(value: unknown): number | null { + return typeof value === "number" && + Number.isSafeInteger(value) && + value > 0 + ? value + : null +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null +} diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index ea75943..9582b77 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -11,9 +11,10 @@ import { type ChatTurnValue, } from "@/domains/chat/service" import { chatTurnPersistence } from "@/domains/chat/chat-turn-persistence" +import { readSourceAssetUrls } from "@/domains/chunks/read" import { startBackgroundReconciliation } from "@/domains/sources/background-reconcile" -import { sourceService } from "@/domains/sources/service" import { sourceWorkflowRuntime } from "@/domains/sources/workflow-runtime" +import { makeKnowhereClientWithParsedStorage } from "@/integrations/knowhere" import { notebookRequestContext } from "@/domains/workspace/request-context" import type { Source } from "@/infrastructure/db/schema" import { isAuthError } from "@/integrations/dashboard/api-key-service" @@ -68,8 +69,30 @@ const answerChatEffect = (input: AnswerChatInput) => apiKey, }), ) - const loadSourceAssetUrls = (source: (typeof sources)[number]) => - sourceService.getParseAssetUrls(workspace.id, source.id) + const { knowledge } = makeKnowhereClientWithParsedStorage(apiKey, { + workspaceId: workspace.id, + }) + const loadSourceAssetUrls = async ( + source: (typeof sources)[number], + ): Promise>> => { + if (source.status !== "ready" || !source.knowhereDocumentId) return {} + + try { + return await readSourceAssetUrls({ + knowledge, + documentId: source.knowhereDocumentId, + revisionKey: source.knowhereJobId, + }) + } catch (error) { + logger.warn("chat: durable parsed asset read failed", { + workspaceId: workspace.id, + sourceId: source.id, + documentId: source.knowhereDocumentId, + error: summarizeUnknownError(error), + }) + return {} + } + } const result: Either.Either = yield* Effect.tryPromise(() => diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 4358e8a..f4df92a 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -18,6 +18,8 @@ const mocks = vi.hoisted(() => ({ loggerInfo: vi.fn(), loggerWarn: vi.fn(), listSourcesForWorkspace: vi.fn(), + makeKnowhereClientWithParsedStorage: vi.fn(), + readChunks: vi.fn(), softDeleteChatThread: vi.fn(), startBackgroundReconciliation: vi.fn(), })) @@ -51,6 +53,11 @@ vi.mock("@/domains/workspace/request-context", () => ({ }, })) +vi.mock("@/integrations/knowhere", () => ({ + makeKnowhereClientWithParsedStorage: + mocks.makeKnowhereClientWithParsedStorage, +})) + vi.mock("@/domains/chat/thread-service", () => ({ chatThreadService: { appendMessage: mocks.appendMessageToThread, @@ -77,6 +84,18 @@ import { chatThreadRouteService } from "./route-threads" describe("chat route services", () => { beforeEach(() => { vi.clearAllMocks() + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: { documents: { listChunks: vi.fn() } }, + knowledge: { readChunks: mocks.readChunks }, + }) + mocks.readChunks.mockResolvedValue({ + document: { localDocumentId: "doc" }, + chunks: [], + page: 1, + pageSize: 200, + totalChunks: 0, + totalPages: 1, + }) }) it("orchestrates a chat turn from request body to response body", async () => { @@ -140,6 +159,80 @@ describe("chat route services", () => { ) }) + it("builds durable citation asset URLs from SDK reads for a ready source", async () => { + const workspace = makeWorkspace() + const client = { retrieval: { query: vi.fn() } } + const readySource = makeSource({ + status: "ready", + knowhereDocumentId: "doc_legacy", + knowhereJobId: "job_1", + }) + const durableUrl = + "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_legacy/job_1/assets/pages/page-1.png" + mocks.readChunks.mockResolvedValue({ + document: { localDocumentId: "doc_legacy" }, + chunks: [ + { + position: 1, + chunkId: "c1", + chunkType: "page", + content: "Page", + readableContent: "Page", + sectionPath: "Page 1", + sourceChunkPath: "Page 1", + filePath: "pages/page-1.png", + assetUrl: durableUrl, + metadata: {}, + }, + ], + page: 1, + pageSize: 200, + totalChunks: 1, + totalPages: 1, + }) + mocks.getAuthenticatedWithClient.mockResolvedValue({ + user: { id: "user_1" }, + workspace, + apiKey: "jwt_123", + client, + }) + mocks.listSourcesForWorkspace.mockResolvedValue([readySource]) + mocks.handleChatTurn.mockImplementation( + async (input: { + readonly loadSourceAssetUrls?: ( + source: Source, + ) => Promise>> + }) => { + const assetUrls = await input.loadSourceAssetUrls?.(readySource) + expect(assetUrls).toEqual({ "pages/page-1.png": durableUrl }) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "message_user", role: "user", content: "Show the page" }, + { id: "message_assistant", role: "assistant", content: "Answer" }, + ], + }) + }, + ) + + const result = await chatAnswerRouteService.answerChat({ + body: { message: "Show the page" }, + }) + + expect(result.status).toBe(200) + expect(mocks.makeKnowhereClientWithParsedStorage).toHaveBeenCalledWith( + "jwt_123", + { workspaceId: workspace.id }, + ) + expect(mocks.readChunks).toHaveBeenCalledWith({ + documentId: "doc_legacy", + revisionKey: "job_1", + page: 1, + pageSize: 200, + assetUrlPolicy: "durable", + }) + }) + it("triggers background reconciliation for parsing sources without blocking chat", async () => { const workspace = makeWorkspace() const client = { retrieval: { query: vi.fn() } } @@ -381,6 +474,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_123", knowhereDocumentId: "doc_1", stagedBlobPathname: null, diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index e931f04..7c0ce92 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -270,6 +270,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_123", knowhereDocumentId: "doc_included", stagedBlobPathname: null, diff --git a/src/domains/chat/types.ts b/src/domains/chat/types.ts index f3f987d..a0469e0 100644 --- a/src/domains/chat/types.ts +++ b/src/domains/chat/types.ts @@ -1,12 +1,12 @@ /** * Chat citation / retrieval hit. Mirrors RetrievalResult from the SDK. - * No chunkId here; retrieval does not expose one. */ export type RetrievalResultView = { readonly content: string readonly chunkType: string readonly score: number | null readonly assetUrl?: string + readonly pageCitationAssetUrl?: string readonly source: { readonly documentId?: string | null readonly sourceFileName?: string | null diff --git a/src/domains/chat/view.ts b/src/domains/chat/view.ts index 9afc4f6..ed39295 100644 --- a/src/domains/chat/view.ts +++ b/src/domains/chat/view.ts @@ -51,6 +51,7 @@ function toPersistedCitationViews(value: unknown): ChatCitationView[] | undefine chunkType: getString(item.chunkType) ?? "text", score: getNumber(item.score) ?? 0, assetUrl: getString(item.assetUrl), + pageCitationAssetUrl: getString(item.pageCitationAssetUrl), description: getString(item.description), source: { documentId: getString(item.source.documentId), @@ -81,6 +82,9 @@ function toPersistedArtifactViews(value: unknown): ChatArtifactView[] | undefine chunkType: getString(item.citation.chunkType) ?? "text", score: getNumber(item.citation.score) ?? 0, assetUrl: getString(item.citation.assetUrl), + pageCitationAssetUrl: getString( + item.citation.pageCitationAssetUrl, + ), description: getString(item.citation.description), source: { documentId: getString(item.citation.source.documentId), diff --git a/src/domains/chunks/index.test.ts b/src/domains/chunks/index.test.ts index 0dfc22f..316bfd6 100644 --- a/src/domains/chunks/index.test.ts +++ b/src/domains/chunks/index.test.ts @@ -578,6 +578,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 100, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_123", knowhereDocumentId: "doc_123", stagedBlobPathname: null, diff --git a/src/domains/chunks/index.ts b/src/domains/chunks/index.ts index 8a285d7..5e1b25a 100644 --- a/src/domains/chunks/index.ts +++ b/src/domains/chunks/index.ts @@ -1,5 +1,5 @@ import { Effect } from "effect" -import type { DocumentChunk } from "@ontos-ai/knowhere-sdk" +import type { DocumentChunk, KnowledgeReadChunk } from "@ontos-ai/knowhere-sdk" import { parsedChunkNormalization } from "./normalization" import type { Source } from "@/infrastructure/db/schema" @@ -179,6 +179,35 @@ export function toParsedChunkView( }) } +/** + * Map an SDK `KnowledgeReadChunk` (from `knowledge.readChunks`) to the view + * model. Asset URLs on durable reads are already hardened SDK-side, so no + * `assetUrlsByFilePath` remap is needed here. + */ +export function toParsedChunkViewFromReadChunk( + chunk: KnowledgeReadChunk, + sourceTitle: string, + documentId?: string, +): ParsedChunkView { + return parsedChunkNormalization.createParsedChunkView({ + chunkId: chunk.chunkId, + documentId, + parserChunkId: chunk.chunkId, + sectionPath: chunk.sectionPath, + chunkType: chunk.chunkType, + contentSource: chunk.contentSource, + content: chunk.content, + metadata: chunk.metadata, + filePathCandidates: [ + chunk.filePath, + chunk.metadata["filePath"], + chunk.metadata["file_path"], + ], + assetUrl: chunk.assetUrl, + sourceTitle, + }) +} + export function resolveCitationChunk( citation: ChatCitationView, chunks: readonly ParsedChunkView[], diff --git a/src/domains/chunks/read.test.ts b/src/domains/chunks/read.test.ts new file mode 100644 index 0000000..1d0786d --- /dev/null +++ b/src/domains/chunks/read.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it, vi } from "vitest" +import type { Knowledge } from "@ontos-ai/knowhere-sdk" + +import { readAllSourceChunks, readSourceChunkPage } from "./read" + +function makeReadChunk(overrides: Record = {}) { + return { + position: 1, + chunkId: "parser_1", + chunkType: "text", + content: "Body", + readableContent: "Body", + sectionPath: "Summary", + sourceChunkPath: "Summary", + filePath: undefined, + assetUrl: undefined, + metadata: {}, + ...overrides, + } +} + +describe("readSourceChunkPage", () => { + it("reads a durable page and maps chunks to the view model", async () => { + const readChunks = vi.fn(async () => ({ + document: { localDocumentId: "doc_1" }, + chunks: [ + makeReadChunk({ + chunkType: "image", + filePath: "images/a.png", + assetUrl: "https://blob.example/images/a.png", + }), + ], + page: 2, + pageSize: 50, + totalChunks: 120, + totalPages: 3, + })) + const knowledge = { readChunks } as unknown as Knowledge + + const result = await readSourceChunkPage({ + knowledge, + source: { documentId: "doc_1", title: "notes.pdf", revisionKey: "rev_1" }, + params: { page: 2, pageSize: 50 }, + }) + + expect(readChunks).toHaveBeenCalledWith({ + documentId: "doc_1", + revisionKey: "rev_1", + page: 2, + pageSize: 50, + assetUrlPolicy: "durable", + }) + expect(result.pagination).toEqual({ + page: 2, + pageSize: 50, + total: 120, + totalPages: 3, + }) + expect(result.chunks[0]).toMatchObject({ + parserChunkId: "parser_1", + documentId: "doc_1", + assetUrl: "https://blob.example/images/a.png", + sourceTitle: "notes.pdf", + }) + }) + + it("omits revisionKey when the source has none", async () => { + const readChunks = vi.fn(async () => ({ + document: { localDocumentId: "doc_1" }, + chunks: [], + page: 1, + pageSize: 50, + totalChunks: 0, + totalPages: 1, + })) + const knowledge = { readChunks } as unknown as Knowledge + + await readSourceChunkPage({ + knowledge, + source: { documentId: "doc_1", title: "notes.pdf", revisionKey: null }, + params: { page: 1, pageSize: 50 }, + }) + + expect(readChunks).toHaveBeenCalledWith({ + documentId: "doc_1", + page: 1, + pageSize: 50, + assetUrlPolicy: "durable", + }) + }) +}) + +describe("readAllSourceChunks", () => { + it("pages the SDK to exhaustion", async () => { + const readChunks = vi + .fn() + .mockResolvedValueOnce({ + document: { localDocumentId: "doc_1" }, + chunks: [makeReadChunk({ chunkId: "c1" })], + page: 1, + pageSize: 200, + totalChunks: 2, + totalPages: 2, + }) + .mockResolvedValueOnce({ + document: { localDocumentId: "doc_1" }, + chunks: [makeReadChunk({ chunkId: "c2" })], + page: 2, + pageSize: 200, + totalChunks: 2, + totalPages: 2, + }) + const knowledge = { readChunks } as unknown as Knowledge + + const chunks = await readAllSourceChunks({ + knowledge, + source: { documentId: "doc_1", title: "notes.pdf", revisionKey: "rev_1" }, + }) + + expect(readChunks).toHaveBeenCalledTimes(2) + expect(chunks.map((chunk) => chunk.parserChunkId)).toEqual(["c1", "c2"]) + }) +}) diff --git a/src/domains/chunks/read.ts b/src/domains/chunks/read.ts new file mode 100644 index 0000000..f375d48 --- /dev/null +++ b/src/domains/chunks/read.ts @@ -0,0 +1,142 @@ +import "server-only" + +import type { Knowledge } from "@ontos-ai/knowhere-sdk" + +import { toParsedChunkViewFromReadChunk, type ChunkPage, type ChunkPageParams } from "@/domains/chunks" +import type { ParsedChunkView } from "@/domains/chunks/types" + +const loadAllPageSize = 200 + +type ReadableSource = { + readonly documentId: string + readonly title: string + readonly revisionKey?: string | null +} + +/** + * Read a single display page of parsed chunks through the SDK. The SDK serves + * from configured Blob storage when fresh and falls back to Knowhere remote + * otherwise, hardening visible asset URLs into durable Blob URLs + * (`assetUrlPolicy: "durable"`) and scheduling a background sync on a miss. + */ +export async function readSourceChunkPage(input: { + readonly knowledge: Knowledge + readonly source: ReadableSource + readonly params: ChunkPageParams +}): Promise { + const response = await input.knowledge.readChunks({ + documentId: input.source.documentId, + ...(input.source.revisionKey ? { revisionKey: input.source.revisionKey } : {}), + page: input.params.page, + pageSize: input.params.pageSize, + assetUrlPolicy: "durable", + }) + + const chunks = response.chunks.map((chunk) => + toParsedChunkViewFromReadChunk(chunk, input.source.title, input.source.documentId), + ) + + return { + chunks, + pagination: { + page: response.page ?? input.params.page, + pageSize: response.pageSize ?? input.params.pageSize, + total: response.totalChunks ?? chunks.length, + totalPages: + response.totalPages ?? + Math.max(1, Math.ceil(chunks.length / input.params.pageSize)), + }, + } +} + +/** + * Read every parsed chunk for a source by paging the SDK to exhaustion. Used by + * the tree view and load-all display mode. + */ +export async function readAllSourceChunks(input: { + readonly knowledge: Knowledge + readonly source: ReadableSource +}): Promise { + const chunks: ParsedChunkView[] = [] + let page = 1 + let totalPages = 1 + + do { + const response = await input.knowledge.readChunks({ + documentId: input.source.documentId, + ...(input.source.revisionKey + ? { revisionKey: input.source.revisionKey } + : {}), + page, + pageSize: loadAllPageSize, + assetUrlPolicy: "durable", + }) + for (const chunk of response.chunks) { + chunks.push( + toParsedChunkViewFromReadChunk( + chunk, + input.source.title, + input.source.documentId, + ), + ) + } + totalPages = Math.max(1, response.totalPages ?? 1) + page += 1 + } while (page <= totalPages) + + return chunks +} + +/** + * Build a `filePath -> durable Blob URL` map for a source by paging durable + * reads to exhaustion. This is the single asset-hardening path for chat: the + * SDK writes any missing asset into Blob during the durable read and returns + * the durable URL, which we index by both the chunk file path and any + * `metadata.pageAssets[].artifactRef`. + */ +export async function readSourceAssetUrls(input: { + readonly knowledge: Knowledge + readonly documentId: string + readonly revisionKey?: string | null +}): Promise>> { + const assetUrlsByFilePath: Record = {} + let page = 1 + let totalPages = 1 + + do { + const response = await input.knowledge.readChunks({ + documentId: input.documentId, + ...(input.revisionKey ? { revisionKey: input.revisionKey } : {}), + page, + pageSize: loadAllPageSize, + assetUrlPolicy: "durable", + }) + for (const chunk of response.chunks) { + if (chunk.filePath && chunk.assetUrl) { + assetUrlsByFilePath[chunk.filePath] = chunk.assetUrl + } + collectPageAssetUrls(chunk.metadata, assetUrlsByFilePath) + } + totalPages = Math.max(1, response.totalPages ?? 1) + page += 1 + } while (page <= totalPages) + + return assetUrlsByFilePath +} + +function collectPageAssetUrls( + metadata: Record, + target: Record, +): void { + const pageAssets = metadata["pageAssets"] + if (!Array.isArray(pageAssets)) return + for (const pageAsset of pageAssets) { + if (typeof pageAsset !== "object" || pageAsset === null) continue + const record = pageAsset as Record + const artifactRef = record["artifactRef"] + const assetUrl = record["assetUrl"] + if (typeof artifactRef === "string" && typeof assetUrl === "string") { + target[artifactRef] = assetUrl + } + } +} diff --git a/src/domains/chunks/server.test.ts b/src/domains/chunks/server.test.ts deleted file mode 100644 index f58f5ea..0000000 --- a/src/domains/chunks/server.test.ts +++ /dev/null @@ -1,440 +0,0 @@ -import { describe, expect, it, vi, type Mock } from "vitest" -import { Effect } from "effect" -import type { DocumentChunk } from "@ontos-ai/knowhere-sdk" - -import type { Source } from "@/infrastructure/db/schema" -import { - loadChunkPageForSource, - loadChunksForSource, -} from "./server" - -describe("server chunk cache", () => { - it("returns upstream chunks on a visible cache miss and warms mirrored assets in the background", async () => { - const warmTasks: Array<() => Promise> = [] - const cacheStore = createCacheStore() - const listChunks = vi.fn(async () => ({ - documentId: "doc_1", - jobResultId: "revision_1", - chunks: [ - makeDocumentChunk({ - id: "image_1", - chunkId: "parser_image_1", - chunkType: "image", - filePath: "images/image-1.png", - assetUrl: "https://knowhere.example/assets/image-1.png", - }), - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })) - const fetchAsset = vi.fn(async () => - new Response("image-body", { - headers: { "content-type": "image/png" }, - }), - ) - - const page = await Effect.runPromise( - loadChunkPageForSource( - makeSource({ knowhereJobId: "revision_1" }), - { documents: { listChunks } }, - { page: 1, pageSize: 1 }, - { - cacheStore, - fetchAsset, - scheduleWarm: (task) => warmTasks.push(task), - workspaceId: "workspace_1", - }, - ), - ) - - expect(page.chunks[0]?.assetUrl).toBe( - "https://knowhere.example/assets/image-1.png", - ) - expect(cacheStore.putMock).not.toHaveBeenCalled() - expect(warmTasks).toHaveLength(1) - - await warmTasks[0]?.() - - expect(fetchAsset).toHaveBeenCalledWith( - "https://knowhere.example/assets/image-1.png", - ) - expect(cacheStore.putMock).toHaveBeenCalledWith( - expect.stringContaining("/chunk-assets/revision_1/"), - Buffer.from("image-body"), - expect.objectContaining({ contentType: "image/png" }), - ) - const cachedPagePut = cacheStore.putMock.mock.calls.find( - ([pathname]) => - typeof pathname === "string" && pathname.endsWith(".json"), - ) - expect(cachedPagePut).toBeDefined() - const cachedPage = JSON.parse(String(cachedPagePut?.[1])) as { - readonly chunks: readonly { readonly assetUrl?: string }[] - } - expect(cachedPage.chunks[0]?.assetUrl).toContain( - "https://blob.example/workspaces/workspace_1/sources/source_1/chunk-assets/revision_1/", - ) - }) - - it("returns a cached visible page after verifying the current Knowhere revision", async () => { - const cachedPage = { - chunks: [ - { - chunkId: "image_1", - documentId: "doc_1", - type: "image", - content: "", - assetUrl: "https://blob.example/image-1.png", - sourceTitle: "notes.pdf", - }, - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - } - const listChunks = vi.fn(async () => ({ - documentId: "doc_1", - jobResultId: "revision_1", - chunks: [], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })) - const cacheStore = createCacheStore({ - get: vi.fn(async () => ({ - statusCode: 200, - stream: createTextStream(JSON.stringify(cachedPage)), - })), - }) - - const page = await Effect.runPromise( - loadChunkPageForSource( - makeSource({ knowhereJobId: "revision_1" }), - { documents: { listChunks } }, - { page: 1, pageSize: 1 }, - { - cacheStore, - workspaceId: "workspace_1", - }, - ), - ) - - expect(page).toEqual(cachedPage) - expect(listChunks).toHaveBeenCalledWith("doc_1", { - page: 1, - pageSize: 1, - includeAssetUrls: false, - }) - expect(cacheStore.getMock).toHaveBeenCalledWith( - expect.stringContaining("/revision_1/visible/page-1-size-1.json"), - { access: "public" }, - ) - }) - - it("ignores old cached pages when Knowhere reports a new job id", async () => { - const warmTasks: Array<() => Promise> = [] - const staleCachedPage = { - chunks: [ - { - chunkId: "stale_1", - documentId: "doc_1", - type: "text", - content: "Old chunk", - sourceTitle: "notes.pdf", - }, - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - } - const cacheStore = createCacheStore({ - get: vi.fn(async (pathname: string) => - pathname.includes("/job_old/") - ? { - statusCode: 200, - stream: createTextStream(JSON.stringify(staleCachedPage)), - } - : null, - ), - }) - const listChunks = vi.fn(async ( - _documentId: string, - params: { readonly includeAssetUrls: boolean }, - ) => ({ - documentId: "doc_1", - jobId: "job_new", - chunks: params.includeAssetUrls - ? [ - makeDocumentChunk({ - id: "text_new", - chunkId: "parser_text_new", - content: "New chunk", - }), - ] - : [], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })) - const onRevisionKey = vi.fn(async () => undefined) - - const page = await Effect.runPromise( - loadChunkPageForSource( - makeSource({ knowhereJobId: "job_old" }), - { documents: { listChunks } }, - { page: 1, pageSize: 1 }, - { - cacheStore, - onRevisionKey, - scheduleWarm: (task) => warmTasks.push(task), - workspaceId: "workspace_1", - }, - ), - ) - - expect(page.chunks[0]?.content).toBe("New chunk") - expect(cacheStore.getMock).toHaveBeenCalledWith( - expect.stringContaining("/job_new/visible/page-1-size-1.json"), - { access: "public" }, - ) - expect(cacheStore.getMock).not.toHaveBeenCalledWith( - expect.stringContaining("/job_old/visible/page-1-size-1.json"), - expect.anything(), - ) - expect(onRevisionKey).toHaveBeenCalledWith("job_new") - expect(warmTasks).toHaveLength(1) - }) - - it("uses structure-only chunk loading for full-tree requests", async () => { - const warmTasks: Array<() => Promise> = [] - const cacheStore = createCacheStore() - const listChunks = vi.fn(async () => ({ - documentId: "doc_1", - jobResultId: "revision_1", - chunks: [ - makeDocumentChunk({ - id: "text_1", - chunkId: "parser_text_1", - content: "A text chunk", - }), - ], - pagination: { - page: 1, - pageSize: 200, - total: 1, - totalPages: 1, - }, - })) - const fetchAsset = vi.fn() - - const chunks = await Effect.runPromise( - loadChunksForSource( - makeSource({ knowhereJobId: "revision_1" }), - { documents: { listChunks } }, - { - cacheStore, - fetchAsset, - scheduleWarm: (task) => warmTasks.push(task), - workspaceId: "workspace_1", - }, - ), - ) - - expect(chunks).toMatchObject([{ chunkId: "text_1" }]) - expect(listChunks).toHaveBeenCalledWith("doc_1", { - page: 1, - pageSize: 200, - includeAssetUrls: false, - }) - expect(fetchAsset).not.toHaveBeenCalled() - expect(warmTasks).toHaveLength(1) - await warmTasks[0]?.() - expect(cacheStore.putMock).toHaveBeenCalledWith( - expect.stringContaining("/structure/page-1-size-200.json"), - expect.any(String), - expect.objectContaining({ contentType: "application/json; charset=utf-8" }), - ) - }) - - it("caches media chunks without previews when Knowhere has no upstream asset URL", async () => { - const warmTasks: Array<() => Promise> = [] - const cacheStore = createCacheStore() - const listChunks = vi.fn(async () => ({ - documentId: "doc_1", - jobResultId: "revision_1", - chunks: [ - makeDocumentChunk({ - id: "image_1", - chunkId: "parser_image_1", - chunkType: "image", - filePath: "images/image-1.png", - assetUrl: null, - }), - ], - pagination: { - page: 1, - pageSize: 1, - total: 1, - totalPages: 1, - }, - })) - const fetchAsset = vi.fn() - - const page = await Effect.runPromise( - loadChunkPageForSource( - makeSource({ knowhereJobId: "revision_1" }), - { documents: { listChunks } }, - { page: 1, pageSize: 1 }, - { - cacheStore, - fetchAsset, - scheduleWarm: (task) => warmTasks.push(task), - workspaceId: "workspace_1", - }, - ), - ) - - expect(page.chunks[0]?.assetUrl).toBeUndefined() - await warmTasks[0]?.() - expect(fetchAsset).not.toHaveBeenCalled() - const cachedPagePut = cacheStore.putMock.mock.calls.find( - ([pathname]) => - typeof pathname === "string" && pathname.endsWith(".json"), - ) - const cachedPage = JSON.parse(String(cachedPagePut?.[1])) as { - readonly chunks: readonly { readonly assetUrl?: string }[] - } - expect(cachedPage.chunks[0]?.assetUrl).toBeUndefined() - }) -}) - -type TestCacheGetResult = - | { - readonly statusCode: 200 - readonly stream: ReadableStream - } - | { - readonly statusCode: 304 - readonly stream: null - } - -type TestCachePutOptions = { - readonly access: "public" - readonly allowOverwrite: boolean - readonly contentType: string - readonly multipart?: boolean -} - -type TestCacheStore = { - readonly get: ( - pathname: string, - options: { readonly access: "public" }, - ) => Promise - readonly put: ( - pathname: string, - body: string | Buffer, - options: TestCachePutOptions, - ) => Promise<{ readonly url: string }> - readonly getMock: TestCacheGetMock - readonly putMock: TestCachePutMock -} - -type TestCacheGetMock = Mock< - ( - pathname: string, - options: { readonly access: "public" }, - ) => Promise -> - -type TestCachePutMock = Mock< - ( - pathname: string, - body: string | Buffer, - options: TestCachePutOptions, - ) => Promise<{ readonly url: string }> -> - -function createCacheStore(overrides: Partial<{ - readonly get: TestCacheGetMock - readonly put: TestCachePutMock -}> = {}): TestCacheStore { - const getMock = - overrides.get ?? - vi.fn(async () => null) - const putMock = - overrides.put ?? - vi.fn(async (pathname: string) => ({ - url: `https://blob.example/${pathname}`, - })) - - return { - get: (pathname, options) => getMock(pathname, options), - put: (pathname, body, options) => putMock(pathname, body, options), - getMock, - putMock, - } -} - -function makeDocumentChunk( - overrides: Partial = {}, -): DocumentChunk { - return { - id: "document_chunk_1", - chunkId: "parser_chunk_1", - chunkType: "text", - content: "Chunk content", - sectionId: null, - sectionPath: null, - sourceChunkPath: null, - filePath: null, - sortOrder: 1, - metadata: {}, - assetUrl: null, - ...overrides, - } -} - -function makeSource(overrides: Partial = {}): Source { - return { - id: "source_1", - workspaceId: "workspace_1", - title: "notes.pdf", - mimeType: "application/pdf", - sizeBytes: 100, - status: "ready", - failureReason: null, - knowhereJobId: "revision_1", - knowhereDocumentId: "doc_1", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-06T00:00:00Z"), - updatedAt: new Date("2026-05-06T00:00:00Z"), - deletedAt: null, - ...overrides, - } -} - -function createTextStream(text: string): ReadableStream { - const stream = new Response(text).body - if (!stream) throw new Error("Response body stream was not created.") - return stream -} diff --git a/src/domains/chunks/server.ts b/src/domains/chunks/server.ts deleted file mode 100644 index a28ee3b..0000000 --- a/src/domains/chunks/server.ts +++ /dev/null @@ -1,739 +0,0 @@ -import "server-only" - -import path from "node:path" -import { createHash } from "node:crypto" -import { get as getBlob, put } from "@vercel/blob" -import { after } from "next/server" -import { Effect } from "effect" -import type { - DocumentChunk, - DocumentChunkListResponse, -} from "@ontos-ai/knowhere-sdk" - -import { - resolveChunkConnectionTargets, - toParsedChunkView, - type ChunkKnowhereClient, - type ChunkPage, - type ChunkPageParams, - type LoadChunksOptions, -} from "@/domains/chunks" -import type { ParsedChunkView } from "@/domains/chunks/types" -import type { Source } from "@/infrastructure/db/schema" -import { logger } from "@/lib/logger" - -type ChunkPageMode = "visible" | "structure" - -type ChunkPageBlobGetResult = - | { - readonly statusCode: 200 - readonly stream: ReadableStream - } - | { - readonly statusCode: 304 - readonly stream: null - } - -type ChunkPageBlobPutOptions = { - readonly access: "public" - readonly allowOverwrite: boolean - readonly contentType: string - readonly multipart?: boolean -} - -type ChunkPageBlobStore = { - readonly get: ( - pathname: string, - options: { readonly access: "public" }, - ) => Promise - readonly put: ( - pathname: string, - body: string | Buffer, - options: ChunkPageBlobPutOptions, - ) => Promise<{ readonly url: string }> -} - -type FetchChunkAsset = (assetUrl: string) => Promise - -type ChunkPageWarmScheduler = (task: () => Promise) => void - -type ServerLoadChunksOptions = LoadChunksOptions & { - readonly workspaceId?: string - readonly cacheStore?: ChunkPageBlobStore - readonly fetchAsset?: FetchChunkAsset - readonly mode?: ChunkPageMode - readonly onRevisionKey?: (revisionKey: string) => Promise - readonly scheduleWarm?: ChunkPageWarmScheduler -} - -type WarmChunkPageCacheInput = { - readonly source: Source - readonly client: ChunkKnowhereClient - readonly params: ChunkPageParams - readonly revisionKey: string - readonly workspaceId: string - readonly cacheStore: ChunkPageBlobStore - readonly fetchAsset: FetchChunkAsset - readonly scheduleWarm: ChunkPageWarmScheduler - readonly startAssetIndex?: number - readonly mirroredAssetUrlsByOriginalUrl?: Readonly> -} - -type MirrorableChunkAsset = { - readonly chunkId: string - readonly chunkType: "image" | "table" - readonly assetUrl: string - readonly filePath?: string -} - -const documentChunkPageSize = 200 -const visibleChunkPageMode: ChunkPageMode = "visible" -const structureChunkPageMode: ChunkPageMode = "structure" -const maximumMirroredAssetsPerWarmStep = 50 -const maximumWarmStepDurationMs = 45_000 -const assetMirrorConcurrency = 10 - -const defaultBlobStore: ChunkPageBlobStore = { - get: (pathname, options) => getBlob(pathname, options), - put: (pathname, body, options) => - put(pathname, body, { - access: options.access, - allowOverwrite: options.allowOverwrite, - contentType: options.contentType, - multipart: options.multipart, - }), -} - -const defaultFetchAsset: FetchChunkAsset = (assetUrl: string) => fetch(assetUrl) - -const defaultScheduleWarm: ChunkPageWarmScheduler = ( - task: () => Promise, -) => { - try { - after(task) - } catch { - void task() - } -} - -export const loadChunksForSource = ( - source: Source, - client: ChunkKnowhereClient, - options: ServerLoadChunksOptions = {}, -) => - Effect.gen(function* () { - if (source.status !== "ready" || !source.knowhereDocumentId) return [] - - const chunks: ParsedChunkView[] = [] - let page = 1 - let totalPages = 1 - - do { - const chunkPage = yield* loadChunkPageForSource(source, client, { - page, - pageSize: documentChunkPageSize, - }, { - ...options, - mode: structureChunkPageMode, - }) - chunks.push(...chunkPage.chunks) - totalPages = chunkPage.pagination.totalPages - page += 1 - } while (page <= totalPages) - - return resolveChunkConnectionTargets(chunks) - }) - -export const loadChunkPageForSource = ( - source: Source, - client: ChunkKnowhereClient, - params: ChunkPageParams, - options: ServerLoadChunksOptions = {}, -) => - Effect.gen(function* () { - const emptyPage = createEmptyChunkPage(params) - if (source.status !== "ready" || !source.knowhereDocumentId) { - return emptyPage - } - - const mode = options.mode ?? visibleChunkPageMode - const workspaceId = options.workspaceId ?? source.workspaceId - const cacheStore = options.cacheStore ?? defaultBlobStore - const includeAssetUrls = mode === visibleChunkPageMode - const revisionProbeResponse = yield* Effect.promise(() => - client.documents.listChunks(source.knowhereDocumentId!, { - page: params.page, - pageSize: params.pageSize, - includeAssetUrls: false, - }), - ) - const probeRevisionKey = getRevisionKey(revisionProbeResponse, source) - if (probeRevisionKey) { - scheduleRevisionKeyUpdate(source, probeRevisionKey, options.onRevisionKey) - const cachedPage = yield* Effect.promise(() => - readCachedChunkPage({ - cacheStore, - documentId: source.knowhereDocumentId!, - mode, - params, - revisionKey: probeRevisionKey, - workspaceId, - }), - ) - if (cachedPage) return cachedPage - } - - const response = includeAssetUrls - ? yield* Effect.promise(() => - client.documents.listChunks(source.knowhereDocumentId!, { - page: params.page, - pageSize: params.pageSize, - includeAssetUrls, - }), - ) - : revisionProbeResponse - const revisionKey = getRevisionKey(response, source) ?? probeRevisionKey - if (revisionKey && revisionKey !== probeRevisionKey) { - scheduleRevisionKeyUpdate(source, revisionKey, options.onRevisionKey) - } - - const chunkPage = createChunkPageFromResponse({ - response, - source, - params, - options: - mode === visibleChunkPageMode - ? { assetUrlsByFilePath: options.assetUrlsByFilePath } - : {}, - }) - - if (revisionKey) { - if (mode === visibleChunkPageMode) { - scheduleChunkPageWarm({ - source, - client, - params, - revisionKey, - workspaceId, - cacheStore, - fetchAsset: options.fetchAsset ?? defaultFetchAsset, - scheduleWarm: options.scheduleWarm ?? defaultScheduleWarm, - }) - } else { - scheduleStructurePageCacheWrite({ - cacheStore, - chunkPage, - documentId: source.knowhereDocumentId, - mode, - params, - revisionKey, - scheduleWarm: options.scheduleWarm ?? defaultScheduleWarm, - workspaceId, - }) - } - } - - return chunkPage - }) - -export async function warmChunkPageCache( - input: WarmChunkPageCacheInput, -): Promise { - const response = await input.client.documents.listChunks( - input.source.knowhereDocumentId!, - { - page: input.params.page, - pageSize: input.params.pageSize, - includeAssetUrls: true, - }, - ) - const assets = collectMirrorableChunkAssets(response.chunks) - const startAssetIndex = input.startAssetIndex ?? 0 - const mirroredAssetUrls = new Map( - Object.entries(input.mirroredAssetUrlsByOriginalUrl ?? {}), - ) - - const stepStartedAt = Date.now() - const assetBatch = assets.slice( - startAssetIndex, - startAssetIndex + maximumMirroredAssetsPerWarmStep, - ) - const mirroredBatch = await mirrorChunkAssets({ - assets: assetBatch, - cacheStore: input.cacheStore, - fetchAsset: input.fetchAsset, - revisionKey: input.revisionKey, - source: input.source, - }) - for (const mirroredAsset of mirroredBatch) { - mirroredAssetUrls.set(mirroredAsset.assetUrl, mirroredAsset.blobUrl) - } - - const nextAssetIndex = startAssetIndex + assetBatch.length - if ( - nextAssetIndex < assets.length && - Date.now() - stepStartedAt < maximumWarmStepDurationMs - ) { - input.scheduleWarm(() => - warmChunkPageCache({ - ...input, - startAssetIndex: nextAssetIndex, - mirroredAssetUrlsByOriginalUrl: Object.fromEntries(mirroredAssetUrls), - }), - ) - return - } - - if (nextAssetIndex < assets.length) { - input.scheduleWarm(() => - warmChunkPageCache({ - ...input, - startAssetIndex: nextAssetIndex, - mirroredAssetUrlsByOriginalUrl: Object.fromEntries(mirroredAssetUrls), - }), - ) - return - } - - const rewrittenChunks = response.chunks.map((chunk) => - rewriteChunkAssetUrl(chunk, mirroredAssetUrls), - ) - const chunkPage = createChunkPageFromResponse({ - response: { - ...response, - chunks: rewrittenChunks, - }, - source: input.source, - params: input.params, - options: {}, - }) - - await writeCachedChunkPage({ - cacheStore: input.cacheStore, - chunkPage, - documentId: input.source.knowhereDocumentId!, - mode: visibleChunkPageMode, - params: input.params, - revisionKey: input.revisionKey, - workspaceId: input.workspaceId, - }) -} - -function scheduleChunkPageWarm(input: WarmChunkPageCacheInput): void { - input.scheduleWarm(async () => { - try { - await warmChunkPageCache(input) - } catch (error) { - logger.warn("chunks: chunk page cache warm failed", { - sourceId: input.source.id, - documentId: input.source.knowhereDocumentId, - page: input.params.page, - pageSize: input.params.pageSize, - revisionKey: input.revisionKey, - error: getErrorMessage(error), - }) - } - }) -} - -function scheduleStructurePageCacheWrite(input: { - readonly cacheStore: ChunkPageBlobStore - readonly chunkPage: ChunkPage - readonly documentId: string - readonly mode: ChunkPageMode - readonly params: ChunkPageParams - readonly revisionKey: string - readonly scheduleWarm: ChunkPageWarmScheduler - readonly workspaceId: string -}): void { - input.scheduleWarm(async () => { - try { - await writeCachedChunkPage(input) - } catch (error) { - logger.warn("chunks: structure chunk page cache write failed", { - documentId: input.documentId, - page: input.params.page, - pageSize: input.params.pageSize, - revisionKey: input.revisionKey, - error: getErrorMessage(error), - }) - } - }) -} - -async function readCachedChunkPage(input: { - readonly cacheStore: ChunkPageBlobStore - readonly documentId: string - readonly mode: ChunkPageMode - readonly params: ChunkPageParams - readonly revisionKey: string - readonly workspaceId: string -}): Promise { - const pathname = getChunkPageCachePathname(input) - const result = await input.cacheStore.get(pathname, { access: "public" }) - if (!result || result.statusCode !== 200) return null - - const text = await new Response(result.stream).text() - return parseCachedChunkPage(text) -} - -async function writeCachedChunkPage(input: { - readonly cacheStore: ChunkPageBlobStore - readonly chunkPage: ChunkPage - readonly documentId: string - readonly mode: ChunkPageMode - readonly params: ChunkPageParams - readonly revisionKey: string - readonly workspaceId: string -}): Promise { - await input.cacheStore.put( - getChunkPageCachePathname(input), - JSON.stringify(input.chunkPage), - { - access: "public", - allowOverwrite: true, - contentType: "application/json; charset=utf-8", - }, - ) -} - -function createChunkPageFromResponse(input: { - readonly response: { - readonly chunks: readonly DocumentChunk[] - readonly pagination?: { - readonly page?: number - readonly pageSize?: number - readonly total?: number - readonly totalPages?: number - } - } - readonly source: Source - readonly params: ChunkPageParams - readonly options: LoadChunksOptions -}): ChunkPage { - const chunks = input.response.chunks.map((chunk) => - toParsedChunkView( - chunk, - input.source.title, - input.source.knowhereDocumentId ?? undefined, - input.options, - ), - ) - - return { - chunks, - pagination: { - page: getFiniteNonNegativeNumber( - input.response.pagination?.page, - input.params.page, - ), - pageSize: getFiniteNonNegativeNumber( - input.response.pagination?.pageSize, - input.params.pageSize, - ), - total: getFiniteNonNegativeNumber( - input.response.pagination?.total, - chunks.length, - ), - totalPages: getFiniteNonNegativeNumber( - input.response.pagination?.totalPages, - Math.ceil(chunks.length / input.params.pageSize), - ), - }, - } -} - -function createEmptyChunkPage(params: ChunkPageParams): ChunkPage { - return { - chunks: [], - pagination: { - page: params.page, - pageSize: params.pageSize, - total: 0, - totalPages: 0, - }, - } -} - -function parseCachedChunkPage(text: string): ChunkPage | null { - try { - const value: unknown = JSON.parse(text) - if (!isRecord(value)) return null - const chunks = value["chunks"] - const pagination = value["pagination"] - if (!Array.isArray(chunks) || !isRecord(pagination)) return null - const page = getNumber(pagination["page"]) - const pageSize = getNumber(pagination["pageSize"]) - const total = getNumber(pagination["total"]) - const totalPages = getNumber(pagination["totalPages"]) - if ( - page === undefined || - pageSize === undefined || - total === undefined || - totalPages === undefined - ) { - return null - } - - return { - chunks: chunks.filter(isParsedChunkView), - pagination: { - page, - pageSize, - total, - totalPages, - }, - } - } catch { - return null - } -} - -async function mirrorChunkAssets(input: { - readonly assets: readonly MirrorableChunkAsset[] - readonly cacheStore: ChunkPageBlobStore - readonly fetchAsset: FetchChunkAsset - readonly revisionKey: string - readonly source: Source -}): Promise< - readonly { - readonly assetUrl: string - readonly blobUrl: string - }[] -> { - return mapWithConcurrency( - input.assets, - assetMirrorConcurrency, - async (asset) => { - const response = await input.fetchAsset(asset.assetUrl) - if (!response.ok) { - throw new Error( - `Chunk asset fetch failed with status ${response.status}.`, - ) - } - - const body = Buffer.from(await response.arrayBuffer()) - const blob = await input.cacheStore.put( - getMirroredAssetPathname({ - asset, - revisionKey: input.revisionKey, - source: input.source, - }), - body, - { - access: "public", - allowOverwrite: true, - contentType: getMirroredAssetContentType(asset, response), - multipart: true, - }, - ) - return { - assetUrl: asset.assetUrl, - blobUrl: blob.url, - } - }, - ) -} - -async function mapWithConcurrency( - inputs: readonly Input[], - concurrency: number, - mapInput: (input: Input) => Promise, -): Promise { - const results: Array = [] - let nextIndex = 0 - const workerCount = Math.min(concurrency, inputs.length) - - async function runWorker(): Promise { - while (nextIndex < inputs.length) { - const index = nextIndex - nextIndex += 1 - const input = inputs[index] - if (input === undefined) return - results[index] = await mapInput(input) - } - } - - await Promise.all( - Array.from({ length: workerCount }, () => runWorker()), - ) - - return results.filter( - (result): result is Output => result !== undefined, - ) -} - -function collectMirrorableChunkAssets( - chunks: readonly DocumentChunk[], -): readonly MirrorableChunkAsset[] { - return chunks.flatMap((chunk): readonly MirrorableChunkAsset[] => { - if (chunk.chunkType !== "image" && chunk.chunkType !== "table") return [] - if (typeof chunk.assetUrl !== "string" || chunk.assetUrl.length === 0) { - return [] - } - - return [ - { - chunkId: chunk.id, - chunkType: chunk.chunkType, - assetUrl: chunk.assetUrl, - ...(chunk.filePath ? { filePath: chunk.filePath } : {}), - }, - ] - }) -} - -function rewriteChunkAssetUrl( - chunk: DocumentChunk, - mirroredAssetUrls: ReadonlyMap, -): DocumentChunk { - if (typeof chunk.assetUrl !== "string") return chunk - const mirroredAssetUrl = mirroredAssetUrls.get(chunk.assetUrl) - return mirroredAssetUrl ? { ...chunk, assetUrl: mirroredAssetUrl } : chunk -} - -function getChunkPageCachePathname(input: { - readonly documentId: string - readonly mode: ChunkPageMode - readonly params: ChunkPageParams - readonly revisionKey: string - readonly workspaceId: string -}): string { - return [ - "workspaces", - encodePathSegment(input.workspaceId), - "chunk-pages", - encodePathSegment(input.documentId), - encodePathSegment(input.revisionKey), - input.mode, - `page-${input.params.page}-size-${input.params.pageSize}.json`, - ].join("/") -} - -function getMirroredAssetPathname(input: { - readonly asset: MirrorableChunkAsset - readonly revisionKey: string - readonly source: Source -}): string { - const basename = getMirroredAssetBasename(input.asset) - return [ - "workspaces", - encodePathSegment(input.source.workspaceId), - "sources", - encodePathSegment(input.source.id), - "chunk-assets", - encodePathSegment(input.revisionKey), - `${hashValue(input.asset.assetUrl)}-${basename}`, - ].join("/") -} - -function getMirroredAssetBasename(asset: MirrorableChunkAsset): string { - const candidate = - asset.filePath ?? - getUrlPathname(asset.assetUrl).split("/").filter(Boolean).at(-1) ?? - `${asset.chunkId}.bin` - return candidate.replace(/[^a-zA-Z0-9._-]+/g, "-") -} - -function getMirroredAssetContentType( - asset: MirrorableChunkAsset, - response: Response, -): string { - const headerContentType = response.headers.get("content-type") - if (headerContentType) return headerContentType - if (asset.chunkType === "table") return "text/html; charset=utf-8" - - const extension = path.extname(asset.filePath ?? getUrlPathname(asset.assetUrl)) - .toLowerCase() - if (extension === ".jpg" || extension === ".jpeg") return "image/jpeg" - if (extension === ".png") return "image/png" - if (extension === ".webp") return "image/webp" - if (extension === ".gif") return "image/gif" - return "application/octet-stream" -} - -function getRevisionKey( - response: Pick, - source: Source, -): string | null { - return ( - getNonEmptyString(response.jobId) ?? - getNonEmptyString(response.jobResultId) ?? - getFallbackRevisionKey(source) - ) -} - -function getFallbackRevisionKey(source: Source): string | null { - return ( - getNonEmptyString(source.knowhereJobId) ?? - getNonEmptyString(source.knowhereDocumentId) - ) -} - -function scheduleRevisionKeyUpdate( - source: Source, - revisionKey: string, - onRevisionKey: ((revisionKey: string) => Promise) | undefined, -): void { - if (!onRevisionKey || source.knowhereJobId === revisionKey) return - - void onRevisionKey(revisionKey).catch((error: unknown) => { - logger.warn("chunks: source revision key update failed", { - sourceId: source.id, - documentId: source.knowhereDocumentId, - revisionKey, - error: getErrorMessage(error), - }) - }) -} - -function getFiniteNonNegativeNumber( - value: number | undefined, - fallback: number, -): number { - return typeof value === "number" && Number.isFinite(value) && value >= 0 - ? value - : fallback -} - -function isParsedChunkView(value: unknown): value is ParsedChunkView { - if (!isRecord(value)) return false - return ( - typeof value["chunkId"] === "string" && - typeof value["type"] === "string" && - typeof value["content"] === "string" && - typeof value["sourceTitle"] === "string" - ) -} - -function getNumber(value: unknown): number | undefined { - return typeof value === "number" && Number.isFinite(value) ? value : undefined -} - -function getNonEmptyString(value: unknown): string | null { - return typeof value === "string" && value.length > 0 ? value : null -} - -function isRecord(value: unknown): value is Readonly> { - return typeof value === "object" && value !== null && !Array.isArray(value) -} - -function getUrlPathname(value: string): string { - try { - return new URL(value).pathname - } catch { - return value.split("?")[0] ?? value - } -} - -function encodePathSegment(value: string): string { - return encodeURIComponent(value) -} - -function hashValue(value: string): string { - return createHash("sha256").update(value).digest("hex").slice(0, 16) -} - -function getErrorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error) -} diff --git a/src/domains/demo/workspace-source-resolution.ts b/src/domains/demo/workspace-source-resolution.ts index 39abc5a..d0918af 100644 --- a/src/domains/demo/workspace-source-resolution.ts +++ b/src/domains/demo/workspace-source-resolution.ts @@ -39,7 +39,7 @@ export function resolveWorkspaceDemoSources( } } -export function getWorkspaceSourcesNeedingKnowhereChunkCount( +export function getWorkspaceSourcesNeedingChunkCount( sources: readonly Source[], ): Source[] { return sources.filter((source) => !source.demoKey) diff --git a/src/domains/sources/counts.test.ts b/src/domains/sources/counts.test.ts index 22cdda9..22f0b8c 100644 --- a/src/domains/sources/counts.test.ts +++ b/src/domains/sources/counts.test.ts @@ -14,6 +14,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 1, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: "doc_1", stagedBlobPathname: null, @@ -29,10 +30,8 @@ function makeSource(overrides: Partial = {}): Source { } describe("countChunksBySourceId", () => { - it("counts chunks only for ready sources with a Knowhere document id", async () => { - const listChunks = vi.fn().mockResolvedValue({ - pagination: { total: 12 }, - }) + it("counts ready source chunks from the document total", async () => { + const listChunks = vi.fn(async () => ({ pagination: { total: 12 } })) const mockClient = { documents: { listChunks }, } as unknown as Knowhere @@ -43,14 +42,18 @@ describe("countChunksBySourceId", () => { countChunksBySourceId( [ makeSource({ id: "ready", knowhereDocumentId: "doc_ready" }), - makeSource({ id: "parsing", status: "parsing", knowhereDocumentId: null }), + makeSource({ + id: "parsing", + status: "parsing", + knowhereDocumentId: null, + }), makeSource({ id: "missing-doc", knowhereDocumentId: null }), ], mockClient, ), ) - expect(listChunks).toHaveBeenCalledOnce() + expect(listChunks).toHaveBeenCalledTimes(1) expect(listChunks).toHaveBeenCalledWith("doc_ready", { page: 1, pageSize: 1, @@ -58,8 +61,10 @@ describe("countChunksBySourceId", () => { expect(counts).toEqual(new Map([["ready", 12]])) }) - it("skips a source count when Knowhere chunks lookup fails", async () => { - const listChunks = vi.fn().mockRejectedValue(new Error("temporary outage")) + it("skips a source count when the document total lookup fails", async () => { + const listChunks = vi.fn(async () => { + throw new Error("temporary outage") + }) const mockClient = { documents: { listChunks }, } as unknown as Knowhere @@ -74,6 +79,7 @@ describe("countChunksBySourceId", () => { ) expect(counts.size).toBe(0) + expect(listChunks).toHaveBeenCalledTimes(1) }) it("does not count materialized demo sources through their copied document id", async () => { diff --git a/src/domains/sources/counts.ts b/src/domains/sources/counts.ts index 310a694..e6e0c9b 100644 --- a/src/domains/sources/counts.ts +++ b/src/domains/sources/counts.ts @@ -1,15 +1,27 @@ import "server-only" -import { Effect, Either } from "effect" +import { Effect } from "effect" import type Knowhere from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" +type CountChunksClient = { + readonly documents: { + listChunks( + documentId: string, + params: { readonly page: number; readonly pageSize: number }, + ): Promise<{ + readonly pagination?: { readonly total?: number } + }> + } +} + export const countChunksBySourceId = ( sources: readonly Source[], client: Knowhere, ) => Effect.gen(function* () { + const countClient = client as unknown as CountChunksClient const readySources = sources.filter( (source) => !source.demoKey && @@ -21,24 +33,10 @@ export const countChunksBySourceId = ( const entries = yield* Effect.all( readySources.map((source) => Effect.gen(function* () { - const documentId = source.knowhereDocumentId - if (!documentId) return [source.id, undefined] as const - - const result = yield* Effect.either( - Effect.tryPromise(() => - client.documents.listChunks(documentId, { - page: 1, - pageSize: 1, - }), - ), - ) - - if (Either.isLeft(result)) return [source.id, undefined] as const - - return [ - source.id, - result.right.pagination.total, - ] as const + const total = yield* Effect.tryPromise(() => + loadSourceChunkCount(countClient, source.knowhereDocumentId!), + ).pipe(Effect.catchAll(() => Effect.succeed(undefined))) + return [source.id, total] as const }), ), { concurrency: "unbounded" }, @@ -65,3 +63,15 @@ export const sourceViewOptionsBySourceId = ( ]), ) }) + +async function loadSourceChunkCount( + client: CountChunksClient, + documentId: string, +): Promise { + const response = await client.documents.listChunks(documentId, { + page: 1, + pageSize: 1, + }) + const total = response.pagination?.total + return typeof total === "number" && Number.isFinite(total) ? total : undefined +} diff --git a/src/domains/sources/parsed-document-blob-storage.test.ts b/src/domains/sources/parsed-document-blob-storage.test.ts new file mode 100644 index 0000000..8a3c427 --- /dev/null +++ b/src/domains/sources/parsed-document-blob-storage.test.ts @@ -0,0 +1,223 @@ +import { describe, expect, it } from "vitest" +import type { + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, + ParsedDocumentSyncProgress, +} from "@ontos-ai/knowhere-sdk" + +import { + BlobParsedDocumentStorage, + type ParsedDocumentBlobStore, +} from "./parsed-document-blob-storage" + +type StoredBlob = { + readonly body: string | Buffer + readonly contentType: string +} + +function createFakeBlobStore(): { + readonly store: ParsedDocumentBlobStore + readonly objects: Map +} { + const objects = new Map() + const toUrl = (pathname: string) => + `https://fake.public.blob.vercel-storage.com/${pathname}` + + const store: ParsedDocumentBlobStore = { + get: async (pathname) => { + const object = objects.get(pathname) + if (!object) return null + const body = + typeof object.body === "string" + ? object.body + : new Uint8Array(object.body) + return { + statusCode: 200, + stream: new Response(body).body as ReadableStream, + } + }, + put: async (pathname, body, options) => { + objects.set(pathname, { body, contentType: options.contentType }) + return { url: toUrl(pathname), pathname } + }, + head: async (pathname) => { + const object = objects.get(pathname) + return object ? { url: toUrl(pathname) } : null + }, + del: async (pathname) => { + objects.delete(pathname) + }, + } + + return { store, objects } +} + +const documentId = "doc_123" +const revisionKey = "job_result_1" + +const manifest: KnowhereParsedSnapshotManifest = { + version: 1, + kind: "knowhere-parsed-result-snapshot", + jobId: "job_1", + revisionKey, + documentId, + sourceFileName: "example.pdf", + totalChunks: 2, + chunkPageSize: 200, + chunkPages: [{ page: 1, pageSize: 200, chunkCount: 2, key: "chunks/page-1.json" }], + assetUrlsByFilePath: {}, + createdAt: "2026-07-04T00:00:00.000Z", +} + +const chunkPage: KnowhereParsedSnapshotChunkPage = { + version: 1, + jobId: "job_1", + revisionKey, + documentId, + sourceFileName: "example.pdf", + page: 1, + pageSize: 200, + total: 2, + totalPages: 1, + chunks: [ + { + id: "c1", + chunkId: "c1", + chunkType: "text", + content: "hello", + sourceChunkPath: "example.pdf", + sortOrder: 0, + metadata: {}, + }, + ], +} + +describe("BlobParsedDocumentStorage", () => { + it("round-trips a manifest keyed by workspace/document/revision", async () => { + const { store, objects } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + await storage.writeManifest({ documentId, revisionKey, manifest }) + + expect([...objects.keys()]).toContain( + "workspaces/ws_1/parsed-documents/doc_123/job_result_1/manifest/current.json", + ) + const read = await storage.readManifest({ documentId, revisionKey }) + expect(read).toEqual(manifest) + }) + + it("round-trips a chunk page", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + await storage.writeChunkPage({ documentId, revisionKey, page: chunkPage }) + const read = await storage.readChunkPage({ documentId, revisionKey, page: 1 }) + expect(read).toEqual(chunkPage) + }) + + it("round-trips sync progress", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + const progress: ParsedDocumentSyncProgress = { + documentId, + revisionKey, + nextChunkPage: 3, + nextAssetIndex: 0, + status: "running", + updatedAt: "2026-07-04T00:00:00.000Z", + } + + await storage.writeSyncProgress(progress) + const read = await storage.readSyncProgress({ documentId, revisionKey }) + expect(read).toEqual(progress) + }) + + it("writes an asset and resolves a durable URL", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + const written = await storage.writeAsset({ + documentId, + revisionKey, + sourcePath: "images/fig-1.png", + body: new Uint8Array([1, 2, 3]), + contentType: "image/png", + }) + expect(written.url).toContain( + "workspaces/ws_1/parsed-documents/doc_123/job_result_1/assets/images/fig-1.png", + ) + + const url = await storage.getAssetUrl({ + documentId, + revisionKey, + sourcePath: "images/fig-1.png", + }) + expect(url).toBe(written.url) + }) + + it("returns null for a missing manifest, chunk page, progress, and asset", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + expect(await storage.readManifest({ documentId, revisionKey })).toBeNull() + expect( + await storage.readChunkPage({ documentId, revisionKey, page: 9 }), + ).toBeNull() + expect(await storage.readSyncProgress({ documentId, revisionKey })).toBeNull() + expect( + await storage.getAssetUrl({ documentId, revisionKey, sourcePath: "x.png" }), + ).toBeNull() + }) + + it("isolates artifacts by revision key so a stale revision does not read", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + await storage.writeManifest({ documentId, revisionKey, manifest }) + const staleRead = await storage.readManifest({ + documentId, + revisionKey: "job_result_2", + }) + expect(staleRead).toBeNull() + }) + + it("rejects traversal in the document id, revision key, and asset path", async () => { + const { store } = createFakeBlobStore() + const storage = new BlobParsedDocumentStorage({ + workspaceId: "ws_1", + blobStore: store, + }) + + await expect( + storage.readManifest({ documentId: "../escape", revisionKey }), + ).rejects.toThrow(/Invalid parsed storage segment/) + await expect( + storage.readManifest({ documentId, revisionKey: "a/b" }), + ).rejects.toThrow(/Invalid parsed storage segment/) + await expect( + storage.getAssetUrl({ + documentId, + revisionKey, + sourcePath: "../../etc/passwd", + }), + ).rejects.toThrow(/Invalid parsed storage path/) + }) +}) diff --git a/src/domains/sources/parsed-document-blob-storage.ts b/src/domains/sources/parsed-document-blob-storage.ts new file mode 100644 index 0000000..23a93a0 --- /dev/null +++ b/src/domains/sources/parsed-document-blob-storage.ts @@ -0,0 +1,291 @@ +import "server-only" + +import { del, get, head, put, BlobNotFoundError } from "@vercel/blob" +import type { + KnowhereParsedSnapshotChunkPage, + KnowhereParsedSnapshotManifest, + ParsedDocumentStorage, + ParsedDocumentStorageAsset, + ParsedDocumentStorageAssetParams, + ParsedDocumentStorageChunkPageParams, + ParsedDocumentStorageDocument, + ParsedDocumentStorageManifestParams, + ParsedDocumentSyncProgress, +} from "@ontos-ai/knowhere-sdk" + +/** + * Vercel Blob backed implementation of the SDK `ParsedDocumentStorage` + * interface. The SDK only ever calls this with `{ documentId, revisionKey }`, + * so the adapter is constructed with the enclosing workspace and derives all + * blob keys from documentId + revisionKey. Keying every artifact under the + * revision key gives the SDK freshness contract for free: a manifest read at a + * revision path always carries a matching `revisionKey`. + * + * The `blobStore` seam mirrors the pattern used elsewhere in the sources + * domain so tests can inject a fake store instead of hitting Vercel Blob. + */ + +const parsedDocumentsDirectoryName = "parsed-documents" +const manifestStoragePath = "manifest/current.json" +const syncProgressStoragePath = "sync-progress.json" +const jsonContentType = "application/json; charset=utf-8" + +type BlobGetResult = + | { + readonly statusCode: 200 + readonly stream: ReadableStream + } + | { + readonly statusCode: 304 + readonly stream: null + } + +type BlobPutResult = { + readonly url: string + readonly pathname: string +} + +type BlobHeadResult = { + readonly url: string +} + +/** + * Minimal blob operations the adapter depends on. The default binds directly to + * `@vercel/blob`; tests provide an in-memory stand-in. + */ +export type ParsedDocumentBlobStore = { + readonly get: ( + pathname: string, + options: { readonly access: "public" }, + ) => Promise + readonly put: ( + pathname: string, + body: string | Buffer, + options: { + readonly access: "public" + readonly allowOverwrite: boolean + readonly contentType: string + readonly multipart?: boolean + }, + ) => Promise + readonly head: (pathname: string) => Promise + readonly del: (pathname: string) => Promise +} + +export type BlobParsedDocumentStorageInput = { + readonly workspaceId: string + readonly blobStore?: ParsedDocumentBlobStore +} + +const vercelBlobStore: ParsedDocumentBlobStore = { + get: (pathname, options) => get(pathname, options), + put: async (pathname, body, options) => { + const blob = await put(pathname, body, { + access: options.access, + addRandomSuffix: false, + allowOverwrite: options.allowOverwrite, + contentType: options.contentType, + multipart: options.multipart, + }) + return { url: blob.url, pathname: blob.pathname } + }, + head: async (pathname) => { + try { + const blob = await head(pathname) + return { url: blob.url } + } catch (error) { + if (error instanceof BlobNotFoundError) return null + throw error + } + }, + del: (pathname) => del(pathname), +} + +export class BlobParsedDocumentStorage implements ParsedDocumentStorage { + private readonly workspaceId: string + private readonly blobStore: ParsedDocumentBlobStore + + constructor(input: BlobParsedDocumentStorageInput) { + this.workspaceId = input.workspaceId + this.blobStore = input.blobStore ?? vercelBlobStore + } + + async readManifest( + params: ParsedDocumentStorageManifestParams, + ): Promise { + return this.readJson( + this.getManifestKey(params.documentId, params.revisionKey), + ) + } + + async writeManifest(params: { + readonly documentId: string + readonly revisionKey: string + readonly manifest: KnowhereParsedSnapshotManifest + }): Promise { + await this.writeJson( + this.getManifestKey(params.documentId, params.revisionKey), + params.manifest, + ) + } + + async readChunkPage( + params: ParsedDocumentStorageChunkPageParams, + ): Promise { + // chunkType filtering happens SDK-side after read; storage returns the full page. + return this.readJson( + this.getChunkPageKey(params.documentId, params.revisionKey, params.page), + ) + } + + async writeChunkPage(params: { + readonly documentId: string + readonly revisionKey: string + readonly page: KnowhereParsedSnapshotChunkPage + }): Promise { + await this.writeJson( + this.getChunkPageKey( + params.documentId, + params.revisionKey, + params.page.page, + ), + params.page, + ) + } + + async writeAsset( + params: ParsedDocumentStorageDocument & ParsedDocumentStorageAsset, + ): Promise<{ readonly sourcePath: string; readonly url?: string }> { + const blob = await this.blobStore.put( + this.getAssetKey(params.documentId, params.revisionKey, params.sourcePath), + Buffer.from(params.body), + { + access: "public", + allowOverwrite: true, + contentType: params.contentType, + multipart: true, + }, + ) + return { sourcePath: params.sourcePath, url: blob.url } + } + + async getAssetUrl( + params: ParsedDocumentStorageAssetParams, + ): Promise { + const result = await this.blobStore.head( + this.getAssetKey(params.documentId, params.revisionKey, params.sourcePath), + ) + return result?.url ?? null + } + + async readSyncProgress( + params: ParsedDocumentStorageDocument, + ): Promise { + return this.readJson( + this.getSyncProgressKey(params.documentId, params.revisionKey), + ) + } + + async writeSyncProgress(params: ParsedDocumentSyncProgress): Promise { + await this.writeJson( + this.getSyncProgressKey(params.documentId, params.revisionKey), + params, + ) + } + + private getRevisionPrefix(documentId: string, revisionKey: string): string { + return [ + "workspaces", + normalizePathSegment(this.workspaceId), + parsedDocumentsDirectoryName, + normalizePathSegment(documentId), + normalizePathSegment(revisionKey), + ].join("/") + } + + private getManifestKey(documentId: string, revisionKey: string): string { + return `${this.getRevisionPrefix(documentId, revisionKey)}/${manifestStoragePath}` + } + + private getChunkPageKey( + documentId: string, + revisionKey: string, + page: number, + ): string { + return `${this.getRevisionPrefix(documentId, revisionKey)}/chunks/page-${page}.json` + } + + private getSyncProgressKey(documentId: string, revisionKey: string): string { + return `${this.getRevisionPrefix(documentId, revisionKey)}/${syncProgressStoragePath}` + } + + private getAssetKey( + documentId: string, + revisionKey: string, + sourcePath: string, + ): string { + return `${this.getRevisionPrefix(documentId, revisionKey)}/assets/${normalizeRelativeStoragePath(sourcePath)}` + } + + private async readJson(key: string): Promise { + const text = await this.readBlobText(key) + if (text === null) return null + try { + return JSON.parse(text) as T + } catch { + return null + } + } + + private async writeJson(key: string, value: unknown): Promise { + await this.blobStore.put(key, JSON.stringify(value), { + access: "public", + allowOverwrite: true, + contentType: jsonContentType, + }) + } + + private async readBlobText(key: string): Promise { + try { + const result = await this.blobStore.get(key, { access: "public" }) + if (!result || result.statusCode !== 200) return null + return await new Response(result.stream).text() + } catch (error) { + if (error instanceof BlobNotFoundError) return null + throw error + } + } +} + +/** + * Reject path segments that could traverse outside the intended prefix. Mirrors + * the SDK `DiskParsedDocumentStorage` guard so blob keys stay well-formed. + */ +function normalizePathSegment(value: string): string { + if ( + value.length === 0 || + value.includes("\0") || + value.includes("/") || + value.includes("\\") || + value === "." || + value === ".." + ) { + throw new Error(`Invalid parsed storage segment: ${value}`) + } + return value +} + +function normalizeRelativeStoragePath(value: string): string { + const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "") + if ( + normalized.length === 0 || + normalized.includes("\0") || + normalized.startsWith("/") || + normalized + .split("/") + .some((part) => part.length === 0 || part === "." || part === "..") + ) { + throw new Error(`Invalid parsed storage path: ${value}`) + } + return normalized +} diff --git a/src/domains/sources/parsed-document-sync-capacity.test.ts b/src/domains/sources/parsed-document-sync-capacity.test.ts new file mode 100644 index 0000000..5800e09 --- /dev/null +++ b/src/domains/sources/parsed-document-sync-capacity.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest" + +import { parsedDocumentSyncCapacityGuard } from "./parsed-document-sync-capacity" + +describe("parsedDocumentSyncCapacityGuard", () => { + it("uses the configured defaults when environment variables are absent", () => { + expect(parsedDocumentSyncCapacityGuard.readPolicy({})).toEqual({ + globalActiveLimit: 100, + workspaceActiveLimit: 5, + documentActiveLimit: 1, + waitSeconds: 60, + }) + }) + + it("reads positive integer capacity settings from the environment", () => { + expect( + parsedDocumentSyncCapacityGuard.readPolicy({ + SYNC_GLOBAL_ACTIVE_LIMIT: "12", + SYNC_WORKSPACE_ACTIVE_LIMIT: "3", + SYNC_DOCUMENT_ACTIVE_LIMIT: "2", + SYNC_WAIT_SECONDS: "90", + }), + ).toEqual({ + globalActiveLimit: 12, + workspaceActiveLimit: 3, + documentActiveLimit: 2, + waitSeconds: 90, + }) + }) + + it("denies capacity by document, workspace, then global cap", () => { + const policy = parsedDocumentSyncCapacityGuard.readPolicy({}) + + expect( + parsedDocumentSyncCapacityGuard.selectDenialReason( + { globalActive: 0, workspaceActive: 0, documentActive: 1 }, + policy, + ), + ).toBe("document") + expect( + parsedDocumentSyncCapacityGuard.selectDenialReason( + { globalActive: 0, workspaceActive: 5, documentActive: 0 }, + policy, + ), + ).toBe("workspace") + expect( + parsedDocumentSyncCapacityGuard.selectDenialReason( + { globalActive: 100, workspaceActive: 0, documentActive: 0 }, + policy, + ), + ).toBe("global") + }) +}) diff --git a/src/domains/sources/parsed-document-sync-capacity.ts b/src/domains/sources/parsed-document-sync-capacity.ts new file mode 100644 index 0000000..cff5584 --- /dev/null +++ b/src/domains/sources/parsed-document-sync-capacity.ts @@ -0,0 +1,304 @@ +import "server-only" + +import { randomUUID } from "node:crypto" + +import { and, eq, isNull, sql } from "drizzle-orm" +import { Effect } from "effect" + +import { databaseRuntime } from "@/domains/workspace/database-runtime" +import { DbClient } from "@/infrastructure/db" +import { parsedDocumentSyncLeases } from "@/infrastructure/db/schema" + +type SyncCapacityPolicy = { + readonly globalActiveLimit: number + readonly workspaceActiveLimit: number + readonly documentActiveLimit: number + readonly waitSeconds: number +} + +type ActiveSyncCounts = { + readonly globalActive: number + readonly workspaceActive: number + readonly documentActive: number +} + +type SyncCapacityDenialReason = "global" | "workspace" | "document" + +type AcquireSyncLeaseInput = { + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly revisionKey?: string + readonly policy?: SyncCapacityPolicy +} + +type AcquireSyncLeaseResult = + | { + readonly kind: "acquired" + readonly leaseToken: string + readonly activeCounts: ActiveSyncCounts + } + | { + readonly kind: "capacity-full" + readonly reason: SyncCapacityDenialReason + readonly waitSeconds: number + readonly activeCounts: ActiveSyncCounts + } + | { + readonly kind: "source-missing" + } + +type ReleaseSyncLeaseInput = { + readonly leaseToken: string + readonly releaseReason: SyncLeaseReleaseReason +} + +type SyncLeaseReleaseReason = "completed" | "incomplete" | "failed" + +type AcquireSyncLeaseRow = { + readonly leaseId: string | null + readonly leaseToken: string | null + readonly hasSource: boolean + readonly globalActive: number | string + readonly workspaceActive: number | string + readonly documentActive: number | string +} + +type RawRowsResult = + | readonly Row[] + | { + readonly rows: readonly Row[] + } + +const defaultGlobalActiveLimit: number = 100 +const defaultWorkspaceActiveLimit: number = 5 +const defaultDocumentActiveLimit: number = 1 +const defaultWaitSeconds: number = 60 +const leaseDurationSeconds: number = 10 * 60 + +const capacityLockClassId: number = 24_071_105 +const capacityLockObjectId: number = 2_607_198 + +function readPolicy( + env: Readonly> = process.env, +): SyncCapacityPolicy { + return { + globalActiveLimit: readPositiveIntegerEnv( + "SYNC_GLOBAL_ACTIVE_LIMIT", + defaultGlobalActiveLimit, + env, + ), + workspaceActiveLimit: readPositiveIntegerEnv( + "SYNC_WORKSPACE_ACTIVE_LIMIT", + defaultWorkspaceActiveLimit, + env, + ), + documentActiveLimit: readPositiveIntegerEnv( + "SYNC_DOCUMENT_ACTIVE_LIMIT", + defaultDocumentActiveLimit, + env, + ), + waitSeconds: readPositiveIntegerEnv( + "SYNC_WAIT_SECONDS", + defaultWaitSeconds, + env, + ), + } +} + +function readPositiveIntegerEnv( + name: string, + defaultValue: number, + env: Readonly>, +): number { + const rawValue = env[name] + if (!rawValue) return defaultValue + + const value = Number(rawValue) + if (!Number.isInteger(value) || value < 1) return defaultValue + + return value +} + +function selectDenialReason( + counts: ActiveSyncCounts, + policy: SyncCapacityPolicy, +): SyncCapacityDenialReason | null { + if (counts.documentActive >= policy.documentActiveLimit) return "document" + if (counts.workspaceActive >= policy.workspaceActiveLimit) return "workspace" + if (counts.globalActive >= policy.globalActiveLimit) return "global" + + return null +} + +const acquireEffect = ( + input: AcquireSyncLeaseInput, +): Effect.Effect => + Effect.gen(function* () { + const db = yield* DbClient + const policy = input.policy ?? readPolicy() + const leaseToken = randomUUID() + const result = yield* Effect.promise(() => + db.execute(sql` + WITH lock AS ( + SELECT pg_advisory_xact_lock(${capacityLockClassId}, ${capacityLockObjectId}) + ), + expired AS ( + UPDATE parsed_document_sync_leases + SET released_at = now(), + release_reason = 'expired', + updated_at = now() + WHERE released_at IS NULL + AND expires_at <= now() + RETURNING id + ), + counts AS ( + SELECT + EXISTS ( + SELECT 1 + FROM sources + WHERE id = ${input.sourceId}::uuid + AND workspace_id = ${input.workspaceId}::uuid + AND deleted_at IS NULL + ) AS "hasSource", + ( + SELECT count(*)::int + FROM parsed_document_sync_leases + WHERE released_at IS NULL + AND expires_at > now() + ) AS "globalActive", + ( + SELECT count(*)::int + FROM parsed_document_sync_leases + WHERE workspace_id = ${input.workspaceId}::uuid + AND released_at IS NULL + AND expires_at > now() + ) AS "workspaceActive", + ( + SELECT count(*)::int + FROM parsed_document_sync_leases + WHERE document_id = ${input.documentId} + AND released_at IS NULL + AND expires_at > now() + ) AS "documentActive" + FROM lock + ), + inserted AS ( + INSERT INTO parsed_document_sync_leases ( + workspace_id, + source_id, + document_id, + revision_key, + lease_token, + expires_at + ) + SELECT + ${input.workspaceId}::uuid, + ${input.sourceId}::uuid, + ${input.documentId}, + ${input.revisionKey ?? null}, + ${leaseToken}, + now() + (${leaseDurationSeconds} * interval '1 second') + FROM counts + WHERE "hasSource" + AND "globalActive" < ${policy.globalActiveLimit} + AND "workspaceActive" < ${policy.workspaceActiveLimit} + AND "documentActive" < ${policy.documentActiveLimit} + RETURNING id AS "leaseId", lease_token AS "leaseToken" + ) + SELECT + (SELECT "leaseId" FROM inserted) AS "leaseId", + (SELECT "leaseToken" FROM inserted) AS "leaseToken", + counts."hasSource", + counts."globalActive", + counts."workspaceActive", + counts."documentActive" + FROM counts + `), + ) + const row = getFirstRow(getRawRows(result)) + if (!row) return { kind: "source-missing" } + if (!row.hasSource) return { kind: "source-missing" } + + const activeCounts = normalizeActiveCounts(row) + if (row.leaseToken) { + return { + kind: "acquired", + leaseToken: row.leaseToken, + activeCounts, + } + } + + return { + kind: "capacity-full", + reason: selectDenialReason(activeCounts, policy) ?? "global", + waitSeconds: policy.waitSeconds, + activeCounts, + } + }) + +const releaseEffect = ( + input: ReleaseSyncLeaseInput, +): Effect.Effect => + Effect.gen(function* () { + const db = yield* DbClient + yield* Effect.promise(() => + db + .update(parsedDocumentSyncLeases) + .set({ + releasedAt: sql`now()`, + releaseReason: input.releaseReason, + updatedAt: sql`now()`, + }) + .where( + and( + eq(parsedDocumentSyncLeases.leaseToken, input.leaseToken), + isNull(parsedDocumentSyncLeases.releasedAt), + ), + ), + ) + }) + +function normalizeActiveCounts(row: AcquireSyncLeaseRow): ActiveSyncCounts { + return { + globalActive: normalizeCount(row.globalActive), + workspaceActive: normalizeCount(row.workspaceActive), + documentActive: normalizeCount(row.documentActive), + } +} + +function normalizeCount(value: number | string): number { + return typeof value === "number" ? value : Number(value) +} + +function getRawRows(value: RawRowsResult): readonly Row[] { + if (isReadonlyArray(value)) return value + return value.rows +} + +function getFirstRow(rows: readonly Row[]): Row | undefined { + return rows[0] +} + +function isReadonlyArray( + value: RawRowsResult, +): value is readonly Row[] { + return Array.isArray(value) +} + +async function acquire( + input: AcquireSyncLeaseInput, +): Promise { + return databaseRuntime.runPromise(acquireEffect(input)) +} + +async function release(input: ReleaseSyncLeaseInput): Promise { + await databaseRuntime.runPromise(releaseEffect(input)) +} + +export const parsedDocumentSyncCapacityGuard = { + acquire, + readPolicy, + release, + selectDenialReason, +} diff --git a/src/domains/sources/parsed-document-sync-scheduler.ts b/src/domains/sources/parsed-document-sync-scheduler.ts new file mode 100644 index 0000000..c0ee4ce --- /dev/null +++ b/src/domains/sources/parsed-document-sync-scheduler.ts @@ -0,0 +1,158 @@ +import "server-only" + +import { Client } from "@upstash/workflow" +import type { ParsedDocumentSyncScheduler } from "@ontos-ai/knowhere-sdk" + +import { logger } from "@/lib/logger" + +/** + * A `ParsedDocumentSyncScheduler` whose `schedule` enqueues a durable QStash + * continuation instead of running the (non-serializable) task closure inline. + * The SDK schedules a background sync on every storage read-miss; the closure + * captures a per-request client we cannot serialize across serverless + * invocations, so we discard it and trigger `/api/sources/parsed-sync`, which + * rebuilds a parsed-storage client and loops `syncParsedDocument` to completion. + * + * The scheduler is pre-bound to a single `{ workspaceId, sourceId, documentId }` + * because the SDK invokes `schedule(task)` with no arguments. `revisionKey` is + * intentionally NOT part of the bound identity — the sync route resolves the + * current revision itself so a scheduler bound before the revision is known + * still enqueues correctly. + */ + +export type ParsedSyncTrigger = (input: { + readonly url: string + readonly body: ParsedSyncPayload + readonly workflowRunId: string + readonly delaySeconds?: number +}) => Promise + +export type ParsedSyncPayload = { + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly apiKey: string + readonly revisionKey?: string + readonly segmentIndex?: number +} + +export type CreateParsedDocumentSyncSchedulerInput = { + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly apiKey: string + readonly revisionKey?: string + readonly trigger?: ParsedSyncTrigger +} + +// Bounded per-document guard: the SDK may schedule on every read-miss within a +// single request. The cooldown collapses duplicate same-process enqueues +// without permanently blocking future syncs. +const triggerCooldownMs: number = 60_000 +const lastTriggeredAtByKey: Map = new Map() + +function resolveBaseURL(): string { + return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" +} + +export function getParsedSyncWorkflowRunId(input: { + readonly documentId: string + readonly revisionKey: string + readonly segmentIndex: number +}): string { + return `${input.documentId}-sync-${input.revisionKey}-${input.segmentIndex}` +} + +const defaultTrigger: ParsedSyncTrigger = async (input) => { + const token = process.env.QSTASH_TOKEN + if (!token) { + throw new Error("QSTASH_TOKEN is required to schedule parsed document sync.") + } + await new Client({ token }).trigger({ + url: input.url, + body: input.body, + workflowRunId: input.workflowRunId, + retries: 3, + delay: input.delaySeconds, + }) +} + +export function getParsedSyncUrl(): string { + return `${resolveBaseURL()}/api/sources/parsed-sync` +} + +/** + * Enqueue a durable parsed-document sync from outside the SDK read path — used + * by the reconcile workflow to hand off to the resumable parsed-sync route, and + * by retry to resume a failed storage sync. Starts at segment 0. + */ +export async function enqueueParsedDocumentSync( + input: { + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly apiKey: string + readonly revisionKey?: string + readonly delaySeconds?: number + }, + trigger: ParsedSyncTrigger = defaultTrigger, +): Promise { + await trigger({ + url: getParsedSyncUrl(), + body: { ...input, segmentIndex: 0 }, + workflowRunId: getParsedSyncWorkflowRunId({ + documentId: input.documentId, + revisionKey: input.revisionKey ?? "initial", + segmentIndex: 0, + }), + delaySeconds: input.delaySeconds, + }) +} + +export function createParsedDocumentSyncScheduler( + input: CreateParsedDocumentSyncSchedulerInput, +): ParsedDocumentSyncScheduler { + const trigger = input.trigger ?? defaultTrigger + const cooldownKey = `${input.documentId}|${input.revisionKey ?? "unknown"}` + + return { + schedule: () => { + const now = Date.now() + const lastTriggeredAt = lastTriggeredAtByKey.get(cooldownKey) + if ( + lastTriggeredAt !== undefined && + now - lastTriggeredAt < triggerCooldownMs + ) { + return + } + lastTriggeredAtByKey.set(cooldownKey, now) + + const url = `${resolveBaseURL()}/api/sources/parsed-sync` + const workflowRunId = getParsedSyncWorkflowRunId({ + documentId: input.documentId, + revisionKey: input.revisionKey ?? "initial", + segmentIndex: 0, + }) + void trigger({ + url, + body: { + workspaceId: input.workspaceId, + sourceId: input.sourceId, + documentId: input.documentId, + apiKey: input.apiKey, + revisionKey: input.revisionKey, + segmentIndex: 0, + }, + workflowRunId, + }).catch((error: unknown) => { + lastTriggeredAtByKey.delete(cooldownKey) + logger.error("parsed-sync-scheduler: failed to enqueue sync", { + workspaceId: input.workspaceId, + sourceId: input.sourceId, + documentId: input.documentId, + error: error instanceof Error ? error.message : String(error), + }) + }) + }, + } +} diff --git a/src/domains/sources/parsed-sync-route-workflow.test.ts b/src/domains/sources/parsed-sync-route-workflow.test.ts new file mode 100644 index 0000000..91095f5 --- /dev/null +++ b/src/domains/sources/parsed-sync-route-workflow.test.ts @@ -0,0 +1,296 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" + +const mocks = vi.hoisted(() => ({ + acquireSyncCapacity: vi.fn(), + makeKnowhereClientWithParsedStorage: vi.fn(), + releaseSyncCapacity: vi.fn(), + updateSyncStatus: vi.fn(), + findInWorkspace: vi.fn(), + markFailed: vi.fn(), + markReady: vi.fn(), + markSourceReadyAfterReconciliation: vi.fn(), + loggerInfo: vi.fn(), + loggerError: vi.fn(), +})) + +vi.mock("@/integrations/knowhere", () => ({ + makeKnowhereClientWithParsedStorage: mocks.makeKnowhereClientWithParsedStorage, +})) + +vi.mock("./workflow-runtime", () => ({ + sourceWorkflowRuntime: { + updateSyncStatus: mocks.updateSyncStatus, + findInWorkspace: mocks.findInWorkspace, + markFailed: mocks.markFailed, + markReady: mocks.markReady, + }, +})) + +vi.mock("./source-reconcile-workflow", () => ({ + markSourceReadyAfterReconciliation: mocks.markSourceReadyAfterReconciliation, +})) + +vi.mock("./parsed-document-sync-capacity", () => ({ + parsedDocumentSyncCapacityGuard: { + acquire: mocks.acquireSyncCapacity, + release: mocks.releaseSyncCapacity, + }, +})) + +vi.mock("@/lib/logger", () => ({ + logger: { info: mocks.loggerInfo, error: mocks.loggerError }, +})) + +import { parsedSyncRouteWorkflow } from "./parsed-sync-route-workflow" + +type RunStep = (id: string, task: () => Promise | T) => Promise + +function createContext(overrides: { url?: string } = {}) { + const run: RunStep = async (_id, task) => task() + return { + run, + url: overrides.url ?? "https://notebook.example/api/sources/parsed-sync", + } +} + +const basePayload = { + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + apiKey: "key_1", + segmentIndex: 0, +} + +describe("parsedSyncRouteWorkflow.runParsedSyncWorkflow", () => { + beforeEach(() => { + mocks.acquireSyncCapacity.mockResolvedValue({ + kind: "acquired", + leaseToken: "lease_1", + activeCounts: { + globalActive: 0, + workspaceActive: 0, + documentActive: 0, + }, + }) + mocks.releaseSyncCapacity.mockResolvedValue(undefined) + mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ + status: "ready", + }) + }) + + afterEach(() => { + vi.clearAllMocks() + }) + + it("marks the source ready when sync completes in one segment", async () => { + const syncParsedDocument = vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_1", + completed: true, + })) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: {}, + knowledge: { syncParsedDocument }, + }) + + await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ + context: createContext(), + payload: basePayload, + }) + + expect(syncParsedDocument).toHaveBeenCalledTimes(1) + expect(syncParsedDocument).toHaveBeenCalledWith({ documentId: "doc_1" }) + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { revisionKey: "rev_1", syncStatus: "completed" }, + ) + expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + }) + expect(mocks.releaseSyncCapacity).toHaveBeenCalledWith({ + leaseToken: "lease_1", + releaseReason: "completed", + }) + }) + + it("marks ready and triggers a continuation when sync is incomplete", async () => { + const syncParsedDocument = vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_1", + completed: false, + })) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: {}, + knowledge: { syncParsedDocument }, + }) + const triggered: Array<{ workflowRunId: string; segmentIndex?: number }> = [] + const restore = parsedSyncRouteWorkflow.setContinuationTriggerForTesting( + async (input) => { + triggered.push({ + workflowRunId: input.workflowRunId, + segmentIndex: input.payload.segmentIndex, + }) + }, + ) + + try { + await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ + context: createContext(), + payload: basePayload, + }) + } finally { + restore() + } + + expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + }) + expect(triggered).toHaveLength(1) + expect(triggered[0]?.segmentIndex).toBe(1) + expect(triggered[0]?.workflowRunId).toBe("doc_1-sync-rev_1-1") + expect(mocks.updateSyncStatus).toHaveBeenLastCalledWith( + "workspace_1", + "source_1", + { revisionKey: "rev_1", syncStatus: "running" }, + ) + expect(mocks.releaseSyncCapacity).toHaveBeenCalledWith({ + leaseToken: "lease_1", + releaseReason: "incomplete", + }) + }) + + it("passes an explicit revisionKey into syncParsedDocument when provided", async () => { + const syncParsedDocument = vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_9", + completed: true, + })) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: {}, + knowledge: { syncParsedDocument }, + }) + + await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ + context: createContext(), + payload: { ...basePayload, revisionKey: "rev_9" }, + }) + + expect(syncParsedDocument).toHaveBeenCalledWith({ + documentId: "doc_1", + revisionKey: "rev_9", + }) + }) + + it("schedules a delayed retry when capacity is full", async () => { + const syncParsedDocument = vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_1", + completed: true, + })) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: {}, + knowledge: { syncParsedDocument }, + }) + mocks.acquireSyncCapacity.mockResolvedValue({ + kind: "capacity-full", + reason: "workspace", + waitSeconds: 60, + activeCounts: { + globalActive: 10, + workspaceActive: 5, + documentActive: 0, + }, + }) + const triggered: Array<{ + readonly workflowRunId: string + readonly segmentIndex?: number + readonly delaySeconds?: number + }> = [] + const restore = parsedSyncRouteWorkflow.setContinuationTriggerForTesting( + async (input) => { + triggered.push({ + workflowRunId: input.workflowRunId, + segmentIndex: input.payload.segmentIndex, + delaySeconds: input.delaySeconds, + }) + }, + ) + + try { + await parsedSyncRouteWorkflow.runParsedSyncWorkflow({ + context: createContext(), + payload: { ...basePayload, revisionKey: "rev_1" }, + }) + } finally { + restore() + } + + expect(syncParsedDocument).not.toHaveBeenCalled() + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { + revisionKey: "rev_1", + syncStatus: "pending", + syncError: null, + }, + ) + expect(triggered).toEqual([ + { + workflowRunId: "doc_1-sync-rev_1-1", + segmentIndex: 1, + delaySeconds: 60, + }, + ]) + expect(mocks.releaseSyncCapacity).not.toHaveBeenCalled() + }) +}) + +describe("parsedSyncRouteWorkflow.markSyncFailedAfterWorkflowFailure", () => { + afterEach(() => { + vi.clearAllMocks() + }) + + it("fails a parsing source with failure_stage storage_sync", async () => { + mocks.findInWorkspace.mockResolvedValue({ id: "source_1", status: "parsing" }) + + await parsedSyncRouteWorkflow.markSyncFailedAfterWorkflowFailure( + { ...basePayload, revisionKey: "rev_1" }, + "boom", + ) + + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { revisionKey: "rev_1", syncStatus: "failed", syncError: "boom" }, + ) + expect(mocks.markFailed).toHaveBeenCalledWith( + "workspace_1", + "source_1", + expect.stringContaining("storage sync failed"), + "parsing", + "storage_sync", + ) + }) + + it("does not fail an already-ready source, only records sync_status", async () => { + mocks.findInWorkspace.mockResolvedValue({ id: "source_1", status: "ready" }) + + await parsedSyncRouteWorkflow.markSyncFailedAfterWorkflowFailure( + basePayload, + "boom", + ) + + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { revisionKey: undefined, syncStatus: "failed", syncError: "boom" }, + ) + expect(mocks.markFailed).not.toHaveBeenCalled() + }) +}) diff --git a/src/domains/sources/parsed-sync-route-workflow.ts b/src/domains/sources/parsed-sync-route-workflow.ts new file mode 100644 index 0000000..6be3b23 --- /dev/null +++ b/src/domains/sources/parsed-sync-route-workflow.ts @@ -0,0 +1,322 @@ +import "server-only" + +import { Client, type WorkflowContext } from "@upstash/workflow" +import type { KnowledgeSyncParsedDocumentResponse } from "@ontos-ai/knowhere-sdk" + +import { makeKnowhereClientWithParsedStorage } from "@/integrations/knowhere" +import { logger } from "@/lib/logger" +import { + getParsedSyncWorkflowRunId, + type ParsedSyncPayload, +} from "./parsed-document-sync-scheduler" +import { parsedDocumentSyncCapacityGuard } from "./parsed-document-sync-capacity" +import { markSourceReadyAfterReconciliation } from "./source-reconcile-workflow" +import { sourceWorkflowRuntime } from "./workflow-runtime" + +type ParsedSyncWorkflowContext = Pick< + WorkflowContext, + "run" | "url" +> + +type NormalizedParsedSyncPayload = { + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly apiKey: string + readonly revisionKey?: string + readonly segmentIndex: number +} + +type ContinuationTriggerInput = { + readonly url: string + readonly payload: ParsedSyncPayload + readonly workflowRunId: string + readonly delaySeconds?: number +} + +type SyncLeaseReleaseReason = "completed" | "incomplete" | "failed" + +// Sync steps per workflow segment. Each `syncParsedDocument` call is bounded by +// the SDK limits (pages + deadline); this caps how many bounded steps we run in +// one serverless invocation before handing off to a fresh continuation. +const maxSyncStepsPerSegment = 4 + +let triggerContinuation: typeof triggerParsedSyncContinuation = + triggerParsedSyncContinuation + +function normalizeParsedSyncPayload( + payload: ParsedSyncPayload, +): NormalizedParsedSyncPayload { + return { + workspaceId: payload.workspaceId, + sourceId: payload.sourceId, + documentId: payload.documentId, + apiKey: payload.apiKey, + revisionKey: payload.revisionKey, + segmentIndex: + typeof payload.segmentIndex === "number" && + Number.isInteger(payload.segmentIndex) && + payload.segmentIndex >= 0 + ? payload.segmentIndex + : 0, + } +} + +async function runParsedSyncWorkflow(input: { + readonly context: ParsedSyncWorkflowContext + readonly payload: NormalizedParsedSyncPayload +}): Promise { + const { context, payload } = input + const { workspaceId, sourceId, documentId, apiKey } = payload + const { knowledge } = makeKnowhereClientWithParsedStorage(apiKey, { + workspaceId, + }) + + const preSyncReady = await context.run( + `source-ready-before-sync-${payload.segmentIndex}`, + async () => + markSourceReadyAfterReconciliation({ + workspaceId, + sourceId, + documentId, + }), + ) + if (preSyncReady.status === "gone") return + + let revisionKey = payload.revisionKey + let completed = false + let releaseReason: SyncLeaseReleaseReason = "incomplete" + + const capacity = await context.run( + `acquire-sync-capacity-${payload.segmentIndex}`, + async () => + parsedDocumentSyncCapacityGuard.acquire({ + workspaceId, + sourceId, + documentId, + revisionKey, + }), + ) + if (capacity.kind === "source-missing") return + if (capacity.kind === "capacity-full") { + await context.run( + `record-capacity-wait-${payload.segmentIndex}`, + async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "pending", + syncError: null, + }), + ) + const nextSegmentIndex = payload.segmentIndex + 1 + await context.run( + `trigger-sync-capacity-retry-${nextSegmentIndex}`, + async () => + triggerContinuation({ + url: context.url, + payload: { + workspaceId, + sourceId, + documentId, + apiKey, + revisionKey, + segmentIndex: nextSegmentIndex, + }, + workflowRunId: getParsedSyncWorkflowRunId({ + documentId, + revisionKey: revisionKey ?? "initial", + segmentIndex: nextSegmentIndex, + }), + delaySeconds: capacity.waitSeconds, + }), + ) + logger.info("parsed-sync: capacity full; retry scheduled", { + sourceId, + documentId, + reason: capacity.reason, + waitSeconds: capacity.waitSeconds, + activeCounts: capacity.activeCounts, + }) + return + } + + try { + for (let step = 0; step < maxSyncStepsPerSegment; step++) { + const result: KnowledgeSyncParsedDocumentResponse = await context.run( + `sync-${payload.segmentIndex}-${step}`, + async () => + knowledge.syncParsedDocument({ + documentId, + ...(revisionKey ? { revisionKey } : {}), + }), + ) + revisionKey = result.revisionKey + + await context.run(`record-progress-${payload.segmentIndex}-${step}`, () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: result.completed ? "completed" : "running", + }), + ) + + if (result.completed) { + completed = true + releaseReason = "completed" + break + } + } + + if (!completed) { + const nextSegmentIndex = payload.segmentIndex + 1 + await context.run( + `trigger-sync-continuation-${nextSegmentIndex}`, + async () => + triggerContinuation({ + url: context.url, + payload: { + workspaceId, + sourceId, + documentId, + apiKey, + revisionKey, + segmentIndex: nextSegmentIndex, + }, + workflowRunId: getParsedSyncWorkflowRunId({ + documentId, + revisionKey: revisionKey ?? "initial", + segmentIndex: nextSegmentIndex, + }), + }), + ) + logger.info("parsed-sync: continuation triggered", { + sourceId, + documentId, + segmentIndex: nextSegmentIndex, + }) + return + } + } catch (error) { + releaseReason = "failed" + throw error + } finally { + await context.run(`release-sync-capacity-${payload.segmentIndex}`, async () => + releaseCapacityLease({ + leaseToken: capacity.leaseToken, + releaseReason, + sourceId, + documentId, + }), + ) + } + + logger.info("parsed-sync: parsed document sync finished", { + sourceId, + documentId, + revisionKey, + status: preSyncReady.status, + }) +} + +async function releaseCapacityLease(input: { + readonly leaseToken: string + readonly releaseReason: SyncLeaseReleaseReason + readonly sourceId: string + readonly documentId: string +}): Promise { + try { + await parsedDocumentSyncCapacityGuard.release({ + leaseToken: input.leaseToken, + releaseReason: input.releaseReason, + }) + } catch (error) { + logger.error("parsed-sync: failed to release capacity lease", { + sourceId: input.sourceId, + documentId: input.documentId, + error: error instanceof Error ? error.message : String(error), + }) + } +} + +async function triggerParsedSyncContinuation( + input: ContinuationTriggerInput, +): Promise { + const token = process.env.QSTASH_TOKEN + if (!token) { + throw new Error("QSTASH_TOKEN is required to continue parsed document sync.") + } + + await new Client({ token }).trigger({ + url: input.url, + body: input.payload, + workflowRunId: input.workflowRunId, + retries: 3, + delay: input.delaySeconds, + }) +} + +async function markSyncFailedAfterWorkflowFailure( + payload: ParsedSyncPayload, + failResponse: string, +): Promise { + const normalized = normalizeParsedSyncPayload(payload) + const reason = getSafeFailureReason(failResponse) + + // Record the storage-sync failure. A source still `parsing` is failed with + // failure_stage=storage_sync so a retry resumes sync without reparsing; an + // already-ready source is left ready (it still serves via remote fallback), + // only its sync_status is marked failed for observability. + await sourceWorkflowRuntime.updateSyncStatus( + normalized.workspaceId, + normalized.sourceId, + { + revisionKey: normalized.revisionKey, + syncStatus: "failed", + syncError: reason, + }, + ) + + const source = await sourceWorkflowRuntime.findInWorkspace( + normalized.workspaceId, + normalized.sourceId, + ) + if (source?.status === "parsing") { + await sourceWorkflowRuntime.markFailed( + normalized.workspaceId, + normalized.sourceId, + `Parsed document storage sync failed: ${reason}`, + "parsing", + "storage_sync", + ) + } + + logger.error("parsed-sync: marked sync failed after workflow failure", { + workspaceId: normalized.workspaceId, + sourceId: normalized.sourceId, + documentId: normalized.documentId, + segmentIndex: normalized.segmentIndex, + sourceStatus: source?.status, + }) +} + +function getSafeFailureReason(value: string): string { + const normalized = value.replace(/\s+/g, " ").trim() + if (normalized.length === 0) return "retry attempts were exhausted." + return normalized.slice(0, 500) +} + +function setContinuationTriggerForTesting( + trigger: typeof triggerParsedSyncContinuation, +): () => void { + const previous = triggerContinuation + triggerContinuation = trigger + return () => { + triggerContinuation = previous + } +} + +export const parsedSyncRouteWorkflow = { + markSyncFailedAfterWorkflowFailure, + normalizeParsedSyncPayload, + runParsedSyncWorkflow, + setContinuationTriggerForTesting, +} diff --git a/src/domains/sources/reconcile.test.ts b/src/domains/sources/reconcile.test.ts index 4c90172..5be0cea 100644 --- a/src/domains/sources/reconcile.test.ts +++ b/src/domains/sources/reconcile.test.ts @@ -20,6 +20,7 @@ function makeSource(overrides: Partial): Source { sizeBytes: 1, status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: null, stagedBlobPathname: null, diff --git a/src/domains/sources/repository.ts b/src/domains/sources/repository.ts index c7c4b6f..3c6d01c 100644 --- a/src/domains/sources/repository.ts +++ b/src/domains/sources/repository.ts @@ -21,7 +21,9 @@ type SourceRepository = { readonly saveParseResultEffect: typeof sourceParseResultRepository.saveParseResultEffect readonly mergeParseAssetUrlsEffect: typeof sourceParseResultRepository.mergeParseAssetUrlsEffect readonly getParseResultProgressEffect: typeof sourceParseResultRepository.getParseResultProgressEffect + readonly getParseSnapshotMetadataEffect: typeof sourceParseResultRepository.getParseSnapshotMetadataEffect readonly getParseAssetUrlsEffect: typeof sourceParseResultRepository.getParseAssetUrlsEffect + readonly updateSyncStatusEffect: typeof sourceParseResultRepository.updateSyncStatusEffect } export const sourceRepository: SourceRepository = { @@ -45,5 +47,8 @@ export const sourceRepository: SourceRepository = { sourceParseResultRepository.mergeParseAssetUrlsEffect, getParseResultProgressEffect: sourceParseResultRepository.getParseResultProgressEffect, + getParseSnapshotMetadataEffect: + sourceParseResultRepository.getParseSnapshotMetadataEffect, getParseAssetUrlsEffect: sourceParseResultRepository.getParseAssetUrlsEffect, + updateSyncStatusEffect: sourceParseResultRepository.updateSyncStatusEffect, } diff --git a/src/domains/sources/retry.test.ts b/src/domains/sources/retry.test.ts index 86c03a1..19eb565 100644 --- a/src/domains/sources/retry.test.ts +++ b/src/domains/sources/retry.test.ts @@ -18,6 +18,7 @@ describe("retrySourceToKnowhereEffect", () => { const parsingSource = makeSource({ status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_retry", knowhereDocumentId: "doc_retry", }) @@ -123,6 +124,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 5, status: "failed", failureReason: "Knowhere upload failed.", + failureStage: null, knowhereJobId: null, knowhereDocumentId: null, stagedBlobPathname: null, diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index 733ccc1..7d97920 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -1,6 +1,8 @@ import { Effect } from "effect" import { demoView } from "@/domains/demo/view" +import { readAllSourceChunks, readSourceChunkPage } from "@/domains/chunks/read" +import { resolveChunkConnectionTargets } from "@/domains/chunks" import type { DemoChunkPage } from "@/integrations/knowhere-demo" import { logger } from "@/lib/logger" import { routeResult } from "@/lib/route-result" @@ -8,7 +10,10 @@ import { decodeRemoteSourceId, findRemoteLibraryDocumentBySourceId, } from "./remote-library" -import { getClientForWorkspace } from "./route-dependencies" +import { + getClientForWorkspace, + getKnowledgeForSource, +} from "./route-dependencies" import { sourceRowRepository } from "./source-row-repository" import type { JsonRouteResult, @@ -23,8 +28,6 @@ type RouteChunksDependencies = Pick< | "ensureApiKeyForWorkspace" | "ensureWorkspace" | "getCurrentUser" - | "loadChunkPageForSource" - | "loadChunksForSource" | "makeKnowhereClient" | "sourceService" > @@ -85,41 +88,42 @@ const loadSourceChunksEffect = ( return demoResult ?? sourceNotFound() } - const client = yield* Effect.tryPromise(() => - getClientForWorkspace(workspace.id, input.cookieHeader, deps), + // Reads never return zero chunks for a ready remote document; the SDK falls + // back to Knowhere when Blob storage is missing or stale. A source that is + // not yet ready has no published document to read. + if (source.status !== "ready" || !source.knowhereDocumentId) { + return sourceSnapshotProcessing(input) + } + + const apiKey = yield* Effect.tryPromise(() => + deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), ) + const knowledge = getKnowledgeForSource({ + apiKey, + workspaceId: workspace.id, + sourceId: source.id, + documentId: source.knowhereDocumentId, + revisionKey: source.knowhereJobId, + }) + const readableSource = { + documentId: source.knowhereDocumentId, + title: source.title, + revisionKey: source.knowhereJobId, + } + if (input.shouldLoadAll) { - const chunks = yield* deps.loadChunksForSource(source, client, { - workspaceId: workspace.id, - onRevisionKey: async (revisionKey) => { - await deps.sourceService.updateSourceRevisionKey( - workspace.id, - source.id, - revisionKey, - ) - }, - }) - return routeResult.ok({ chunks }) + const chunks = yield* Effect.tryPromise(() => + readAllSourceChunks({ knowledge, source: readableSource }), + ) + return routeResult.ok({ chunks: resolveChunkConnectionTargets(chunks) }) } - const assetUrlsByFilePath = yield* Effect.tryPromise(() => - deps.sourceService.getParseAssetUrls(workspace.id, source.id), - ) - const chunkPage = yield* deps.loadChunkPageForSource( - source, - client, - input.pageParams, - { - assetUrlsByFilePath, - workspaceId: workspace.id, - onRevisionKey: async (revisionKey) => { - await deps.sourceService.updateSourceRevisionKey( - workspace.id, - source.id, - revisionKey, - ) - }, - }, + const chunkPage = yield* Effect.tryPromise(() => + readSourceChunkPage({ + knowledge, + source: readableSource, + params: input.pageParams, + }), ) return routeResult.ok(chunkPage) }) @@ -137,6 +141,9 @@ const loadRemoteChunkPageEffect = ( const workspace = yield* Effect.tryPromise(() => deps.ensureWorkspace(user.id), ) + const apiKey = yield* Effect.tryPromise(() => + deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), + ) const client = yield* Effect.tryPromise(() => getClientForWorkspace(workspace.id, input.cookieHeader, deps), ) @@ -159,35 +166,34 @@ const loadRemoteChunkPageEffect = ( revisionKey: remoteDocument.revisionKey ?? null, }), ) + const documentId = source.knowhereDocumentId ?? remoteDocument.documentId + + const knowledge = getKnowledgeForSource({ + apiKey, + workspaceId: workspace.id, + sourceId: source.id, + documentId, + revisionKey: source.knowhereJobId ?? remoteDocument.revisionKey ?? null, + }) + const readableSource = { + documentId, + title: source.title, + revisionKey: source.knowhereJobId ?? remoteDocument.revisionKey ?? null, + } if (input.shouldLoadAll) { - const chunks = yield* deps.loadChunksForSource(source, client, { - workspaceId: workspace.id, - onRevisionKey: async (revisionKey) => { - await deps.sourceService.updateSourceRevisionKey( - workspace.id, - source.id, - revisionKey, - ) - }, - }) - return routeResult.ok({ chunks }) + const chunks = yield* Effect.tryPromise(() => + readAllSourceChunks({ knowledge, source: readableSource }), + ) + return routeResult.ok({ chunks: resolveChunkConnectionTargets(chunks) }) } - const chunkPage = yield* deps.loadChunkPageForSource( - source, - client, - input.pageParams, - { - workspaceId: workspace.id, - onRevisionKey: async (revisionKey) => { - await deps.sourceService.updateSourceRevisionKey( - workspace.id, - source.id, - revisionKey, - ) - }, - }, + const chunkPage = yield* Effect.tryPromise(() => + readSourceChunkPage({ + knowledge, + source: readableSource, + params: input.pageParams, + }), ) return routeResult.ok(chunkPage) }) @@ -294,4 +300,41 @@ function sourceNotFound(): JsonRouteResult<{ readonly message: string }> { return routeResult.error(404, "Source not found.") } +function sourceSnapshotProcessing( + input: LoadSourceChunksInput, +): JsonRouteResult<{ + readonly chunks: [] + readonly pagination?: { + readonly page: number + readonly pageSize: number + readonly total: 0 + readonly totalPages: 0 + } + readonly message: string +}> { + if (input.shouldLoadAll) { + return routeResult.ok( + { + chunks: [], + message: "Source is still being prepared.", + }, + 202, + ) + } + + return routeResult.ok( + { + chunks: [], + pagination: { + page: input.pageParams.page, + pageSize: input.pageParams.pageSize, + total: 0, + totalPages: 0, + }, + message: "Source is still being prepared.", + }, + 202, + ) +} + export { createRouteChunks } diff --git a/src/domains/sources/route-dependencies.ts b/src/domains/sources/route-dependencies.ts index 25313ce..a39635f 100644 --- a/src/domains/sources/route-dependencies.ts +++ b/src/domains/sources/route-dependencies.ts @@ -1,17 +1,18 @@ import "server-only" import { del } from "@vercel/blob" +import type { Knowledge } from "@ontos-ai/knowhere-sdk" -import { - loadChunkPageForSource, - loadChunksForSource, -} from "@/domains/chunks/server" import { ensureApiKeyForWorkspace } from "@/integrations/dashboard/api-key-service" -import { makeKnowhereClient as makeDefaultKnowhereClient } from "@/integrations/knowhere" +import { + makeKnowhereClient as makeDefaultKnowhereClient, + makeKnowhereClientWithParsedStorage, +} from "@/integrations/knowhere" import { knowhereDemoApi } from "@/integrations/knowhere-demo" import { getCurrentUser, requireUser } from "@/infrastructure/auth" import { workspaceService } from "@/domains/workspace/service" import { sourceViewOptionsBySourceId as getDefaultSourceViewOptionsBySourceId } from "./counts" +import { createParsedDocumentSyncScheduler } from "./parsed-document-sync-scheduler" import { reconcileSourcesForWorkspace as reconcileDefaultSourcesForWorkspace } from "./reconcile" import { sourceWorkflowRuntime } from "./workflow-runtime" import { sourceService as defaultSourceService } from "./service" @@ -32,8 +33,6 @@ const defaultDependencies: SourceRouteServiceDependencies = { sources, client as ReturnType, ), - loadChunkPageForSource, - loadChunksForSource, makeKnowhereClient: (apiKey: string) => makeDefaultKnowhereClient(apiKey) as SourceRouteKnowhereClient, listSourcesForWorkspace: sourceWorkflowRuntime.listForWorkspace, @@ -45,11 +44,9 @@ const defaultDependencies: SourceRouteServiceDependencies = { requireUser, sourceService: { findInWorkspace: defaultSourceService.findInWorkspace, - getParseAssetUrls: defaultSourceService.getParseAssetUrls, hideDemoSource: defaultSourceService.hideDemoSource, listHiddenDemoSourceIds: defaultSourceService.listHiddenDemoSourceIds, localizeRemoteDocument: defaultSourceService.localizeRemoteDocument, - updateSourceRevisionKey: defaultSourceService.updateSourceRevisionKey, softDelete: defaultSourceService.softDelete, upsertMaterializedDemoSource: defaultSourceService.upsertMaterializedDemoSource, @@ -88,4 +85,35 @@ async function getClientForWorkspace( return deps.makeKnowhereClient(apiKey) } -export { createSourceRouteDependencies, getClientForWorkspace } +/** + * Build a `Knowledge` configured with Vercel-Blob parsed storage plus a + * QStash-backed background-sync scheduler bound to this source's document. + * Chunk reads go through the returned `knowledge`; on a storage miss the SDK + * serves from Knowhere remote and the scheduler enqueues a durable backfill. + */ +function getKnowledgeForSource(input: { + readonly apiKey: string + readonly workspaceId: string + readonly sourceId: string + readonly documentId: string + readonly revisionKey?: string | null +}): Knowledge { + const scheduler = createParsedDocumentSyncScheduler({ + workspaceId: input.workspaceId, + sourceId: input.sourceId, + documentId: input.documentId, + apiKey: input.apiKey, + revisionKey: input.revisionKey ?? undefined, + }) + const { knowledge } = makeKnowhereClientWithParsedStorage(input.apiKey, { + workspaceId: input.workspaceId, + scheduler, + }) + return knowledge +} + +export { + createSourceRouteDependencies, + getClientForWorkspace, + getKnowledgeForSource, +} diff --git a/src/domains/sources/route-listing.ts b/src/domains/sources/route-listing.ts index 9476285..35c0c72 100644 --- a/src/domains/sources/route-listing.ts +++ b/src/domains/sources/route-listing.ts @@ -3,7 +3,7 @@ import { Effect } from "effect" import { demoView } from "@/domains/demo/view" import { getMaterializedDemoSourceViewOptionsBySourceId, - getWorkspaceSourcesNeedingKnowhereChunkCount, + getWorkspaceSourcesNeedingChunkCount, resolveWorkspaceDemoSources, } from "@/domains/demo/workspace-source-resolution" import { routeResult } from "@/lib/route-result" @@ -96,8 +96,8 @@ const listSourcesEffect = ( client, localSources: demoSourceResolution.workspaceSources, }) - const sourcesNeedingKnowhereChunkCount = - getWorkspaceSourcesNeedingKnowhereChunkCount(workspaceSources) + const sourcesNeedingChunkCount = + getWorkspaceSourcesNeedingChunkCount(workspaceSources) const materializedDemoSourceOptions = getMaterializedDemoSourceViewOptionsBySourceId(workspaceSources, catalog) yield* Effect.sync(() => @@ -111,7 +111,7 @@ const listSourcesEffect = ( }), ) const sourceOptions = yield* deps.getSourceViewOptionsBySourceId( - sourcesNeedingKnowhereChunkCount, + sourcesNeedingChunkCount, client, ) const hiddenDemoSourceIds = new Set( diff --git a/src/domains/sources/route-retry.test.ts b/src/domains/sources/route-retry.test.ts new file mode 100644 index 0000000..fc1316c --- /dev/null +++ b/src/domains/sources/route-retry.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it, vi } from "vitest" + +import { createRouteRetry } from "./route-retry" +import type { Source, Workspace } from "@/infrastructure/db/schema" + +const workspace: Workspace = { + id: "workspace_1", + userId: "user_1", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), +} + +function makeSource(overrides: Partial = {}): Source { + return { + id: "source_1", + workspaceId: "workspace_1", + title: "notes.pdf", + mimeType: "application/pdf", + sizeBytes: 5, + status: "failed", + failureReason: "boom", + failureStage: null, + knowhereJobId: "job_1", + knowhereDocumentId: "doc_1", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + ...overrides, + } +} + +function baseDeps() { + return { + ensureApiKeyForWorkspace: vi.fn(async () => "jwt_123"), + ensureWorkspace: vi.fn(async () => workspace), + makeKnowhereClient: vi.fn(), + requireUser: vi.fn(async () => ({ id: "user_1", email: null, name: null })), + sourceService: { + retrySourceToKnowhere: vi.fn(), + }, + } +} + +describe("createRouteRetry", () => { + it("resumes parsed sync for a storage_sync failure without reparsing", async () => { + const storageSyncFailed = makeSource({ failureStage: "storage_sync" }) + const resumedSource = makeSource({ status: "parsing", failureStage: null }) + const resumeParsedSync = vi.fn(async () => resumedSource) + const deps = baseDeps() + const retry = createRouteRetry({ + ...deps, + resumeParsedSync, + sourceService: { + findInWorkspace: vi.fn(async () => storageSyncFailed), + retrySourceToKnowhere: deps.sourceService.retrySourceToKnowhere, + }, + } as unknown as Parameters[0]) + + const result = await retry.retrySource({ + cookieHeader: "session=abc", + sourceId: "source_1", + }) + + expect(result.status).toBe(200) + expect(resumeParsedSync).toHaveBeenCalledWith({ + workspace, + source: storageSyncFailed, + apiKey: "jwt_123", + }) + expect(deps.sourceService.retrySourceToKnowhere).not.toHaveBeenCalled() + }) + + it("reparses a plain parse failure with a saved original blob", async () => { + const parseFailed = makeSource({ + failureStage: "parse", + originalBlobUrl: "https://blob.example/source-uploads/u/document.pdf", + originalBlobPathname: "source-uploads/u/document.pdf", + }) + const parsingSource = makeSource({ status: "parsing", failureStage: null }) + const resumeParsedSync = vi.fn() + const retrySourceToKnowhere = vi.fn(async () => parsingSource) + const deps = baseDeps() + const retry = createRouteRetry({ + ...deps, + resumeParsedSync, + makeKnowhereClient: vi.fn(() => ({})), + sourceService: { + findInWorkspace: vi.fn(async () => parseFailed), + retrySourceToKnowhere, + }, + } as unknown as Parameters[0]) + + const result = await retry.retrySource({ + cookieHeader: "session=abc", + sourceId: "source_1", + }) + + expect(result.status).toBe(200) + expect(resumeParsedSync).not.toHaveBeenCalled() + expect(retrySourceToKnowhere).toHaveBeenCalled() + }) +}) diff --git a/src/domains/sources/route-retry.ts b/src/domains/sources/route-retry.ts index 68464bb..094c91c 100644 --- a/src/domains/sources/route-retry.ts +++ b/src/domains/sources/route-retry.ts @@ -1,8 +1,12 @@ import { Effect } from "effect" +import { logger } from "@/lib/logger" import { routeResult } from "@/lib/route-result" import { startBackgroundReconciliation } from "./background-reconcile" +import { enqueueParsedDocumentSync } from "./parsed-document-sync-scheduler" +import { sourceWorkflowRuntime } from "./workflow-runtime" import { toSourceView } from "./view" +import type { Source, Workspace } from "@/infrastructure/db/schema" import type { JsonRouteResult, RetrySourceBody, @@ -17,7 +21,9 @@ type RouteRetryDependencies = Pick< | "makeKnowhereClient" | "requireUser" | "sourceService" -> +> & { + readonly resumeParsedSync?: typeof resumeParsedSync +} type RouteRetry = { readonly retrySource: ( @@ -51,6 +57,23 @@ const retrySourceEffect = ( if (source.status !== "failed") { return routeResult.error(409, "Only failed sources can be retried.") } + + // A storage-sync failure keeps the parsed Knowhere document intact — resume + // the parsed-document sync from the stored revision instead of reparsing. + if (source.failureStage === "storage_sync" && source.knowhereDocumentId) { + const apiKey = yield* Effect.tryPromise(() => + deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), + ) + const resumedSource = yield* Effect.tryPromise(() => + (deps.resumeParsedSync ?? resumeParsedSync)({ + workspace, + source, + apiKey, + }), + ) + return routeResult.ok({ source: toSourceView(resumedSource) }) + } + if (!source.originalBlobUrl || !source.originalBlobPathname) { return routeResult.error( 409, @@ -75,4 +98,50 @@ const retrySourceEffect = ( return routeResult.ok({ source: toSourceView(retriedSource) }) }) -export { createRouteRetry } +/** + * Resume a parsed-document storage sync that failed after a successful parse. + * Moves the source back to `parsing` (clearing the failure stage), resets the + * sync status, and re-enqueues the resumable parsed-sync workflow from the + * stored document + revision — no reparse. + */ +async function resumeParsedSync(input: { + readonly workspace: Workspace + readonly source: Source + readonly apiKey: string +}): Promise { + const { workspace, source, apiKey } = input + const documentId = source.knowhereDocumentId + if (!documentId) return source + + const revisionKey = source.knowhereJobId ?? undefined + const parsingSource = await sourceWorkflowRuntime.markParsing( + workspace.id, + source.id, + revisionKey ?? documentId, + documentId, + "failed", + ) + if (!parsingSource) return source + + await sourceWorkflowRuntime.updateSyncStatus(workspace.id, source.id, { + revisionKey, + syncStatus: "running", + syncError: null, + }) + await enqueueParsedDocumentSync({ + workspaceId: workspace.id, + sourceId: source.id, + documentId, + apiKey, + revisionKey, + }) + logger.info("sources: resumed parsed document storage sync on retry", { + workspaceId: workspace.id, + sourceId: source.id, + documentId, + revisionKey, + }) + return parsingSource +} + +export { createRouteRetry, resumeParsedSync } diff --git a/src/domains/sources/route-service.test.ts b/src/domains/sources/route-service.test.ts index 3d11b96..26e60cb 100644 --- a/src/domains/sources/route-service.test.ts +++ b/src/domains/sources/route-service.test.ts @@ -22,6 +22,7 @@ const source: Source = { sizeBytes: 5, status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: null, stagedBlobPathname: null, @@ -119,6 +120,51 @@ describe("source route service", () => { expect(listHiddenDemoSourceIds).toHaveBeenCalledWith(workspace.id); }); + it("returns processing without building a client when a source is not ready", async () => { + const parsingSource: Source = { + ...source, + id: "00000000-0000-4000-8000-000000000002", + knowhereDocumentId: "doc_legacy", + knowhereJobId: null, + status: "parsing", + }; + const makeKnowhereClient = vi.fn(); + const service = createSourceRouteService({ + ensureApiKeyForWorkspace: vi.fn(async () => "jwt_123"), + ensureWorkspace: vi.fn(async () => workspace), + getCurrentUser: vi.fn(async () => ({ + id: "user_1", + email: null, + name: null, + })), + makeKnowhereClient, + sourceService: { + findInWorkspace: vi.fn(async () => parsingSource), + }, + }); + + const result = await service.loadSourceChunks({ + cookieHeader: "session=abc", + sourceId: parsingSource.id, + shouldLoadAll: false, + pageParams: { page: 1, pageSize: 50 }, + }); + + expect(result).toEqual({ + status: 202, + body: { + chunks: [], + pagination: { + page: 1, + pageSize: 50, + total: 0, + totalPages: 0, + }, + message: "Source is still being prepared.", + }, + }); + }); + it("lists shared default and legacy namespace documents as lightweight remote sources", async () => { const localReadySource: Source = { ...source, @@ -801,6 +847,7 @@ describe("source route service", () => { ...failedSource, status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_retry", }; const knowhereClient = { diff --git a/src/domains/sources/route-types.ts b/src/domains/sources/route-types.ts index 7303ec3..96bd80c 100644 --- a/src/domains/sources/route-types.ts +++ b/src/domains/sources/route-types.ts @@ -3,10 +3,6 @@ import type { ChunkPage, ChunkPageParams, } from "@/domains/chunks" -import type { - loadChunkPageForSource, - loadChunksForSource, -} from "@/domains/chunks/server" import type { ParsedChunkView } from "@/domains/chunks/types" import type { SourceStatus, SourceView } from "@/domains/sources/types" import type { AuthUser } from "@/infrastructure/auth" @@ -179,10 +175,6 @@ type SourceWorkflowService = { workspaceId: string, sourceId: string, ) => Promise - readonly getParseAssetUrls: ( - workspaceId: string, - sourceId: string, - ) => Promise>> readonly hideDemoSource: ( workspaceId: string, demoSourceId: string, @@ -200,11 +192,6 @@ type SourceWorkflowService = { readonly revisionKey?: string | null }, ) => Promise - readonly updateSourceRevisionKey: ( - workspaceId: string, - sourceId: string, - revisionKey: string, - ) => Promise readonly upsertMaterializedDemoSource: ( workspaceId: string, input: { @@ -240,8 +227,6 @@ type SourceRouteServiceDependencies = { sources: readonly Source[], client: SourceRouteKnowhereClient, ) => ReturnType - readonly loadChunkPageForSource: typeof loadChunkPageForSource - readonly loadChunksForSource: typeof loadChunksForSource readonly makeKnowhereClient: (apiKey: string) => SourceRouteKnowhereClient readonly listSourcesForWorkspace: (workspaceId: string) => Promise readonly reconcileSourcesForWorkspace: ( diff --git a/src/domains/sources/service.ts b/src/domains/sources/service.ts index 0d7c7f6..4fcbfd5 100644 --- a/src/domains/sources/service.ts +++ b/src/domains/sources/service.ts @@ -17,20 +17,11 @@ type SourceService = { workspaceId: string, sourceId: string, ) => Promise - readonly getParseAssetUrls: ( - workspaceId: string, - sourceId: string, - ) => Promise>> readonly listForWorkspace: (workspaceId: string) => Promise readonly localizeRemoteDocument: ( workspaceId: string, input: Parameters[1], ) => Promise - readonly updateSourceRevisionKey: ( - workspaceId: string, - sourceId: string, - revisionKey: string, - ) => Promise readonly listHiddenDemoSourceIds: (workspaceId: string) => Promise readonly hideDemoSource: ( workspaceId: string, @@ -105,12 +96,10 @@ const retrySourceToKnowhere: SourceService["retrySourceToKnowhere"] = ( export const sourceService: SourceService = { findInWorkspace: sourceWorkflowRuntime.findInWorkspace, - getParseAssetUrls: sourceWorkflowRuntime.getParseAssetUrls, hideDemoSource: sourceWorkflowRuntime.hideDemoSource, listHiddenDemoSourceIds: sourceWorkflowRuntime.listHiddenDemoSourceIds, listForWorkspace: sourceWorkflowRuntime.listForWorkspace, localizeRemoteDocument: sourceWorkflowRuntime.localizeRemoteDocument, - updateSourceRevisionKey: sourceWorkflowRuntime.updateRevisionKey, softDelete: sourceWorkflowRuntime.softDelete, upsertMaterializedDemoSource: sourceWorkflowRuntime.upsertMaterializedDemoSource, diff --git a/src/domains/sources/source-parse-result-repository.ts b/src/domains/sources/source-parse-result-repository.ts index 8a5711e..6ddc1e2 100644 --- a/src/domains/sources/source-parse-result-repository.ts +++ b/src/domains/sources/source-parse-result-repository.ts @@ -12,14 +12,46 @@ import { sourceRowRepository } from "./source-row-repository" type SaveSourceParseResultInput = { readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string + readonly snapshotManifestKey?: string + readonly revisionKey?: string + readonly syncStatus?: SourceParseSyncStatus + readonly syncError?: string | null readonly assetUrlsByFilePath: Readonly> } +export type SourceParseSyncStatus = + | "pending" + | "running" + | "completed" + | "failed" + type SourceParseResultProgress = { readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null + readonly revisionKey?: string | null + readonly syncStatus?: string | null + readonly syncError?: string | null + readonly assetUrlsByFilePath: Readonly> +} + +export type SourceParseSnapshotMetadata = { + readonly resultBlobUrl: string + readonly snapshotManifestUrl?: string | null + readonly snapshotManifestKey?: string | null + readonly revisionKey?: string | null + readonly syncStatus?: string | null + readonly syncError?: string | null readonly assetUrlsByFilePath: Readonly> } +type UpdateSyncStatusInput = { + readonly revisionKey?: string + readonly syncStatus: SourceParseSyncStatus + readonly syncError?: string | null +} + type SourceParseResultRepository = { readonly saveParseResultEffect: ( workspaceId: string, @@ -35,10 +67,19 @@ type SourceParseResultRepository = { workspaceId: string, sourceId: string, ) => Effect.Effect + readonly getParseSnapshotMetadataEffect: ( + workspaceId: string, + sourceId: string, + ) => Effect.Effect readonly getParseAssetUrlsEffect: ( workspaceId: string, sourceId: string, ) => Effect.Effect>, never, DbClient> + readonly updateSyncStatusEffect: ( + workspaceId: string, + sourceId: string, + input: UpdateSyncStatusInput, + ) => Effect.Effect } export function buildAtomicAssetUrlsMergeSql( @@ -62,12 +103,22 @@ const saveParseResultEffect: SourceParseResultRepository["saveParseResultEffect" .values({ sourceId, resultBlobUrl: input.resultBlobUrl, + snapshotManifestUrl: input.snapshotManifestUrl, + snapshotManifestKey: input.snapshotManifestKey, + revisionKey: input.revisionKey, + syncStatus: input.syncStatus, + syncError: input.syncError, assetUrls: input.assetUrlsByFilePath, }) .onConflictDoUpdate({ target: sourceParseResults.sourceId, set: { resultBlobUrl: input.resultBlobUrl, + snapshotManifestUrl: input.snapshotManifestUrl, + snapshotManifestKey: input.snapshotManifestKey, + revisionKey: input.revisionKey, + syncStatus: input.syncStatus, + syncError: input.syncError, assetUrls: input.assetUrlsByFilePath, updatedAt: sql`now()`, }, @@ -93,12 +144,22 @@ const mergeParseAssetUrlsEffect: SourceParseResultRepository["mergeParseAssetUrl .values({ sourceId, resultBlobUrl: input.resultBlobUrl, + snapshotManifestUrl: input.snapshotManifestUrl, + snapshotManifestKey: input.snapshotManifestKey, + revisionKey: input.revisionKey, + syncStatus: input.syncStatus, + syncError: input.syncError, assetUrls: input.assetUrlsByFilePath, }) .onConflictDoUpdate({ target: sourceParseResults.sourceId, set: { resultBlobUrl: input.resultBlobUrl, + snapshotManifestUrl: input.snapshotManifestUrl, + snapshotManifestKey: input.snapshotManifestKey, + revisionKey: input.revisionKey, + syncStatus: input.syncStatus, + syncError: input.syncError, assetUrls: buildAtomicAssetUrlsMergeSql( input.assetUrlsByFilePath, ), @@ -124,6 +185,11 @@ const getParseResultProgressEffect: SourceParseResultRepository["getParseResultP db .select({ resultBlobUrl: sourceParseResults.resultBlobUrl, + snapshotManifestUrl: sourceParseResults.snapshotManifestUrl, + snapshotManifestKey: sourceParseResults.snapshotManifestKey, + revisionKey: sourceParseResults.revisionKey, + syncStatus: sourceParseResults.syncStatus, + syncError: sourceParseResults.syncError, assetUrls: sourceParseResults.assetUrls, }) .from(sourceParseResults) @@ -134,11 +200,54 @@ const getParseResultProgressEffect: SourceParseResultRepository["getParseResultP if (!progress) return null return { - resultBlobUrl: progress.resultBlobUrl, + resultBlobUrl: progress.resultBlobUrl ?? "", + snapshotManifestUrl: progress.snapshotManifestUrl, + snapshotManifestKey: progress.snapshotManifestKey, + revisionKey: progress.revisionKey, + syncStatus: progress.syncStatus, + syncError: progress.syncError, assetUrlsByFilePath: progress.assetUrls, } }) +const updateSyncStatusEffect: SourceParseResultRepository["updateSyncStatusEffect"] = + (workspaceId: string, sourceId: string, input: UpdateSyncStatusInput) => + Effect.gen(function* () { + const db = yield* DbClient + const source = yield* Effect.promise(() => + sourceRowRepository.findInWorkspaceWithDb(db, workspaceId, sourceId), + ) + if (!source) return null + + // Upsert: a read-miss backfill may set sync status before any parsed + // snapshot row exists, so insert a bare row when one is not present yet. + const [result] = yield* Effect.promise(() => + db + .insert(sourceParseResults) + .values({ + sourceId, + revisionKey: input.revisionKey, + syncStatus: input.syncStatus, + syncError: input.syncError ?? null, + assetUrls: {}, + }) + .onConflictDoUpdate({ + target: sourceParseResults.sourceId, + set: { + ...(input.revisionKey !== undefined + ? { revisionKey: input.revisionKey } + : {}), + syncStatus: input.syncStatus, + syncError: input.syncError ?? null, + updatedAt: sql`now()`, + }, + }) + .returning(), + ) + + return result ?? null + }) + const getParseAssetUrlsEffect: SourceParseResultRepository["getParseAssetUrlsEffect"] = (workspaceId: string, sourceId: string) => Effect.gen(function* () { @@ -159,9 +268,18 @@ const getParseAssetUrlsEffect: SourceParseResultRepository["getParseAssetUrlsEff return row[0]?.assetUrls ?? {} }) +const getParseSnapshotMetadataEffect: SourceParseResultRepository["getParseSnapshotMetadataEffect"] = + (workspaceId: string, sourceId: string) => + Effect.gen(function* () { + const progress = yield* getParseResultProgressEffect(workspaceId, sourceId) + return progress + }) + export const sourceParseResultRepository: SourceParseResultRepository = { saveParseResultEffect, mergeParseAssetUrlsEffect, getParseResultProgressEffect, + getParseSnapshotMetadataEffect, getParseAssetUrlsEffect, + updateSyncStatusEffect, } diff --git a/src/domains/sources/source-reconcile-route-workflow.test.ts b/src/domains/sources/source-reconcile-route-workflow.test.ts index 0841c54..ee415ff 100644 --- a/src/domains/sources/source-reconcile-route-workflow.test.ts +++ b/src/domains/sources/source-reconcile-route-workflow.test.ts @@ -1,11 +1,15 @@ -import { afterEach, describe, expect, it, vi } from "vitest" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ + acquireSyncCapacity: vi.fn(), + enqueueParsedDocumentSync: vi.fn(), + releaseSyncCapacity: vi.fn(), + updateSyncStatus: vi.fn(), + markFailed: vi.fn(), loggerError: vi.fn(), loggerInfo: vi.fn(), loggerWarn: vi.fn(), - makeKnowhereClient: vi.fn(), - markFailed: vi.fn(), + makeKnowhereClientWithParsedStorage: vi.fn(), markSourceReadyAfterReconciliation: vi.fn(), pollSourceReconciliation: vi.fn(), })) @@ -18,11 +22,24 @@ vi.mock("@/domains/sources/source-reconcile-workflow", () => ({ vi.mock("@/domains/sources/workflow-runtime", () => ({ sourceWorkflowRuntime: { markFailed: mocks.markFailed, + updateSyncStatus: mocks.updateSyncStatus, + }, +})) + +vi.mock("./parsed-document-sync-scheduler", () => ({ + enqueueParsedDocumentSync: mocks.enqueueParsedDocumentSync, +})) + +vi.mock("./parsed-document-sync-capacity", () => ({ + parsedDocumentSyncCapacityGuard: { + acquire: mocks.acquireSyncCapacity, + release: mocks.releaseSyncCapacity, }, })) vi.mock("@/integrations/knowhere", () => ({ - makeKnowhereClient: mocks.makeKnowhereClient, + makeKnowhereClientWithParsedStorage: + mocks.makeKnowhereClientWithParsedStorage, })) vi.mock("@/lib/logger", () => ({ @@ -35,7 +52,49 @@ vi.mock("@/lib/logger", () => ({ import { sourceReconcileRouteWorkflow } from "./source-reconcile-route-workflow" +const activeCounts = { + globalActive: 0, + workspaceActive: 0, + documentActive: 0, +} + +function createClient(overrides: { + syncParsedDocument?: ReturnType + jobResultId?: string + jobId?: string +}) { + const listChunks = vi.fn(async () => ({ + jobResultId: overrides.jobResultId ?? "rev_1", + jobId: overrides.jobId ?? "job_1", + })) + return { + client: { jobs: {}, documents: { listChunks } }, + knowledge: { + syncParsedDocument: + overrides.syncParsedDocument ?? + vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_1", + completed: true, + })), + }, + listChunks, + } +} + describe("sourceReconcileRouteWorkflow", () => { + beforeEach(() => { + mocks.acquireSyncCapacity.mockResolvedValue({ + kind: "acquired", + leaseToken: "lease_1", + activeCounts, + }) + mocks.releaseSyncCapacity.mockResolvedValue(undefined) + mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ + status: "ready", + }) + }) + afterEach(() => { vi.clearAllMocks() }) @@ -57,7 +116,7 @@ describe("sourceReconcileRouteWorkflow", () => { }) }) - it("marks the source ready when Knowhere publishes a document id", async () => { + it("marks the source ready then syncs the parsed document", async () => { const context = createWorkflowContext() const continuations: ContinuationTriggerInput[] = [] const restore = @@ -66,16 +125,16 @@ describe("sourceReconcileRouteWorkflow", () => { continuations.push(input) }, ) - const client = { jobs: {} } - mocks.makeKnowhereClient.mockReturnValue(client) + const wired = createClient({}) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: wired.client, + knowledge: wired.knowledge, + }) mocks.pollSourceReconciliation.mockResolvedValue({ kind: "ready-to-prepare", jobId: "job_1", documentId: "doc_1", }) - mocks.markSourceReadyAfterReconciliation.mockResolvedValue({ - status: "ready", - }) try { await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ @@ -90,14 +149,178 @@ describe("sourceReconcileRouteWorkflow", () => { restore() } + expect(wired.knowledge.syncParsedDocument).toHaveBeenCalledWith({ + documentId: "doc_1", + revisionKey: "rev_1", + }) + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { revisionKey: "rev_1", syncStatus: "completed" }, + ) expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ workspaceId: "workspace_1", sourceId: "source_1", documentId: "doc_1", }) + expect(mocks.acquireSyncCapacity).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + revisionKey: "rev_1", + }) + expect(mocks.releaseSyncCapacity).toHaveBeenCalledWith({ + leaseToken: "lease_1", + releaseReason: "completed", + }) + expect(mocks.enqueueParsedDocumentSync).not.toHaveBeenCalled() expect(continuations).toEqual([]) }) + it("hands off to parsed-sync when sync is incomplete", async () => { + const context = createWorkflowContext() + const syncParsedDocument = vi.fn(async () => ({ + documentId: "doc_1", + revisionKey: "rev_1", + completed: false, + })) + const wired = createClient({ syncParsedDocument }) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: wired.client, + knowledge: wired.knowledge, + }) + mocks.pollSourceReconciliation.mockResolvedValue({ + kind: "ready-to-prepare", + jobId: "job_1", + documentId: "doc_1", + }) + + await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ + context, + payload: sourceReconcileRouteWorkflow.normalizeReconcilePayload({ + workspaceId: "workspace_1", + sourceId: "source_1", + apiKey: "jwt_1", + }), + }) + + expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + }) + expect(mocks.enqueueParsedDocumentSync).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + apiKey: "jwt_1", + revisionKey: "rev_1", + }) + expect(mocks.releaseSyncCapacity).toHaveBeenCalledWith({ + leaseToken: "lease_1", + releaseReason: "incomplete", + }) + }) + + it("records sync failure when Blob sync throws", async () => { + const context = createWorkflowContext() + const syncParsedDocument = vi.fn(async () => { + throw new Error("blob write failed") + }) + const wired = createClient({ syncParsedDocument }) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: wired.client, + knowledge: wired.knowledge, + }) + mocks.pollSourceReconciliation.mockResolvedValue({ + kind: "ready-to-prepare", + jobId: "job_1", + documentId: "doc_1", + }) + + await expect( + sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ + context, + payload: sourceReconcileRouteWorkflow.normalizeReconcilePayload({ + workspaceId: "workspace_1", + sourceId: "source_1", + apiKey: "jwt_1", + }), + }), + ).rejects.toThrow("blob write failed") + + expect(mocks.updateSyncStatus).toHaveBeenCalledWith( + "workspace_1", + "source_1", + { + revisionKey: "rev_1", + syncStatus: "failed", + syncError: "blob write failed", + }, + ) + expect(mocks.markFailed).not.toHaveBeenCalled() + expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + }) + expect(mocks.releaseSyncCapacity).toHaveBeenCalledWith( + { + leaseToken: "lease_1", + releaseReason: "failed", + }, + ) + }) + + it("marks the source ready and schedules delayed sync when capacity is full", async () => { + const context = createWorkflowContext() + const wired = createClient({}) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: wired.client, + knowledge: wired.knowledge, + }) + mocks.acquireSyncCapacity.mockResolvedValue({ + kind: "capacity-full", + reason: "document", + waitSeconds: 60, + activeCounts: { + globalActive: 10, + workspaceActive: 1, + documentActive: 1, + }, + }) + mocks.pollSourceReconciliation.mockResolvedValue({ + kind: "ready-to-prepare", + jobId: "job_1", + documentId: "doc_1", + }) + + await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ + context, + payload: sourceReconcileRouteWorkflow.normalizeReconcilePayload({ + workspaceId: "workspace_1", + sourceId: "source_1", + apiKey: "jwt_1", + }), + }) + + expect(mocks.markSourceReadyAfterReconciliation).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + }) + expect(wired.knowledge.syncParsedDocument).not.toHaveBeenCalled() + expect(mocks.enqueueParsedDocumentSync).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceId: "source_1", + documentId: "doc_1", + apiKey: "jwt_1", + revisionKey: "rev_1", + delaySeconds: 60, + }) + expect(mocks.releaseSyncCapacity).not.toHaveBeenCalled() + }) + it("triggers a fresh poll run when Knowhere is still running after the segment budget", async () => { const context = createWorkflowContext() const continuations: ContinuationTriggerInput[] = [] @@ -107,7 +330,10 @@ describe("sourceReconcileRouteWorkflow", () => { continuations.push(input) }, ) - mocks.makeKnowhereClient.mockReturnValue({ jobs: {} }) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: { jobs: {}, documents: { listChunks: vi.fn() } }, + knowledge: { syncParsedDocument: vi.fn() }, + }) mocks.pollSourceReconciliation.mockResolvedValue({ kind: "waiting", jobId: "job_1", diff --git a/src/domains/sources/source-reconcile-route-workflow.ts b/src/domains/sources/source-reconcile-route-workflow.ts index a1e732a..1e45127 100644 --- a/src/domains/sources/source-reconcile-route-workflow.ts +++ b/src/domains/sources/source-reconcile-route-workflow.ts @@ -6,8 +6,10 @@ import { markSourceReadyAfterReconciliation, pollSourceReconciliation, } from "@/domains/sources/source-reconcile-workflow" -import { makeKnowhereClient } from "@/integrations/knowhere" +import { makeKnowhereClientWithParsedStorage } from "@/integrations/knowhere" import { logger } from "@/lib/logger" +import { enqueueParsedDocumentSync } from "./parsed-document-sync-scheduler" +import { parsedDocumentSyncCapacityGuard } from "./parsed-document-sync-capacity" import { sourceWorkflowRuntime } from "./workflow-runtime" type ReconcilePayload = { @@ -39,9 +41,12 @@ type ContinuationTriggerInput = { readonly workflowRunId: string } +type SyncLeaseReleaseReason = "completed" | "incomplete" | "failed" + const maxPollAttempts = 25 const initialDelaySeconds = 3 const maxDelaySeconds = 30 +const maxSyncStepsPerReconcile = 4 let triggerContinuation: typeof triggerReconcileContinuation = triggerReconcileContinuation @@ -52,7 +57,9 @@ async function runPollAndMirrorWorkflow(input: { }): Promise { const { context, payload } = input const { workspaceId, sourceId, apiKey } = payload - const client = makeKnowhereClient(apiKey) + const { client, knowledge } = makeKnowhereClientWithParsedStorage(apiKey, { + workspaceId, + }) let delay = initialDelaySeconds let completedJob: { readonly jobId: string @@ -118,6 +125,21 @@ async function runPollAndMirrorWorkflow(input: { return } + const revisionKey = await context.run("resolve-revision-key", async () => { + const firstPage = await client.documents.listChunks( + jobToPrepare.documentId, + { page: 1, pageSize: 1, includeAssetUrls: false }, + ) + return firstPage.jobResultId ?? firstPage.jobId ?? jobToPrepare.jobId + }) + await context.run("record-sync-pending", async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "pending", + syncError: null, + }), + ) + const ready = await context.run("source-ready", async () => markSourceReadyAfterReconciliation({ workspaceId, @@ -125,13 +147,144 @@ async function runPollAndMirrorWorkflow(input: { documentId: jobToPrepare.documentId, }), ) + if (ready.status === "gone") return + + const capacity = await context.run("acquire-sync-capacity", async () => + parsedDocumentSyncCapacityGuard.acquire({ + workspaceId, + sourceId, + documentId: jobToPrepare.documentId, + revisionKey, + }), + ) + if (capacity.kind === "source-missing") return + if (capacity.kind === "capacity-full") { + await context.run("enqueue-capacity-retry", async () => + enqueueParsedDocumentSync({ + workspaceId, + sourceId, + documentId: jobToPrepare.documentId, + apiKey, + revisionKey, + delaySeconds: capacity.waitSeconds, + }), + ) + logger.info("workflow: parsed storage sync delayed by capacity guard", { + sourceId, + documentId: jobToPrepare.documentId, + revisionKey, + reason: capacity.reason, + waitSeconds: capacity.waitSeconds, + activeCounts: capacity.activeCounts, + }) + return + } + + let syncCompleted = false + let releaseReason: SyncLeaseReleaseReason = "incomplete" + try { + await context.run("record-sync-running", async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "running", + syncError: null, + }), + ) + + for (let step = 0; step < maxSyncStepsPerReconcile; step++) { + const result = await context.run(`parsed-sync-${step}`, async () => { + try { + return await knowledge.syncParsedDocument({ + documentId: jobToPrepare.documentId, + revisionKey, + }) + } catch (error) { + await sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "failed", + syncError: getErrorMessage(error), + }) + throw error + } + }) + if (result.completed) { + syncCompleted = true + releaseReason = "completed" + break + } + } + + if (!syncCompleted) { + await context.run("record-sync-progress", async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "running", + }), + ) + await context.run("enqueue-parsed-sync-continuation", async () => + enqueueParsedDocumentSync({ + workspaceId, + sourceId, + documentId: jobToPrepare.documentId, + apiKey, + revisionKey, + }), + ) + logger.info("workflow: parsed storage sync handed off to parsed-sync", { + sourceId, + documentId: jobToPrepare.documentId, + revisionKey, + }) + return + } + } catch (error) { + releaseReason = "failed" + throw error + } finally { + await context.run("release-sync-capacity", async () => + releaseCapacityLease({ + leaseToken: capacity.leaseToken, + releaseReason, + sourceId, + documentId: jobToPrepare.documentId, + }), + ) + } + + await context.run("record-sync-completed", async () => + sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { + revisionKey, + syncStatus: "completed", + }), + ) logger.info("workflow: source parse reconciliation finished", { sourceId, jobId: jobToPrepare.jobId, + revisionKey, status: ready.status, }) } +async function releaseCapacityLease(input: { + readonly leaseToken: string + readonly releaseReason: SyncLeaseReleaseReason + readonly sourceId: string + readonly documentId: string +}): Promise { + try { + await parsedDocumentSyncCapacityGuard.release({ + leaseToken: input.leaseToken, + releaseReason: input.releaseReason, + }) + } catch (error) { + logger.error("workflow: failed to release sync capacity lease", { + sourceId: input.sourceId, + documentId: input.documentId, + error: getErrorMessage(error), + }) + } +} + function normalizeReconcilePayload( payload: ReconcilePayload, ): NormalizedReconcilePayload { @@ -208,6 +361,10 @@ function getSafeFailureReason(value: string): string { return normalized.slice(0, 500) } +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + export const sourceReconcileRouteWorkflow = { getPollWorkflowRunId, markSourceFailedAfterWorkflowFailure, diff --git a/src/domains/sources/source-reconcile-workflow.test.ts b/src/domains/sources/source-reconcile-workflow.test.ts index 3f0632e..5757da8 100644 --- a/src/domains/sources/source-reconcile-workflow.test.ts +++ b/src/domains/sources/source-reconcile-workflow.test.ts @@ -3,6 +3,7 @@ import type { JobResult } from "@ontos-ai/knowhere-sdk" import type { Source, Workspace } from "@/infrastructure/db/schema" import { + markSourceFailedAfterReconciliation, markSourceReadyAfterReconciliation, pollSourceReconciliation, } from "./source-reconcile-workflow" @@ -23,6 +24,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 1, status: "parsing", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: null, stagedBlobPathname: null, @@ -197,3 +199,34 @@ describe("markSourceReadyAfterReconciliation", () => { ) }) }) + +describe("markSourceFailedAfterReconciliation", () => { + it("marks failed and cleans staged uploads after fatal preparation errors", async () => { + const source = makeSource({ + stagedBlobPathname: "source-uploads/upload_1/document.pdf", + }) + const repository = createRepository(source) + const blobStore = { + deleteStagedSourceBlob: vi.fn(async () => undefined), + } + + const result = await markSourceFailedAfterReconciliation({ + workspaceId: workspace.id, + sourceId: "source_1", + reason: "Page citation asset preparation failed.", + repository, + blobStore, + }) + + expect(result).toEqual({ status: "failed" }) + expect(repository.markFailed).toHaveBeenCalledWith( + workspace.id, + "source_1", + "Page citation asset preparation failed.", + "parsing", + ) + expect(blobStore.deleteStagedSourceBlob).toHaveBeenCalledWith( + "source-uploads/upload_1/document.pdf", + ) + }) +}) diff --git a/src/domains/sources/source-reconcile-workflow.ts b/src/domains/sources/source-reconcile-workflow.ts index 0355350..4571038 100644 --- a/src/domains/sources/source-reconcile-workflow.ts +++ b/src/domains/sources/source-reconcile-workflow.ts @@ -55,6 +55,14 @@ export type MarkSourceReadyAfterReconciliationInput = { readonly blobStore?: SourceReconcileWorkflowBlobStore } +export type MarkSourceFailedAfterReconciliationInput = { + readonly workspaceId: string + readonly sourceId: string + readonly reason: string + readonly repository?: SourceReconcileWorkflowRepository + readonly blobStore?: SourceReconcileWorkflowBlobStore +} + export type PollSourceReconciliationResult = | { readonly kind: "waiting" @@ -75,6 +83,10 @@ export type MarkSourceReadyAfterReconciliationResult = { readonly status: string } +export type MarkSourceFailedAfterReconciliationResult = { + readonly status: string +} + export async function pollSourceReconciliation({ workspaceId, sourceId, @@ -165,6 +177,27 @@ export async function markSourceReadyAfterReconciliation({ return { status: readySource.status } } +export async function markSourceFailedAfterReconciliation({ + workspaceId, + sourceId, + reason, + repository = sourceWorkflowRuntime, + blobStore = vercelBlobStore, +}: MarkSourceFailedAfterReconciliationInput): Promise { + const source = await repository.findInWorkspace(workspaceId, sourceId) + if (!source) return { status: "gone" } + if (source.status !== "parsing") return { status: source.status } + + await failSourceAndCleanup({ + workspaceId, + source, + reason, + repository, + blobStore, + }) + return { status: "failed" } +} + async function failSourceAndCleanup(input: { readonly workspaceId: string readonly source: Source diff --git a/src/domains/sources/source-row-repository.ts b/src/domains/sources/source-row-repository.ts index 77c56dd..2c06765 100644 --- a/src/domains/sources/source-row-repository.ts +++ b/src/domains/sources/source-row-repository.ts @@ -26,6 +26,7 @@ type SourceUpdate = Partial< | "sizeBytes" | "status" | "failureReason" + | "failureStage" | "knowhereJobId" | "knowhereDocumentId" | "stagedBlobPathname" @@ -83,6 +84,7 @@ type SourceRowRepository = { sourceId: string, reason: string, requiredStatus?: string, + failureStage?: string, ) => Effect.Effect readonly clearStagedBlobEffect: ( workspaceId: string, @@ -193,6 +195,7 @@ const markParsingEffect: SourceRowRepository["markParsingEffect"] = ( knowhereJobId: jobId, knowhereDocumentId: documentId, failureReason: null, + failureStage: null, }, requiredStatus) const markReadyEffect: SourceRowRepository["markReadyEffect"] = ( @@ -204,6 +207,7 @@ const markReadyEffect: SourceRowRepository["markReadyEffect"] = ( status: "ready", knowhereDocumentId: documentId, failureReason: null, + failureStage: null, }, "parsing") const updateRevisionKeyEffect: SourceRowRepository["updateRevisionKeyEffect"] = ( @@ -220,10 +224,12 @@ const markFailedEffect: SourceRowRepository["markFailedEffect"] = ( sourceId: string, reason: string, requiredStatus?: string, + failureStage?: string, ) => updateInWorkspaceEffect(workspaceId, sourceId, { status: "failed", failureReason: reason, + failureStage: failureStage ?? null, }, requiredStatus) const clearStagedBlobEffect: SourceRowRepository["clearStagedBlobEffect"] = ( diff --git a/src/domains/sources/types.ts b/src/domains/sources/types.ts index e1abe9e..8479f87 100644 --- a/src/domains/sources/types.ts +++ b/src/domains/sources/types.ts @@ -48,7 +48,7 @@ export type SourceView = { readonly originalFile?: SourceOriginalFileView /** Official Library metadata when this row is an API-owned catalog item. */ readonly officialLibrary?: SourceOfficialLibraryView - /** Count from the Knowhere chunks API, not a local aggregate. */ + /** Count from the Notebook parsed snapshot manifest when available. */ readonly chunkCount?: number /** User opt-out for this query session. Drives excludeDocumentIds. */ readonly excludedFromQuery?: boolean diff --git a/src/domains/sources/upload.test.ts b/src/domains/sources/upload.test.ts index 912ab9d..77c9022 100644 --- a/src/domains/sources/upload.test.ts +++ b/src/domains/sources/upload.test.ts @@ -23,6 +23,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 12, status: "uploading", failureReason: null, + failureStage: null, knowhereJobId: null, knowhereDocumentId: null, stagedBlobPathname: null, diff --git a/src/domains/sources/view.test.ts b/src/domains/sources/view.test.ts index e854c29..cc5b223 100644 --- a/src/domains/sources/view.test.ts +++ b/src/domains/sources/view.test.ts @@ -12,6 +12,7 @@ function makeSource(overrides: Partial = {}): Source { sizeBytes: 1, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: "doc_1", stagedBlobPathname: null, diff --git a/src/domains/sources/workflow-runtime.test.ts b/src/domains/sources/workflow-runtime.test.ts index 3cc714a..08e9853 100644 --- a/src/domains/sources/workflow-runtime.test.ts +++ b/src/domains/sources/workflow-runtime.test.ts @@ -31,6 +31,7 @@ function makeSource(status: Source["status"]): Source { sizeBytes: 1024, status, failureReason: status === "failed" ? "failed" : null, + failureStage: null, knowhereJobId: status === "parsing" ? "job_1" : null, knowhereDocumentId: status === "ready" ? "document_1" : null, stagedBlobPathname: null, diff --git a/src/domains/sources/workflow-runtime.ts b/src/domains/sources/workflow-runtime.ts index bbec158..24464e0 100644 --- a/src/domains/sources/workflow-runtime.ts +++ b/src/domains/sources/workflow-runtime.ts @@ -2,6 +2,7 @@ import "server-only" import { databaseRuntime } from "@/domains/workspace/database-runtime" import { sourceRepository } from "./repository" +import type { SourceParseSnapshotMetadata } from "./source-parse-result-repository" import type { Source, SourceParseResult } from "@/infrastructure/db/schema" import type { UploadSourceRepository } from "./upload" @@ -17,6 +18,10 @@ type SaveSourceParseResultInput = Parameters< typeof sourceRepository.saveParseResultEffect >[2] +type UpdateSyncStatusInput = Parameters< + typeof sourceRepository.updateSyncStatusEffect +>[2] + type UpsertMaterializedDemoSourceInput = Parameters< typeof sourceRepository.upsertMaterializedDemoSourceEffect >[1] @@ -38,6 +43,7 @@ type UploadRepositoryRuntime = { sourceId: string, reason: string, requiredStatus?: string, + failureStage?: string, ) => Promise } @@ -60,10 +66,11 @@ type SourceWorkflowRuntime = UploadRepositoryRuntime & { readonly getParseResultProgress: ( workspaceId: string, sourceId: string, - ) => Promise<{ - readonly resultBlobUrl: string - readonly assetUrlsByFilePath: Readonly> - } | null> + ) => Promise + readonly getParseSnapshotMetadata: ( + workspaceId: string, + sourceId: string, + ) => Promise readonly listForWorkspace: (workspaceId: string) => Promise readonly localizeRemoteDocument: ( workspaceId: string, @@ -94,6 +101,11 @@ type SourceWorkflowRuntime = UploadRepositoryRuntime & { sourceId: string, input: SaveSourceParseResultInput, ) => Promise + readonly updateSyncStatus: ( + workspaceId: string, + sourceId: string, + input: UpdateSyncStatusInput, + ) => Promise readonly softDelete: ( workspaceId: string, sourceId: string, @@ -189,9 +201,16 @@ const markFailed: SourceWorkflowRuntime["markFailed"] = ( sourceId: string, reason: string, requiredStatus?: string, + failureStage?: string, ) => databaseRuntime.runPromise( - sourceRepository.markFailedEffect(workspaceId, sourceId, reason, requiredStatus), + sourceRepository.markFailedEffect( + workspaceId, + sourceId, + reason, + requiredStatus, + failureStage, + ), ) const clearStagedBlob: SourceWorkflowRuntime["clearStagedBlob"] = ( @@ -234,6 +253,15 @@ const mergeParseAssetUrls: SourceWorkflowRuntime["mergeParseAssetUrls"] = ( sourceRepository.mergeParseAssetUrlsEffect(workspaceId, sourceId, input), ) +const updateSyncStatus: SourceWorkflowRuntime["updateSyncStatus"] = ( + workspaceId: string, + sourceId: string, + input: UpdateSyncStatusInput, +) => + databaseRuntime.runPromise( + sourceRepository.updateSyncStatusEffect(workspaceId, sourceId, input), + ) + const getParseResultProgress: SourceWorkflowRuntime["getParseResultProgress"] = (workspaceId: string, sourceId: string) => databaseRuntime.runPromise( @@ -248,6 +276,12 @@ const getParseAssetUrls: SourceWorkflowRuntime["getParseAssetUrls"] = ( sourceRepository.getParseAssetUrlsEffect(workspaceId, sourceId), ) +const getParseSnapshotMetadata: SourceWorkflowRuntime["getParseSnapshotMetadata"] = + (workspaceId: string, sourceId: string) => + databaseRuntime.runPromise( + sourceRepository.getParseSnapshotMetadataEffect(workspaceId, sourceId), + ) + function createUploadRepository( runtime: UploadRepositoryRuntime = sourceWorkflowRuntime, ): UploadSourceRepository { @@ -287,6 +321,7 @@ export const sourceWorkflowRuntime: SourceWorkflowRuntime = { findInWorkspace, getParseAssetUrls, getParseResultProgress, + getParseSnapshotMetadata, hideDemoSource, listForWorkspace, listHiddenDemoSourceIds, @@ -298,5 +333,6 @@ export const sourceWorkflowRuntime: SourceWorkflowRuntime = { mergeParseAssetUrls, saveParseResult, softDelete, + updateSyncStatus, upsertMaterializedDemoSource, } diff --git a/src/domains/workspace/client.test.ts b/src/domains/workspace/client.test.ts index 42e8c65..bc19bba 100644 --- a/src/domains/workspace/client.test.ts +++ b/src/domains/workspace/client.test.ts @@ -63,6 +63,33 @@ describe("workspaceClient", () => { }) }) + it("preserves source chunk processing messages", async () => { + mockRouteClient.getJson.mockResolvedValue({ + chunks: [], + message: "Source parsed snapshot is still being prepared.", + pagination: { + page: 1, + pageSize: 50, + total: 0, + totalPages: 0, + }, + }) + + const page = await workspaceClient.fetchChunkPage("source_1", 1) + + expect(page).toEqual({ + chunks: [], + isProcessing: true, + message: "Source parsed snapshot is still being prepared.", + pagination: { + page: 1, + pageSize: 50, + total: 0, + totalPages: 0, + }, + }) + }) + it("throws materialization route errors instead of treating them as empty sources", async () => { mockRouteClient.postJsonWithStatus.mockResolvedValue({ status: 502, diff --git a/src/domains/workspace/client.ts b/src/domains/workspace/client.ts index 9a7b3cd..a650b6f 100644 --- a/src/domains/workspace/client.ts +++ b/src/domains/workspace/client.ts @@ -24,6 +24,8 @@ const workspaceClientConfig = { type SourceChunksResponse = { chunks?: ParsedChunkView[] + isProcessing?: boolean + message?: string pagination?: { page: number pageSize: number @@ -128,6 +130,8 @@ async function fetchChunkPage( return { chunks: Array.isArray(body.chunks) ? body.chunks : [], + ...(typeof body.message === "string" ? { message: body.message } : {}), + ...(typeof body.message === "string" ? { isProcessing: true } : {}), pagination: body.pagination, } } diff --git a/src/domains/workspace/initial-state.test.ts b/src/domains/workspace/initial-state.test.ts index 9ea24e3..23bd50f 100644 --- a/src/domains/workspace/initial-state.test.ts +++ b/src/domains/workspace/initial-state.test.ts @@ -647,6 +647,7 @@ function makeSource( sizeBytes: 1024, status: "ready", failureReason: null, + failureStage: null, knowhereJobId: "job_1", knowhereDocumentId: "document_1", stagedBlobPathname: null, diff --git a/src/domains/workspace/initial-state.ts b/src/domains/workspace/initial-state.ts index 368d43b..851a34b 100644 --- a/src/domains/workspace/initial-state.ts +++ b/src/domains/workspace/initial-state.ts @@ -7,7 +7,7 @@ import type { ParsedChunkView } from "@/domains/chunks/types" import { demoView } from "@/domains/demo/view" import { getMaterializedDemoSourceViewOptionsBySourceId, - getWorkspaceSourcesNeedingKnowhereChunkCount, + getWorkspaceSourcesNeedingChunkCount, resolveWorkspaceDemoSources, } from "@/domains/demo/workspace-source-resolution" import { chatThreadService } from "@/domains/chat/thread-service" @@ -331,8 +331,8 @@ export const loadWorkspaceShellInitialStateEffect = ( localSources: demoSourceResolution.workspaceSources, }), ) - const sourcesNeedingKnowhereChunkCount = - getWorkspaceSourcesNeedingKnowhereChunkCount(workspaceSources) + const sourcesNeedingChunkCount = + getWorkspaceSourcesNeedingChunkCount(workspaceSources) const materializedDemoSourceOptions = getMaterializedDemoSourceViewOptionsBySourceId( workspaceSources, @@ -354,7 +354,7 @@ export const loadWorkspaceShellInitialStateEffect = ( operation: "sourceViewOptionsBySourceId", }, deps.sourceViewOptionsBySourceId( - sourcesNeedingKnowhereChunkCount, + sourcesNeedingChunkCount, client, ), ) diff --git a/src/domains/workspace/integration.test.ts b/src/domains/workspace/integration.test.ts index 797cdbf..8b6d80b 100644 --- a/src/domains/workspace/integration.test.ts +++ b/src/domains/workspace/integration.test.ts @@ -145,7 +145,7 @@ describeIfDb("workspace helpers — integration", () => { markSourceReady: sourceWorkflowRuntime.markReady, markSourceFailed: sourceWorkflowRuntime.markFailed, saveSourceParseResult: sourceWorkflowRuntime.saveParseResult, - getParseAssetUrls: sourceService.getParseAssetUrls, + getParseAssetUrls: sourceWorkflowRuntime.getParseAssetUrls, hideDemoSource: sourceService.hideDemoSource, listHiddenDemoSourceIds: sourceService.listHiddenDemoSourceIds, upsertMaterializedDemoSource: @@ -331,6 +331,7 @@ describeIfDb("workspace helpers — integration", () => { chunkType: "text", score: 0.91, assetUrl: "https://assets.example/doc.pdf", + pageCitationAssetUrl: "https://assets.example/page-4.png", description: "intro summary", source: { documentId: "doc_123", @@ -353,6 +354,7 @@ describeIfDb("workspace helpers — integration", () => { chunkType: "text", score: 0.91, assetUrl: "https://assets.example/doc.pdf", + pageCitationAssetUrl: "https://assets.example/page-4.png", description: "intro summary", source: { documentId: "doc_123", diff --git a/src/domains/workspace/persistence.test.ts b/src/domains/workspace/persistence.test.ts index 47534d1..1c9be69 100644 --- a/src/domains/workspace/persistence.test.ts +++ b/src/domains/workspace/persistence.test.ts @@ -207,6 +207,7 @@ describe("chatRepository", () => { chunkType: "text", score: 0.99, assetUrl: "https://assets.example/doc.pdf", + pageCitationAssetUrl: "https://assets.example/page-4.png", description: "intro", source: { documentId: "doc_1", @@ -225,6 +226,7 @@ describe("chatRepository", () => { chunkType: "text", score: 0.99, assetUrl: "https://assets.example/doc.pdf", + pageCitationAssetUrl: "https://assets.example/page-4.png", description: "intro", source: { documentId: "doc_1", diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index 720a303..4e9470f 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -17,8 +17,8 @@ import { * - Postgres stores only metadata, status, Knowhere IDs, and chat * threads/messages. * - It does NOT store file bytes or chunk copies in Postgres. Original - * uploads and parsed media artifacts live in Blob storage; chunks are - * fetched on demand from Knowhere's chunks API. + * uploads and parsed-source snapshots live in Blob storage; retrieval + * stays upstream in Knowhere. * * Soft delete: * - Every user-visible resource has a nullable `deleted_at` timestamp. @@ -66,10 +66,12 @@ export type NewWorkspace = typeof workspaces.$inferInsert; * - `size_bytes` — original upload size (for display + quota) * - `status` — lifecycle: uploading | parsing | ready | failed * - `failure_reason` — human-readable error text when status=failed + * - `failure_stage` — which stage failed: parse | storage_sync; drives + * whether a retry reparses or only resumes storage sync * - `knowhere_job_id` — set once the parse job is created - * - `knowhere_document_id` — set when parsing completes; the sole handle - * used to fetch chunks and to exclude a source - * from a retrieval query + * - `knowhere_document_id` — set when parsing completes; used to import + * parsed snapshots and to exclude a source from a + * retrieval query * - `original_blob_*` — public Blob pointer for the original upload preview * and download path * - `staged_blob_*` — legacy temporary Blob staging pointer retained for @@ -95,6 +97,7 @@ export const sources = pgTable( sizeBytes: bigint("size_bytes", { mode: "number" }).notNull(), status: text("status").notNull(), failureReason: text("failure_reason"), + failureStage: text("failure_stage"), knowhereJobId: text("knowhere_job_id"), knowhereDocumentId: text("knowhere_document_id"), stagedBlobPathname: text("staged_blob_pathname"), @@ -161,10 +164,19 @@ export type NewDemoSourceVisibility = typeof demoSourceVisibilities.$inferInsert /** * Notebook-owned parse-result artifact index for one source. * - * Knowhere's chunk list currently may omit media asset URLs, while parsed chunk - * metadata still points to ZIP-relative files like `images/image-1.jpg`. - * This table stores the Notebook Blob copy of the result ZIP plus a - * file-path-to-public-URL map for those extracted parsed artifacts. + * Blob is the Notebook-owned read model for parsed chunks after source + * reconciliation completes. The parsed snapshot itself (manifest, chunk pages, + * assets, and resumable sync progress) lives in Vercel Blob under + * `workspaces/{ws}/parsed-documents/{documentId}/{revisionKey}/...`, managed by + * the SDK `ParsedDocumentStorage`. This row records: + * - `revision_key` — current parsed revision (jobResultId ?? jobId); the + * storage fast-path key passed into SDK reads + * - `sync_status` — pending | running | completed | failed for the + * background/parse-time storage sync + * - `sync_error` — last storage-sync error detail when sync_status=failed + * - `result_blob_url` / `snapshot_manifest_*` — legacy columns retained for + * rows written by the pre-migration manifest format + * - `asset_urls` — legacy file-path-to-public-URL map for older rows */ export const sourceParseResults = pgTable( "source_parse_results", @@ -174,7 +186,12 @@ export const sourceParseResults = pgTable( .notNull() .references(() => sources.id, { onDelete: "cascade" }) .unique(), - resultBlobUrl: text("result_blob_url").notNull(), + resultBlobUrl: text("result_blob_url"), + snapshotManifestUrl: text("snapshot_manifest_url"), + snapshotManifestKey: text("snapshot_manifest_key"), + revisionKey: text("revision_key"), + syncStatus: text("sync_status"), + syncError: text("sync_error"), assetUrls: jsonb("asset_urls") .$type>>() .notNull(), @@ -191,6 +208,61 @@ export const sourceParseResults = pgTable( export type SourceParseResult = typeof sourceParseResults.$inferSelect; export type NewSourceParseResult = typeof sourceParseResults.$inferInsert; +/** + * Durable active-work leases for Notebook-owned parsed-document Blob sync. + * + * The sync workers acquire one row before calling the Knowhere SDK's + * Vercel-Blob mirror. Active rows (`released_at IS NULL`) are counted globally, + * per workspace, and per document so Vercel can scale route invocations without + * allowing one user or one document to consume all sync capacity. Expired rows + * are released during the next acquire attempt; normal workers release in a + * `finally` block after their bounded sync segment exits. + */ +export const parsedDocumentSyncLeases = pgTable( + "parsed_document_sync_leases", + { + id: uuid("id").primaryKey().defaultRandom(), + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + sourceId: uuid("source_id") + .notNull() + .references(() => sources.id, { onDelete: "cascade" }), + documentId: text("document_id").notNull(), + revisionKey: text("revision_key"), + leaseToken: text("lease_token").notNull(), + acquiredAt: timestamp("acquired_at", { withTimezone: true }) + .notNull() + .defaultNow(), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + releasedAt: timestamp("released_at", { withTimezone: true }), + releaseReason: text("release_reason"), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + uniqueIndex("parsed_document_sync_leases_token_idx").on(t.leaseToken), + index("parsed_document_sync_leases_active_idx") + .on(t.expiresAt) + .where(sql`released_at IS NULL`), + index("parsed_document_sync_leases_workspace_active_idx") + .on(t.workspaceId) + .where(sql`released_at IS NULL`), + index("parsed_document_sync_leases_document_active_idx") + .on(t.documentId) + .where(sql`released_at IS NULL`), + ], +); + +export type ParsedDocumentSyncLease = + typeof parsedDocumentSyncLeases.$inferSelect; +export type NewParsedDocumentSyncLease = + typeof parsedDocumentSyncLeases.$inferInsert; + /** * A chat thread is a conversation within a workspace. `demo_key` is retained * for legacy seeded demo conversations. diff --git a/src/integrations/knowhere.ts b/src/integrations/knowhere.ts index 81f6726..c0d074d 100644 --- a/src/integrations/knowhere.ts +++ b/src/integrations/knowhere.ts @@ -1,5 +1,35 @@ import Knowhere from "@ontos-ai/knowhere-sdk" +import type { + Knowledge, + ParsedDocumentStorageLimits, + ParsedDocumentSyncScheduler, +} from "@ontos-ai/knowhere-sdk" import { logger } from "@/lib/logger" +import { BlobParsedDocumentStorage } from "@/domains/sources/parsed-document-blob-storage" + +/** + * Vercel-safe defaults for parsed-document reads and background sync. Each read + * miss and each background sync step is bounded by page count and a deadline so + * a single serverless invocation stays well under the platform ceiling; the + * SDK returns `completed:false` and the caller re-enqueues to continue. + */ +const defaultParsedStorageLimits: ParsedDocumentStorageLimits = { + chunkPageSize: 200, + remotePageSize: 100, + maxPagesPerSync: 10, + maxAssetsPerSync: 20, + syncDeadlineMs: 8000, + grepMaxPages: 50, + grepDeadlineMs: 8000, + outlineMaxPages: 50, + outlineDeadlineMs: 8000, +} + +type ParsedStorageOptions = { + readonly workspaceId: string + readonly scheduler?: ParsedDocumentSyncScheduler + readonly limits?: ParsedDocumentStorageLimits +} /** * Create a Knowhere client with the given API key. @@ -14,6 +44,29 @@ export function makeKnowhereClient(apiKey: string): Knowhere { return wrapKnowhereClient(client) } +/** + * Create a Knowhere client plus a `Knowledge` configured with a Vercel-Blob + * `ParsedDocumentStorage`. Reads through `knowledge` serve from Blob first and + * fall back to Knowhere remote transparently; the `scheduler` (when provided) + * backfills Blob in the background. Use `client` for retrieval/documents/jobs. + * + * `withParsedStorage` is invoked through the logging Proxy so `this` binds to + * the real Knowledge; the returned Knowledge wraps the unwrapped inner client + * (its internal `documents.listChunks` calls are not logged, which is fine). + */ +export function makeKnowhereClientWithParsedStorage( + apiKey: string, + options: ParsedStorageOptions, +): { readonly client: Knowhere; readonly knowledge: Knowledge } { + const client = makeKnowhereClient(apiKey) + const knowledge = client.knowledge.withParsedStorage({ + storage: new BlobParsedDocumentStorage({ workspaceId: options.workspaceId }), + scheduler: options.scheduler, + limits: options.limits ?? defaultParsedStorageLimits, + }) + return { client, knowledge } +} + function wrapKnowhereClient(client: Knowhere): Knowhere { return new Proxy(client, { get(target, prop, receiver) {