From fd2854be50b1225477f80ecdc79c8f3e05dc9a3a Mon Sep 17 00:00:00 2001 From: Oleg Isonen Date: Sat, 18 Jul 2026 23:35:28 +0100 Subject: [PATCH 01/14] feat: add queryable Assets resources --- .../asset-query-form-utils.test.ts | 97 ++ .../settings-panel/asset-query-form-utils.ts | 78 ++ .../settings-panel/asset-query-form.tsx | 550 +++++++++++ .../settings-panel/resource-panel.tsx | 57 +- .../app/routes/rest.resources-loader.ts | 25 +- .../app/services/api-router.server.test.ts | 122 +++ .../builder/app/services/api-router.server.ts | 163 +++- .../assets-field-catalog.server.test.ts | 67 ++ .../$resources/assets-field-catalog.server.ts | 51 + .../assets-index-status.server.test.ts | 73 ++ .../$resources/assets-index-status.server.ts | 88 ++ .../$resources/assets-query.server.test.ts | 166 ++++ .../shared/$resources/assets-query.server.ts | 130 +++ .../shared/$resources/assets.server.test.ts | 49 + apps/builder/app/shared/asset-client.ts | 23 + .../shared/asset-resource-api.client.test.ts | 54 ++ .../app/shared/asset-resource-api.client.ts | 76 ++ .../app/shared/code-editor-groq.test.ts | 59 ++ apps/builder/app/shared/code-editor.tsx | 55 +- apps/builder/app/shared/db/canvas.server.ts | 56 +- .../app/shared/groq-completion.test.ts | 220 +++++ apps/builder/app/shared/groq-completion.ts | 132 +++ .../builder/app/shared/resource-utils.test.ts | 28 + .../sync/patch/patch-service.server.test.ts | 112 +++ .../shared/sync/patch/patch-service.server.ts | 92 +- .../docs/asset-resource-architecture.md | 694 ++++++++++++++ apps/builder/package.json | 2 + packages/asset-resource/package.json | 44 + .../asset-resource/scripts/benchmark-groq.mjs | 104 +++ .../scripts/benchmark-index.mjs | 88 ++ .../scripts/benchmark-scale.mjs | 260 ++++++ packages/asset-resource/src/blog-e2e.test.ts | 125 +++ .../src/candidate-selection.test.ts | 105 +++ .../asset-resource/src/candidate-selection.ts | 196 ++++ packages/asset-resource/src/canonical.test.ts | 172 ++++ packages/asset-resource/src/canonical.ts | 182 ++++ .../asset-resource/src/field-catalog.test.ts | 356 +++++++ packages/asset-resource/src/field-catalog.ts | 276 ++++++ packages/asset-resource/src/hydration.test.ts | 258 ++++++ packages/asset-resource/src/hydration.ts | 320 +++++++ .../asset-resource/src/index-storage.test.ts | 87 ++ packages/asset-resource/src/index-storage.ts | 76 ++ packages/asset-resource/src/index.test.ts | 265 ++++++ packages/asset-resource/src/index.ts | 175 ++++ packages/asset-resource/src/markdown.test.ts | 198 ++++ packages/asset-resource/src/markdown.ts | 521 +++++++++++ .../src/published-runtime.test.ts | 178 ++++ .../asset-resource/src/published-runtime.ts | 270 ++++++ .../src/query-validation.test.ts | 100 ++ .../asset-resource/src/query-validation.ts | 185 ++++ .../asset-resource/src/resource-index.test.ts | 270 ++++++ packages/asset-resource/src/resource-index.ts | 152 +++ .../asset-resource/src/scale-fixture.test.ts | 22 + packages/asset-resource/src/scale-fixture.ts | 39 + packages/asset-resource/src/sha256.ts | 9 + packages/asset-resource/src/stable-json.ts | 44 + .../src/transport-parity.test.ts | 106 +++ .../asset-resource/src/worker-assets.test.ts | 84 ++ packages/asset-resource/tsconfig.dts.json | 9 + packages/asset-resource/tsconfig.json | 3 + packages/asset-uploader/package.json | 2 + .../asset-uploader/src/asset-patch-core.ts | 27 +- .../src/builder-api-data-boundary.test.ts | 63 ++ .../src/canonical-metadata-backfill.test.ts | 787 ++++++++++++++++ .../src/canonical-metadata-backfill.ts | 551 +++++++++++ .../canonical-metadata-persistence.test.ts | 177 ++++ .../src/canonical-metadata-persistence.ts | 178 ++++ packages/asset-uploader/src/client.ts | 28 +- packages/asset-uploader/src/clients/fs/fs.ts | 7 + .../src/clients/fs/immutable-object.test.ts | 40 + .../src/clients/fs/immutable-object.ts | 33 + .../src/clients/fs/read.test.ts | 58 ++ .../asset-uploader/src/clients/fs/read.ts | 54 ++ .../src/clients/s3/immutable-object.test.ts | 126 +++ .../src/clients/s3/immutable-object.ts | 111 +++ .../src/clients/s3/index-audit.ts | 55 ++ .../src/clients/s3/read.test.ts | 68 ++ .../asset-uploader/src/clients/s3/read.ts | 61 ++ packages/asset-uploader/src/clients/s3/s3.ts | 75 ++ .../asset-uploader/src/field-catalog.test.ts | 78 ++ packages/asset-uploader/src/field-catalog.ts | 35 + packages/asset-uploader/src/index.server.ts | 19 + packages/asset-uploader/src/patch.test.ts | 70 +- packages/asset-uploader/src/patch.ts | 6 +- .../asset-uploader/src/query-preview.test.ts | 208 +++++ packages/asset-uploader/src/query-preview.ts | 70 ++ .../src/resource-index-build.test.ts | 184 ++++ .../src/resource-index-build.ts | 125 +++ .../resource-index-garbage-collection.test.ts | 91 ++ .../src/resource-index-garbage-collection.ts | 77 ++ .../src/resource-index-lifecycle.test.ts | 108 +++ .../src/resource-index-lifecycle.ts | 89 ++ .../src/resource-index-maintenance.test.ts | 100 ++ .../src/resource-index-maintenance.ts | 61 ++ .../src/resource-index-observability.test.ts | 40 + .../src/resource-index-observability.ts | 74 ++ .../src/resource-index-orphan-audit.test.ts | 65 ++ .../src/resource-index-orphan-audit.ts | 74 ++ .../src/resource-index-persistence.ts | 171 ++++ .../src/resource-index-publication.test.ts | 61 ++ .../src/resource-index-publication.ts | 84 ++ .../src/resource-index-rebuild.test.ts | 79 ++ .../src/resource-index-rebuild.ts | 44 + .../src/resource-index-scheduler.test.ts | 55 ++ .../src/resource-index-scheduler.ts | 91 ++ .../src/resource-index-snapshot.test.ts | 95 ++ .../src/resource-index-snapshot.ts | 160 ++++ ...esource-index-status-authorization.test.ts | 35 + .../src/resource-index-status.test.ts | 44 + .../src/resource-index-status.ts | 78 ++ packages/asset-uploader/src/upload.test.ts | 130 ++- packages/asset-uploader/src/upload.ts | 20 +- packages/cli/package.json | 1 + packages/cli/src/docs/api-use-cases.md | 25 + packages/cli/src/docs/manual-llm.md | 4 + packages/cli/src/prebuild.test.ts | 160 +++- packages/cli/src/prebuild.ts | 71 +- packages/cli/src/template-placeholders.d.ts | 6 + .../defaults/app/asset-resource-fetch.ts | 60 ++ .../defaults/app/route-templates/html.tsx | 17 +- .../defaults/app/route-templates/text.tsx | 17 +- .../defaults/app/route-templates/xml.tsx | 17 +- packages/cli/templates/defaults/package.json | 1 + .../react-router/app/asset-resource-fetch.ts | 60 ++ .../react-router/app/route-templates/html.tsx | 17 +- .../react-router/app/route-templates/text.tsx | 17 +- .../react-router/app/route-templates/xml.tsx | 17 +- .../cli/templates/react-router/package.json | 1 + .../ssg/app/route-templates/html/+data.ts | 3 +- packages/feature-flags/src/flags.ts | 3 + packages/http-client/src/index.ts | 76 ++ .../supabase/tests/asset-file-metadata.sql | 143 +++ .../supabase/tests/asset-resource-indexes.sql | 244 +++++ .../migration.sql | 26 + .../migration.sql | 110 +++ .../migration.sql | 101 ++ .../migration.sql | 139 +++ .../migration.sql | 35 + .../migration.sql | 30 + .../migration.sql | 54 ++ .../migration.sql | 36 + .../migration.sql | 87 ++ .../migration.sql | 100 ++ packages/prisma-client/prisma/schema.prisma | 92 +- .../runtime-operation-contracts.ts | 870 ++++++++++++++++++ packages/project-build/src/mcp.ts | 59 ++ .../src/runtime/asset-resources.ts | 409 ++++++++ packages/project-build/src/runtime/data.ts | 18 +- .../src/runtime/output-schemas.ts | 24 + .../src/runtime/registry.test.ts | 179 ++++ .../project-build/src/runtime/registry.ts | 65 ++ .../server-only-router-operation-metadata.ts | 209 +++++ .../src/builder-api/operation-docs.ts | 74 ++ packages/protocol/src/schema.ts | 17 +- packages/sdk/src/index.ts | 1 + packages/sdk/src/resource-loader.test.ts | 214 ++++- packages/sdk/src/resource-loader.ts | 113 ++- packages/sdk/src/schema.ts | 1 + .../sdk/src/schema/asset-resource.test.ts | 354 +++++++ packages/sdk/src/schema/asset-resource.ts | 381 ++++++++ .../src/context/router.server.test.ts | 32 + .../src/context/router.server.ts | 6 +- pnpm-lock.yaml | 348 ++++--- 163 files changed, 18745 insertions(+), 219 deletions(-) create mode 100644 apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts create mode 100644 apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts create mode 100644 apps/builder/app/builder/features/settings-panel/asset-query-form.tsx create mode 100644 apps/builder/app/shared/$resources/assets-field-catalog.server.test.ts create mode 100644 apps/builder/app/shared/$resources/assets-field-catalog.server.ts create mode 100644 apps/builder/app/shared/$resources/assets-index-status.server.test.ts create mode 100644 apps/builder/app/shared/$resources/assets-index-status.server.ts create mode 100644 apps/builder/app/shared/$resources/assets-query.server.test.ts create mode 100644 apps/builder/app/shared/$resources/assets-query.server.ts create mode 100644 apps/builder/app/shared/$resources/assets.server.test.ts create mode 100644 apps/builder/app/shared/asset-resource-api.client.test.ts create mode 100644 apps/builder/app/shared/asset-resource-api.client.ts create mode 100644 apps/builder/app/shared/code-editor-groq.test.ts create mode 100644 apps/builder/app/shared/groq-completion.test.ts create mode 100644 apps/builder/app/shared/groq-completion.ts create mode 100644 apps/builder/docs/asset-resource-architecture.md create mode 100644 packages/asset-resource/package.json create mode 100644 packages/asset-resource/scripts/benchmark-groq.mjs create mode 100644 packages/asset-resource/scripts/benchmark-index.mjs create mode 100644 packages/asset-resource/scripts/benchmark-scale.mjs create mode 100644 packages/asset-resource/src/blog-e2e.test.ts create mode 100644 packages/asset-resource/src/candidate-selection.test.ts create mode 100644 packages/asset-resource/src/candidate-selection.ts create mode 100644 packages/asset-resource/src/canonical.test.ts create mode 100644 packages/asset-resource/src/canonical.ts create mode 100644 packages/asset-resource/src/field-catalog.test.ts create mode 100644 packages/asset-resource/src/field-catalog.ts create mode 100644 packages/asset-resource/src/hydration.test.ts create mode 100644 packages/asset-resource/src/hydration.ts create mode 100644 packages/asset-resource/src/index-storage.test.ts create mode 100644 packages/asset-resource/src/index-storage.ts create mode 100644 packages/asset-resource/src/index.test.ts create mode 100644 packages/asset-resource/src/index.ts create mode 100644 packages/asset-resource/src/markdown.test.ts create mode 100644 packages/asset-resource/src/markdown.ts create mode 100644 packages/asset-resource/src/published-runtime.test.ts create mode 100644 packages/asset-resource/src/published-runtime.ts create mode 100644 packages/asset-resource/src/query-validation.test.ts create mode 100644 packages/asset-resource/src/query-validation.ts create mode 100644 packages/asset-resource/src/resource-index.test.ts create mode 100644 packages/asset-resource/src/resource-index.ts create mode 100644 packages/asset-resource/src/scale-fixture.test.ts create mode 100644 packages/asset-resource/src/scale-fixture.ts create mode 100644 packages/asset-resource/src/sha256.ts create mode 100644 packages/asset-resource/src/stable-json.ts create mode 100644 packages/asset-resource/src/transport-parity.test.ts create mode 100644 packages/asset-resource/src/worker-assets.test.ts create mode 100644 packages/asset-resource/tsconfig.dts.json create mode 100644 packages/asset-resource/tsconfig.json create mode 100644 packages/asset-uploader/src/builder-api-data-boundary.test.ts create mode 100644 packages/asset-uploader/src/canonical-metadata-backfill.test.ts create mode 100644 packages/asset-uploader/src/canonical-metadata-backfill.ts create mode 100644 packages/asset-uploader/src/canonical-metadata-persistence.test.ts create mode 100644 packages/asset-uploader/src/canonical-metadata-persistence.ts create mode 100644 packages/asset-uploader/src/clients/fs/immutable-object.test.ts create mode 100644 packages/asset-uploader/src/clients/fs/immutable-object.ts create mode 100644 packages/asset-uploader/src/clients/fs/read.test.ts create mode 100644 packages/asset-uploader/src/clients/fs/read.ts create mode 100644 packages/asset-uploader/src/clients/s3/immutable-object.test.ts create mode 100644 packages/asset-uploader/src/clients/s3/immutable-object.ts create mode 100644 packages/asset-uploader/src/clients/s3/index-audit.ts create mode 100644 packages/asset-uploader/src/clients/s3/read.test.ts create mode 100644 packages/asset-uploader/src/clients/s3/read.ts create mode 100644 packages/asset-uploader/src/field-catalog.test.ts create mode 100644 packages/asset-uploader/src/field-catalog.ts create mode 100644 packages/asset-uploader/src/query-preview.test.ts create mode 100644 packages/asset-uploader/src/query-preview.ts create mode 100644 packages/asset-uploader/src/resource-index-build.test.ts create mode 100644 packages/asset-uploader/src/resource-index-build.ts create mode 100644 packages/asset-uploader/src/resource-index-garbage-collection.test.ts create mode 100644 packages/asset-uploader/src/resource-index-garbage-collection.ts create mode 100644 packages/asset-uploader/src/resource-index-lifecycle.test.ts create mode 100644 packages/asset-uploader/src/resource-index-lifecycle.ts create mode 100644 packages/asset-uploader/src/resource-index-maintenance.test.ts create mode 100644 packages/asset-uploader/src/resource-index-maintenance.ts create mode 100644 packages/asset-uploader/src/resource-index-observability.test.ts create mode 100644 packages/asset-uploader/src/resource-index-observability.ts create mode 100644 packages/asset-uploader/src/resource-index-orphan-audit.test.ts create mode 100644 packages/asset-uploader/src/resource-index-orphan-audit.ts create mode 100644 packages/asset-uploader/src/resource-index-persistence.ts create mode 100644 packages/asset-uploader/src/resource-index-publication.test.ts create mode 100644 packages/asset-uploader/src/resource-index-publication.ts create mode 100644 packages/asset-uploader/src/resource-index-rebuild.test.ts create mode 100644 packages/asset-uploader/src/resource-index-rebuild.ts create mode 100644 packages/asset-uploader/src/resource-index-scheduler.test.ts create mode 100644 packages/asset-uploader/src/resource-index-scheduler.ts create mode 100644 packages/asset-uploader/src/resource-index-snapshot.test.ts create mode 100644 packages/asset-uploader/src/resource-index-snapshot.ts create mode 100644 packages/asset-uploader/src/resource-index-status-authorization.test.ts create mode 100644 packages/asset-uploader/src/resource-index-status.test.ts create mode 100644 packages/asset-uploader/src/resource-index-status.ts create mode 100644 packages/cli/src/template-placeholders.d.ts create mode 100644 packages/cli/templates/defaults/app/asset-resource-fetch.ts create mode 100644 packages/cli/templates/react-router/app/asset-resource-fetch.ts create mode 100644 packages/postgrest/supabase/tests/asset-file-metadata.sql create mode 100644 packages/postgrest/supabase/tests/asset-resource-indexes.sql create mode 100644 packages/prisma-client/prisma/migrations/20260718120000_asset_file_metadata/migration.sql create mode 100644 packages/prisma-client/prisma/migrations/20260718121000_serialize_asset_file_metadata/migration.sql create mode 100644 packages/prisma-client/prisma/migrations/20260718122000_asset_resource_indexes/migration.sql create mode 100644 packages/prisma-client/prisma/migrations/20260718123000_activate_asset_resource_index/migration.sql create mode 100644 packages/prisma-client/prisma/migrations/20260718124000_fail_asset_resource_index_build/migration.sql create mode 100644 packages/prisma-client/prisma/migrations/20260718125000_cancel_asset_resource_index_build/migration.sql create mode 100644 packages/prisma-client/prisma/migrations/20260718126000_check_asset_resource_publication/migration.sql create mode 100644 packages/prisma-client/prisma/migrations/20260718127000_delete_asset_resource_index_query/migration.sql create mode 100644 packages/prisma-client/prisma/migrations/20260718128000_asset_resource_index_references/migration.sql create mode 100644 packages/prisma-client/prisma/migrations/20260718129000_asset_resource_index_gc/migration.sql create mode 100644 packages/project-build/src/runtime/asset-resources.ts create mode 100644 packages/sdk/src/schema/asset-resource.test.ts create mode 100644 packages/sdk/src/schema/asset-resource.ts diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts new file mode 100644 index 000000000000..0e32e83f6493 --- /dev/null +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "vitest"; +import { computeExpression } from "@webstudio-is/project-build/runtime"; +import { + createAssetQueryResourceBody, + getAssetIndexStatusLabel, + getAssetFileTypeGroqPredicate, + isEmptyAssetQueryResult, + parseAssetQueryResourceBody, +} from "./asset-query-form-utils"; + +describe("asset query resource body", () => { + test("preserves runtime parameter bindings as expressions", () => { + const body = createAssetQueryResourceBody({ + query: "*[properties.slug == $slug]", + parameters: [ + { name: "slug", value: "$ws$dataSource$routeSlug" }, + { name: "locale", value: '"en"' }, + ], + resultLimit: 1, + }); + + expect( + computeExpression(body, new Map([["routeSlug", "hello-world"]])) + ).toEqual({ + query: "*[properties.slug == $slug]", + parameters: { slug: "hello-world", locale: "en" }, + resultLimit: 1, + content: { mode: "none" }, + }); + expect(parseAssetQueryResourceBody(body)).toEqual({ + queryExpression: '"*[properties.slug == $slug]"', + resultLimitExpression: "1", + contentExpression: '{ "mode": "none" }', + parameters: [ + { name: "slug", value: "$ws$dataSource$routeSlug" }, + { name: "locale", value: '"en"' }, + ], + }); + }); + + test("serializes result and selected-file hydration limits", () => { + const body = createAssetQueryResourceBody({ + query: "*[0]{_id, revision, contentRef}", + parameters: [], + resultLimit: 1, + contentExpression: JSON.stringify({ + mode: "range", + offset: 100, + length: 500, + }), + }); + + expect(computeExpression(body, new Map())).toEqual({ + query: "*[0]{_id, revision, contentRef}", + parameters: {}, + resultLimit: 1, + content: { mode: "range", offset: 100, length: 500 }, + }); + expect(parseAssetQueryResourceBody(body)).toMatchObject({ + resultLimitExpression: "1", + contentExpression: '{"mode":"range","offset":100,"length":500}', + }); + }); + + test("classifies index and empty-result preview states", () => { + const base = { + resourceId: "posts", + queryHash: "query-revision", + assetRevision: "asset-revision", + updatedAt: "2026-07-18T12:00:00.000Z", + }; + expect(getAssetIndexStatusLabel(undefined)).toContain("No index"); + expect(getAssetIndexStatusLabel({ ...base, state: "indexing" })).toContain( + "first index" + ); + expect(getAssetIndexStatusLabel({ ...base, state: "stale" })).toContain( + "stale" + ); + expect(getAssetIndexStatusLabel({ ...base, state: "failed" })).toContain( + "failed" + ); + expect( + getAssetIndexStatusLabel({ + ...base, + state: "active", + activeRevision: "active-revision", + }) + ).toContain("active"); + expect(isEmptyAssetQueryResult([])).toBe(true); + expect(isEmptyAssetQueryResult(null)).toBe(true); + expect(isEmptyAssetQueryResult({})).toBe(false); + }); + + test("file-type helpers produce visible GROQ source", () => { + expect(getAssetFileTypeGroqPredicate("md")).toBe('extension == "md"'); + }); +}); diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts new file mode 100644 index 000000000000..ed3bc2532ac3 --- /dev/null +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts @@ -0,0 +1,78 @@ +import { + generateObjectExpression, + parseObjectExpression, + type AssetResourceIndexStatus, +} from "@webstudio-is/sdk"; + +export type AssetQueryParameterBinding = { + name: string; + value: string; +}; + +export const parseAssetQueryResourceBody = (body: string | undefined) => { + const fields = parseObjectExpression(body ?? ""); + const parameters = parseObjectExpression(fields.get("parameters") ?? ""); + return { + queryExpression: fields.get("query"), + resultLimitExpression: fields.get("resultLimit"), + contentExpression: fields.get("content"), + parameters: Array.from(parameters, ([name, value]) => ({ name, value })), + }; +}; + +export const createAssetQueryResourceBody = ({ + query, + parameters, + resultLimit = 100, + contentExpression = '{ "mode": "none" }', +}: { + query: string; + parameters: readonly AssetQueryParameterBinding[]; + resultLimit?: number; + contentExpression?: string; +}) => + generateObjectExpression( + new Map([ + ["query", JSON.stringify(query)], + [ + "parameters", + generateObjectExpression( + new Map( + parameters + .filter(({ name }) => name.trim().length > 0) + .map(({ name, value }) => [name, value]) + ) + ), + ], + ["resultLimit", JSON.stringify(resultLimit)], + ["content", contentExpression], + ]) + ); + +export const getAssetIndexStatusLabel = ( + status: AssetResourceIndexStatus | undefined +) => { + if (status === undefined) { + return "No index has been created yet."; + } + if (status.state === "indexing") { + return status.activeRevision === undefined + ? "Building the first index…" + : "Building a replacement index; the last active revision is preserved."; + } + if (status.state === "stale") { + return "The saved query index is stale."; + } + if (status.state === "failed") { + return status.activeRevision === undefined + ? "The index build failed." + : "The replacement build failed; the last active revision is preserved."; + } + return "The query index is active."; +}; + +export const isEmptyAssetQueryResult = (result: unknown) => + result === null || (Array.isArray(result) && result.length === 0); + +export const getAssetFileTypeGroqPredicate = (extension: string) => + `extension == ${JSON.stringify(extension)}`; diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx b/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx new file mode 100644 index 000000000000..58d28cad5d47 --- /dev/null +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx @@ -0,0 +1,550 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { useStore } from "@nanostores/react"; +import { + assetResourceContentOptions, + assetResourceLimits, + assetResourceIndexStatus, + assetResourceQueryResponse, + builderAssetFieldCatalog, + isLiteralExpression, + type AssetResourceContentOptions, + type AssetResourceIndexStatus, + type BuilderAssetFieldCatalog, + type Resource, +} from "@webstudio-is/sdk"; +import { + Button, + Flex, + Grid, + InputField, + Label, + Select, + SmallIconButton, + Switch, + Text, +} from "@webstudio-is/design-system"; +import { PlusIcon, TrashIcon } from "@webstudio-is/icons"; +import { getAssetResourceReferencedFieldPaths } from "@webstudio-is/asset-resource"; +import { $assets } from "~/shared/sync/data-stores"; +import { + BindingControl, + BindingPopover, + evaluateExpressionWithinScope, +} from "~/builder/shared/binding-popover"; +import { ExpressionEditor } from "~/builder/shared/expression-editor"; +import { CodeEditor } from "~/shared/code-editor"; +import type { EditorApi } from "~/shared/code-editor-base"; +import { + loadBuilderAssetFieldCatalog, + loadBuilderAssetIndexStatus, + previewBuilderAssetQuery, +} from "~/shared/asset-resource-api.client"; +import { + createAssetQueryResourceBody, + getAssetFileTypeGroqPredicate, + getAssetIndexStatusLabel, + isEmptyAssetQueryResult, + parseAssetQueryResourceBody, + type AssetQueryParameterBinding, +} from "./asset-query-form-utils"; + +const AssetQueryParameters = ({ + scope, + aliases, + parameters, + onChange, +}: { + scope: Record; + aliases: Map; + parameters: AssetQueryParameterBinding[]; + onChange: (parameters: AssetQueryParameterBinding[]) => void; +}) => ( + + + + } + onClick={() => onChange([...parameters, { name: "", value: '""' }])} + /> + + + {parameters.map((parameter, index) => ( + + { + const next = [...parameters]; + next[index] = { ...parameter, name: event.target.value }; + onChange(next); + }} + /> + +
+ { + const next = [...parameters]; + next[index] = { ...parameter, value }; + onChange(next); + }} + onChangeComplete={(value) => { + const next = [...parameters]; + next[index] = { ...parameter, value }; + onChange(next); + }} + /> +
+ { + const next = [...parameters]; + next[index] = { ...parameter, value }; + onChange(next); + }} + onRemove={(value) => { + const next = [...parameters]; + next[index] = { ...parameter, value: JSON.stringify(value) }; + onChange(next); + }} + /> + + } + onClick={() => + onChange(parameters.filter((_, item) => item !== index)) + } + /> + + ))} + {parameters.length === 0 && ( + + No runtime parameters + + )} + + +); + +const contentModeLabels: Record = { + none: "No content", + full: "Complete text", + range: "Byte range", + "markdown-body": "Markdown body", +}; + +const AssetQueryOptions = ({ + resultLimit, + content, + onResultLimitChange, + onContentChange, +}: { + resultLimit: number; + content: AssetResourceContentOptions; + onResultLimitChange: (value: number) => void; + onContentChange: (value: AssetResourceContentOptions) => void; +}) => ( + + + + { + if (Number.isNaN(event.target.valueAsNumber) === false) { + onResultLimitChange(event.target.valueAsNumber); + } + }} + /> + + + + + aria-label="Selected-file content" + options={["none", "full", "range", "markdown-body"]} + getLabel={(mode: AssetResourceContentOptions["mode"]) => + contentModeLabels[mode] + } + value={content.mode} + onChange={(mode) => + onContentChange( + mode === "range" ? { mode, offset: 0, length: 16 * 1024 } : { mode } + ) + } + /> + + {(content.mode === "full" || content.mode === "markdown-body") && ( + + + { + if (Number.isNaN(event.target.valueAsNumber) === false) { + onContentChange({ + mode: content.mode, + maxBytes: event.target.valueAsNumber, + }); + } + }} + /> + + )} + {content.mode === "range" && ( + + + + { + if (Number.isNaN(event.target.valueAsNumber) === false) { + onContentChange({ + ...content, + offset: event.target.valueAsNumber, + }); + } + }} + /> + + + + { + if (Number.isNaN(event.target.valueAsNumber) === false) { + onContentChange({ + ...content, + length: event.target.valueAsNumber, + }); + } + }} + /> + + + )} + +); + +const AssetQueryPreview = ({ + resourceId, + query, + parameters, + scope, + resultLimit, + content, + indexStatus, + onIndexStatusChange, +}: { + resourceId?: string; + query: string; + parameters: AssetQueryParameterBinding[]; + scope: Record; + resultLimit: number; + content: AssetResourceContentOptions; + indexStatus?: AssetResourceIndexStatus; + onIndexStatusChange: (status: AssetResourceIndexStatus | undefined) => void; +}) => { + const [previewState, setPreviewState] = useState< + | { type: "idle" } + | { type: "loading" } + | { type: "error"; message: string } + | { type: "success"; result: unknown } + >({ type: "idle" }); + + useEffect(() => { + if (resourceId === undefined) { + onIndexStatusChange(undefined); + return; + } + let ignore = false; + loadBuilderAssetIndexStatus(resourceId) + .then((response) => { + if (ignore) { + return; + } + const data = response.data as { status?: unknown }; + const parsed = assetResourceIndexStatus.safeParse(data.status); + onIndexStatusChange(parsed.success ? parsed.data : undefined); + }) + .catch(() => { + if (ignore === false) { + onIndexStatusChange(undefined); + } + }); + return () => { + ignore = true; + }; + }, [onIndexStatusChange, resourceId]); + + const preview = async () => { + setPreviewState({ type: "loading" }); + try { + const response = await previewBuilderAssetQuery({ + query, + parameters: Object.fromEntries( + parameters + .filter(({ name }) => name.trim().length > 0) + .map(({ name, value }) => [ + name, + evaluateExpressionWithinScope(value, scope), + ]) + ), + resultLimit, + content, + }); + const parsed = assetResourceQueryResponse.safeParse(response.data); + if (parsed.success === false) { + setPreviewState({ + type: "error", + message: "The preview response was invalid.", + }); + return; + } + if (parsed.data.ok === false) { + setPreviewState({ type: "error", message: parsed.data.error.message }); + return; + } + setPreviewState({ type: "success", result: parsed.data.result }); + } catch { + setPreviewState({ type: "error", message: "The query preview failed." }); + } + }; + const isEmpty = + previewState.type === "success" && + isEmptyAssetQueryResult(previewState.result); + + return ( + + + + + + + {getAssetIndexStatusLabel(indexStatus)} + + {previewState.type === "error" && ( + {previewState.message} + )} + {isEmpty && The query returned no results.} + {previewState.type === "success" && isEmpty === false && ( + {}} + onChangeComplete={() => {}} + /> + )} + + ); +}; + +export const AssetQueryForm = ({ + resource, + scope, + aliases, + enabled, + enabledId, + onEnabledChange, +}: { + resource?: Resource; + scope: Record; + aliases: Map; + enabled: boolean; + enabledId: string; + onEnabledChange: (enabled: boolean) => void; +}) => { + const assets = useStore($assets); + const parsedBody = useMemo( + () => parseAssetQueryResourceBody(resource?.body), + [resource?.body] + ); + const initialQuery = evaluateExpressionWithinScope( + parsedBody.queryExpression ?? "", + {} + ); + const [query, setQuery] = useState( + typeof initialQuery === "string" + ? initialQuery + : '*[_type == "asset.file"] | order(_id asc) [0...20]' + ); + const [parameters, setParameters] = useState(parsedBody.parameters); + const parsedResultLimit = evaluateExpressionWithinScope( + parsedBody.resultLimitExpression ?? "", + {} + ); + const [resultLimit, setResultLimit] = useState( + typeof parsedResultLimit === "number" ? parsedResultLimit : 100 + ); + const parsedContent = assetResourceContentOptions.safeParse( + evaluateExpressionWithinScope(parsedBody.contentExpression ?? "", {}) + ); + const [content, setContent] = useState( + parsedContent.success ? parsedContent.data : { mode: "none" } + ); + const [fieldCatalog, setFieldCatalog] = useState(); + const [indexStatus, setIndexStatus] = useState(); + + useEffect(() => { + if (enabled === false) { + return; + } + let ignore = false; + loadBuilderAssetFieldCatalog() + .then((response) => { + if (ignore) { + return; + } + const parsed = builderAssetFieldCatalog.safeParse(response.data); + setFieldCatalog(parsed.success ? parsed.data : undefined); + }) + .catch(() => { + if (ignore === false) { + setFieldCatalog(undefined); + } + }); + return () => { + ignore = true; + }; + }, [assets, enabled]); + + const editorApiRef = useRef(); + const groqCompletion = useMemo(() => { + let resourceFieldPaths: Set | undefined; + if (indexStatus?.activeRevision !== undefined) { + try { + resourceFieldPaths = new Set( + getAssetResourceReferencedFieldPaths(query) + ); + } catch { + // Keep project catalog completions while the query is temporarily invalid. + } + } + return { + catalog: fieldCatalog, + parameterNames: Array.from( + new Set( + parameters + .map(({ name }) => name) + .filter((name) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) + ) + ), + resourceFieldPaths, + }; + }, [fieldCatalog, indexStatus?.activeRevision, parameters, query]); + + return ( + <> + + + + + {enabled && ( + <> + + + + + + + + + + + + + + + )} + + ); +}; diff --git a/apps/builder/app/builder/features/settings-panel/resource-panel.tsx b/apps/builder/app/builder/features/settings-panel/resource-panel.tsx index 0adc756f6244..3dd083872b0e 100644 --- a/apps/builder/app/builder/features/settings-panel/resource-panel.tsx +++ b/apps/builder/app/builder/features/settings-panel/resource-panel.tsx @@ -30,6 +30,7 @@ import { sitemapResourceUrl, currentDateResourceUrl, assetsResourceUrl, + assetsQueryResourceUrl, } from "@webstudio-is/sdk/runtime"; import { Box, @@ -46,6 +47,7 @@ import { theme, } from "@webstudio-is/design-system"; import { TrashIcon, InfoCircleIcon, PlusIcon } from "@webstudio-is/icons"; +import { isFeatureEnabled } from "@webstudio-is/feature-flags"; import { humanizeString } from "~/shared/string-utils"; import { $variableValuesByInstanceSelector } from "~/shared/nano-states"; import { $dataSources } from "~/shared/sync/data-stores"; @@ -76,6 +78,7 @@ import { type ResourceBodyInputType, } from "@webstudio-is/project-build/runtime"; import { parseCurl, type CurlRequest } from "./curl"; +import { AssetQueryForm } from "./asset-query-form"; export const UrlField = ({ scope, @@ -912,13 +915,22 @@ export const SystemResourceForm = forwardRef< undefined | PanelApi, { variable?: DataSource } >(({ variable }, ref) => { + const { scope, aliases } = useResourceScope({ variable }); const resources = useStore($resources); const resource = variable?.type === "resource" ? resources.get(variable.resourceId) : undefined; + const isStoredAssetQuery = + resource?.url === JSON.stringify(assetsQueryResourceUrl); + const assetsLocalResource = { + label: "Assets", + value: JSON.stringify(assetsResourceUrl), + description: + "Loads all project assets by default, with optional GROQ querying and selected-file content.", + }; const localResources = [ { label: "Sitemap", @@ -931,22 +943,26 @@ export const SystemResourceForm = forwardRef< description: "Provides current date information (year, month, day) normalized to midnight UTC. Time components are set to 00:00:00 to prevent React hydration errors.", }, - { - label: "Assets", - value: JSON.stringify(assetsResourceUrl), - description: - "Resource that loads the list of assets of the current project.", - }, + assetsLocalResource, ]; const [localResource, setLocalResource] = useState(() => { + if (isStoredAssetQuery) { + return assetsLocalResource; + } return ( localResources.find( (localResource) => localResource.value === resource?.url ) ?? localResources[0] ); }); - + const isAssetsResource = + localResource.value === JSON.stringify(assetsResourceUrl); + const canConfigureAssetQuery = + isFeatureEnabled("assetResource") || isStoredAssetQuery; + const [isAssetQueryEnabled, setIsAssetQueryEnabled] = + useState(isStoredAssetQuery); + const isAssetQuery = isAssetsResource && isAssetQueryEnabled; useImperativeHandle(ref, () => ({ save: (formData) => { // preserve existing instance scope when edit @@ -973,11 +989,24 @@ export const SystemResourceForm = forwardRef< })); const resourceId = useId(); + const assetQueryEnabledId = useId(); return ( <> - - + + + {configurationError !== undefined && ( + {configurationError} + )} { }; type PanelApi = { - save: (formData: FormData) => void; + save: (formData: FormData) => void | false; }; type BodyType = ResourceBodyInputType; @@ -965,6 +965,9 @@ export const SystemResourceForm = forwardRef< const isAssetQuery = isAssetsResource && isAssetQueryEnabled; useImperativeHandle(ref, () => ({ save: (formData) => { + if (formData.get("asset-query-valid") === "false") { + return false; + } // preserve existing instance scope when edit const scopeInstanceId = variable?.scopeInstanceId ?? $selectedInstance.get()?.id; diff --git a/apps/builder/app/builder/features/settings-panel/variable-popover.tsx b/apps/builder/app/builder/features/settings-panel/variable-popover.tsx index 691e83ce87f4..9655e4077c5c 100644 --- a/apps/builder/app/builder/features/settings-panel/variable-popover.tsx +++ b/apps/builder/app/builder/features/settings-panel/variable-popover.tsx @@ -282,7 +282,7 @@ const TypeField = ({ }; type PanelApi = { - save: (formData: FormData) => void; + save: (formData: FormData) => void | false; }; const ParameterForm = forwardRef< @@ -916,10 +916,10 @@ const VariablePopoverContent = ({ nameElement.checkValidity() ) { const formData = new FormData(event.currentTarget); - panelRef.current?.save(formData); + const saved = panelRef.current?.save(formData); // close popover whenever new variable is created // to prevent creating duplicated variable - if (variable === undefined) { + if (variable === undefined && saved !== false) { onClose(); } } diff --git a/apps/builder/app/services/api-router.server.ts b/apps/builder/app/services/api-router.server.ts index 1e7a4b6f085c..be5c4aa65421 100644 --- a/apps/builder/app/services/api-router.server.ts +++ b/apps/builder/app/services/api-router.server.ts @@ -745,6 +745,7 @@ export const apiRouter = router({ ), astNodes: validated.astNodes, astDepth: validated.astDepth, + datasetScans: validated.datasetScans, }; } catch (error) { return throwAssetQueryApiError(error); diff --git a/apps/builder/app/shared/$resources/assets-query.server.test.ts b/apps/builder/app/shared/$resources/assets-query.server.test.ts index 69202ffbf517..b20dd36101cb 100644 --- a/apps/builder/app/shared/$resources/assets-query.server.test.ts +++ b/apps/builder/app/shared/$resources/assets-query.server.test.ts @@ -132,7 +132,7 @@ describe("asset query preview system resource", () => { }); }); - test("maps query validation and timeout errors", async () => { + test("maps query validation and execution errors", async () => { vi.mocked(previewAssetResourceQuery).mockRejectedValueOnce( new AssetResourceQueryValidationError({ code: "INVALID_QUERY", @@ -150,17 +150,17 @@ describe("asset query preview system resource", () => { vi.mocked(previewAssetResourceQuery).mockRejectedValueOnce( new AssetResourceQueryExecutionError({ - code: "QUERY_TIMEOUT", - message: "Timed out", + code: "RESULT_LIMIT_EXCEEDED", + message: "Too many results", }) ); - const timeout = await loader({ + const excessive = await loader({ request: outerRequest(), resourceRequest: innerRequest({ query: "*[]" }), }); - expect(timeout.status).toBe(504); - await expect(timeout.json()).resolves.toMatchObject({ - error: { code: "QUERY_TIMEOUT" }, + expect(excessive.status).toBe(400); + await expect(excessive.json()).resolves.toMatchObject({ + error: { code: "RESULT_LIMIT_EXCEEDED" }, }); }); }); diff --git a/apps/builder/app/shared/$resources/assets-query.server.ts b/apps/builder/app/shared/$resources/assets-query.server.ts index d9d3af6ed2fa..b1494f8f11c7 100644 --- a/apps/builder/app/shared/$resources/assets-query.server.ts +++ b/apps/builder/app/shared/$resources/assets-query.server.ts @@ -109,7 +109,7 @@ export const loader = async ({ code: error.code, message: error.message, details: error.details, - status: error.code === "QUERY_TIMEOUT" ? 504 : 400, + status: 400, }); } if (error instanceof AssetResourceHydrationError) { @@ -117,7 +117,7 @@ export const loader = async ({ code: error.code, message: error.message, details: error.details, - status: error.code === "PROTECTED_CONTENT" ? 403 : 400, + status: 400, }); } return failure({ diff --git a/apps/builder/app/shared/db/canvas.server.ts b/apps/builder/app/shared/db/canvas.server.ts index 5371435501aa..66e02f3c2b51 100644 --- a/apps/builder/app/shared/db/canvas.server.ts +++ b/apps/builder/app/shared/db/canvas.server.ts @@ -13,6 +13,7 @@ import { loadCanonicalAssetFileEntries, loadAssetResourceIndexSnapshots, getAssetResourceQuery, + reconcileAssetResourceIndexesForPublication, synchronizeCanonicalAssets, } from "@webstudio-is/asset-uploader/index.server"; import type { AppContext } from "@webstudio-is/trpc-interface/index.server"; @@ -171,7 +172,6 @@ const addProjectMetadata = async ( return query === undefined ? [] : [{ resourceId: resource.id, query }]; }); let assetResourceIndexes: PublishedProjectBundle["assetResourceIndexes"]; - let protectedAssetIds: PublishedProjectBundle["protectedAssetIds"]; if (assetQueryResources.length > 0) { const assetClient = createAssetClient(); await synchronizeCanonicalAssets({ @@ -183,25 +183,34 @@ const addProjectMetadata = async ( client: context.postgrest.client, projectId: project.id, }); - protectedAssetIds = canonicalEntries - .filter( - ({ document }) => - document.properties.draft === true || - document.properties.private === true - ) - .map(({ assetId }) => assetId); + const indexedResources = await Promise.all( + assetQueryResources.map(async ({ resourceId, query }) => ({ + resourceId, + query, + queryHash: await computeAssetResourceQueryHash(query), + })) + ); + const expectedAssetRevision = + await computeCanonicalAssetRevision(canonicalEntries); + await reconcileAssetResourceIndexesForPublication({ + client: context.postgrest.client, + store: assetClient.resourceIndexStore, + projectId: project.id, + resources: indexedResources, + entries: canonicalEntries, + assetRevision: expectedAssetRevision, + }); assetResourceIndexes = await loadAssetResourceIndexSnapshots({ client: context.postgrest.client, projectId: project.id, - resources: await Promise.all( - assetQueryResources.map(async ({ resourceId, query }) => ({ - resourceId, - queryHash: await computeAssetResourceQueryHash(query), - })) - ), + resources: indexedResources, read: assetClient.readFile, - expectedAssetRevision: - await computeCanonicalAssetRevision(canonicalEntries), + referenceId: data.build.id, + garbageCollectionStore: + assetClient.resourceIndexStore.delete === undefined + ? undefined + : { delete: assetClient.resourceIndexStore.delete }, + expectedAssetRevision, }); } @@ -212,7 +221,6 @@ const addProjectMetadata = async ( projectDomain: project.domain, projectTitle: project.title, assetResourceIndexes, - protectedAssetIds, }; }; diff --git a/apps/builder/app/shared/resource-utils.test.ts b/apps/builder/app/shared/resource-utils.test.ts index 4707aa12d21d..170b0afa8151 100644 --- a/apps/builder/app/shared/resource-utils.test.ts +++ b/apps/builder/app/shared/resource-utils.test.ts @@ -237,4 +237,21 @@ describe("getResourceKey - pure function tests", () => { expect(new Set(keys).size).toBe(requests.length); }); + + test("asset query cache separates resource identities", () => { + const request: ResourceRequest = { + resourceId: "posts", + name: "assets-query", + control: "system", + method: "post", + url: "/$resources/assets/query", + searchParams: [], + headers: [], + body: { query: "*[]" }, + }; + + expect(getResourceKey(request)).not.toBe( + getResourceKey({ ...request, resourceId: "other-posts" }) + ); + }); }); diff --git a/apps/builder/app/shared/resource-utils.ts b/apps/builder/app/shared/resource-utils.ts index 8b196ff5848c..094e730f9d75 100644 --- a/apps/builder/app/shared/resource-utils.ts +++ b/apps/builder/app/shared/resource-utils.ts @@ -6,6 +6,7 @@ export const getResourceKey = (resource: ResourceRequest) => { return hash( JSON.stringify([ // explicitly list all fields to keep hash stable + ...(resource.resourceId === undefined ? [] : [resource.resourceId]), resource.name, resource.method, resource.url, diff --git a/apps/builder/app/shared/resources.ts b/apps/builder/app/shared/resources.ts index b2a6eed53b13..3fb74329c049 100644 --- a/apps/builder/app/shared/resources.ts +++ b/apps/builder/app/shared/resources.ts @@ -1,5 +1,10 @@ import { atom, computed } from "nanostores"; -import type { DataSource, Resource, ResourceRequest } from "@webstudio-is/sdk"; +import { + isStoredAssetQueryResource, + type DataSource, + type Resource, + type ResourceRequest, +} from "@webstudio-is/sdk"; import { restResourcesLoader } from "./router-utils"; import { computeExpression } from "@webstudio-is/project-build/runtime"; import { fetch } from "./fetch.client"; @@ -107,6 +112,9 @@ export const computeResourceRequest = ( values: Map ): ResourceRequest => { const request: ResourceRequest = { + ...(isStoredAssetQueryResource(resource) + ? { resourceId: resource.id } + : {}), name: resource.name, method: resource.method, url: computeExpression(resource.url, values), diff --git a/apps/builder/app/shared/sync/patch/patch-service.server.test.ts b/apps/builder/app/shared/sync/patch/patch-service.server.test.ts index 9a037b756f33..3b0465bec17f 100644 --- a/apps/builder/app/shared/sync/patch/patch-service.server.test.ts +++ b/apps/builder/app/shared/sync/patch/patch-service.server.test.ts @@ -6,6 +6,7 @@ const createContentModeCapabilities = vi.hoisted(() => vi.fn(() => ({ capabilities: true })) ); const synchronizeAssetResourceIndexQueries = vi.hoisted(() => vi.fn()); +const synchronizeCanonicalAsset = vi.hoisted(() => vi.fn()); const synchronizeAllCanonicalAssetStandardMetadata = vi.hoisted(() => vi.fn()); const updateAssetResourceIndexesAfterCanonicalChange = vi.hoisted(() => vi.fn() @@ -26,6 +27,7 @@ vi.mock("./patch-auth.server", () => ({ })); vi.mock("@webstudio-is/asset-uploader/index.server", () => ({ synchronizeAssetResourceIndexQueries, + synchronizeCanonicalAsset, synchronizeAllCanonicalAssetStandardMetadata, updateAssetResourceIndexesAfterCanonicalChange, })); @@ -116,6 +118,10 @@ describe("applyPatchRequest", () => { deletedResourceIds: [], updatedResourceIds: [], }); + synchronizeCanonicalAsset.mockReset().mockResolvedValue({ + status: "indexed", + revision: "revision-1", + }); synchronizeAllCanonicalAssetStandardMetadata .mockReset() .mockResolvedValue(0); @@ -209,6 +215,56 @@ describe("applyPatchRequest", () => { ); }); + test("indexes added assets before rebuilding resource indexes", async () => { + const assetEntry = { + ...patch.entries[0], + transaction: { + id: "tx-asset-add", + payload: [ + { + namespace: "assets", + patches: [ + { + op: "add", + path: ["asset-1"], + value: { id: "asset-1", name: "post.md" }, + }, + ], + }, + ], + } as never, + }; + authorizePatchEntries.mockResolvedValue({ + authorized: [{ entry: assetEntry, context: { writer: 1 } }], + rejected: [], + }); + patchLoadedBuild.mockImplementation(async ({ build }) => ({ + status: "ok", + version: 4, + build: { ...build, version: 4 }, + })); + + await expect( + applyPatchRequest(createContext(), { ...patch, entries: [assetEntry] }) + ).resolves.toMatchObject({ status: "ok" }); + + expect(synchronizeCanonicalAsset).toHaveBeenCalledWith({ + client: expect.anything(), + assetClient: createAssetClient.mock.results[0]?.value, + projectId: "project-1", + assetId: "asset-1", + }); + expect(updateAssetResourceIndexesAfterCanonicalChange).toHaveBeenCalledWith( + expect.objectContaining({ + projectId: "project-1", + changedAssetIds: ["asset-1"], + }) + ); + expect(synchronizeCanonicalAsset.mock.invocationCallOrder[0]).toBeLessThan( + updateAssetResourceIndexesAfterCanonicalChange.mock.invocationCallOrder[0] + ); + }); + test("returns per-entry partial results while applying authorized entries", async () => { authorizePatchEntries.mockResolvedValue({ authorized: [{ entry: patch.entries[1], context: { writer: 2 } }], diff --git a/apps/builder/app/shared/sync/patch/patch-service.server.ts b/apps/builder/app/shared/sync/patch/patch-service.server.ts index 00b10c32c0f9..438ff0afd01e 100644 --- a/apps/builder/app/shared/sync/patch/patch-service.server.ts +++ b/apps/builder/app/shared/sync/patch/patch-service.server.ts @@ -17,6 +17,7 @@ import type { } from "./patch-normalize.server"; import { synchronizeAllCanonicalAssetStandardMetadata, + synchronizeCanonicalAsset, synchronizeAssetResourceIndexQueries, updateAssetResourceIndexesAfterCanonicalChange, } from "@webstudio-is/asset-uploader/index.server"; @@ -397,6 +398,7 @@ export const applyPatchRequest = async ( ); if (assetChanges.length > 0) { try { + const assetClient = createAssetClient(); const hasFolderChanges = assetChanges.some( ({ namespace }) => namespace === "assetFolders" ); @@ -406,6 +408,27 @@ export const applyPatchRequest = async ( projectId: patch.projectId, }); } + const addedAssetIds = [ + ...new Set( + assetChanges + .filter(({ namespace }) => namespace === "assets") + .flatMap(({ patches }) => + patches.flatMap(({ op, path }) => + op === "add" && path.length === 1 && typeof path[0] === "string" + ? [path[0]] + : [] + ) + ) + ), + ]; + for (const assetId of addedAssetIds) { + await synchronizeCanonicalAsset({ + client: context.postgrest.client, + assetClient, + projectId: patch.projectId, + assetId, + }); + } const changedAssetIds = [ ...new Set( assetChanges.flatMap(({ patches }) => @@ -417,7 +440,7 @@ export const applyPatchRequest = async ( ]; await updateAssetResourceIndexesAfterCanonicalChange({ client: context.postgrest.client, - store: createAssetClient().resourceIndexStore, + store: assetClient.resourceIndexStore, projectId: patch.projectId, changedAssetIds: changedAssetIds.length === 0 ? ["project-assets"] : changedAssetIds, diff --git a/apps/builder/docs/asset-resource-architecture.md b/apps/builder/docs/asset-resource-architecture.md index 33269ed2a9b1..6a896b94ece3 100644 --- a/apps/builder/docs/asset-resource-architecture.md +++ b/apps/builder/docs/asset-resource-architecture.md @@ -263,51 +263,26 @@ object access, separate from public asset-delivery storage. V1 does not deduplicate index objects across resources: `resourceId` is part of both object identity and the revision primary key. Cleanup therefore never -shares one object accidentally. Durable resource, Builder-build, and deployment -reference rows still provide an exact per-revision owner count; cross-resource -content-addressed deduplication can use those counts if introduced later. - -## Draft and private publication policy - -V1 treats publication visibility as a mandatory policy applied before GROQ, -not as a convention that each resource query must remember to implement. - -Markdown frontmatter classifies a file as follows: - -| Frontmatter | Builder preview | Public deployment | -| --------------------- | -------------------------------------------------------------- | ----------------------------------------------------------------------- | -| No visibility flag | Available to authorized project viewers | Eligible for indexes and static asset materialization | -| `draft: true` | Available to authorized project viewers with a draft indicator | Excluded from public indexes and public static assets | -| `private: true` | Available only through authenticated project APIs | Excluded from public indexes, static assets, and public hydration paths | -| Both flags are `true` | Uses the stricter `private` behavior | Excluded | - -Only the Boolean value `true` activates either flag. Other observed types are -reported as field-catalog mixed-type diagnostics and do not accidentally hide -or publish based on truthiness. A project can opt into stricter validation, -but a query can never opt out of publication exclusion. - -The publication snapshot applies the policy to canonical metadata before -building deployable resource indexes. It also filters Markdown -materialization, so knowing an old content reference or generated filename -cannot retrieve excluded bytes from the deployment. Public index manifests, -checksums, logs, and errors must not reveal excluded frontmatter values. - -Builder previews execute through authenticated project APIs and may select -drafts. Private files require the same project authorization used to read the -underlying asset and cannot use a public deployment URL for hydration. Preview -responses are private and non-cacheable by shared caches. - -This policy applies to Markdown content selected by Assets resource queries. -The legacy Assets system resource and publication behavior for existing image, -font, and generic-file references remain compatible. If a published page or -resource would require an excluded Markdown file, publication fails with the -asset ID and a visibility error instead of silently deploying it or producing -a broken public URL. - -Changing either visibility flag changes the canonical asset revision and -invalidates every affected resource index. Re-publishing removes newly -excluded files and metadata from the new immutable deployment; existing -immutable deployments follow their original snapshot until retired. +shares one object accidentally. The active state protects the current revision, +and publication holds a temporary `BUILD` reference while it reads an immutable +snapshot. Published deployments contain their own static copy and no longer +depend on the private object. After activation, query deletion, and snapshot +release, best-effort cleanup claims unreferenced revisions, deletes their private +objects, and then removes their database rows. Garbage claims are 15-minute +leases. A later cleanup worker rotates and resumes an expired claim, so a crash +before or after object deletion cannot permanently strand the revision row. + +## Schemaless frontmatter and publication + +Frontmatter fields have no built-in publication meaning. Names such as `draft`, +`private`, `published`, or `status` are ordinary values below `properties` and +remain available to GROQ like any other user-defined field. + +All uploaded assets follow the existing asset publication behavior. To exclude +documents from a blog resource, the user must express that policy in its query, +for example `*[properties.draft != true]`. Index construction and content +hydration never apply an additional hidden filter. This keeps the resource +schemaless and lets projects choose their own field names and publication model. ## V1 limits and errors @@ -318,11 +293,11 @@ single source for Builder, indexer, generated runtime, and test limits. V1 uses: | ----------------------------- | -------------------------- | | Query source | 32 KiB UTF-8 | | Query syntax tree | 1,000 nodes, depth 64 | -| Query evaluation | 250 ms | +| Dataset scans per query | 1 | | Runtime parameters | 32 values, 64 KiB JSON | | Results | 100 default, 1,000 maximum | | Serialized result | 1 MiB | -| Candidate documents | 5,000 | +| Candidate documents | 1,000 | | Serialized index | 16 MiB | | Frontmatter per file | 64 KiB | | Frontmatter structure | Depth 8, 256 fields | @@ -336,33 +311,32 @@ single source for Builder, indexer, generated runtime, and test limits. V1 uses: Byte limits are measured after UTF-8 encoding; JSON limits use the serialized UTF-8 representation. A request can lower result or hydration limits but cannot raise these ceilings. Index and query work stops before allocating or returning -data beyond a boundary. Query AST checks protect the synchronous evaluator; -the runtime deadline is an additional guard and not the only complexity -control. +data beyond a boundary. Query AST checks protect the synchronous evaluator. V1 +permits one dataset scan per query and evaluates at most 1,000 candidate +documents, avoiding nested dataset scans and unbounded cross-products that +cannot be interrupted safely. All endpoint failures use `AssetResourceQueryFailure`, with `ok: false`, a stable error `code`, safe human-readable `message`, `retryable`, optional safe JSON `details`, and any known query/index/asset revisions. Responses never -include source content, private frontmatter, storage credentials, or internal +include source content, frontmatter values, storage credentials, or internal stack traces. The status and retry contract is: -| HTTP | Codes | Retry behavior | -| ------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------- | -| 400 | `INVALID_REQUEST`, `INVALID_QUERY`, `MISSING_PARAMETER`, `QUERY_COMPLEXITY_EXCEEDED`, `RESULT_LIMIT_EXCEEDED`, `CONTENT_IDENTITY_REQUIRED` | Change the resource or request | -| 401/403 | `FORBIDDEN`, `PROTECTED_CONTENT` | Reauthenticate or change publication visibility | -| 404 | `NOT_FOUND`, `INDEX_NOT_FOUND` | Rebuild or correct the referenced resource | -| 409 | `STALE_INDEX`, `INDEX_BUILD_FAILED` | Retry only when `retryable` is true after indexing | -| 413 | `RESULT_SIZE_EXCEEDED`, `CONTENT_LIMIT_EXCEEDED` | Lower the result or hydration request | -| 415 | `CONTENT_NOT_TEXT`, `CONTENT_DECODING_FAILED` | Use an asset URL or supported UTF-8 content | -| 504 | `QUERY_TIMEOUT` | Simplify the query; automatic retry is false by default | -| 500 | `INTERNAL_ERROR` | Retry only when explicitly marked retryable | +| HTTP | Codes | Retry behavior | +| ------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------- | +| 400 | `INVALID_REQUEST`, `INVALID_QUERY`, `MISSING_PARAMETER`, `QUERY_COMPLEXITY_EXCEEDED`, `RESULT_LIMIT_EXCEEDED`, `CONTENT_IDENTITY_REQUIRED` | Change the resource or request | +| 401/403 | `FORBIDDEN` | Reauthenticate | +| 404 | `NOT_FOUND`, `INDEX_NOT_FOUND` | Rebuild or correct the referenced resource | +| 409 | `STALE_INDEX`, `INDEX_BUILD_FAILED` | Retry only when `retryable` is true after indexing | +| 413 | `RESULT_SIZE_EXCEEDED`, `CONTENT_LIMIT_EXCEEDED` | Lower the result or hydration request | +| 415 | `CONTENT_NOT_TEXT`, `CONTENT_DECODING_FAILED` | Use an asset URL or supported UTF-8 content | +| 500 | `INTERNAL_ERROR` | Retry only when explicitly marked retryable | Builder UI states and generated applications branch on `code`, never parse the message. Validation errors may include bounded field paths and limits in -`details`. Missing parameters list names only; protected-content errors expose -the asset ID but no metadata or content. +`details`. Missing parameters list names only. ## GROQ feasibility and cost @@ -490,6 +464,12 @@ options in the request body. The adapter applies schema defaults before the generic resource serializer runs, so Builder preview and generated apps send identical JSON rather than environment-specific query strings. +Saved query resources also carry their resource ID through the internal +transport. Published runtimes use that identity together with the query hash, +so two resources may intentionally use identical GROQ without one manifest +entry shadowing the other. Requests without an identity remain compatible when +the published manifest contains only one matching query. + Both cache layers hash the complete normalized POST body. Builder `getResourceKey` and generated-runtime `getResourceCacheKey` have collision tests that independently vary the GROQ query, runtime parameters, pinned index @@ -564,6 +544,12 @@ The event consumer must be idempotent and revision-aware. Client-side `invalidateAssets` remains responsible only for refreshing the legacy Builder resource; it is not a durability boundary for index maintenance. +Asset mutations commit independently from derived-index maintenance. A derived +failure therefore never turns an already committed upload or content revision +into a failed primary mutation. Publication synchronizes canonical metadata and +rebuilds a missing, stale, or failed index for the current saved query before +snapshotting it, providing a retry boundary after transient maintenance errors. + After canonical metadata commits, v1 conservatively treats every saved asset query in that project as affected because GROQ may depend on collection count, ordering, or any schema-less field. The maintenance pass loads persisted @@ -591,6 +577,10 @@ The editor provides: - Explicit preview execution and active, indexing, stale, failed, empty, and limit-error states. +The form rejects invalid GROQ, missing, invalid, or duplicate parameter +bindings, and content/result limits outside the runtime contract before saving +the resource. + A listing resource should leave content hydration off: ```groq @@ -628,16 +618,28 @@ TypeScript contains only the deployment ID and immutable index paths. The generated route runtime resolves those paths through Cloudflare's `ASSETS` binding when present, with a same-origin static fetch fallback for other -platforms. Parsed immutable indexes are cached per isolate. Opt-in query-result -cache keys include deployment ID, resource ID, index revision, complete request -body (query, parameters, limits, and hydration), and therefore cannot collide -across revisions or runtime parameters. - -Public index construction excludes Boolean `draft: true` and `private: true` -documents before GROQ candidate selection. Prebuild materializes only Markdown -content references retained by public snapshots. Builder preview remains an -authenticated canonical-metadata path and can apply the documented draft and -private authorization behavior independently. +platforms. Parsed immutable indexes are cached per isolate by deployment ID, +resource ID, index revision, and public path. Multiple domains that point to the +same deployment reuse that immutable data. Domains or previews backed by +different deployments have separate cache identities, even when their public +paths happen to match. Opt-in query-result cache keys include deployment ID, +resource ID, index revision, complete request body (query, parameters, limits, +and hydration), and therefore cannot collide across deployments, revisions, or +runtime parameters. Cache API reads and writes are best-effort optimizations; +cache unavailability never changes a successful query into a runtime failure. + +Vike SSG evaluates the same published query runtime while prerendering. Its +server-only adapter reads immutable indexes and ranged Markdown bytes from the +generated `public` directory, so prerendering requires neither a live HTTP +endpoint nor bundling content into JavaScript. Range hydration uses positioned +file reads rather than loading the complete source asset before slicing it. + +Public index construction applies only the configured GROQ candidate selection. +Prebuild materializes Markdown through the same static asset pipeline as other +assets; frontmatter fields do not silently remove files from a deployment. For +dynamic applications without query-enabled Assets resources, prebuild emits a +no-op generated adapter with no import of the GROQ runtime, keeping it out of +the application bundle. ## Scale benchmark baseline diff --git a/fixtures/react-router-cloudflare/.webstudio/data.json b/fixtures/react-router-cloudflare/.webstudio/data.json index ee40c9f7671b..da5b1dcdba75 100644 --- a/fixtures/react-router-cloudflare/.webstudio/data.json +++ b/fixtures/react-router-cloudflare/.webstudio/data.json @@ -1,5 +1,5 @@ { - "bundleVersion": "bundle-15dvk07", + "bundleVersion": "bundle-1vq244u", "build": { "id": "f565d527-32e7-4731-bc71-aca9e9574587", "projectId": "d845c167-ea07-4875-b08d-83e97c09dcce", diff --git a/fixtures/react-router-docker/.webstudio/data.json b/fixtures/react-router-docker/.webstudio/data.json index ee40c9f7671b..da5b1dcdba75 100644 --- a/fixtures/react-router-docker/.webstudio/data.json +++ b/fixtures/react-router-docker/.webstudio/data.json @@ -1,5 +1,5 @@ { - "bundleVersion": "bundle-15dvk07", + "bundleVersion": "bundle-1vq244u", "build": { "id": "f565d527-32e7-4731-bc71-aca9e9574587", "projectId": "d845c167-ea07-4875-b08d-83e97c09dcce", diff --git a/fixtures/react-router-netlify/.webstudio/data.json b/fixtures/react-router-netlify/.webstudio/data.json index ee40c9f7671b..da5b1dcdba75 100644 --- a/fixtures/react-router-netlify/.webstudio/data.json +++ b/fixtures/react-router-netlify/.webstudio/data.json @@ -1,5 +1,5 @@ { - "bundleVersion": "bundle-15dvk07", + "bundleVersion": "bundle-1vq244u", "build": { "id": "f565d527-32e7-4731-bc71-aca9e9574587", "projectId": "d845c167-ea07-4875-b08d-83e97c09dcce", diff --git a/fixtures/react-router-vercel/.webstudio/data.json b/fixtures/react-router-vercel/.webstudio/data.json index ee40c9f7671b..da5b1dcdba75 100644 --- a/fixtures/react-router-vercel/.webstudio/data.json +++ b/fixtures/react-router-vercel/.webstudio/data.json @@ -1,5 +1,5 @@ { - "bundleVersion": "bundle-15dvk07", + "bundleVersion": "bundle-1vq244u", "build": { "id": "f565d527-32e7-4731-bc71-aca9e9574587", "projectId": "d845c167-ea07-4875-b08d-83e97c09dcce", diff --git a/fixtures/ssg-cloudflare-pages/.webstudio/data.json b/fixtures/ssg-cloudflare-pages/.webstudio/data.json index ccb3f06c247c..ede55011fb25 100644 --- a/fixtures/ssg-cloudflare-pages/.webstudio/data.json +++ b/fixtures/ssg-cloudflare-pages/.webstudio/data.json @@ -1,5 +1,5 @@ { - "bundleVersion": "bundle-15dvk07", + "bundleVersion": "bundle-1vq244u", "build": { "id": "a2e8de30-03d5-4514-a3a6-406b3266a3af", "projectId": "d845c167-ea07-4875-b08d-83e97c09dcce", diff --git a/fixtures/ssg-netlify-by-project-id/.webstudio/data.json b/fixtures/ssg-netlify-by-project-id/.webstudio/data.json index 2c863e6d444d..dc941d0a68b7 100644 --- a/fixtures/ssg-netlify-by-project-id/.webstudio/data.json +++ b/fixtures/ssg-netlify-by-project-id/.webstudio/data.json @@ -1,5 +1,5 @@ { - "bundleVersion": "bundle-15dvk07", + "bundleVersion": "bundle-1vq244u", "build": { "id": "8f0fdff2-e76b-4f86-9203-ea7df5f1143c", "projectId": "8a7358b1-7de3-459d-b7b1-56dddfb6ce1e", diff --git a/fixtures/webstudio-cloudflare-template/.webstudio/data.json b/fixtures/webstudio-cloudflare-template/.webstudio/data.json index ccb3f06c247c..ede55011fb25 100644 --- a/fixtures/webstudio-cloudflare-template/.webstudio/data.json +++ b/fixtures/webstudio-cloudflare-template/.webstudio/data.json @@ -1,5 +1,5 @@ { - "bundleVersion": "bundle-15dvk07", + "bundleVersion": "bundle-1vq244u", "build": { "id": "a2e8de30-03d5-4514-a3a6-406b3266a3af", "projectId": "d845c167-ea07-4875-b08d-83e97c09dcce", diff --git a/fixtures/webstudio-features/.webstudio/data.json b/fixtures/webstudio-features/.webstudio/data.json index 299a0e8d123b..faf765e512eb 100644 --- a/fixtures/webstudio-features/.webstudio/data.json +++ b/fixtures/webstudio-features/.webstudio/data.json @@ -1,5 +1,5 @@ { - "bundleVersion": "bundle-15dvk07", + "bundleVersion": "bundle-1vq244u", "build": { "id": "1b66ee06-8ea5-4420-9a0e-e0d3a67aca32", "projectId": "cddc1d44-af37-4cb6-a430-d300cf6f932d", diff --git a/packages/asset-resource/src/hydration.test.ts b/packages/asset-resource/src/hydration.test.ts index ba4d12b76625..6d67fe2e5ab2 100644 --- a/packages/asset-resource/src/hydration.test.ts +++ b/packages/asset-resource/src/hydration.test.ts @@ -1,7 +1,6 @@ import { describe, expect, test, vi } from "vitest"; import { assetResourceLimits, type AssetFileDocument } from "@webstudio-is/sdk"; import { - AssetResourceHydrationError, hydrateAssetResourceResult, type AssetResourceContentReader, } from "./hydration"; @@ -207,7 +206,7 @@ describe("selected asset content hydration", () => { expect(read).not.toHaveBeenCalled(); }); - test("rejects binary, invalid UTF-8, and protected content clearly", async () => { + test("rejects binary and invalid UTF-8 while treating frontmatter as data", async () => { const binary = createDocument("binary", "png", { name: "binary.png", extension: "png", @@ -245,14 +244,8 @@ describe("selected asset content hydration", () => { options: { mode: "full" }, read, }) - ).rejects.toBeInstanceOf(AssetResourceHydrationError); - await expect( - hydrateAssetResourceResult({ - result: identity(protectedDocument), - documents: [protectedDocument], - options: { mode: "full" }, - read, - }) - ).rejects.toMatchObject({ code: "PROTECTED_CONTENT" }); + ).resolves.toMatchObject({ + content: { private: { text: "secret" } }, + }); }); }); diff --git a/packages/asset-resource/src/hydration.ts b/packages/asset-resource/src/hydration.ts index 024cd7029eb0..33842d375618 100644 --- a/packages/asset-resource/src/hydration.ts +++ b/packages/asset-resource/src/hydration.ts @@ -12,8 +12,7 @@ export class AssetResourceHydrationError extends Error { | "CONTENT_IDENTITY_REQUIRED" | "CONTENT_NOT_TEXT" | "CONTENT_DECODING_FAILED" - | "CONTENT_LIMIT_EXCEEDED" - | "PROTECTED_CONTENT"; + | "CONTENT_LIMIT_EXCEEDED"; readonly details?: Record; constructor({ @@ -174,13 +173,11 @@ export const hydrateAssetResourceResult = async ({ documents, options, read, - allowProtected = false, }: { result: unknown; documents: readonly AssetFileDocument[]; options: AssetResourceContentOptions; read: AssetResourceContentReader; - allowProtected?: boolean; }) => { if (options.mode === "none") { return { content: {}, hydratedFileCount: 0, hydratedBytes: 0 }; @@ -212,13 +209,6 @@ export const hydrateAssetResourceResult = async ({ details: { assetId: identity._id }, }); } - if (document.properties.private === true && allowProtected === false) { - throw new AssetResourceHydrationError({ - code: "PROTECTED_CONTENT", - message: "Selected asset content is protected", - details: { assetId: identity._id }, - }); - } if (isTextDocument(document) === false) { throw new AssetResourceHydrationError({ code: "CONTENT_NOT_TEXT", diff --git a/packages/asset-resource/src/index-storage.ts b/packages/asset-resource/src/index-storage.ts index eabb357c9723..2aefbdcea97e 100644 --- a/packages/asset-resource/src/index-storage.ts +++ b/packages/asset-resource/src/index-storage.ts @@ -14,6 +14,7 @@ export type ImmutableAssetResourceIndexStore = { status: "created" | "exists"; checksum: string; }>; + delete?: (key: string) => Promise<"deleted" | "missing">; }; const encodeKeySegment = (value: string) => { @@ -66,3 +67,7 @@ export const persistAssetResourceIndex = async ({ status: result.status, }; }; + +export type AssetResourceIndexGarbageCollectionStore = { + delete: NonNullable; +}; diff --git a/packages/asset-resource/src/index.test.ts b/packages/asset-resource/src/index.test.ts index bfba06bba03c..2ac4de23c87c 100644 --- a/packages/asset-resource/src/index.test.ts +++ b/packages/asset-resource/src/index.test.ts @@ -1,11 +1,7 @@ import { describe, expect, test, vi } from "vitest"; import { assetResourceLimits, type AssetFileDocument } from "@webstudio-is/sdk"; import { loadResource } from "@webstudio-is/sdk/runtime"; -import { - AssetResourceQueryExecutionError, - createAssetResourceRequest, - executeAssetResourceQuery, -} from "./index"; +import { createAssetResourceRequest, executeAssetResourceQuery } from "./index"; const createDocument = ( id: string, @@ -172,26 +168,4 @@ describe("bounded asset resource execution", () => { }) ).rejects.toMatchObject({ code: "RESULT_SIZE_EXCEEDED" }); }); - - test("rejects execution that exceeds the runtime limit", async () => { - const now = vi.fn().mockReturnValueOnce(0).mockReturnValueOnce(251); - await expect( - executeAssetResourceQuery({ - request: { - query: "*[0]", - parameters: {}, - resultLimit: 1, - content: { mode: "none" }, - }, - documents, - queryHash: "query-revision", - indexRevision: "index-revision", - assetRevision: "asset-revision", - now, - }) - ).rejects.toMatchObject({ - name: AssetResourceQueryExecutionError.name, - code: "QUERY_TIMEOUT", - }); - }); }); diff --git a/packages/asset-resource/src/index.ts b/packages/asset-resource/src/index.ts index bb13d23d9362..6b7274fd227d 100644 --- a/packages/asset-resource/src/index.ts +++ b/packages/asset-resource/src/index.ts @@ -10,6 +10,10 @@ import { } from "@webstudio-is/sdk"; import { assetsQueryResourceUrl } from "@webstudio-is/sdk/runtime"; import { validateAssetResourceQuery } from "./query-validation"; +import { + hydrateAssetResourceResult, + type AssetResourceContentReader, +} from "./hydration"; export * from "./markdown"; export * from "./canonical"; @@ -52,7 +56,6 @@ const evaluateAssetResourceQuery = async ({ export class AssetResourceQueryExecutionError extends Error { readonly code: | "MISSING_PARAMETER" - | "QUERY_TIMEOUT" | "RESULT_LIMIT_EXCEEDED" | "RESULT_SIZE_EXCEEDED"; readonly details?: Record; @@ -86,14 +89,12 @@ export const executeAssetResourceQuery = async ({ queryHash, indexRevision, assetRevision, - now = Date.now, }: { request: AssetResourceQueryRequest; documents: readonly AssetFileDocument[]; queryHash: string; indexRevision: string; assetRevision: string; - now?: () => number; }) => { const validated = validateAssetResourceQuery(request.query); for (const parameterName of validated.parameterNames) { @@ -116,24 +117,11 @@ export const executeAssetResourceQuery = async ({ }); } - const startedAt = now(); const rawResult = await evaluateAssetResourceQuery({ tree: validated.tree, documents, parameters: request.parameters, }); - const elapsedMs = now() - startedAt; - if (elapsedMs > assetResourceLimits.queryRuntimeMs) { - throw new AssetResourceQueryExecutionError({ - code: "QUERY_TIMEOUT", - message: "Asset resource query exceeded the runtime limit", - details: { - elapsedMs, - runtimeLimitMs: assetResourceLimits.queryRuntimeMs, - }, - }); - } - const result = rawResult ?? null; const resultCount = getResultCount(result); if (resultCount > request.resultLimit) { @@ -169,3 +157,33 @@ export const executeAssetResourceQuery = async ({ }, }); }; + +export const executeAndHydrateAssetResourceQuery = async ({ + read, + ...input +}: Parameters[0] & { + read?: AssetResourceContentReader; +}) => { + const response = await executeAssetResourceQuery(input); + if (input.request.content.mode === "none") { + return response; + } + if (read === undefined) { + throw new Error("Asset content reader is required for hydration"); + } + const hydration = await hydrateAssetResourceResult({ + result: response.result, + documents: input.documents, + options: input.request.content, + read, + }); + return { + ...response, + content: hydration.content, + meta: { + ...response.meta, + hydratedFileCount: hydration.hydratedFileCount, + hydratedBytes: hydration.hydratedBytes, + }, + }; +}; diff --git a/packages/asset-resource/src/published-runtime.test.ts b/packages/asset-resource/src/published-runtime.test.ts index bf5ee3da446e..1ac075f265f0 100644 --- a/packages/asset-resource/src/published-runtime.test.ts +++ b/packages/asset-resource/src/published-runtime.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; import { createAssetResourceIndex } from "./resource-index"; import { __testing, + createGeneratedAssetResourceFetch, createPublishedAssetResourceFetch, getPublishedAssetResourceCacheKey, } from "./published-runtime"; @@ -78,6 +79,12 @@ const createQueryRequest = (content: unknown = { mode: "none" }) => }), }); +const withResourceId = (request: Request, resourceId: string) => { + const headers = new Headers(request.headers); + headers.set("x-webstudio-resource-id", resourceId); + return new Request(request, { headers }); +}; + describe("published asset resource runtime", () => { beforeEach(() => __testing.clearParsedIndexCache()); @@ -116,6 +123,159 @@ describe("published asset resource runtime", () => { }); }); + test("uses the generated runtime adapter for asset queries and delegates other requests", async () => { + const { index, manifest } = await createRuntime(); + const assetBindingFetch = vi.fn(async (request: Request) => { + if (new URL(request.url).pathname === manifest[0].indexPath) { + return Response.json(index); + } + return new Response(null, { status: 404 }); + }); + const fallback = vi.fn(async () => new Response("fallback")); + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: new Request("https://site.example/blog/post"), + context: { + cloudflare: { env: { ASSETS: { fetch: assetBindingFetch } } }, + }, + deploymentId: "build-1", + manifest, + fallback, + }); + + const queryResponse = await generatedFetch(createQueryRequest()); + expect(await queryResponse.json()).toMatchObject({ + ok: true, + result: { _id: "post-1", title: "Post" }, + }); + expect(assetBindingFetch).toHaveBeenCalledOnce(); + + const fallbackResponse = await generatedFetch( + "https://external.example/data" + ); + expect(await fallbackResponse.text()).toBe("fallback"); + expect(fallback).toHaveBeenCalledOnce(); + }); + + test("isolates parsed indexes for deployments that use the same public path", async () => { + const first = await createRuntime(); + const secondIndex = await createAssetResourceIndex({ + format: "webstudio-resource-index", + version: 1, + resourceId: "posts", + query, + assetRevision: `sha256:${"b".repeat(64)}`, + queryMode: "parameterized", + parameterNames: ["slug"], + documents: [ + { + ...first.index.documents[0], + revision: `sha256:${"b".repeat(64)}`, + properties: { slug: "post", title: "Other deployment" }, + }, + ], + }); + const secondManifest = [ + { + ...first.manifest[0], + revision: secondIndex.integrity.checksum, + assetRevision: secondIndex.assetRevision, + }, + ]; + const secondFetchAsset = vi.fn(async () => Response.json(secondIndex)); + const secondRuntime = createPublishedAssetResourceFetch({ + deploymentId: "build-2", + manifest: secondManifest, + fetchAsset: secondFetchAsset, + }); + + expect( + await (await first.runtimeFetch(createQueryRequest()))?.json() + ).toMatchObject({ result: { title: "Post" } }); + expect( + await (await secondRuntime(createQueryRequest()))?.json() + ).toMatchObject({ result: { title: "Other deployment" } }); + expect(secondFetchAsset).toHaveBeenCalledOnce(); + }); + + test("selects identical queries by resource identity", async () => { + const first = await createRuntime(); + const secondIndex = await createAssetResourceIndex({ + format: "webstudio-resource-index", + version: 1, + resourceId: "other-posts", + query, + assetRevision: revision, + queryMode: "parameterized", + parameterNames: ["slug"], + documents: first.index.documents, + }); + const manifest = [ + first.manifest[0], + { + resourceId: "other-posts", + revision: secondIndex.integrity.checksum, + queryHash: secondIndex.queryHash, + assetRevision: secondIndex.assetRevision, + indexPath: "/resource-indexes/other-posts.json", + }, + ]; + const fetchAsset = vi.fn(async (path: string) => + Response.json( + path === "/resource-indexes/other-posts.json" + ? secondIndex + : first.index + ) + ); + const runtimeFetch = createPublishedAssetResourceFetch({ + deploymentId: "build-1", + manifest, + fetchAsset, + }); + + const ambiguous = await runtimeFetch(createQueryRequest()); + expect(await ambiguous?.json()).toMatchObject({ + ok: false, + error: { code: "INVALID_REQUEST" }, + }); + const selected = await runtimeFetch( + withResourceId(createQueryRequest(), "other-posts") + ); + expect((await selected?.json())?.ok).toBe(true); + expect(fetchAsset).toHaveBeenCalledWith( + "/resource-indexes/other-posts.json" + ); + }); + + test("executes successfully when the optional result cache fails", async () => { + const { manifest, fetchAsset } = await createRuntime(); + const cache = { + match: vi.fn(async () => { + throw new Error("Cache read failed"); + }), + put: vi.fn(async () => { + throw new Error("Cache write failed"); + }), + }; + const runtimeFetch = createPublishedAssetResourceFetch({ + deploymentId: "build-1", + manifest, + fetchAsset, + cache, + }); + const request = createQueryRequest(); + request.headers.set("cache-control", "public, max-age=60"); + + const response = await runtimeFetch(request); + + expect(response?.status).toBe(200); + expect(await response?.json()).toMatchObject({ + ok: true, + result: { title: "Post" }, + }); + expect(cache.match).toHaveBeenCalledOnce(); + expect(cache.put).toHaveBeenCalledOnce(); + }); + test("rejects stale revisions and keys caches by deployment, revision, parameters, and hydration", async () => { const { runtimeFetch, manifest } = await createRuntime(); const staleRequest = createQueryRequest(); diff --git a/packages/asset-resource/src/published-runtime.ts b/packages/asset-resource/src/published-runtime.ts index 20f3092d3dab..a9b19212caea 100644 --- a/packages/asset-resource/src/published-runtime.ts +++ b/packages/asset-resource/src/published-runtime.ts @@ -2,18 +2,17 @@ import { assetResourceQueryFailure, assetResourceQueryRequest, type AssetResourceQueryFailure, - type AssetResourceQuerySuccess, } from "@webstudio-is/sdk"; -import { assetsQueryResourceUrl } from "@webstudio-is/sdk/runtime"; +import { + assetResourceIdHeader, + assetsQueryResourceUrl, +} from "@webstudio-is/sdk/runtime"; import { AssetResourceQueryExecutionError, computeAssetResourceQueryHash, - executeAssetResourceQuery, + executeAndHydrateAssetResourceQuery, } from "./index"; -import { - AssetResourceHydrationError, - hydrateAssetResourceResult, -} from "./hydration"; +import { AssetResourceHydrationError } from "./hydration"; import { verifyAssetResourceIndex } from "./resource-index"; export type PublishedAssetResourceManifestEntry = { @@ -29,19 +28,35 @@ export type PublishedAssetFetch = ( init?: RequestInit ) => Promise; +type AssetBinding = { + fetch: (request: Request) => Promise; +}; + const parsedIndexCache = new Map< string, Promise>> >(); +const getParsedIndexCacheKey = ({ + deploymentId, + entry, +}: { + deploymentId: string; + entry: PublishedAssetResourceManifestEntry; +}) => + `${deploymentId}\n${entry.resourceId}\n${entry.revision}\n${entry.indexPath}`; + const loadIndex = ({ + deploymentId, entry, fetchAsset, }: { + deploymentId: string; entry: PublishedAssetResourceManifestEntry; fetchAsset: PublishedAssetFetch; }) => { - let pending = parsedIndexCache.get(entry.indexPath); + const cacheKey = getParsedIndexCacheKey({ deploymentId, entry }); + let pending = parsedIndexCache.get(cacheKey); if (pending === undefined) { pending = (async () => { const response = await fetchAsset(entry.indexPath); @@ -59,8 +74,8 @@ const loadIndex = ({ } return index; })(); - parsedIndexCache.set(entry.indexPath, pending); - pending.catch(() => parsedIndexCache.delete(entry.indexPath)); + parsedIndexCache.set(cacheKey, pending); + pending.catch(() => parsedIndexCache.delete(cacheKey)); } return pending; }; @@ -130,9 +145,15 @@ export const createPublishedAssetResourceFetch = ({ fetchAsset: PublishedAssetFetch; cache?: Pick; }) => { - const entriesByQueryHash = new Map( - manifest.map((entry) => [entry.queryHash, entry]) - ); + const entriesByQueryHash = new Map< + string, + PublishedAssetResourceManifestEntry[] + >(); + for (const entry of manifest) { + const entries = entriesByQueryHash.get(entry.queryHash) ?? []; + entries.push(entry); + entriesByQueryHash.set(entry.queryHash, entries); + } return async ( input: RequestInfo | URL, init?: RequestInit @@ -146,7 +167,9 @@ export const createPublishedAssetResourceFetch = ({ } let parsedRequest; try { - parsedRequest = assetResourceQueryRequest.parse(await request.json()); + parsedRequest = assetResourceQueryRequest.parse( + await request.clone().json() + ); } catch { return failure({ code: "INVALID_REQUEST", @@ -155,12 +178,25 @@ export const createPublishedAssetResourceFetch = ({ }); } const queryHash = await computeAssetResourceQueryHash(parsedRequest.query); - const entry = entriesByQueryHash.get(queryHash); + const matchingEntries = entriesByQueryHash.get(queryHash) ?? []; + const resourceId = request.headers.get(assetResourceIdHeader); + const entry = + resourceId === null + ? matchingEntries.length === 1 + ? matchingEntries[0] + : undefined + : matchingEntries.find((entry) => entry.resourceId === resourceId); if (entry === undefined) { return failure({ - code: "INDEX_NOT_FOUND", - message: "No published index matches this asset resource query", - status: 404, + code: + resourceId === null && matchingEntries.length > 1 + ? "INVALID_REQUEST" + : "INDEX_NOT_FOUND", + message: + resourceId === null && matchingEntries.length > 1 + ? "Asset resource identity is required for this query" + : "No published index matches this asset resource query", + status: resourceId === null && matchingEntries.length > 1 ? 400 : 404, }); } if ( @@ -184,25 +220,22 @@ export const createPublishedAssetResourceFetch = ({ request, }); if (cacheKey !== undefined) { - const cached = await selectedCache?.match(cacheKey); + const cached = await selectedCache + ?.match(cacheKey) + .catch(() => undefined); if (cached !== undefined) { return new Response(cached.body, cached); } } try { - const index = await loadIndex({ entry, fetchAsset }); - const response = await executeAssetResourceQuery({ + const index = await loadIndex({ deploymentId, entry, fetchAsset }); + const result = await executeAndHydrateAssetResourceQuery({ request: parsedRequest, documents: index.documents, queryHash: index.queryHash, indexRevision: entry.revision, assetRevision: index.assetRevision, - }); - const hydration = await hydrateAssetResourceResult({ - result: response.result, - documents: index.documents, - options: parsedRequest.content, read: async (contentRef, range) => { const headers = new Headers(); if (range !== undefined && range.length > 0) { @@ -226,22 +259,15 @@ export const createPublishedAssetResourceFetch = ({ }; }, }); - const result: AssetResourceQuerySuccess = { - ...response, - content: hydration.content, - meta: { - ...response.meta, - hydratedFileCount: hydration.hydratedFileCount, - hydratedBytes: hydration.hydratedBytes, - }, - }; const resultResponse = jsonResponse(result); if (cacheKey !== undefined) { resultResponse.headers.set( "cache-control", request.headers.get("cache-control") as string ); - await cache?.put(cacheKey, resultResponse.clone()); + await cache + ?.put(cacheKey, resultResponse.clone()) + .catch(() => undefined); } return resultResponse; } catch (error) { @@ -252,7 +278,7 @@ export const createPublishedAssetResourceFetch = ({ return failure({ code: error.code, message: error.message, - status: error.code === "PROTECTED_CONTENT" ? 403 : 400, + status: 400, }); } return failure({ @@ -265,6 +291,59 @@ export const createPublishedAssetResourceFetch = ({ }; }; +const getAssetBinding = (context: unknown): AssetBinding | undefined => { + if (typeof context !== "object" || context === null) { + return; + } + const cloudflare = Reflect.get(context, "cloudflare"); + if (typeof cloudflare !== "object" || cloudflare === null) { + return; + } + const env = Reflect.get(cloudflare, "env"); + if (typeof env !== "object" || env === null) { + return; + } + const assets = Reflect.get(env, "ASSETS"); + if ( + typeof assets === "object" && + assets !== null && + typeof Reflect.get(assets, "fetch") === "function" + ) { + return assets as AssetBinding; + } +}; + +export const createGeneratedAssetResourceFetch = async ({ + request, + context, + deploymentId, + manifest, + fallback, +}: { + request: Request; + context: unknown; + deploymentId: string; + manifest: readonly PublishedAssetResourceManifestEntry[]; + fallback: typeof fetch; +}): Promise => { + const binding = getAssetBinding(context); + const fetchAsset = (path: string, init?: RequestInit) => { + const assetRequest = new Request(new URL(path, request.url), init); + return binding?.fetch(assetRequest) ?? fetch(assetRequest); + }; + const cache = await globalThis.caches + ?.open(`webstudio-assets-${deploymentId}`) + .catch(() => undefined); + const fetchResource = createPublishedAssetResourceFetch({ + deploymentId, + manifest, + fetchAsset, + cache, + }); + return async (input, init) => + (await fetchResource(input, init)) ?? fallback(input, init); +}; + export const __testing = { clearParsedIndexCache: () => parsedIndexCache.clear(), }; diff --git a/packages/asset-resource/src/query-validation.test.ts b/packages/asset-resource/src/query-validation.test.ts index 282bc5c509b3..1b61bc674be0 100644 --- a/packages/asset-resource/src/query-validation.test.ts +++ b/packages/asset-resource/src/query-validation.test.ts @@ -30,6 +30,7 @@ describe("asset resource query validation", () => { parameterNames: ["slug"], astNodes: expect.any(Number), astDepth: expect.any(Number), + datasetScans: 1, }); }); @@ -84,6 +85,13 @@ describe("asset resource query validation", () => { ); }); + test("rejects more than one dataset scan", () => { + expectValidationCode( + () => validateAssetResourceQuery(`{"first": *[], "second": *[]}`), + "QUERY_COMPLEXITY_EXCEEDED" + ); + }); + test("extracts complete referenced asset field paths", () => { expect( getAssetResourceReferencedFieldPaths( diff --git a/packages/asset-resource/src/query-validation.ts b/packages/asset-resource/src/query-validation.ts index b23924a91129..dc76dba1fdb0 100644 --- a/packages/asset-resource/src/query-validation.ts +++ b/packages/asset-resource/src/query-validation.ts @@ -33,9 +33,13 @@ const isAstNode = (value: unknown): value is AstNode => const measureAst = (tree: ExprNode) => { let nodes = 0; let maxDepth = 0; + let datasetScans = 0; const visit = (node: AstNode, depth: number) => { nodes += 1; maxDepth = Math.max(maxDepth, depth); + if (node.type === "Everything") { + datasetScans += 1; + } if ( nodes > assetResourceLimits.queryAstNodes || maxDepth > assetResourceLimits.queryAstDepth || @@ -58,7 +62,7 @@ const measureAst = (tree: ExprNode) => { } }; visit(tree, 1); - return { nodes, depth: maxDepth }; + return { nodes, depth: maxDepth, datasetScans }; }; export type ValidatedAssetResourceQuery = { @@ -67,6 +71,7 @@ export type ValidatedAssetResourceQuery = { queryMode: "static" | "parameterized"; astNodes: number; astDepth: number; + datasetScans: number; }; export const validateAssetResourceQuery = ( @@ -102,7 +107,8 @@ export const validateAssetResourceQuery = ( const ast = measureAst(tree); if ( ast.nodes > assetResourceLimits.queryAstNodes || - ast.depth > assetResourceLimits.queryAstDepth + ast.depth > assetResourceLimits.queryAstDepth || + ast.datasetScans > assetResourceLimits.queryDatasetScans ) { throw new AssetResourceQueryValidationError({ code: "QUERY_COMPLEXITY_EXCEEDED", @@ -112,6 +118,8 @@ export const validateAssetResourceQuery = ( astNodeLimit: assetResourceLimits.queryAstNodes, astDepth: ast.depth, astDepthLimit: assetResourceLimits.queryAstDepth, + datasetScans: ast.datasetScans, + datasetScanLimit: assetResourceLimits.queryDatasetScans, }, }); } @@ -132,6 +140,7 @@ export const validateAssetResourceQuery = ( queryMode: parameterNames.length === 0 ? "static" : "parameterized", astNodes: ast.nodes, astDepth: ast.depth, + datasetScans: ast.datasetScans, }; }; diff --git a/packages/asset-resource/src/resource-index.test.ts b/packages/asset-resource/src/resource-index.test.ts index 9a0ab96c911f..79aeea28f0a9 100644 --- a/packages/asset-resource/src/resource-index.test.ts +++ b/packages/asset-resource/src/resource-index.test.ts @@ -76,7 +76,7 @@ describe("resource index build", () => { await expect(verifyAssetResourceIndex(index)).resolves.toEqual(index); }); - test("excludes draft and private documents before building public candidates", async () => { + test("treats all frontmatter fields as schemaless query data", async () => { const entries = [ createCanonicalAssetFileEntry({ projectId: "project-1", @@ -101,8 +101,12 @@ describe("resource index build", () => { entries, }); - expect(index.documents.map(({ _id }) => _id)).toEqual(["published"]); - expect(serializeAssetResourceIndex(index)).not.toContain("secret"); + expect(index.documents.map(({ _id }) => _id)).toEqual([ + "draft", + "private", + "published", + ]); + expect(serializeAssetResourceIndex(index)).toContain("secret"); }); test("rejects mixed projects, inconsistent identities, and duplicate assets", async () => { diff --git a/packages/asset-resource/src/resource-index.ts b/packages/asset-resource/src/resource-index.ts index 19f6422b36eb..97b0e12abca0 100644 --- a/packages/asset-resource/src/resource-index.ts +++ b/packages/asset-resource/src/resource-index.ts @@ -20,10 +20,6 @@ const compareStrings = (left: string, right: string) => { return 0; }; -export const isPublicAssetResourceDocument = ( - document: AssetResourceIndexV1["documents"][number] -) => document.properties.draft !== true && document.properties.private !== true; - export type AssetResourceIndexInput = Omit< AssetResourceIndexV1, "parameterNames" | "documents" @@ -125,9 +121,7 @@ export const buildAssetResourceIndex = async ({ const selection = await selectAssetResourceCandidates({ tree: validatedQuery.tree, - documents: entries - .map(({ document }) => document) - .filter(isPublicAssetResourceDocument), + documents: entries.map(({ document }) => document), }); return await createAssetResourceIndex({ format: "webstudio-resource-index", diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.test.ts b/packages/asset-uploader/src/canonical-metadata-backfill.test.ts index 571c3c9c8bf5..793030e7a5c1 100644 --- a/packages/asset-uploader/src/canonical-metadata-backfill.test.ts +++ b/packages/asset-uploader/src/canonical-metadata-backfill.test.ts @@ -166,6 +166,50 @@ describe("canonical asset metadata synchronization", () => { ); }); + test("indexes an empty Markdown file without a storage range read", async () => { + const readFile = vi.fn(); + let document: Record | undefined; + server.use( + db.get("Asset", () => + json([ + { + id: "empty", + projectId: "project-1", + filename: "empty.md", + folderId: null, + file: { + name: "empty.md", + size: 0, + updatedAt: "2026-07-18T01:00:00.000Z", + status: "UPLOADED", + }, + }, + ]) + ), + db.get("AssetFolder", () => json([])), + db.post("rpc/replace_asset_file_metadata", async ({ request }) => { + document = ((await request.json()) as ReplaceMetadataRpcArgs) + .p_document; + return json(true); + }) + ); + + await expect( + synchronizeCanonicalAsset({ + projectId: "project-1", + assetId: "empty", + client: testContext.postgrest.client, + assetClient: { uploadFile: vi.fn(), readFile }, + }) + ).resolves.toMatchObject({ status: "indexed" }); + expect(readFile).not.toHaveBeenCalled(); + expect(document).toMatchObject({ + properties: {}, + size: 0, + contentRef: "empty.md", + }); + }); + test("creates a stable revision from immutable storage identity", () => { expect( createAssetContentRevision({ diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.ts b/packages/asset-uploader/src/canonical-metadata-backfill.ts index 35f9fa653c2d..e7d58c417a59 100644 --- a/packages/asset-uploader/src/canonical-metadata-backfill.ts +++ b/packages/asset-uploader/src/canonical-metadata-backfill.ts @@ -147,7 +147,7 @@ const createCanonicalDocument = ({ name, ...(extension === undefined ? {} : { extension }), ...(folderId === undefined ? {} : { folderId, folderNames }), - mimeType: getMimeTypeByFilename(name), + mimeType: getMimeTypeByFilename(asset.file.name), size: asset.file.size, revision, contentRef: asset.file.name, @@ -187,14 +187,15 @@ const indexCanonicalAsset = async ({ let excerpt: string | undefined; if (markdownExtension.test(asset.file.name)) { const prefixLength = Math.min( - Math.max(1, asset.file.size), + asset.file.size, assetResourceLimits.hydratedFileBytes ); - const stored = await assetClient.readFile(asset.file.name, { - offset: 0, - length: prefixLength, - }); - const bytes = await readPrefix(stored.data, prefixLength); + const bytes = + prefixLength === 0 + ? new Uint8Array() + : await assetClient + .readFile(asset.file.name, { offset: 0, length: prefixLength }) + .then((stored) => readPrefix(stored.data, prefixLength)); const markdown = await Promise.all([ extractMarkdownFrontmatter(bytes), extractMarkdownBodyAndExcerpt(bytes), diff --git a/packages/asset-uploader/src/clients/fs/immutable-object.test.ts b/packages/asset-uploader/src/clients/fs/immutable-object.test.ts index 64755e2da27a..3a24e67ee2c1 100644 --- a/packages/asset-uploader/src/clients/fs/immutable-object.test.ts +++ b/packages/asset-uploader/src/clients/fs/immutable-object.test.ts @@ -36,5 +36,7 @@ describe("filesystem immutable resource index storage", () => { await expect( readFile(join(directory, "projects/one/index.json"), "utf8") ).resolves.toBe("one"); + await expect(store.delete?.(object.key)).resolves.toBe("deleted"); + await expect(store.delete?.(object.key)).resolves.toBe("missing"); }); }); diff --git a/packages/asset-uploader/src/clients/fs/immutable-object.ts b/packages/asset-uploader/src/clients/fs/immutable-object.ts index bf3940fafa37..3af87d466d9d 100644 --- a/packages/asset-uploader/src/clients/fs/immutable-object.ts +++ b/packages/asset-uploader/src/clients/fs/immutable-object.ts @@ -1,33 +1,54 @@ -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, readFile, unlink, writeFile } from "node:fs/promises"; import { dirname, resolve, sep } from "node:path"; import type { ImmutableAssetResourceIndexStore } from "@webstudio-is/asset-resource"; export const createFsImmutableResourceIndexStore = ( directory: string -): ImmutableAssetResourceIndexStore => ({ - putIfAbsent: async (object) => { +): ImmutableAssetResourceIndexStore => { + const resolvePath = (key: string) => { const root = resolve(directory); - const path = resolve(root, object.key); + const path = resolve(root, key); if (path === root || path.startsWith(`${root}${sep}`) === false) { throw new Error("Resource index path escapes immutable storage"); } - await mkdir(dirname(path), { recursive: true }); - try { - await writeFile(path, object.data, { flag: "wx" }); - return { status: "created", checksum: object.checksum }; - } catch (error) { - if ( - error instanceof Error && - "code" in error && - error.code === "EEXIST" - ) { - const existing = await readFile(path); - if (existing.equals(Buffer.from(object.data)) === false) { - throw new Error("Immutable resource index already has other bytes"); + return path; + }; + return { + putIfAbsent: async (object) => { + const path = resolvePath(object.key); + await mkdir(dirname(path), { recursive: true }); + try { + await writeFile(path, object.data, { flag: "wx" }); + return { status: "created", checksum: object.checksum }; + } catch (error) { + if ( + error instanceof Error && + "code" in error && + error.code === "EEXIST" + ) { + const existing = await readFile(path); + if (existing.equals(Buffer.from(object.data)) === false) { + throw new Error("Immutable resource index already has other bytes"); + } + return { status: "exists", checksum: object.checksum }; } - return { status: "exists", checksum: object.checksum }; + throw error; } - throw error; - } - }, -}); + }, + delete: async (key) => { + try { + await unlink(resolvePath(key)); + return "deleted"; + } catch (error) { + if ( + error instanceof Error && + "code" in error && + error.code === "ENOENT" + ) { + return "missing"; + } + throw error; + } + }, + }; +}; diff --git a/packages/asset-uploader/src/clients/s3/immutable-object.test.ts b/packages/asset-uploader/src/clients/s3/immutable-object.test.ts index f92c539aa173..4b630bc1b850 100644 --- a/packages/asset-uploader/src/clients/s3/immutable-object.test.ts +++ b/packages/asset-uploader/src/clients/s3/immutable-object.test.ts @@ -1,6 +1,9 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import type { SignatureV4 } from "@smithy/signature-v4"; -import { putImmutableObjectToS3 } from "./immutable-object"; +import { + deleteImmutableObjectFromS3, + putImmutableObjectToS3, +} from "./immutable-object"; afterEach(() => vi.unstubAllGlobals()); @@ -99,4 +102,25 @@ describe("immutable S3 object persistence", () => { }) ).rejects.toThrow("has no checksum"); }); + + test("deletes an immutable object idempotently", async () => { + const { signer, sign } = createSigner(); + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 204 })) + .mockResolvedValueOnce(new Response(null, { status: 404 })); + vi.stubGlobal("fetch", fetch); + const input = { + signer, + endpoint: "https://storage.example", + bucket: "private-indexes", + key: object.key, + }; + + await expect(deleteImmutableObjectFromS3(input)).resolves.toBe("deleted"); + await expect(deleteImmutableObjectFromS3(input)).resolves.toBe("missing"); + expect(sign).toHaveBeenCalledWith( + expect.objectContaining({ method: "DELETE" }) + ); + }); }); diff --git a/packages/asset-uploader/src/clients/s3/immutable-object.ts b/packages/asset-uploader/src/clients/s3/immutable-object.ts index 8e6e769436e8..b0600c22fd5c 100644 --- a/packages/asset-uploader/src/clients/s3/immutable-object.ts +++ b/packages/asset-uploader/src/clients/s3/immutable-object.ts @@ -71,3 +71,41 @@ export const putImmutableObjectToS3 = async ({ } return { status: "exists", checksum }; }; + +export const deleteImmutableObjectFromS3 = async ({ + signer, + endpoint, + bucket, + key, +}: { + signer: SignatureV4; + endpoint: string; + bucket: string; + key: string; +}) => { + const url = new URL( + `/${bucket}/${extendedEncodeURIComponent(key)}`, + endpoint + ); + const request = await signer.sign({ + method: "DELETE", + protocol: url.protocol, + hostname: url.hostname, + path: url.pathname, + headers: { + "x-amz-date": new Date().toISOString(), + "x-amz-content-sha256": "UNSIGNED-PAYLOAD", + }, + }); + const response = await fetch(url, { + method: request.method, + headers: request.headers, + }); + if (response.status === 404) { + return "missing" as const; + } + if (response.ok === false) { + throw new Error("Cannot delete immutable resource index"); + } + return "deleted" as const; +}; diff --git a/packages/asset-uploader/src/clients/s3/s3.ts b/packages/asset-uploader/src/clients/s3/s3.ts index 55c4ffd5e106..e6e31f63c8e0 100644 --- a/packages/asset-uploader/src/clients/s3/s3.ts +++ b/packages/asset-uploader/src/clients/s3/s3.ts @@ -6,7 +6,10 @@ import type { } from "../../client"; import { uploadToS3 } from "./upload"; import { readFromS3 } from "./read"; -import { putImmutableObjectToS3 } from "./immutable-object"; +import { + deleteImmutableObjectFromS3, + putImmutableObjectToS3, +} from "./immutable-object"; type S3ClientOptions = { endpoint: string; @@ -63,6 +66,13 @@ export const createS3Client = ( bucket: options.bucket, object, }), + delete: (key) => + deleteImmutableObjectFromS3({ + signer, + endpoint: options.endpoint, + bucket: options.bucket, + key, + }), }, uploadFile, readFile: (name, range) => diff --git a/packages/asset-uploader/src/delete.test.ts b/packages/asset-uploader/src/delete.test.ts deleted file mode 100644 index a73992fc1254..000000000000 --- a/packages/asset-uploader/src/delete.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { describe, test, expect } from "vitest"; -import { - createTestServer, - db, - json, - empty, - testContext, -} from "@webstudio-is/postgrest/testing"; -import type { AppContext } from "@webstudio-is/trpc-interface/index.server"; -import { AuthorizationError } from "@webstudio-is/trpc-interface/index.server"; -import { deleteAssets } from "./delete"; - -const server = createTestServer(); - -const uid = () => `proj-${Math.random().toString(36).slice(2)}`; - -const createContext = (): AppContext => - ({ - ...testContext, - authorization: { type: "user", userId: "user-1" }, - getOwnerPlanFeatures: async () => ({}), - }) as unknown as AppContext; - -/** hasProjectPermit: direct ownership check returns a row for a given projectId */ -const ownershipHandler = db.get("Project", ({ request }) => { - const url = new URL(request.url); - if (url.searchParams.has("userId")) { - return json({ id: url.searchParams.get("id")?.replace("eq.", "") }); - } - return json(null); -}); - -describe("deleteAssets (msw)", () => { - test("throws AuthorizationError when caller lacks edit access", async () => { - const projectId = uid(); - server.use( - db.get("Project", () => json(null)), - db.get("WorkspaceProjectAuthorization", () => json([])) - ); - - await expect( - deleteAssets({ ids: ["asset-1"], projectId }, createContext()) - ).rejects.toThrow(AuthorizationError); - }); - - test("throws when no assets found", async () => { - const projectId = uid(); - server.use( - ownershipHandler, - db.get("Asset", () => json([])) - ); - - await expect( - deleteAssets({ ids: ["asset-1"], projectId }, createContext()) - ).rejects.toThrow("Assets not found"); - }); - - test("deletes assets and marks unused file as deleted", async () => { - const projectId = uid(); - const assetRow = { - id: "asset-1", - projectId, - name: "photo.jpg", - file: { name: "photo.jpg" }, - }; - let patchedProject = false; - let deletedAsset = false; - - server.use( - ownershipHandler, - db.get("Asset", ({ request }) => { - const url = new URL(request.url); - // usage check (in name filter) — no other assets use this file - if (url.searchParams.get("name")) { - return json([]); - } - return json([assetRow]); - }), - db.patch("Project", () => { - patchedProject = true; - return json({ id: projectId }); - }), - db.delete("Asset", () => { - deletedAsset = true; - return empty({ status: 204 }); - }), - db.patch("File", () => empty({ status: 204 })) - ); - - await deleteAssets({ ids: ["asset-1"], projectId }, createContext()); - - expect(patchedProject).toBe(true); - expect(deletedAsset).toBe(true); - }); - - test("does not mark file deleted when another asset still uses it", async () => { - const projectId = uid(); - const assetRow = { - id: "asset-1", - projectId, - name: "photo.jpg", - file: { name: "photo.jpg" }, - }; - let markedFileDeleted = false; - let assetGetCount = 0; - - server.use( - ownershipHandler, - db.get("Asset", () => { - assetGetCount += 1; - // First call: load the assets to delete. Second call: usage check. - if (assetGetCount === 1) { - return json([assetRow]); - } - // usage check — another asset still references the file - return json([{ name: "photo.jpg" }]); - }), - db.patch("Project", () => json({ id: projectId })), - db.delete("Asset", () => empty({ status: 204 })), - db.patch("File", () => { - markedFileDeleted = true; - return empty({ status: 204 }); - }) - ); - - await deleteAssets({ ids: ["asset-1"], projectId }, createContext()); - - expect(markedFileDeleted).toBe(false); - }); -}); diff --git a/packages/asset-uploader/src/delete.ts b/packages/asset-uploader/src/delete.ts deleted file mode 100644 index 83562287caef..000000000000 --- a/packages/asset-uploader/src/delete.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { - authorizeProject, - type AppContext, - AuthorizationError, -} from "@webstudio-is/trpc-interface/index.server"; -import type { Asset } from "@webstudio-is/sdk"; -import { deleteAssetsWithClient } from "./asset-patch-core"; - -export const deleteAssets = async ( - props: { - ids: Array; - projectId: string; - }, - context: AppContext -): Promise => { - const canDelete = await authorizeProject.hasProjectPermit( - { projectId: props.projectId, permit: "edit" }, - context - ); - - if (canDelete === false) { - throw new AuthorizationError( - "You don't have access to delete this project assets" - ); - } - - await deleteAssetsWithClient(props, context.postgrest.client); -}; diff --git a/packages/asset-uploader/src/index.server.ts b/packages/asset-uploader/src/index.server.ts index dab28ce33c7b..b588f23fa4eb 100644 --- a/packages/asset-uploader/src/index.server.ts +++ b/packages/asset-uploader/src/index.server.ts @@ -2,7 +2,6 @@ export * from "./db"; export * from "./content-hash"; export * from "./utils/size-limiter"; export * from "./upload"; -export * from "./delete"; export * from "./patch"; export * from "./asset-patch-core"; export * from "./folder-persistence"; @@ -21,3 +20,4 @@ export * from "./resource-index-status"; export * from "./resource-index-rebuild"; export * from "./resource-index-snapshot"; export * from "./resource-index-lifecycle"; +export * from "./resource-index-garbage-collection"; diff --git a/packages/asset-uploader/src/query-preview.ts b/packages/asset-uploader/src/query-preview.ts index e61704d82097..bed31130c317 100644 --- a/packages/asset-uploader/src/query-preview.ts +++ b/packages/asset-uploader/src/query-preview.ts @@ -1,8 +1,7 @@ import { computeAssetResourceQueryHash, computeCanonicalAssetRevision, - executeAssetResourceQuery, - hydrateAssetResourceResult, + executeAndHydrateAssetResourceQuery, } from "@webstudio-is/asset-resource"; import type { AssetResourceQueryRequest } from "@webstudio-is/sdk"; import type { AssetClient } from "./client"; @@ -38,33 +37,12 @@ export const previewAssetResourceQuery = async ({ projectId, }); const assetRevision = await computeCanonicalAssetRevision(entries); - const response = await executeAssetResourceQuery({ + return await executeAndHydrateAssetResourceQuery({ request, documents: entries.map(({ document }) => document), queryHash: await computeAssetResourceQueryHash(request.query), indexRevision: `metadata:${assetRevision}`, assetRevision, + read: assetClient?.readFile, }); - if (request.content.mode === "none") { - return response; - } - if (assetClient === undefined) { - throw new Error("Asset client is required for content hydration"); - } - const hydration = await hydrateAssetResourceResult({ - result: response.result, - documents: entries.map(({ document }) => document), - options: request.content, - read: assetClient.readFile, - allowProtected: true, - }); - return { - ...response, - content: hydration.content, - meta: { - ...response.meta, - hydratedFileCount: hydration.hydratedFileCount, - hydratedBytes: hydration.hydratedBytes, - }, - }; }; diff --git a/packages/asset-uploader/src/resource-index-build.test.ts b/packages/asset-uploader/src/resource-index-build.test.ts index 70421237d94c..10fb94069c96 100644 --- a/packages/asset-uploader/src/resource-index-build.test.ts +++ b/packages/asset-uploader/src/resource-index-build.test.ts @@ -52,9 +52,15 @@ describe("resource index build and activation", () => { p_resource_id: "posts", p_revision: expect.stringMatching(/^sha256:/), p_checksum: expect.stringMatching(/^sha256:/), - p_object_key: expect.stringContaining("projects/project-1/resources/posts"), + p_object_key: expect.stringContaining( + "projects/project-1/resources/posts" + ), }); return json(true); + }), + db.post("rpc/claim_asset_resource_index_garbage", () => { + events.push("cleanup"); + return json([]); }) ); const result = await buildPersistAndActivateAssetResourceIndex({ @@ -64,6 +70,7 @@ describe("resource index build and activation", () => { events.push("persist"); return { status: "created", checksum }; }, + delete: async () => "deleted", }, projectId: "project-1", resourceId: "posts", @@ -71,7 +78,7 @@ describe("resource index build and activation", () => { entries: [entry], }); - expect(events).toEqual(["begin", "persist", "activate"]); + expect(events).toEqual(["begin", "persist", "activate", "cleanup"]); expect(result.index.documents).toHaveLength(1); }); diff --git a/packages/asset-uploader/src/resource-index-build.ts b/packages/asset-uploader/src/resource-index-build.ts index d73de1b87296..532406e9dc45 100644 --- a/packages/asset-uploader/src/resource-index-build.ts +++ b/packages/asset-uploader/src/resource-index-build.ts @@ -1,9 +1,6 @@ import { buildAssetResourceIndex, - computeAssetResourceQueryHash, - computeCanonicalAssetRevision, persistAssetResourceIndex, - validateAssetResourceQuery, type CanonicalAssetFileEntry, type ImmutableAssetResourceIndexStore, } from "@webstudio-is/asset-resource"; @@ -14,6 +11,7 @@ import { cancelAssetResourceIndexBuild, failAssetResourceIndexBuild, } from "./resource-index-persistence"; +import { collectAssetResourceIndexGarbageBestEffort } from "./resource-index-garbage-collection"; export class AssetResourceIndexBuildSupersededError extends Error { constructor() { @@ -53,27 +51,23 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ signal?: AbortSignal; }) => { assertNotCancelled(signal); - validateAssetResourceQuery(query); - const queryHash = await computeAssetResourceQueryHash(query); - const assetRevision = await computeCanonicalAssetRevision(entries); + const index = await buildAssetResourceIndex({ + projectId, + resourceId, + query, + entries, + }); + assertNotCancelled(signal); await beginAssetResourceIndexBuild({ client, projectId, resourceId, query, - queryHash, - assetRevision, + queryHash: index.queryHash, + assetRevision: index.assetRevision, }); try { - assertNotCancelled(signal); - const index = await buildAssetResourceIndex({ - projectId, - resourceId, - query, - entries, - }); - assertNotCancelled(signal); const persisted = await persistAssetResourceIndex({ store, projectId, @@ -93,6 +87,12 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ if (activated === false) { throw new AssetResourceIndexBuildSupersededError(); } + if (store.delete !== undefined) { + await collectAssetResourceIndexGarbageBestEffort({ + client, + store: { delete: store.delete }, + }); + } return { index, persisted }; } catch (error) { if (error instanceof AssetResourceIndexBuildCancelledError) { @@ -100,8 +100,8 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ client, projectId, resourceId, - queryHash, - assetRevision, + queryHash: index.queryHash, + assetRevision: index.assetRevision, }); throw error; } @@ -110,8 +110,8 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ client, projectId, resourceId, - queryHash, - assetRevision, + queryHash: index.queryHash, + assetRevision: index.assetRevision, }); } catch (stateError) { throw new AggregateError( diff --git a/packages/asset-uploader/src/resource-index-garbage-collection.test.ts b/packages/asset-uploader/src/resource-index-garbage-collection.test.ts new file mode 100644 index 000000000000..5ce3991e8b89 --- /dev/null +++ b/packages/asset-uploader/src/resource-index-garbage-collection.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, test, vi } from "vitest"; +import { + createTestServer, + db, + json, + testContext, +} from "@webstudio-is/postgrest/testing"; +import { collectAssetResourceIndexGarbage } from "./resource-index-garbage-collection"; + +const server = createTestServer(); +const candidate = { + projectId: "project-1", + resourceId: "posts", + revision: `sha256:${"a".repeat(64)}`, + objectKey: "projects/project-1/resources/posts/index.json", + gcClaimId: "claim-1", +}; + +describe("resource index garbage collection", () => { + test("deletes claimed objects and finalizes their rows", async () => { + server.use( + db.post("rpc/claim_asset_resource_index_garbage", async ({ request }) => { + expect(await request.json()).toEqual({ + p_before: "2026-07-20T00:00:00.000Z", + p_limit: 10, + }); + return json([candidate]); + }), + db.post("rpc/finalize_asset_resource_index_garbage", () => json(true)) + ); + const remove = vi.fn(async () => "deleted" as const); + + await expect( + collectAssetResourceIndexGarbage({ + client: testContext.postgrest.client, + store: { delete: remove }, + now: new Date("2026-07-20T00:00:00Z"), + limit: 10, + }) + ).resolves.toEqual({ claimed: 1, deleted: 1, missing: 0 }); + expect(remove).toHaveBeenCalledWith(candidate.objectKey); + }); + + test("releases a claim after object storage fails", async () => { + server.use( + db.post("rpc/claim_asset_resource_index_garbage", () => + json([candidate]) + ), + db.post("rpc/release_asset_resource_index_garbage_claim", () => + json(true) + ) + ); + + await expect( + collectAssetResourceIndexGarbage({ + client: testContext.postgrest.client, + store: { + delete: async () => { + throw new Error("R2 unavailable"); + }, + }, + }) + ).rejects.toThrow("Failed to collect 1"); + }); + + test("keeps a claim when database finalization fails after deletion", async () => { + const release = vi.fn(() => json(true)); + server.use( + db.post("rpc/claim_asset_resource_index_garbage", () => + json([candidate]) + ), + db.post("rpc/finalize_asset_resource_index_garbage", () => + json({ message: "database unavailable" }, { status: 500 }) + ), + db.post("rpc/release_asset_resource_index_garbage_claim", release) + ); + const remove = vi.fn(async () => "deleted" as const); + + await expect( + collectAssetResourceIndexGarbage({ + client: testContext.postgrest.client, + store: { delete: remove }, + }) + ).rejects.toThrow("Failed to collect 1"); + expect(remove).toHaveBeenCalledOnce(); + expect(release).not.toHaveBeenCalled(); + }); + + test("treats an already missing object as collected", async () => { + server.use( + db.post("rpc/claim_asset_resource_index_garbage", () => + json([candidate]) + ), + db.post("rpc/finalize_asset_resource_index_garbage", () => json(true)) + ); + + await expect( + collectAssetResourceIndexGarbage({ + client: testContext.postgrest.client, + store: { delete: async () => "missing" }, + }) + ).resolves.toEqual({ claimed: 1, deleted: 0, missing: 1 }); + }); +}); diff --git a/packages/asset-uploader/src/resource-index-garbage-collection.ts b/packages/asset-uploader/src/resource-index-garbage-collection.ts new file mode 100644 index 000000000000..ba4c7119b603 --- /dev/null +++ b/packages/asset-uploader/src/resource-index-garbage-collection.ts @@ -0,0 +1,95 @@ +import type { AssetResourceIndexGarbageCollectionStore } from "@webstudio-is/asset-resource"; +import type { Client } from "@webstudio-is/postgrest/index.server"; +import { assertPostgrestSuccess } from "./patch-utils"; + +export const collectAssetResourceIndexGarbage = async ({ + client, + store, + limit = 100, + now = new Date(), +}: { + client: Client; + store: AssetResourceIndexGarbageCollectionStore; + limit?: number; + now?: Date; +}) => { + if (Number.isSafeInteger(limit) === false || limit <= 0 || limit > 1000) { + throw new Error("Resource index garbage collection limit is invalid"); + } + const candidates = await client.rpc("claim_asset_resource_index_garbage", { + p_before: now.toISOString(), + p_limit: limit, + }); + assertPostgrestSuccess(candidates); + + let deleted = 0; + let missing = 0; + const failures: unknown[] = []; + for (const candidate of candidates.data ?? []) { + let objectDeleted = false; + try { + const status = await store.delete(candidate.objectKey); + objectDeleted = true; + const finalized = await client.rpc( + "finalize_asset_resource_index_garbage", + { + p_project_id: candidate.projectId, + p_resource_id: candidate.resourceId, + p_revision: candidate.revision, + p_gc_claim_id: candidate.gcClaimId, + } + ); + assertPostgrestSuccess(finalized); + if (finalized.data !== true) { + throw new Error("Resource index garbage claim could not be finalized"); + } + if (status === "deleted") { + deleted += 1; + } else { + missing += 1; + } + } catch (error) { + // Once storage deletion succeeds, keep the claim so a later worker can + // retry the idempotent delete and database finalization. Releasing it + // would expose a revision row whose object no longer exists. + if (objectDeleted === false) { + const released = await client.rpc( + "release_asset_resource_index_garbage_claim", + { + p_project_id: candidate.projectId, + p_resource_id: candidate.resourceId, + p_revision: candidate.revision, + p_gc_claim_id: candidate.gcClaimId, + } + ); + assertPostgrestSuccess(released); + } + failures.push(error); + } + } + if (failures.length > 0) { + throw new AggregateError( + failures, + `Failed to collect ${failures.length} resource index objects` + ); + } + return { + claimed: (candidates.data ?? []).length, + deleted, + missing, + }; +}; + +export const collectAssetResourceIndexGarbageBestEffort = async ({ + client, + store, +}: { + client: Client; + store: AssetResourceIndexGarbageCollectionStore; +}) => { + try { + await collectAssetResourceIndexGarbage({ client, store }); + } catch (error) { + console.error("Asset resource index cleanup failed", error); + } +}; diff --git a/packages/asset-uploader/src/resource-index-lifecycle.ts b/packages/asset-uploader/src/resource-index-lifecycle.ts index 633579dfed09..382e6de7afc2 100644 --- a/packages/asset-uploader/src/resource-index-lifecycle.ts +++ b/packages/asset-uploader/src/resource-index-lifecycle.ts @@ -1,29 +1,12 @@ -import { parseObjectExpression, type Resource } from "@webstudio-is/sdk"; +import { getAssetResourceQuery, type Resource } from "@webstudio-is/sdk"; +export { getAssetResourceQuery } from "@webstudio-is/sdk"; import type { Client } from "@webstudio-is/postgrest/index.server"; import type { AssetClientWithResourceIndexStore } from "./client"; import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; import { buildPersistAndActivateAssetResourceIndex } from "./resource-index-build"; import { deleteAssetResourceIndexQuery } from "./resource-index-persistence"; - -const assetQueryUrl = "/$resources/assets/query"; - -export const getAssetResourceQuery = (resource: Resource) => { - try { - if (JSON.parse(resource.url) !== assetQueryUrl) { - return; - } - const body = parseObjectExpression(resource.body ?? ""); - const queryExpression = body.get("query"); - if (queryExpression === undefined) { - return; - } - const query = JSON.parse(queryExpression); - return typeof query === "string" && query.trim() !== "" ? query : undefined; - } catch { - return; - } -}; +import { collectAssetResourceIndexGarbageBestEffort } from "./resource-index-garbage-collection"; export const synchronizeAssetResourceIndexQueries = async ({ client, @@ -65,6 +48,15 @@ export const synchronizeAssetResourceIndexQueries = async ({ for (const resourceId of deletedResourceIds) { await deleteAssetResourceIndexQuery({ client, projectId, resourceId }); } + if ( + deletedResourceIds.length > 0 && + assetClient.resourceIndexStore.delete !== undefined + ) { + await collectAssetResourceIndexGarbageBestEffort({ + client, + store: { delete: assetClient.resourceIndexStore.delete }, + }); + } if (changed.length === 0) { return { deletedResourceIds, updatedResourceIds: [] }; } diff --git a/packages/asset-uploader/src/resource-index-maintenance.test.ts b/packages/asset-uploader/src/resource-index-maintenance.test.ts index d1917fc23637..7574c3687b14 100644 --- a/packages/asset-uploader/src/resource-index-maintenance.test.ts +++ b/packages/asset-uploader/src/resource-index-maintenance.test.ts @@ -1,12 +1,19 @@ import { describe, expect, test, vi } from "vitest"; -import { createCanonicalAssetFileEntry } from "@webstudio-is/asset-resource"; +import { + computeAssetResourceQueryHash, + computeCanonicalAssetRevision, + createCanonicalAssetFileEntry, +} from "@webstudio-is/asset-resource"; import { createTestServer, db, json, testContext, } from "@webstudio-is/postgrest/testing"; -import { updateAssetResourceIndexesAfterCanonicalChange } from "./resource-index-maintenance"; +import { + reconcileAssetResourceIndexesForPublication, + updateAssetResourceIndexesAfterCanonicalChange, +} from "./resource-index-maintenance"; const server = createTestServer(); const document = { @@ -40,9 +47,9 @@ describe("incremental resource index maintenance", () => { const activated: string[] = []; server.use( db.get("AssetResourceIndexState", ({ request }) => { - expect(new URL(request.url).searchParams.get("projectId")).toBe( - "eq.project-1" - ); + const search = new URL(request.url).searchParams; + expect(search.get("projectId")).toBe("eq.project-1"); + expect(search.get("deletedAt")).toBe("is.null"); return json([ { resourceId: "listing", query: `*[extension == "md"]` }, { resourceId: "post", query: `*[properties.slug == $slug]` }, @@ -97,4 +104,73 @@ describe("incremental resource index maintenance", () => { ).resolves.toEqual({ changedAssetIds: [], updatedResourceIds: [] }); expect(putIfAbsent).not.toHaveBeenCalled(); }); + + test("does not rebuild deleted resource queries", async () => { + let metadataLoaded = false; + server.use( + db.get("AssetResourceIndexState", ({ request }) => { + const search = new URL(request.url).searchParams; + return search.get("deletedAt") === "is.null" + ? json([]) + : json([{ resourceId: "deleted", query: "*[]" }]); + }), + db.get("AssetFileMetadata", () => { + metadataLoaded = true; + return json([metadataRow]); + }) + ); + const putIfAbsent = vi.fn(); + + await expect( + updateAssetResourceIndexesAfterCanonicalChange({ + client: testContext.postgrest.client, + store: { putIfAbsent }, + projectId: "project-1", + changedAssetIds: ["post-1"], + }) + ).resolves.toEqual({ + changedAssetIds: ["post-1"], + updatedResourceIds: [], + }); + expect(metadataLoaded).toBe(false); + expect(putIfAbsent).not.toHaveBeenCalled(); + }); + + test("repairs a stale current-query index before publication", async () => { + const query = `*[extension == "md"]`; + const queryHash = await computeAssetResourceQueryHash(query); + const assetRevision = await computeCanonicalAssetRevision([entry]); + server.use( + db.get("AssetResourceIndexState", () => + json([ + { + resourceId: "posts", + queryHash, + assetRevision: `sha256:${"0".repeat(64)}`, + buildStatus: "FAILED", + activeRevision: null, + deletedAt: null, + }, + ]) + ), + db.post("rpc/begin_asset_resource_index_build", () => json(null)), + db.post("rpc/activate_asset_resource_index", () => json(true)) + ); + const putIfAbsent = vi.fn(async ({ checksum }) => ({ + status: "created" as const, + checksum, + })); + + await expect( + reconcileAssetResourceIndexesForPublication({ + client: testContext.postgrest.client, + store: { putIfAbsent }, + projectId: "project-1", + resources: [{ resourceId: "posts", query, queryHash }], + entries: [entry], + assetRevision, + }) + ).resolves.toEqual(["posts"]); + expect(putIfAbsent).toHaveBeenCalledOnce(); + }); }); diff --git a/packages/asset-uploader/src/resource-index-maintenance.ts b/packages/asset-uploader/src/resource-index-maintenance.ts index fbca3fd52522..dbdfa77e9661 100644 --- a/packages/asset-uploader/src/resource-index-maintenance.ts +++ b/packages/asset-uploader/src/resource-index-maintenance.ts @@ -1,8 +1,43 @@ -import type { ImmutableAssetResourceIndexStore } from "@webstudio-is/asset-resource"; +import type { + CanonicalAssetFileEntry, + ImmutableAssetResourceIndexStore, +} from "@webstudio-is/asset-resource"; import type { Client } from "@webstudio-is/postgrest/index.server"; +import type { AssetClient } from "./client"; import { assertPostgrestSuccess } from "./patch-utils"; +import { synchronizeCanonicalAsset } from "./canonical-metadata-backfill"; import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; -import { buildPersistAndActivateAssetResourceIndex } from "./resource-index-build"; +import { + AssetResourceIndexBuildSupersededError, + buildPersistAndActivateAssetResourceIndex, +} from "./resource-index-build"; + +export const synchronizeAssetResourceStateAfterAssetChange = async ({ + client, + assetClient, + projectId, + assetId, +}: { + client: Client; + assetClient: AssetClient; + projectId: string; + assetId: string; +}) => { + await synchronizeCanonicalAsset({ + projectId, + assetId, + client, + assetClient, + }); + if (assetClient.resourceIndexStore !== undefined) { + await updateAssetResourceIndexesAfterCanonicalChange({ + client, + store: assetClient.resourceIndexStore, + projectId, + changedAssetIds: [assetId], + }); + } +}; export const updateAssetResourceIndexesAfterCanonicalChange = async ({ client, @@ -23,6 +58,7 @@ export const updateAssetResourceIndexesAfterCanonicalChange = async ({ .from("AssetResourceIndexState") .select("resourceId, query") .eq("projectId", projectId) + .is("deletedAt", null) .order("resourceId"); assertPostgrestSuccess(stateResult); const resources = stateResult.data ?? []; @@ -59,3 +95,80 @@ export const updateAssetResourceIndexesAfterCanonicalChange = async ({ } return { changedAssetIds: changedIds, updatedResourceIds }; }; + +export const reconcileAssetResourceIndexesForPublication = async ({ + client, + store, + projectId, + resources, + entries, + assetRevision, +}: { + client: Client; + store: ImmutableAssetResourceIndexStore; + projectId: string; + resources: readonly { + resourceId: string; + query: string; + queryHash: string; + }[]; + entries: readonly CanonicalAssetFileEntry[]; + assetRevision: string; +}) => { + if (resources.length === 0) { + return []; + } + const statesResult = await client + .from("AssetResourceIndexState") + .select( + "resourceId, queryHash, assetRevision, buildStatus, activeRevision, deletedAt" + ) + .eq("projectId", projectId) + .in( + "resourceId", + [...new Set(resources.map(({ resourceId }) => resourceId))].sort() + ); + assertPostgrestSuccess(statesResult); + const states = new Map( + (statesResult.data ?? []).map((state) => [state.resourceId, state]) + ); + const rebuilt: string[] = []; + for (const resource of resources) { + const state = states.get(resource.resourceId); + // A different query belongs to a historical build and must not replace the + // Builder's current active query. + if ( + state !== undefined && + (state.deletedAt !== null || state.queryHash !== resource.queryHash) + ) { + continue; + } + if ( + state !== undefined && + state.buildStatus === "ACTIVE" && + state.activeRevision !== null && + state.assetRevision === assetRevision + ) { + continue; + } + try { + await buildPersistAndActivateAssetResourceIndex({ + client, + store, + projectId, + resourceId: resource.resourceId, + query: resource.query, + entries, + }); + } catch (error) { + // A concurrent build of the same identity may activate first. The + // snapshot validation immediately after reconciliation remains the + // authority for whether publication can proceed. + if (error instanceof AssetResourceIndexBuildSupersededError === false) { + throw error; + } + } + rebuilt.push(resource.resourceId); + } + return rebuilt; +}; diff --git a/packages/asset-uploader/src/resource-index-rebuild.test.ts b/packages/asset-uploader/src/resource-index-rebuild.test.ts index 346e6518e114..0f02dd0b9911 100644 --- a/packages/asset-uploader/src/resource-index-rebuild.test.ts +++ b/packages/asset-uploader/src/resource-index-rebuild.test.ts @@ -66,7 +66,14 @@ describe("explicit resource index rebuild", () => { }); test("fails clearly when the resource has no persisted index state", async () => { - server.use(db.get("AssetResourceIndexState", () => json(null))); + server.use( + db.get("AssetResourceIndexState", ({ request }) => { + expect(new URL(request.url).searchParams.get("deletedAt")).toBe( + "is.null" + ); + return json(null); + }) + ); await expect( rebuildAssetResourceIndex({ client: testContext.postgrest.client, @@ -76,4 +83,26 @@ describe("explicit resource index rebuild", () => { }) ).rejects.toBeInstanceOf(AssetResourceIndexNotFoundError); }); + + test("treats a deleted resource index state as not found", async () => { + const putIfAbsent = vi.fn(); + server.use( + db.get("AssetResourceIndexState", ({ request }) => { + const search = new URL(request.url).searchParams; + return search.get("deletedAt") === "is.null" + ? json(null) + : json({ query: "*[]" }); + }) + ); + + await expect( + rebuildAssetResourceIndex({ + client: testContext.postgrest.client, + store: { putIfAbsent }, + projectId: "project-1", + resourceId: "deleted", + }) + ).rejects.toBeInstanceOf(AssetResourceIndexNotFoundError); + expect(putIfAbsent).not.toHaveBeenCalled(); + }); }); diff --git a/packages/asset-uploader/src/resource-index-rebuild.ts b/packages/asset-uploader/src/resource-index-rebuild.ts index 7ec7d3ef35df..9ce7999745d4 100644 --- a/packages/asset-uploader/src/resource-index-rebuild.ts +++ b/packages/asset-uploader/src/resource-index-rebuild.ts @@ -27,6 +27,7 @@ export const rebuildAssetResourceIndex = async ({ .select("query") .eq("projectId", projectId) .eq("resourceId", resourceId) + .is("deletedAt", null) .maybeSingle(); assertPostgrestSuccess(state); if (state.data === null) { diff --git a/packages/asset-uploader/src/resource-index-snapshot.test.ts b/packages/asset-uploader/src/resource-index-snapshot.test.ts index 648554a1dfab..56bb476fe6a2 100644 --- a/packages/asset-uploader/src/resource-index-snapshot.test.ts +++ b/packages/asset-uploader/src/resource-index-snapshot.test.ts @@ -48,8 +48,13 @@ describe("publication resource index snapshots", () => { objectKey: "private/posts.json", }, ]); + const rpc = vi.fn(async (name: string, _parameters?: unknown) => ({ + data: name === "remove_asset_resource_index_reference" ? 1 : null, + error: null, + })); const client = { from: vi.fn().mockReturnValueOnce(states).mockReturnValueOnce(revisions), + rpc, }; const bytes = new TextEncoder().encode(serializeAssetResourceIndex(index)); const read = vi.fn(async () => ({ @@ -69,12 +74,24 @@ describe("publication resource index snapshots", () => { { resourceId: "posts", queryHash: index.queryHash }, ], read, + referenceId: "build-1", + garbageCollectionStore: { delete: async () => "deleted" }, expectedAssetRevision: index.assetRevision, }) ).resolves.toEqual([ { resourceId: "posts", revision: index.integrity.checksum, index }, ]); expect(read).toHaveBeenCalledWith("private/posts.json"); + expect(rpc.mock.calls.map(([name]) => name)).toEqual([ + "add_asset_resource_index_reference", + "remove_asset_resource_index_reference", + "claim_asset_resource_index_garbage", + ]); + const addReference = rpc.mock.calls[0][1] as { p_reference_id: string }; + const removeReference = rpc.mock.calls[1][1] as { p_reference_id: string }; + expect(addReference.p_reference_id).toBe(removeReference.p_reference_id); + expect(addReference.p_reference_id).toMatch(/^build-1:/); + expect(addReference.p_reference_id).not.toBe("build-1"); }); test("blocks missing, inactive, and inconsistent active revisions", async () => { @@ -88,6 +105,7 @@ describe("publication resource index snapshots", () => { { resourceId: "posts", queryHash: `sha256:${"b".repeat(64)}` }, ], read: vi.fn(), + referenceId: "build-1", expectedAssetRevision: `sha256:${"a".repeat(64)}`, }) ).rejects.toMatchObject({ resourceId: "posts" }); diff --git a/packages/asset-uploader/src/resource-index-snapshot.ts b/packages/asset-uploader/src/resource-index-snapshot.ts index 01a57a75bf07..38c1698fbf36 100644 --- a/packages/asset-uploader/src/resource-index-snapshot.ts +++ b/packages/asset-uploader/src/resource-index-snapshot.ts @@ -2,9 +2,17 @@ import { assetResourceLimits, type AssetResourceIndexV1, } from "@webstudio-is/sdk"; -import { verifyAssetResourceIndex } from "@webstudio-is/asset-resource"; +import { + verifyAssetResourceIndex, + type AssetResourceIndexGarbageCollectionStore, +} from "@webstudio-is/asset-resource"; import type { Client, Database } from "@webstudio-is/postgrest/index.server"; import { assertPostgrestSuccess } from "./patch-utils"; +import { + addAssetResourceIndexReference, + removeAssetResourceIndexReferences, +} from "./resource-index-persistence"; +import { collectAssetResourceIndexGarbageBestEffort } from "./resource-index-garbage-collection"; type StateRow = Pick< Database["public"]["Tables"]["AssetResourceIndexState"]["Row"], @@ -70,13 +78,18 @@ export const loadAssetResourceIndexSnapshots = async ({ resources, read, expectedAssetRevision, + referenceId, + garbageCollectionStore, }: { client: Client; projectId: string; resources: readonly { resourceId: string; queryHash: string }[]; read: (name: string) => Promise<{ data: AsyncIterable }>; expectedAssetRevision: string; + referenceId: string; + garbageCollectionStore?: AssetResourceIndexGarbageCollectionStore; }): Promise => { + const operationReferenceId = `${referenceId}:${crypto.randomUUID()}`; const expectedQueryHashes = new Map( resources.map(({ resourceId, queryHash }) => [resourceId, queryHash]) ); @@ -125,36 +138,64 @@ export const loadAssetResourceIndexSnapshots = async ({ revisions.map((row) => [`${row.resourceId}:${row.revision}`, row]) ); - return await Promise.all( - uniqueResourceIds.map(async (resourceId) => { + try { + for (const resourceId of uniqueResourceIds) { const state = statesByResource.get(resourceId) as StateRow & { activeRevision: string; }; - const revision = revisionByIdentity.get( - `${resourceId}:${state.activeRevision}` - ); - if (revision === undefined) { - throw new AssetResourceIndexSnapshotError( - resourceId, - "Active asset resource index revision is missing" - ); - } - const index = await readIndex({ objectKey: revision.objectKey, read }); - if ( - index.resourceId !== resourceId || - index.queryHash !== state.queryHash || - index.queryHash !== revision.queryHash || - index.assetRevision !== state.assetRevision || - index.assetRevision !== revision.assetRevision || - index.integrity.checksum !== state.activeRevision || - revision.revision !== state.activeRevision - ) { - throw new AssetResourceIndexSnapshotError( - resourceId, - "Active asset resource index identity is inconsistent" + await addAssetResourceIndexReference({ + client, + projectId, + resourceId, + revision: state.activeRevision, + type: "BUILD", + referenceId: operationReferenceId, + }); + } + return await Promise.all( + uniqueResourceIds.map(async (resourceId) => { + const state = statesByResource.get(resourceId) as StateRow & { + activeRevision: string; + }; + const revision = revisionByIdentity.get( + `${resourceId}:${state.activeRevision}` ); - } - return { resourceId, revision: state.activeRevision, index }; - }) - ); + if (revision === undefined) { + throw new AssetResourceIndexSnapshotError( + resourceId, + "Active asset resource index revision is missing" + ); + } + const index = await readIndex({ objectKey: revision.objectKey, read }); + if ( + index.resourceId !== resourceId || + index.queryHash !== state.queryHash || + index.queryHash !== revision.queryHash || + index.assetRevision !== state.assetRevision || + index.assetRevision !== revision.assetRevision || + index.integrity.checksum !== state.activeRevision || + revision.revision !== state.activeRevision + ) { + throw new AssetResourceIndexSnapshotError( + resourceId, + "Active asset resource index identity is inconsistent" + ); + } + return { resourceId, revision: state.activeRevision, index }; + }) + ); + } finally { + await removeAssetResourceIndexReferences({ + client, + projectId, + type: "BUILD", + referenceId: operationReferenceId, + }); + if (garbageCollectionStore !== undefined) { + await collectAssetResourceIndexGarbageBestEffort({ + client, + store: garbageCollectionStore, + }); + } + } }; diff --git a/packages/asset-uploader/src/revision.test.ts b/packages/asset-uploader/src/revision.test.ts index 400365383a81..35352f6c1102 100644 --- a/packages/asset-uploader/src/revision.test.ts +++ b/packages/asset-uploader/src/revision.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import { http, HttpResponse } from "msw"; import { createTestServer, @@ -17,6 +17,12 @@ import { const { getRevisionFilename } = __testing__; const server = createTestServer(); +const readEmptyFile = async () => ({ + data: { + async *[Symbol.asyncIterator]() {}, + }, +}); + const createContext = (): AppContext => ({ ...testContext, @@ -82,7 +88,24 @@ describe("asset content revisions", () => { let swapInput: unknown; server.use( ownershipHandler, - db.get("Asset", () => json(assetRow)), + db.get("Asset", ({ request }) => { + const url = new URL(request.url); + if (url.searchParams.has("file.status")) { + return json([ + { + ...assetRow, + name: revisionName, + file: { + ...oldFile, + name: revisionName, + size: 7, + updatedAt: "2026-07-18T00:00:01.000Z", + }, + }, + ]); + } + return json(assetRow); + }), db.post("File", async ({ request }) => { const input = (await request.json()) as { name: string }; revisionName = input.name; @@ -116,7 +139,10 @@ describe("asset content revisions", () => { swapInput = await request.json(); return HttpResponse.json("updated"); } - ) + ), + db.get("AssetFolder", () => json([])), + db.post("rpc/replace_asset_file_metadata", () => json(true)), + db.get("AssetResourceIndexState", () => json([])) ); const asset = await updateAssetContent( @@ -135,6 +161,7 @@ describe("asset content revisions", () => { ); return { format: "json", size: 7, meta: {} }; }, + readFile: readEmptyFile, }, createContext() ); @@ -157,6 +184,118 @@ describe("asset content revisions", () => { }); }); + test("reindexes Markdown metadata after replacing its content", async () => { + const source = "---\ntitle: Updated post\n---\n\nUpdated body"; + const sourceBytes = new TextEncoder().encode(source); + const markdownFile = { + ...oldFile, + name: "post_old-revision.md", + format: "md", + size: 3, + }; + const markdownAsset = { + ...assetRow, + name: markdownFile.name, + filename: "post", + file: markdownFile, + }; + let revisionName = ""; + let canonicalDocument: Record | undefined; + let indexStatesLoaded = false; + server.use( + ownershipHandler, + db.get("Asset", ({ request }) => { + const url = new URL(request.url); + if (url.searchParams.has("file.status")) { + return json([ + { + ...markdownAsset, + name: revisionName, + file: { + ...markdownFile, + name: revisionName, + size: sourceBytes.byteLength, + updatedAt: "2026-07-18T00:00:01.000Z", + }, + }, + ]); + } + return json(markdownAsset); + }), + db.post("File", async ({ request }) => { + revisionName = ((await request.json()) as { name: string }).name; + return empty({ status: 201 }); + }), + db.get("File", () => + json({ + ...markdownFile, + name: revisionName, + format: "file", + status: "UPLOADING", + }) + ), + db.patch("File", () => + json({ + ...markdownFile, + name: revisionName, + size: sourceBytes.byteLength, + updatedAt: "2026-07-18T00:00:01.000Z", + }) + ), + http.post("http://test-postgrest/rpc/swap_asset_file", () => + HttpResponse.json("updated") + ), + db.get("AssetFolder", () => json([])), + db.post("rpc/replace_asset_file_metadata", async ({ request }) => { + canonicalDocument = ( + (await request.json()) as { + p_document: Record; + } + ).p_document; + return json(true); + }), + db.get("AssetResourceIndexState", () => { + indexStatesLoaded = true; + return json([]); + }) + ); + + await updateAssetContent( + { + assetId: "asset", + projectId: "project", + expectedName: markdownFile.name, + data: new Blob([source]).stream(), + }, + { + uploadFile: async () => ({ + format: "md", + size: sourceBytes.byteLength, + meta: {}, + }), + readFile: async (name) => { + expect(name).toBe(revisionName); + return { data: new Blob([source]).stream() }; + }, + resourceIndexStore: { + putIfAbsent: async () => { + throw new Error("No query indexes should be built"); + }, + }, + }, + createContext() + ); + + expect(canonicalDocument).toMatchObject({ + _id: "asset", + name: "post", + contentRef: revisionName, + properties: { title: "Updated post" }, + excerpt: "Updated body", + }); + expect(indexStatesLoaded).toBe(true); + }); + test("rejects a stale revision before uploading", async () => { server.use( ownershipHandler, @@ -175,6 +314,7 @@ describe("asset content revisions", () => { uploadFile: async () => { throw new Error("upload should not run"); }, + readFile: readEmptyFile, }, createContext() ) @@ -235,6 +375,7 @@ describe("asset content revisions", () => { }, { uploadFile: async () => ({ format: "json", size: 7, meta: {} }), + readFile: readEmptyFile, }, createContext() ) @@ -242,12 +383,28 @@ describe("asset content revisions", () => { expect(discardedRevision).toBe(true); }); - test("accepts a revision when the swap commits without a response", async () => { + test("accepts a committed revision when response and index maintenance fail", async () => { let assetLoadCount = 0; let revisionName = ""; server.use( ownershipHandler, - db.get("Asset", () => { + db.get("Asset", ({ request }) => { + const url = new URL(request.url); + if (url.searchParams.has("file.status")) { + return json([ + { + ...assetRow, + name: revisionName, + file: { + ...oldFile, + name: revisionName, + format: "json", + size: 7, + updatedAt: "2026-07-18T00:00:00.000Z", + }, + }, + ]); + } assetLoadCount += 1; return json( assetLoadCount === 1 ? assetRow : { ...assetRow, name: revisionName } @@ -276,8 +433,13 @@ describe("asset content revisions", () => { ), http.post("http://test-postgrest/rpc/swap_asset_file", () => HttpResponse.error() + ), + db.get("AssetFolder", () => json([])), + db.post("rpc/replace_asset_file_metadata", () => + json({ message: "index database unavailable" }, { status: 500 }) ) ); + const log = vi.spyOn(console, "error").mockImplementation(() => {}); await expect( updateAssetContent( @@ -289,6 +451,7 @@ describe("asset content revisions", () => { }, { uploadFile: async () => ({ format: "json", size: 7, meta: {} }), + readFile: readEmptyFile, }, createContext() ) @@ -297,5 +460,10 @@ describe("asset content revisions", () => { name: expect.stringMatching(/^settings_.+\.json$/), format: "json", }); + expect(log).toHaveBeenCalledWith( + "Asset revision resource synchronization failed", + expect.anything() + ); + log.mockRestore(); }); }); diff --git a/packages/asset-uploader/src/revision.ts b/packages/asset-uploader/src/revision.ts index 0223ce79a67e..b9846ea98c90 100644 --- a/packages/asset-uploader/src/revision.ts +++ b/packages/asset-uploader/src/revision.ts @@ -9,12 +9,13 @@ import { type AppContext, } from "@webstudio-is/trpc-interface/index.server"; import type { Client } from "@webstudio-is/postgrest/index.server"; -import type { AssetUploadClient } from "./client"; +import type { AssetClient } from "./client"; import { uploadFileData } from "./upload"; import { createUniqueAssetFilename } from "./utils/get-unique-filename"; import { sanitizeS3Key } from "./utils/sanitize-s3-key"; import { formatAsset } from "./utils/format-asset"; import { assertPostgrestSuccess } from "./patch-utils"; +import { synchronizeAssetResourceStateAfterAssetChange } from "./resource-index-maintenance"; export class AssetRevisionConflictError extends Error {} @@ -103,7 +104,7 @@ export const updateAssetContent = async ( expectedName: string; data: ReadableStream; }, - assetClient: AssetUploadClient, + assetClient: AssetClient, context: AppContext ): Promise => { const canEdit = await authorizeProject.hasProjectPermit( @@ -181,6 +182,7 @@ export const updateAssetContent = async ( context.postgrest.client ); } catch (error) { + let swapCommitted = false; try { const current = await loadAsset({ assetId, @@ -188,19 +190,34 @@ export const updateAssetContent = async ( client: context.postgrest.client, }); if (current.name === revisionName) { - return revision; + swapCommitted = true; + } else { + const discardedRevision = await context.postgrest.client + .from("File") + .update({ isDeleted: true }) + .eq("name", revisionName); + assertPostgrestSuccess(discardedRevision); } - const discardedRevision = await context.postgrest.client - .from("File") - .update({ isDeleted: true }) - .eq("name", revisionName); - assertPostgrestSuccess(discardedRevision); } catch (cleanupError) { console.error("Unable to discard an asset revision", cleanupError); } - throw error; + if (swapCommitted === false) { + throw error; + } } + try { + await synchronizeAssetResourceStateAfterAssetChange({ + client: context.postgrest.client, + assetClient, + projectId, + assetId, + }); + } catch (error) { + // The immutable revision swap has already committed. Keep the primary + // mutation successful; publication reconciles stale query indexes. + console.error("Asset revision resource synchronization failed", error); + } return revision; }; diff --git a/packages/asset-uploader/src/upload.test.ts b/packages/asset-uploader/src/upload.test.ts index f0f1a5d8c18a..c3efd6f79bfb 100644 --- a/packages/asset-uploader/src/upload.test.ts +++ b/packages/asset-uploader/src/upload.test.ts @@ -847,6 +847,73 @@ describe("uploadFile", () => { }); }); + test("keeps a committed upload when resource synchronization fails", async () => { + const source = "Post body"; + const uploadedFile = { + name: "post.md", + format: "md", + size: source.length, + status: "UPLOADED", + uploaderProjectId: "project-1", + isDeleted: false, + description: null, + meta: "{}", + createdAt: "2026-07-18T00:00:00.000Z", + updatedAt: "2026-07-18T01:00:00.000Z", + }; + server.use( + db.get("File", () => json({ ...uploadedFile, status: "UPLOADING" })), + db.patch("File", () => json(uploadedFile)), + db.get("Asset", ({ request }) => { + const url = new URL(request.url); + if (url.searchParams.has("name")) { + return json({ + id: "asset-1", + projectId: "project-1", + filename: null, + description: null, + folderId: null, + }); + } + return json([ + { + id: "asset-1", + projectId: "project-1", + filename: null, + folderId: null, + file: uploadedFile, + }, + ]); + }), + db.get("AssetFolder", () => json([])) + ); + const logError = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect( + uploadFile( + "post.md", + new Blob([source]).stream(), + { + uploadFile: async () => ({ + format: "md", + size: source.length, + meta: {}, + }), + readFile: async () => { + throw new Error("Storage read failed"); + }, + }, + createContext(), + undefined + ) + ).resolves.toMatchObject({ id: "asset-1", name: "post.md" }); + expect(logError).toHaveBeenCalledWith( + "Uploaded asset resource synchronization failed", + expect.any(Error) + ); + logError.mockRestore(); + }); + test("returns uploaded asset with reserved asset row id", async () => { server.use( db.get("File", () => @@ -902,7 +969,16 @@ describe("uploadFile", () => { }); }), db.get("AssetFolder", () => json([])), - db.post("rpc/replace_asset_file_metadata", () => json(true)) + db.post("rpc/replace_asset_file_metadata", async ({ request }) => { + const body = (await request.json()) as { + p_document: { extension: string; mimeType: string }; + }; + expect(body.p_document).toMatchObject({ + extension: "png", + mimeType: "image/png", + }); + return json(true); + }) ); await expect( diff --git a/packages/asset-uploader/src/upload.ts b/packages/asset-uploader/src/upload.ts index 5e0e9107f056..f78f8a1686d4 100644 --- a/packages/asset-uploader/src/upload.ts +++ b/packages/asset-uploader/src/upload.ts @@ -19,8 +19,7 @@ import { sanitizeS3Key } from "./utils/sanitize-s3-key"; import { formatAsset } from "./utils/format-asset"; import { assertPostgrestSuccess } from "./patch-utils"; import type { UploadTicket } from "./types"; -import { synchronizeCanonicalAsset } from "./canonical-metadata-backfill"; -import { updateAssetResourceIndexesAfterCanonicalChange } from "./resource-index-maintenance"; +import { synchronizeAssetResourceStateAfterAssetChange } from "./resource-index-maintenance"; type UploadData = { projectId: string; @@ -527,19 +526,15 @@ export const uploadFile = async ( await cleanupUploadError(name, context, onUploadError); throw error; } - await synchronizeCanonicalAsset({ - projectId: uploadedAsset.projectId, - assetId: uploadedAsset.id, - client: context.postgrest.client, - assetClient: client, - }); - if (client.resourceIndexStore !== undefined) { - await updateAssetResourceIndexesAfterCanonicalChange({ + try { + await synchronizeAssetResourceStateAfterAssetChange({ client: context.postgrest.client, - store: client.resourceIndexStore, + assetClient: client, projectId: uploadedAsset.projectId, - changedAssetIds: [uploadedAsset.id], + assetId: uploadedAsset.id, }); + } catch (error) { + console.error("Uploaded asset resource synchronization failed", error); } return uploadedAsset; }; diff --git a/packages/cli/src/prebuild.test.ts b/packages/cli/src/prebuild.test.ts index 043d17193c81..de29b1a37d98 100644 --- a/packages/cli/src/prebuild.test.ts +++ b/packages/cli/src/prebuild.test.ts @@ -11,21 +11,49 @@ import { import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { tmpdir } from "node:os"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; import { bundleVersion } from "@webstudio-is/protocol"; import { createAssetResourceIndex } from "@webstudio-is/asset-resource"; +import { createAssetQueryResourceBody } from "@webstudio-is/sdk"; import { generateRedirectsModule, materializeAssetResourceIndexes, prebuild, } from "./prebuild"; +import { + createSsgAssetResourceFetch, + fetchSsgPublicAsset, +} from "../templates/ssg/app/asset-resource-fetch"; const originalCwd = process.cwd(); +const execFileAsync = promisify(execFile); const originalFetch = globalThis.fetch; let tempDir: string; let consoleInfo: ReturnType; const rootFolderId = "root"; const elementComponent = "ws:element"; const slowPrebuildTestTimeout = 15_000; + +const runGeneratedCommand = async ( + command: "react-router" | "vite" | "vike", + args: string[] +) => { + const env = { ...process.env }; + for (const name of Object.keys(env)) { + if (name.startsWith("VITEST")) { + delete env[name]; + } + } + env.NODE_ENV = "production"; + env.NODE_OPTIONS = "--conditions=webstudio"; + env.WEBSTUDIO_LOCAL_CLI_BOOTSTRAPPED = "1"; + await execFileAsync(join(originalCwd, `node_modules/.bin/${command}`), args, { + cwd: tempDir, + env, + }); +}; + type Redirects = Array<{ old: string; new: string; status?: "301" | "302" }>; type GeneratedRouteModule = { loader: (args: { request: Request }) => Response | Promise; @@ -329,6 +357,107 @@ test("materializes immutable resource indexes as public JSON with a reference-on expect(manifestModule).toContain("/resource-indexes/"); expect(manifestModule).not.toContain('"documents"'); expect(manifestModule).not.toContain('"properties"'); + const runtimeModule = await readFile( + "app/__generated__/$resources.asset-query-runtime.ts", + "utf8" + ); + expect(runtimeModule).toContain('from "@webstudio-is/asset-resource"'); + expect(runtimeModule).not.toContain('"documents"'); + expect(runtimeModule).not.toContain('"properties"'); +}); + +test("executes and hydrates an asset query from SSG public files", async () => { + const source = "# Prerendered post\n"; + const query = + '*[properties.slug == $slug][0]{_id, revision, contentRef, "title": properties.title}'; + const index = await createAssetResourceIndex({ + format: "webstudio-resource-index", + version: 1, + resourceId: "post", + query, + assetRevision: `sha256:${"a".repeat(64)}`, + queryMode: "parameterized", + parameterNames: ["slug"], + documents: [ + { + _id: "post-1", + _type: "asset.file", + name: "post.md", + path: "blog/post.md", + key: "post", + extension: "md", + mimeType: "text/markdown", + size: new TextEncoder().encode(source).byteLength, + revision: "post-revision", + contentRef: "post.md", + properties: { slug: "post", title: "Prerendered post" }, + }, + ], + }); + await mkdir("public/assets", { recursive: true }); + await mkdir("app/__generated__", { recursive: true }); + await writeFile("public/assets/post.md", source, "utf8"); + await materializeAssetResourceIndexes({ + snapshots: [ + { + resourceId: index.resourceId, + revision: index.integrity.checksum, + index, + }, + ], + publicDirectory: "public", + generatedDirectory: "app/__generated__", + deploymentId: "build-1", + }); + const [indexFile] = await getFilePaths("public/resource-indexes"); + const indexPath = `/${indexFile.slice("public/".length)}`; + await expect((await fetchSsgPublicAsset(indexPath)).json()).resolves.toEqual( + index + ); + await expect( + ( + await fetchSsgPublicAsset("/assets/post.md", { + headers: { range: `bytes=0-${source.length - 1}` }, + }) + ).text() + ).resolves.toBe(source); + const runtimeFetch = createSsgAssetResourceFetch({ + deploymentId: "build-1", + manifest: [ + { + resourceId: index.resourceId, + revision: index.integrity.checksum, + queryHash: index.queryHash, + assetRevision: index.assetRevision, + indexPath, + }, + ], + }); + + const response = await runtimeFetch( + new Request("https://site.example/$resources/assets/query", { + method: "POST", + body: JSON.stringify({ + query, + parameters: { slug: "post" }, + resultLimit: 1, + content: { mode: "full" }, + }), + }) + ); + + expect({ + status: response?.status, + body: await response?.json(), + }).toMatchObject({ + status: 200, + body: { + ok: true, + result: { _id: "post-1", title: "Prerendered post" }, + content: { "post-1": { text: source } }, + meta: { hydratedFileCount: 1 }, + }, + }); }); describe("prebuild", () => { @@ -652,7 +781,6 @@ describe("prebuild", () => { index, }, ], - protectedAssetIds: ["draft.md"], }; await writeSiteData( siteData as unknown as ReturnType @@ -669,8 +797,8 @@ describe("prebuild", () => { await expect(readFile("public/assets/post.md", "utf8")).resolves.toBe( source ); - await expect(readFile("public/assets/draft.md", "utf8")).rejects.toThrow( - "ENOENT" + await expect(readFile("public/assets/draft.md", "utf8")).resolves.toBe( + "draft secret" ); const manifest = await readFile( "app/__generated__/$resources.asset-query-manifest.ts", @@ -678,9 +806,9 @@ describe("prebuild", () => { ); expect(manifest).not.toContain("Published post"); expect(manifest).not.toContain("draft secret"); - const assetRuntime = await readFile("app/asset-resource-fetch.ts", "utf8"); - expect(assetRuntime).toContain('Reflect.get(env, "ASSETS")'); - expect(assetRuntime).toContain("binding?.fetch(assetRequest)"); + await expect( + readFile("app/asset-resource-fetch.ts", "utf8") + ).rejects.toThrow("ENOENT"); expect( (await getFilePaths("app/routes")).filter((path) => path.includes("$slug") @@ -691,6 +819,15 @@ describe("prebuild", () => { test("uses pass-through images in the base react-router template", async () => { await prebuild({ assets: false, template: ["react-router"] }); + const route = await readFile("app/routes/_index.tsx", "utf8"); + expect(route).toContain("$resources.asset-query-runtime"); + expect(route).not.toContain("@webstudio-is/asset-resource"); + const assetQueryRuntime = await readFile( + "app/__generated__/$resources.asset-query-runtime.ts", + "utf8" + ); + expect(assetQueryRuntime).not.toContain("@webstudio-is/asset-resource"); + expect(assetQueryRuntime).toContain("=> fallback"); await expect(readFile("app/constants.mjs", "utf8")).resolves.toContain( "return props.src" ); @@ -702,6 +839,25 @@ describe("prebuild", () => { expect(packageJson.dependencies).not.toHaveProperty("ipx"); }); + test("omits the asset query runtime from dynamic app bundles without asset queries", async () => { + await prebuild({ assets: false, template: ["react-router"] }); + await symlink(join(originalCwd, "node_modules"), "node_modules", "dir"); + + await runGeneratedCommand("react-router", ["build"]); + + const serverBundle = ( + await Promise.all( + ( + await getFilePaths("build/server") + ) + .filter((path) => path.endsWith(".js")) + .map((path) => readFile(path, "utf8")) + ) + ).join("\n"); + expect(serverBundle).not.toContain("@webstudio-is/asset-resource"); + expect(serverBundle).not.toContain("groq-js"); + }, 30_000); + test("keeps IPX image optimization in the react-router Docker overlay", async () => { await prebuild({ assets: false, @@ -770,8 +926,121 @@ describe("prebuild", () => { await expect( readFile("app/__generated__/[blog].$slug._index.tsx", "utf8") ).resolves.toContain("export { Page }"); + await expect(readFile("pages/index/+data.ts", "utf8")).resolves.toContain( + "createSsgAssetResourceFetch" + ); + const packageJson = JSON.parse(await readFile("package.json", "utf8")); + expect(packageJson.dependencies).toHaveProperty( + "@webstudio-is/asset-resource" + ); }); + test("prerenders SSG pages with asset query data", async () => { + const query = '*[]{"title": properties.title}'; + const index = await createAssetResourceIndex({ + format: "webstudio-resource-index", + version: 1, + resourceId: "posts", + query, + assetRevision: `sha256:${"a".repeat(64)}`, + queryMode: "static", + parameterNames: [], + documents: [ + { + _id: "post-1", + _type: "asset.file", + name: "post.md", + path: "blog/post.md", + key: "post", + extension: "md", + mimeType: "text/markdown", + size: 1, + revision: "post-revision", + contentRef: "post.md", + properties: { title: "Prerendered post" }, + }, + ], + }); + const siteData = { + ...createSiteData(), + assetResourceIndexes: [ + { + resourceId: index.resourceId, + revision: index.integrity.checksum, + index, + }, + ], + }; + siteData.build.resources = [ + [ + "posts", + { + id: "posts", + name: "Posts", + control: "system", + method: "post", + url: '"/$resources/assets/query"', + headers: [], + body: createAssetQueryResourceBody({ + query, + parameters: [], + resultLimit: 10, + }), + }, + ], + ] as never; + siteData.build.dataSources = [ + [ + "posts-data", + { + id: "posts-data", + type: "resource", + name: "posts", + resourceId: "posts", + scopeInstanceId: "root", + }, + ], + ] as never; + await writeSiteData( + siteData as unknown as ReturnType + ); + await prebuild({ assets: false, template: ["ssg"] }); + await symlink(join(originalCwd, "node_modules"), "node_modules", "dir"); + const pageModule = (await import( + `${ + pathToFileURL(join(tempDir, "pages/index/+data.ts")).href + }?test=${crypto.randomUUID()}` + )) as { + data: (context: { + urlOriginal: string; + headers: Record; + routeParams: Record; + }) => Promise<{ resources: Record }>; + }; + + const pageData = await pageModule.data({ + urlOriginal: "/", + headers: { host: "example.com" }, + routeParams: {}, + }); + + expect(pageData.resources).toMatchObject({ + Posts: { + ok: true, + status: 200, + data: { + ok: true, + result: [{ title: "Prerendered post" }], + }, + }, + }); + await runGeneratedCommand("vite", ["build"]); + await runGeneratedCommand("vike", ["prerender"]); + await expect(readFile("dist/client/index.html", "utf8")).resolves.toContain( + "" + ); + }, 30_000); + test("generates html, xml, and text document routes", async () => { await writeSiteData( createSiteData({ diff --git a/packages/cli/src/prebuild.ts b/packages/cli/src/prebuild.ts index a525b1648859..e3851756a9fa 100644 --- a/packages/cli/src/prebuild.ts +++ b/packages/cli/src/prebuild.ts @@ -135,7 +135,40 @@ const readAssetBaseUrl = async (constantsPath: string) => { }; const getResourceIndexPublicPath = (resourceId: string, revision: string) => - `/resource-indexes/${encodeURIComponent(resourceId)}.${encodeURIComponent(revision)}.json`; + `/resource-indexes/${encodeURIComponent(resourceId)}.${encodeURIComponent( + revision + )}.json`; + +export const generateAssetQueryRuntimeModule = ({ + deploymentId, + manifest, +}: { + deploymentId: string; + manifest: readonly { + resourceId: string; + revision: string; + queryHash: string; + assetRevision: string; + indexPath: string; + }[]; +}) => { + const inputType = `{ + request: Request; + context: unknown; + fallback: typeof fetch; + }`; + if (manifest.length === 0) { + return `export const createGeneratedAssetResourceFetch = async ({ fallback }: ${inputType}): Promise => fallback;\n`; + } + return `import { createGeneratedAssetResourceFetch as createRuntimeFetch } from "@webstudio-is/asset-resource"; + +const deploymentId = ${JSON.stringify(deploymentId)}; +const manifest = ${JSON.stringify(manifest, null, 2)}; + +export const createGeneratedAssetResourceFetch = ({ request, context, fallback }: ${inputType}) => + createRuntimeFetch({ request, context, deploymentId, manifest, fallback }); +`; +}; export const materializeAssetResourceIndexes = async ({ snapshots, @@ -176,13 +209,23 @@ export const materializeAssetResourceIndexes = async ({ } await writeFile( join(generatedDirectory, "$resources.asset-query-manifest.ts"), - `export const assetQueryDeploymentId = ${JSON.stringify(deploymentId)};\nexport const assetQueryManifest = ${JSON.stringify( + `export const assetQueryDeploymentId = ${JSON.stringify( + deploymentId + )};\nexport const assetQueryManifest = ${JSON.stringify( manifest.map(({ index: _index, ...entry }) => entry), null, 2 )};\n`, "utf8" ); + await writeFile( + join(generatedDirectory, "$resources.asset-query-runtime.ts"), + generateAssetQueryRuntimeModule({ + deploymentId, + manifest: manifest.map(({ index: _index, ...entry }) => entry), + }), + "utf8" + ); }; const writeWsAuthResources = async ( @@ -285,7 +328,9 @@ const generateRedirectFallbackRoute = (runtime: "remix" | "react-router") => { runtime === "react-router" ? "react-router" : "@remix-run/server-runtime"; return ` - import { type LoaderFunctionArgs } from ${JSON.stringify(loaderFunctionArgs)}; + import { type LoaderFunctionArgs } from ${JSON.stringify( + loaderFunctionArgs + )}; import { redirectRequest } from "../redirect-url"; // @todo think about how to make __generated__ typeable // @ts-ignore @@ -396,7 +441,10 @@ export const prebuild = async (options: { const parsedSiteData = publishedProjectBundle.safeParse(loadedSiteData); if (parsedSiteData.success === false) { throw new Error( - `Project bundle is invalid, please make sure the project is synced. Invalid fields: ${formatZodIssues(parsedSiteData.error.issues, loadedSiteData)}` + `Project bundle is invalid, please make sure the project is synced. Invalid fields: ${formatZodIssues( + parsedSiteData.error.issues, + loadedSiteData + )}` ); } const siteData = parsedSiteData.data; @@ -707,7 +755,9 @@ export const prebuild = async (options: { export const projectDomain = ${JSON.stringify(siteData.projectDomain)}; - export const lastPublished = "${new Date(siteData.build.createdAt).toISOString()}"; + export const lastPublished = "${new Date( + siteData.build.createdAt + ).toISOString()}"; export const siteName = ${JSON.stringify(projectMeta?.siteName)}; @@ -748,7 +798,9 @@ export const prebuild = async (options: { } export const CustomCode = () => { - return (<>${projectMeta?.code ? htmlToJsx(projectMeta.code) : ""}); + return (<>${ + projectMeta?.code ? htmlToJsx(projectMeta.code) : "" + }); } ` : "" @@ -810,6 +862,14 @@ export const prebuild = async (options: { file ) ) + .replaceAll( + "__ASSET_QUERY_RUNTIME__", + importFrom(`./app/__generated__/$resources.asset-query-runtime`, file) + ) + .replaceAll( + "__ASSET_RESOURCE_FETCH__", + importFrom("./app/asset-resource-fetch", file) + ) .replaceAll( "__AUTH__", importFrom(`./app/__generated__/$resources.wsauth.server`, file) @@ -891,12 +951,8 @@ export const prebuild = async (options: { if (options.assets === true && siteData.assets.length > 0) { const downloading = createProgress(); downloading.start("Downloading fonts and images"); - const protectedAssetIds = new Set(siteData.protectedAssetIds ?? []); - const assetsToMaterialize = siteData.assets.filter( - (asset) => protectedAssetIds.has(asset.id) === false - ); await materializeAssetFiles({ - assets: assetsToMaterialize, + assets: siteData.assets, continueOnError: true, origin: siteData.origin || "", sourceAssetsDirectory: join(buildRoot, LOCAL_ASSETS_DIR), diff --git a/packages/cli/src/template-placeholders.d.ts b/packages/cli/src/template-placeholders.d.ts index 3a265d85a669..34678a644bdb 100644 --- a/packages/cli/src/template-placeholders.d.ts +++ b/packages/cli/src/template-placeholders.d.ts @@ -4,3 +4,23 @@ declare module "__ASSET_QUERY_MANIFEST__" { export const assetQueryDeploymentId: string; export const assetQueryManifest: PublishedAssetResourceManifestEntry[]; } + +declare module "__ASSET_QUERY_RUNTIME__" { + export const createGeneratedAssetResourceFetch: (options: { + request: Request; + context: unknown; + fallback: typeof fetch; + }) => Promise; +} + +declare module "__ASSET_RESOURCE_FETCH__" { + import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource"; + + export const createSsgAssetResourceFetch: (options: { + deploymentId: string; + manifest: readonly PublishedAssetResourceManifestEntry[]; + }) => ( + input: RequestInfo | URL, + init?: RequestInit + ) => Promise; +} diff --git a/packages/cli/templates/defaults/app/asset-resource-fetch.ts b/packages/cli/templates/defaults/app/asset-resource-fetch.ts deleted file mode 100644 index ef2efac3f650..000000000000 --- a/packages/cli/templates/defaults/app/asset-resource-fetch.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource"; - -type AssetBinding = { - fetch: (request: Request) => Promise; -}; - -const getAssetBinding = (context: unknown): AssetBinding | undefined => { - if (typeof context !== "object" || context === null) { - return; - } - const cloudflare = Reflect.get(context, "cloudflare"); - if (typeof cloudflare !== "object" || cloudflare === null) { - return; - } - const env = Reflect.get(cloudflare, "env"); - if (typeof env !== "object" || env === null) { - return; - } - const assets = Reflect.get(env, "ASSETS"); - if ( - typeof assets === "object" && - assets !== null && - typeof Reflect.get(assets, "fetch") === "function" - ) { - return assets as AssetBinding; - } -}; - -export const createGeneratedAssetResourceFetch = async ({ - request, - context, - deploymentId, - manifest, - fallback, -}: { - request: Request; - context: unknown; - deploymentId: string; - manifest: Parameters< - typeof createPublishedAssetResourceFetch - >[0]["manifest"]; - fallback: typeof fetch; -}): Promise => { - const binding = getAssetBinding(context); - const fetchAsset = (path: string, init?: RequestInit) => { - const assetRequest = new Request(new URL(path, request.url), init); - return binding?.fetch(assetRequest) ?? fetch(assetRequest); - }; - const cache = globalThis.caches - ? await caches.open(`webstudio-assets-${deploymentId}`) - : undefined; - const fetchResource = createPublishedAssetResourceFetch({ - deploymentId, - manifest, - fetchAsset, - cache, - }); - return async (input, init) => - (await fetchResource(input, init)) ?? fallback(input, init); -}; diff --git a/packages/cli/templates/defaults/app/route-templates/html.tsx b/packages/cli/templates/defaults/app/route-templates/html.tsx index 0106cb59c700..3a88a7080889 100644 --- a/packages/cli/templates/defaults/app/route-templates/html.tsx +++ b/packages/cli/templates/defaults/app/route-templates/html.tsx @@ -46,11 +46,7 @@ import css from "__CSS__?url"; import { sitemap } from "__SITEMAP__"; import { assets } from "__ASSETS__"; import { authRoutes } from "__AUTH__"; -import { - assetQueryDeploymentId, - assetQueryManifest, -} from "__ASSET_QUERY_MANIFEST__"; -import { createGeneratedAssetResourceFetch } from "../asset-resource-fetch"; +import { createGeneratedAssetResourceFetch } from "__ASSET_QUERY_RUNTIME__"; const authenticateProductionRequest = (request: Request) => { const host = @@ -131,8 +127,6 @@ export const loader = async (arg: LoaderFunctionArgs) => { const generatedFetch = await createGeneratedAssetResourceFetch({ request: arg.request, context: arg.context, - deploymentId: assetQueryDeploymentId, - manifest: assetQueryManifest, fallback: customFetch, }); const resources = await loadResources( diff --git a/packages/cli/templates/defaults/app/route-templates/text.tsx b/packages/cli/templates/defaults/app/route-templates/text.tsx index 3a557d38e02f..d8cec689df08 100644 --- a/packages/cli/templates/defaults/app/route-templates/text.tsx +++ b/packages/cli/templates/defaults/app/route-templates/text.tsx @@ -6,11 +6,7 @@ import { getPageMeta, getRemixParams, getResources } from "__SERVER__"; import { sitemap } from "__SITEMAP__"; import { assets } from "__ASSETS__"; import { authRoutes } from "__AUTH__"; -import { - assetQueryDeploymentId, - assetQueryManifest, -} from "__ASSET_QUERY_MANIFEST__"; -import { createGeneratedAssetResourceFetch } from "../asset-resource-fetch"; +import { createGeneratedAssetResourceFetch } from "__ASSET_QUERY_RUNTIME__"; const authenticateProductionRequest = (request: Request) => { const host = @@ -90,8 +86,6 @@ export const loader = async (arg: LoaderFunctionArgs) => { const generatedFetch = await createGeneratedAssetResourceFetch({ request: arg.request, context: arg.context, - deploymentId: assetQueryDeploymentId, - manifest: assetQueryManifest, fallback: customFetch, }); const resources = await loadResources( diff --git a/packages/cli/templates/defaults/app/route-templates/xml.tsx b/packages/cli/templates/defaults/app/route-templates/xml.tsx index dd7ecf10c78b..2f91078a73ef 100644 --- a/packages/cli/templates/defaults/app/route-templates/xml.tsx +++ b/packages/cli/templates/defaults/app/route-templates/xml.tsx @@ -12,11 +12,7 @@ import { assetBaseUrl, imageLoader } from "__CONSTANTS__"; import { sitemap } from "__SITEMAP__"; import { assets } from "__ASSETS__"; import { authRoutes } from "__AUTH__"; -import { - assetQueryDeploymentId, - assetQueryManifest, -} from "__ASSET_QUERY_MANIFEST__"; -import { createGeneratedAssetResourceFetch } from "../asset-resource-fetch"; +import { createGeneratedAssetResourceFetch } from "__ASSET_QUERY_RUNTIME__"; const authenticateProductionRequest = (request: Request) => { const host = @@ -98,8 +94,6 @@ export const loader = async (arg: LoaderFunctionArgs) => { const generatedFetch = await createGeneratedAssetResourceFetch({ request: arg.request, context: arg.context, - deploymentId: assetQueryDeploymentId, - manifest: assetQueryManifest, fallback: customFetch, }); const resources = await loadResources( diff --git a/packages/cli/templates/react-router/app/asset-resource-fetch.ts b/packages/cli/templates/react-router/app/asset-resource-fetch.ts deleted file mode 100644 index ef2efac3f650..000000000000 --- a/packages/cli/templates/react-router/app/asset-resource-fetch.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource"; - -type AssetBinding = { - fetch: (request: Request) => Promise; -}; - -const getAssetBinding = (context: unknown): AssetBinding | undefined => { - if (typeof context !== "object" || context === null) { - return; - } - const cloudflare = Reflect.get(context, "cloudflare"); - if (typeof cloudflare !== "object" || cloudflare === null) { - return; - } - const env = Reflect.get(cloudflare, "env"); - if (typeof env !== "object" || env === null) { - return; - } - const assets = Reflect.get(env, "ASSETS"); - if ( - typeof assets === "object" && - assets !== null && - typeof Reflect.get(assets, "fetch") === "function" - ) { - return assets as AssetBinding; - } -}; - -export const createGeneratedAssetResourceFetch = async ({ - request, - context, - deploymentId, - manifest, - fallback, -}: { - request: Request; - context: unknown; - deploymentId: string; - manifest: Parameters< - typeof createPublishedAssetResourceFetch - >[0]["manifest"]; - fallback: typeof fetch; -}): Promise => { - const binding = getAssetBinding(context); - const fetchAsset = (path: string, init?: RequestInit) => { - const assetRequest = new Request(new URL(path, request.url), init); - return binding?.fetch(assetRequest) ?? fetch(assetRequest); - }; - const cache = globalThis.caches - ? await caches.open(`webstudio-assets-${deploymentId}`) - : undefined; - const fetchResource = createPublishedAssetResourceFetch({ - deploymentId, - manifest, - fetchAsset, - cache, - }); - return async (input, init) => - (await fetchResource(input, init)) ?? fallback(input, init); -}; diff --git a/packages/cli/templates/react-router/app/route-templates/html.tsx b/packages/cli/templates/react-router/app/route-templates/html.tsx index a54045479f91..ea30c7265f27 100644 --- a/packages/cli/templates/react-router/app/route-templates/html.tsx +++ b/packages/cli/templates/react-router/app/route-templates/html.tsx @@ -45,11 +45,7 @@ import css from "__CSS__?url"; import { sitemap } from "__SITEMAP__"; import { assets } from "__ASSETS__"; import { authRoutes } from "__AUTH__"; -import { - assetQueryDeploymentId, - assetQueryManifest, -} from "__ASSET_QUERY_MANIFEST__"; -import { createGeneratedAssetResourceFetch } from "../asset-resource-fetch"; +import { createGeneratedAssetResourceFetch } from "__ASSET_QUERY_RUNTIME__"; const authenticateProductionRequest = (request: Request) => { const host = @@ -130,8 +126,6 @@ export const loader = async (arg: LoaderFunctionArgs) => { const generatedFetch = await createGeneratedAssetResourceFetch({ request: arg.request, context: arg.context, - deploymentId: assetQueryDeploymentId, - manifest: assetQueryManifest, fallback: customFetch, }); const resources = await loadResources( diff --git a/packages/cli/templates/react-router/app/route-templates/text.tsx b/packages/cli/templates/react-router/app/route-templates/text.tsx index d89f207c0858..88e3391e08cc 100644 --- a/packages/cli/templates/react-router/app/route-templates/text.tsx +++ b/packages/cli/templates/react-router/app/route-templates/text.tsx @@ -6,11 +6,7 @@ import { getPageMeta, getRemixParams, getResources } from "__SERVER__"; import { sitemap } from "__SITEMAP__"; import { assets } from "__ASSETS__"; import { authRoutes } from "__AUTH__"; -import { - assetQueryDeploymentId, - assetQueryManifest, -} from "__ASSET_QUERY_MANIFEST__"; -import { createGeneratedAssetResourceFetch } from "../asset-resource-fetch"; +import { createGeneratedAssetResourceFetch } from "__ASSET_QUERY_RUNTIME__"; const authenticateProductionRequest = (request: Request) => { const host = @@ -90,8 +86,6 @@ export const loader = async (arg: LoaderFunctionArgs) => { const generatedFetch = await createGeneratedAssetResourceFetch({ request: arg.request, context: arg.context, - deploymentId: assetQueryDeploymentId, - manifest: assetQueryManifest, fallback: customFetch, }); const resources = await loadResources( diff --git a/packages/cli/templates/react-router/app/route-templates/xml.tsx b/packages/cli/templates/react-router/app/route-templates/xml.tsx index 5927dff8e2a9..5cecf6a16f32 100644 --- a/packages/cli/templates/react-router/app/route-templates/xml.tsx +++ b/packages/cli/templates/react-router/app/route-templates/xml.tsx @@ -12,11 +12,7 @@ import { assetBaseUrl, imageLoader } from "__CONSTANTS__"; import { sitemap } from "__SITEMAP__"; import { assets } from "__ASSETS__"; import { authRoutes } from "__AUTH__"; -import { - assetQueryDeploymentId, - assetQueryManifest, -} from "__ASSET_QUERY_MANIFEST__"; -import { createGeneratedAssetResourceFetch } from "../asset-resource-fetch"; +import { createGeneratedAssetResourceFetch } from "__ASSET_QUERY_RUNTIME__"; const authenticateProductionRequest = (request: Request) => { const host = @@ -98,8 +94,6 @@ export const loader = async (arg: LoaderFunctionArgs) => { const generatedFetch = await createGeneratedAssetResourceFetch({ request: arg.request, context: arg.context, - deploymentId: assetQueryDeploymentId, - manifest: assetQueryManifest, fallback: customFetch, }); const resources = await loadResources( diff --git a/packages/cli/templates/ssg/app/asset-resource-fetch.ts b/packages/cli/templates/ssg/app/asset-resource-fetch.ts new file mode 100644 index 000000000000..5cbceb67cd60 --- /dev/null +++ b/packages/cli/templates/ssg/app/asset-resource-fetch.ts @@ -0,0 +1,96 @@ +import { open } from "node:fs/promises"; +import { resolve, sep } from "node:path"; +import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource"; + +const resolvePublicAsset = (path: string) => { + const publicDirectory = resolve("public"); + const pathname = new URL(path, "https://webstudio.local").pathname; + const filePath = resolve(publicDirectory, `.${pathname}`); + if (filePath.startsWith(`${publicDirectory}${sep}`) === false) { + throw new Error("Static asset path is outside the public directory"); + } + return filePath; +}; + +const getByteRange = (header: string | null, size: number) => { + if (header?.startsWith("bytes=") !== true) { + return { offset: 0, length: size, partial: false }; + } + const values = header.slice("bytes=".length).split("-"); + const offset = Number(values[0]); + const requestedEnd = Number(values[1]); + if ( + values.length !== 2 || + Number.isSafeInteger(offset) === false || + Number.isSafeInteger(requestedEnd) === false || + offset < 0 || + requestedEnd < offset || + offset >= size + ) { + return; + } + const end = Math.min(requestedEnd, size - 1); + return { offset, length: end - offset + 1, partial: true }; +}; + +export const fetchSsgPublicAsset = async (path: string, init?: RequestInit) => { + let file; + try { + file = await open(resolvePublicAsset(path), "r"); + } catch (error) { + if ( + typeof error === "object" && + error !== null && + Reflect.get(error, "code") === "ENOENT" + ) { + return new Response("Not found", { status: 404 }); + } + throw error; + } + try { + const { size } = await file.stat(); + const range = getByteRange( + new Headers(init?.headers).get("range"), + size + ); + if (range === undefined) { + return new Response("Range not satisfiable", { + status: 416, + headers: { "content-range": `bytes */${size}` }, + }); + } + const body = range.partial + ? await file + .read(new Uint8Array(range.length), 0, range.length, range.offset) + .then(({ buffer, bytesRead }) => buffer.subarray(0, bytesRead)) + : await file.readFile(); + const headers = new Headers({ "content-length": String(body.length) }); + if (range.partial) { + headers.set( + "content-range", + `bytes ${range.offset}-${range.offset + body.length - 1}/${size}` + ); + } + return new Response(Uint8Array.from(body).buffer, { + status: range.partial ? 206 : 200, + headers, + }); + } finally { + await file.close(); + } +}; + +export const createSsgAssetResourceFetch = ({ + deploymentId, + manifest, +}: { + deploymentId: string; + manifest: Parameters< + typeof createPublishedAssetResourceFetch + >[0]["manifest"]; +}) => + createPublishedAssetResourceFetch({ + deploymentId, + manifest, + fetchAsset: fetchSsgPublicAsset, + }); diff --git a/packages/cli/templates/ssg/app/route-templates/html/+data.ts b/packages/cli/templates/ssg/app/route-templates/html/+data.ts index baed1a7e5e26..44fce4c306fb 100644 --- a/packages/cli/templates/ssg/app/route-templates/html/+data.ts +++ b/packages/cli/templates/ssg/app/route-templates/html/+data.ts @@ -2,8 +2,22 @@ import type { PageContextServer } from "vike/types"; import { isLocalResource, loadResources } from "@webstudio-is/sdk/runtime"; import { getPageMeta, getResources } from "__SERVER__"; import { assets } from "__ASSETS__"; +import { + assetQueryDeploymentId, + assetQueryManifest, +} from "__ASSET_QUERY_MANIFEST__"; +import { createSsgAssetResourceFetch } from "__ASSET_RESOURCE_FETCH__"; -const customFetch: typeof fetch = (input, init) => { +const fetchAssetResource = createSsgAssetResourceFetch({ + deploymentId: assetQueryDeploymentId, + manifest: assetQueryManifest, +}); + +const customFetch: typeof fetch = async (input, init) => { + const assetResourceResponse = await fetchAssetResource(input, init); + if (assetResourceResponse !== undefined) { + return assetResourceResponse; + } if (typeof input !== "string") { return fetch(input, init); } @@ -23,13 +37,13 @@ const customFetch: typeof fetch = (input, init) => { }; const response = new Response(JSON.stringify(data)); response.headers.set("content-type", "application/json; charset=utf-8"); - return Promise.resolve(response); + return response; } if (isLocalResource(input, "assets")) { const response = new Response(JSON.stringify(assets)); response.headers.set("content-type", "application/json; charset=utf-8"); - return Promise.resolve(response); + return response; } return fetch(input, init); diff --git a/packages/cli/templates/ssg/package.json b/packages/cli/templates/ssg/package.json index 87a4f46d5b20..ce3826a3a1fb 100644 --- a/packages/cli/templates/ssg/package.json +++ b/packages/cli/templates/ssg/package.json @@ -9,6 +9,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@webstudio-is/asset-resource": "0.0.0-webstudio-version", "@webstudio-is/image": "0.0.0-webstudio-version", "@webstudio-is/react-sdk": "0.0.0-webstudio-version", "@webstudio-is/sdk": "0.0.0-webstudio-version", diff --git a/packages/cli/templates/ssg/vite.config.ts b/packages/cli/templates/ssg/vite.config.ts index 8e2bf4e3d6d2..83d1624d67ab 100644 --- a/packages/cli/templates/ssg/vite.config.ts +++ b/packages/cli/templates/ssg/vite.config.ts @@ -2,14 +2,17 @@ import { defineConfig } from "vite"; import react from "@vitejs/plugin-react"; import vike from "vike/plugin"; +const sourceConditions = + process.env.WEBSTUDIO_LOCAL_CLI_BOOTSTRAPPED === "1" ? ["webstudio"] : []; + export default defineConfig({ plugins: [react(), vike()], resolve: { - conditions: ["browser", "development|production"], + conditions: [...sourceConditions, "browser", "development|production"], }, ssr: { resolve: { - conditions: ["node", "development|production"], + conditions: [...sourceConditions, "node", "development|production"], }, }, }); diff --git a/packages/postgrest/supabase/tests/asset-file-metadata.sql b/packages/postgrest/supabase/tests/asset-file-metadata.sql index 540caa0a00c9..f548b9ee9767 100644 --- a/packages/postgrest/supabase/tests/asset-file-metadata.sql +++ b/packages/postgrest/supabase/tests/asset-file-metadata.sql @@ -139,5 +139,49 @@ SELECT is( 'Stale cleanup removed the canonical row' ); +INSERT INTO "File" ("name", "format", "size", "updatedAt", "status") +VALUES ('metadata-image.png', 'png', 20, '2026-07-18T12:00:00Z', 'UPLOADED'); + +INSERT INTO "Asset" ("id", "projectId", "name", "filename") +VALUES ( + 'metadata-image-asset', + 'metadata-test-project', + 'metadata-image.png', + 'cover.png' +); + +SELECT is( + replace_asset_file_metadata( + 'metadata-test-project', + 'metadata-image-asset', + 'image-revision', + '{"_id":"metadata-image-asset","revision":"image-revision"}'::JSONB, + '[]'::JSONB, + '{"storageName":"metadata-image.png","fileUpdatedAt":"2026-07-18T12:00:00Z","fileSize":20,"filename":"cover.png","folderId":null}'::JSONB + ), + TRUE, + 'Canonical metadata supports any uploaded asset type' +); + +SELECT is( + delete_stale_asset_file_metadata( + 'metadata-test-project', + ARRAY['metadata-image-asset'] + ), + 0, + 'Stale cleanup preserves metadata for an uploaded non-Markdown asset' +); + +SELECT is( + ( + SELECT count(*)::INTEGER + FROM "AssetFileMetadata" + WHERE "projectId" = 'metadata-test-project' + AND "assetId" = 'metadata-image-asset' + ), + 1, + 'Non-Markdown canonical metadata remains available' +); + SELECT * FROM finish(); ROLLBACK; diff --git a/packages/postgrest/supabase/tests/asset-resource-indexes.sql b/packages/postgrest/supabase/tests/asset-resource-indexes.sql index c55bb9af5f8c..8c884496b193 100644 --- a/packages/postgrest/supabase/tests/asset-resource-indexes.sql +++ b/packages/postgrest/supabase/tests/asset-resource-indexes.sql @@ -197,6 +197,35 @@ SELECT is( 'The revision becomes collectable after its last reference is removed' ); +SELECT is( + ( + SELECT count(*)::INTEGER + FROM claim_asset_resource_index_garbage('2100-01-01', 10) + ), + 0, + 'A fresh garbage claim cannot be claimed by another worker' +); + +UPDATE "AssetResourceIndexRevision" +SET "gcStartedAt" = CURRENT_TIMESTAMP - INTERVAL '16 minutes' +WHERE "projectId" = 'resource-index-test-project' + AND "resourceId" = 'posts'; + +CREATE TEMP TABLE reclaimed_resource_index_garbage AS +SELECT * FROM claim_asset_resource_index_garbage('2100-01-01', 10); + +SELECT is( + (SELECT count(*)::INTEGER FROM reclaimed_resource_index_garbage), + 1, + 'An expired garbage claim is reclaimed after its lease' +); + +SELECT isnt( + (SELECT "gcClaimId" FROM reclaimed_resource_index_garbage), + (SELECT "gcClaimId" FROM claimed_resource_index_garbage), + 'Reclaiming an expired garbage claim rotates its claim id' +); + SELECT is( finalize_asset_resource_index_garbage( (SELECT "projectId" FROM claimed_resource_index_garbage), @@ -204,16 +233,27 @@ SELECT is( (SELECT revision FROM claimed_resource_index_garbage), (SELECT "gcClaimId" FROM claimed_resource_index_garbage) ), + FALSE, + 'An expired claim cannot finalize after another worker reclaims it' +); + +SELECT is( + finalize_asset_resource_index_garbage( + (SELECT "projectId" FROM reclaimed_resource_index_garbage), + (SELECT "resourceId" FROM reclaimed_resource_index_garbage), + (SELECT revision FROM reclaimed_resource_index_garbage), + (SELECT "gcClaimId" FROM reclaimed_resource_index_garbage) + ), TRUE, 'The matching garbage claim finalizes revision cleanup' ); SELECT is( finalize_asset_resource_index_garbage( - (SELECT "projectId" FROM claimed_resource_index_garbage), - (SELECT "resourceId" FROM claimed_resource_index_garbage), - (SELECT revision FROM claimed_resource_index_garbage), - (SELECT "gcClaimId" FROM claimed_resource_index_garbage) + (SELECT "projectId" FROM reclaimed_resource_index_garbage), + (SELECT "resourceId" FROM reclaimed_resource_index_garbage), + (SELECT revision FROM reclaimed_resource_index_garbage), + (SELECT "gcClaimId" FROM reclaimed_resource_index_garbage) ), FALSE, 'Repeated cleanup is idempotent' diff --git a/packages/prisma-client/prisma/migrations/20260718121000_serialize_asset_file_metadata/migration.sql b/packages/prisma-client/prisma/migrations/20260718121000_serialize_asset_file_metadata/migration.sql index e4a7d2011f8f..f099a3a8122c 100644 --- a/packages/prisma-client/prisma/migrations/20260718121000_serialize_asset_file_metadata/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718121000_serialize_asset_file_metadata/migration.sql @@ -95,7 +95,6 @@ BEGIN WHERE asset."projectId" = p_project_id AND asset."id" = asset_id AND file."status" = 'UPLOADED' - AND file."name" ~* '\.md$' ) THEN DELETE FROM public."AssetFileMetadata" WHERE "projectId" = p_project_id diff --git a/packages/prisma-client/prisma/migrations/20260718129000_asset_resource_index_gc/migration.sql b/packages/prisma-client/prisma/migrations/20260718129000_asset_resource_index_gc/migration.sql index 6a673eaa427b..4f95925d75dd 100644 --- a/packages/prisma-client/prisma/migrations/20260718129000_asset_resource_index_gc/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718129000_asset_resource_index_gc/migration.sql @@ -20,7 +20,12 @@ BEGIN SELECT candidate.ctid FROM public."AssetResourceIndexRevision" AS candidate WHERE candidate."unreferencedAt" <= p_before - AND candidate."gcClaimId" IS NULL + AND ( + candidate."gcClaimId" IS NULL + -- Object deletion is idempotent, so an interrupted worker's claim can + -- safely be rotated and resumed after its lease expires. + OR candidate."gcStartedAt" <= CURRENT_TIMESTAMP - INTERVAL '15 minutes' + ) AND NOT EXISTS ( SELECT 1 FROM public."AssetResourceIndexReference" AS reference WHERE reference."projectId" = candidate."projectId" diff --git a/packages/project-build/src/runtime/asset-resources.ts b/packages/project-build/src/runtime/asset-resources.ts index b143fcf0f1ea..384dc65e61b7 100644 --- a/packages/project-build/src/runtime/asset-resources.ts +++ b/packages/project-build/src/runtime/asset-resources.ts @@ -2,9 +2,11 @@ import { assetResourceContentOptions, assetResourceLimits, assetResourceQueryRequest, + createAssetQueryResourceBody, decodeDataSourceVariable, - generateObjectExpression, - parseObjectExpression, + isAssetsResource as isSdkAssetsResource, + isStoredAssetQueryResource, + parseAssetQueryResourceBody, SYSTEM_VARIABLE_ID, transpileExpression, type Resource, @@ -23,7 +25,6 @@ import { } from "./data"; import { throwBuilderRuntimeError } from "./errors"; import { paginateOutput, paginatedOutputInputSchema } from "./output"; -import { getStaticStringLiteral } from "./text-replacement"; const parameterName = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/); @@ -125,24 +126,15 @@ export const createAssetResourceBody = ({ resultLimit, content, }: AssetResourceConfiguration) => - generateObjectExpression( - new Map([ - ["query", JSON.stringify(query)], - [ - "parameters", - generateObjectExpression( - new Map( - parameters.map(({ name, value }) => [ - name, - normalizeExpression(value), - ]) - ) - ), - ], - ["resultLimit", JSON.stringify(resultLimit)], - ["content", JSON.stringify(content)], - ]) - ); + createAssetQueryResourceBody({ + query, + parameters: parameters.map(({ name, value }) => ({ + name, + value: normalizeExpression(value), + })), + resultLimit, + contentExpression: JSON.stringify(content), + }); const parseJsonExpression = (expression: string | undefined) => { if (expression === undefined) { @@ -155,16 +147,8 @@ const parseJsonExpression = (expression: string | undefined) => { } }; -const isStoredAssetQueryResource = (resource: Resource) => - resource.control === "system" && - resource.method === "post" && - getStaticStringLiteral(resource.url) === assetsQueryResourceUrl; - export const isAssetsResource = (resource: Resource) => - isStoredAssetQueryResource(resource) || - (resource.control === "system" && - resource.method === "get" && - getStaticStringLiteral(resource.url) === assetsResourceUrl); + isSdkAssetsResource(resource); export const parseAssetResourceConfiguration = ( resource: Resource @@ -172,16 +156,12 @@ export const parseAssetResourceConfiguration = ( if (isStoredAssetQueryResource(resource) === false) { return; } - const fields = parseObjectExpression(resource.body ?? ""); - const parameterFields = parseObjectExpression(fields.get("parameters") ?? ""); + const fields = parseAssetQueryResourceBody(resource.body); const parsed = storedAssetQueryConfigurationInput.safeParse({ - query: parseJsonExpression(fields.get("query")), - parameters: Array.from(parameterFields, ([name, value]) => ({ - name, - value, - })), - resultLimit: parseJsonExpression(fields.get("resultLimit")), - content: parseJsonExpression(fields.get("content")), + query: parseJsonExpression(fields.queryExpression), + parameters: fields.parameters, + resultLimit: parseJsonExpression(fields.resultLimitExpression), + content: parseJsonExpression(fields.contentExpression), }); if (parsed.success === false) { return; diff --git a/packages/protocol/src/schema.ts b/packages/protocol/src/schema.ts index 61b8289a4639..33b946d78e79 100644 --- a/packages/protocol/src/schema.ts +++ b/packages/protocol/src/schema.ts @@ -87,7 +87,6 @@ export const publishedProjectBundle = projectBundle.extend({ }) ) .optional(), - protectedAssetIds: z.array(z.string().min(1)).optional(), }); export type PublishedProjectBundle = z.infer; diff --git a/packages/sdk/src/asset-resource-config.test.ts b/packages/sdk/src/asset-resource-config.test.ts new file mode 100644 index 000000000000..8e56897b4d28 --- /dev/null +++ b/packages/sdk/src/asset-resource-config.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from "vitest"; +import { + createAssetQueryResourceBody, + getAssetResourceQuery, + isAssetsResource, + isStoredAssetQueryResource, + parseAssetQueryResourceBody, +} from "./asset-resource-config"; +import type { Resource } from "./schema/resources"; + +const createResource = (overrides: Partial = {}): Resource => ({ + id: "posts", + name: "Posts", + control: "system", + method: "post", + url: '"/$resources/assets/query"', + headers: [], + ...overrides, +}); + +describe("asset query resource configuration", () => { + test("round-trips query fields and expression parameters", () => { + const body = createAssetQueryResourceBody({ + query: "*[properties.slug == $slug]", + parameters: [{ name: "slug", value: "$ws$dataSource$routeSlug" }], + resultLimit: 1, + }); + + expect(parseAssetQueryResourceBody(body)).toEqual({ + queryExpression: '"*[properties.slug == $slug]"', + parameters: [{ name: "slug", value: "$ws$dataSource$routeSlug" }], + resultLimitExpression: "1", + contentExpression: '{ "mode": "none" }', + }); + expect(getAssetResourceQuery(createResource({ body }))).toBe( + "*[properties.slug == $slug]" + ); + }); + + test("only recognizes exact system Assets resource contracts", () => { + expect(isStoredAssetQueryResource(createResource())).toBe(true); + expect(isAssetsResource(createResource())).toBe(true); + expect( + isAssetsResource( + createResource({ method: "get", url: '"/$resources/assets"' }) + ) + ).toBe(true); + expect(isAssetsResource(createResource({ control: undefined }))).toBe( + false + ); + expect( + isAssetsResource( + createResource({ url: '"/$resources/assets/query/extra"' }) + ) + ).toBe(false); + expect( + isAssetsResource(createResource({ url: ' "/$resources/assets/query"' })) + ).toBe(false); + }); +}); diff --git a/packages/sdk/src/asset-resource-config.ts b/packages/sdk/src/asset-resource-config.ts new file mode 100644 index 000000000000..61bf25b35d87 --- /dev/null +++ b/packages/sdk/src/asset-resource-config.ts @@ -0,0 +1,86 @@ +import { generateObjectExpression, parseObjectExpression } from "./expression"; +import { assetsQueryResourceUrl, assetsResourceUrl } from "./resource-loader"; +import type { Resource } from "./schema/resources"; + +export type AssetQueryParameterBinding = { + name: string; + value: string; +}; + +const getStaticStringLiteral = (expression: string) => { + try { + const value = JSON.parse(expression); + return typeof value === "string" && JSON.stringify(value) === expression + ? value + : undefined; + } catch { + return; + } +}; + +export const parseAssetQueryResourceBody = (body: string | undefined) => { + const fields = parseObjectExpression(body ?? ""); + const parameters = parseObjectExpression(fields.get("parameters") ?? ""); + return { + queryExpression: fields.get("query"), + resultLimitExpression: fields.get("resultLimit"), + contentExpression: fields.get("content"), + parameters: Array.from(parameters, ([name, value]) => ({ name, value })), + }; +}; + +export const createAssetQueryResourceBody = ({ + query, + parameters, + resultLimit = 100, + contentExpression = '{ "mode": "none" }', +}: { + query: string; + parameters: readonly AssetQueryParameterBinding[]; + resultLimit?: number; + contentExpression?: string; +}) => + generateObjectExpression( + new Map([ + ["query", JSON.stringify(query)], + [ + "parameters", + generateObjectExpression( + new Map( + parameters + .filter(({ name }) => name.trim().length > 0) + .map(({ name, value }) => [name, value]) + ) + ), + ], + ["resultLimit", JSON.stringify(resultLimit)], + ["content", contentExpression], + ]) + ); + +export const isStoredAssetQueryResource = (resource: Resource) => + resource.control === "system" && + resource.method === "post" && + getStaticStringLiteral(resource.url) === assetsQueryResourceUrl; + +export const isAssetsResource = (resource: Resource) => + isStoredAssetQueryResource(resource) || + (resource.control === "system" && + resource.method === "get" && + getStaticStringLiteral(resource.url) === assetsResourceUrl); + +export const getAssetResourceQuery = (resource: Resource) => { + if (isStoredAssetQueryResource(resource) === false) { + return; + } + const { queryExpression } = parseAssetQueryResourceBody(resource.body); + if (queryExpression === undefined) { + return; + } + try { + const query = JSON.parse(queryExpression); + return typeof query === "string" && query.trim() !== "" ? query : undefined; + } catch { + return; + } +}; diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 21b1738b695d..8a5c8160faa0 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -18,6 +18,7 @@ export * from "./schema/component-meta"; export * from "./assets"; export * from "./asset-folder-hierarchy"; export * from "./asset-folder-normalization"; +export * from "./asset-resource-config"; export * from "./core-metas"; export * from "./instances-utils"; export * from "./page-utils"; diff --git a/packages/sdk/src/resource-loader.test.ts b/packages/sdk/src/resource-loader.test.ts index 8caaedd425a2..195f4b5e60b3 100644 --- a/packages/sdk/src/resource-loader.test.ts +++ b/packages/sdk/src/resource-loader.test.ts @@ -335,6 +335,29 @@ describe("loadResource", () => { ]), }); }); + test("identifies generated Assets query requests by resource", async () => { + mockFetch.mockResolvedValue(Response.json({ ok: true })); + + await loadResource(mockFetch, { + resourceId: "posts-resource", + name: "Posts", + control: "system", + url: "/$resources/assets/query", + searchParams: [], + method: "post", + headers: [{ name: "content-type", value: "application/json" }], + body: { query: "*[]" }, + }); + + expect(mockFetch).toHaveBeenCalledWith("/$resources/assets/query", { + method: "post", + headers: new Headers([ + ["content-type", "application/json"], + ["x-webstudio-resource-id", "posts-resource"], + ]), + body: JSON.stringify({ query: "*[]" }), + }); + }); }); describe("getResourceCacheKey", () => { diff --git a/packages/sdk/src/resource-loader.ts b/packages/sdk/src/resource-loader.ts index 1e3f6ab6aa08..8aff44061901 100644 --- a/packages/sdk/src/resource-loader.ts +++ b/packages/sdk/src/resource-loader.ts @@ -29,6 +29,7 @@ export const assetsResourceUrl = `/${LOCAL_RESOURCE_PREFIX}/assets`; export const assetsQueryResourceUrl = `/${LOCAL_RESOURCE_PREFIX}/assets/query`; export const assetsFieldCatalogResourceUrl = `/${LOCAL_RESOURCE_PREFIX}/assets/field-catalog`; export const assetsIndexStatusResourceUrl = `/${LOCAL_RESOURCE_PREFIX}/assets/index-status`; +export const assetResourceIdHeader = "x-webstudio-resource-id"; export type ResourceLoadOptions = { signal?: AbortSignal; @@ -98,6 +99,13 @@ export const loadResource = async ( serializeValue(value), ]) ); + if ( + resourceRequest.resourceId !== undefined && + resourceRequest.control === "system" && + isLocalResource(href, "assets/query") + ) { + requestHeaders.set(assetResourceIdHeader, resourceRequest.resourceId); + } const requestInit: RequestInit = { method, headers: requestHeaders, diff --git a/packages/sdk/src/resources-generator.test.tsx b/packages/sdk/src/resources-generator.test.tsx index f5724d554df0..81f2e0b95842 100644 --- a/packages/sdk/src/resources-generator.test.tsx +++ b/packages/sdk/src/resources-generator.test.tsx @@ -67,6 +67,36 @@ test("generate resources loader", () => { `); }); +test("generates resource identity for an Assets query", () => { + const generated = generateResources({ + scope: createScope(), + page: { rootInstanceId: "body" } as Page, + dataSources: toMap([ + { + id: "postsVariable", + scopeInstanceId: "body", + type: "resource", + name: "Posts", + resourceId: "postsResource", + }, + ]), + resources: toMap([ + { + id: "postsResource", + name: "Posts", + control: "system" as const, + url: '"/$resources/assets/query"', + method: "post" as const, + headers: [], + body: '{ query: "*[]" }', + }, + ]), + props: new Map(), + }); + + expect(generated).toContain('resourceId: "postsResource"'); +}); + test("generate variable and use in resources loader", () => { expect( generateResources({ diff --git a/packages/sdk/src/resources-generator.ts b/packages/sdk/src/resources-generator.ts index c435b0aba95a..9cf14d1f5905 100644 --- a/packages/sdk/src/resources-generator.ts +++ b/packages/sdk/src/resources-generator.ts @@ -5,6 +5,7 @@ import type { Props } from "./schema/props"; import type { Instance, Instances } from "./schema/instances"; import type { Scope } from "./scope"; import { generateExpression, SYSTEM_VARIABLE_ID } from "./expression"; +import { isStoredAssetQueryResource } from "./asset-resource-config"; export const generateResources = ({ scope, @@ -29,6 +30,9 @@ export const generateResources = ({ // call resource by bound variable name const resourceName = scope.getName(resource.id, resource.name); generatedRequest += ` const ${resourceName}: ResourceRequest = {\n`; + if (isStoredAssetQueryResource(resource)) { + generatedRequest += ` resourceId: ${JSON.stringify(resource.id)},\n`; + } generatedRequest += ` name: ${JSON.stringify(resource.name)},\n`; if (resource.control !== undefined) { generatedRequest += ` control: "${resource.control}",\n`; diff --git a/packages/sdk/src/schema/asset-resource.ts b/packages/sdk/src/schema/asset-resource.ts index 0e312fc33eae..93580a4002e0 100644 --- a/packages/sdk/src/schema/asset-resource.ts +++ b/packages/sdk/src/schema/asset-resource.ts @@ -130,13 +130,13 @@ export const assetResourceLimits = { queryBytes: 32 * 1024, queryAstNodes: 1000, queryAstDepth: 64, - queryRuntimeMs: 250, + queryDatasetScans: 1, parameterCount: 32, parameterBytes: 64 * 1024, defaultResultCount: 100, resultCount: 1000, resultBytes: 1024 * 1024, - candidateDocuments: 5000, + candidateDocuments: 1000, indexBytes: 16 * 1024 * 1024, frontmatterBytes: 64 * 1024, frontmatterDepth: 8, @@ -337,14 +337,12 @@ export const assetResourceErrorCode = z.enum([ "INDEX_NOT_FOUND", "INDEX_BUILD_FAILED", "QUERY_COMPLEXITY_EXCEEDED", - "QUERY_TIMEOUT", "RESULT_LIMIT_EXCEEDED", "RESULT_SIZE_EXCEEDED", "CONTENT_IDENTITY_REQUIRED", "CONTENT_NOT_TEXT", "CONTENT_DECODING_FAILED", "CONTENT_LIMIT_EXCEEDED", - "PROTECTED_CONTENT", "INTERNAL_ERROR", ]); diff --git a/packages/sdk/src/schema/resources.ts b/packages/sdk/src/schema/resources.ts index 91391ebb756d..0a831d0748b5 100644 --- a/packages/sdk/src/schema/resources.ts +++ b/packages/sdk/src/schema/resources.ts @@ -40,6 +40,7 @@ export type Resource = z.infer; // evaluated variant of resource export const resourceRequest = z.object({ + resourceId: z.string().optional(), name: z.string(), control: z.optional(z.union([z.literal("system"), z.literal("graphql")])), method: method, From fec582b5a7d1dcfe5e32662ec3ea13747acdf192 Mon Sep 17 00:00:00 2001 From: Oleg Isonen Date: Mon, 20 Jul 2026 21:02:38 +0100 Subject: [PATCH 07/14] fix: simplify asset resource synchronization --- .../asset-query-form-utils.test.ts | 8 +++ .../settings-panel/asset-query-form-utils.ts | 5 +- .../settings-panel/resource-panel.tsx | 2 +- .../asset-resource/src/published-runtime.ts | 4 +- .../asset-resource/src/resource-index.test.ts | 18 +++++++ packages/asset-resource/src/resource-index.ts | 5 +- .../src/canonical-metadata-backfill.test.ts | 23 ++++---- .../src/canonical-metadata-backfill.ts | 15 +++--- packages/asset-uploader/src/patch.test.ts | 2 +- .../src/resource-index-build.ts | 3 ++ .../src/resource-index-lifecycle.ts | 3 ++ .../src/resource-index-maintenance.ts | 10 ++-- packages/asset-uploader/src/revision.test.ts | 2 +- packages/asset-uploader/src/upload.test.ts | 52 +++++-------------- packages/asset-uploader/src/upload.ts | 19 ++----- .../sdk/src/asset-resource-config.test.ts | 11 ++++ packages/sdk/src/asset-resource-config.ts | 8 ++- 17 files changed, 106 insertions(+), 84 deletions(-) diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts index d1f85810416a..06517b0a742b 100644 --- a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts @@ -121,6 +121,14 @@ describe("asset query resource body", () => { content: { mode: "range", offset: 0, length: 10 }, }) ).toBeUndefined(); + expect( + getAssetQueryConfigurationError({ + query: "*[properties.slug == $slug]", + parameters: [{ name: " slug " }], + resultLimit: 1, + content: { mode: "none" }, + }) + ).toBeUndefined(); expect( getAssetQueryConfigurationError({ query: "*[properties.slug == $slug]", diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts index da5f684f59ca..a8d3307e5ff1 100644 --- a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts @@ -1,6 +1,7 @@ import { assetResourceContentOptions, assetResourceLimits, + normalizeAssetQueryParameterName, type AssetResourceContentOptions, type AssetResourceIndexStatus, } from "@webstudio-is/sdk"; @@ -59,7 +60,9 @@ export const getAssetQueryConfigurationError = ({ if (parameters.length > assetResourceLimits.parameterCount) { return `Use at most ${assetResourceLimits.parameterCount} runtime parameters.`; } - const names = parameters.map(({ name }) => name.trim()); + const names = parameters.map(({ name }) => + normalizeAssetQueryParameterName(name) + ); if (names.some((name) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) === false)) { return "Runtime parameter names must be valid identifiers."; } diff --git a/apps/builder/app/builder/features/settings-panel/resource-panel.tsx b/apps/builder/app/builder/features/settings-panel/resource-panel.tsx index 9a0dab7a3a6c..917d7d4d1fb8 100644 --- a/apps/builder/app/builder/features/settings-panel/resource-panel.tsx +++ b/apps/builder/app/builder/features/settings-panel/resource-panel.tsx @@ -356,7 +356,7 @@ const HeaderPair = ({ value={name} onChange={(event) => onChange(event.target.value, value)} /> - + - input instanceof Request - ? new Request(input, init) - : new Request(input, init); + new Request(input, init); export const getPublishedAssetResourceCacheKey = async ({ deploymentId, diff --git a/packages/asset-resource/src/resource-index.test.ts b/packages/asset-resource/src/resource-index.test.ts index 79aeea28f0a9..8bb0ba176bb8 100644 --- a/packages/asset-resource/src/resource-index.test.ts +++ b/packages/asset-resource/src/resource-index.test.ts @@ -76,6 +76,24 @@ describe("resource index build", () => { await expect(verifyAssetResourceIndex(index)).resolves.toEqual(index); }); + test("reuses a canonical revision prepared for a batch", async () => { + const entry = createCanonicalAssetFileEntry({ + projectId: "project-1", + document: createDocument("post", {}), + }); + const assetRevision = `sha256:${"a".repeat(64)}`; + + const index = await buildAssetResourceIndex({ + projectId: "project-1", + resourceId: "resource-1", + query: "*[]", + entries: [entry], + assetRevision, + }); + + expect(index.assetRevision).toBe(assetRevision); + }); + test("treats all frontmatter fields as schemaless query data", async () => { const entries = [ createCanonicalAssetFileEntry({ diff --git a/packages/asset-resource/src/resource-index.ts b/packages/asset-resource/src/resource-index.ts index 97b0e12abca0..42e1c5eed9f2 100644 --- a/packages/asset-resource/src/resource-index.ts +++ b/packages/asset-resource/src/resource-index.ts @@ -95,11 +95,13 @@ export const buildAssetResourceIndex = async ({ resourceId, query, entries, + assetRevision, }: { projectId: string; resourceId: string; query: string; entries: readonly CanonicalAssetFileEntry[]; + assetRevision?: string; }) => { const validatedQuery = validateAssetResourceQuery(query); const assetIds = new Set(); @@ -128,7 +130,8 @@ export const buildAssetResourceIndex = async ({ version: 1, resourceId, query, - assetRevision: await computeCanonicalAssetRevision(entries), + assetRevision: + assetRevision ?? (await computeCanonicalAssetRevision(entries)), queryMode: selection.queryMode, parameterNames: selection.parameterNames, documents: selection.documents, diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.test.ts b/packages/asset-uploader/src/canonical-metadata-backfill.test.ts index 793030e7a5c1..6378b934047e 100644 --- a/packages/asset-uploader/src/canonical-metadata-backfill.test.ts +++ b/packages/asset-uploader/src/canonical-metadata-backfill.test.ts @@ -55,7 +55,7 @@ describe("canonical asset metadata synchronization", () => { { id: "one", projectId: "project-1", - filename: "one.md", + filename: "one", folderId: "blog", file: { name: "stored-one.md", @@ -167,6 +167,7 @@ describe("canonical asset metadata synchronization", () => { }); test("indexes an empty Markdown file without a storage range read", async () => { + const storageName = "empty_abcdefghijklmnopqrstu.md"; const readFile = vi.fn(); let document: Record | undefined; server.use( @@ -175,10 +176,10 @@ describe("canonical asset metadata synchronization", () => { { id: "empty", projectId: "project-1", - filename: "empty.md", + filename: "empty", folderId: null, file: { - name: "empty.md", + name: storageName, size: 0, updatedAt: "2026-07-18T01:00:00.000Z", status: "UPLOADED", @@ -204,9 +205,11 @@ describe("canonical asset metadata synchronization", () => { ).resolves.toMatchObject({ status: "indexed" }); expect(readFile).not.toHaveBeenCalled(); expect(document).toMatchObject({ + name: "empty.md", + path: "empty.md", properties: {}, size: 0, - contentRef: "empty.md", + contentRef: storageName, }); }); @@ -253,7 +256,7 @@ describe("canonical asset metadata synchronization", () => { { id: "asset-1", projectId: "project-1", - filename: "renamed.md", + filename: "renamed", folderId: "blog", file: { name: "stored.md", @@ -305,7 +308,7 @@ describe("canonical asset metadata synchronization", () => { storageName: "stored.md", fileUpdatedAt: "2026-07-18T04:00:00.000Z", fileSize: 35, - filename: "renamed.md", + filename: "renamed", folderId: "blog", }, document: { @@ -418,7 +421,7 @@ describe("canonical asset metadata synchronization", () => { { id: "asset-1", projectId: "project-1", - filename: "new.md", + filename: "new", folderId: "blog", file: { name: "stored.md", @@ -477,7 +480,7 @@ describe("canonical asset metadata synchronization", () => { { id: "stable", projectId: "project-1", - filename: "stable.md", + filename: "stable", folderId: null, file: { name: "stable.md", @@ -489,7 +492,7 @@ describe("canonical asset metadata synchronization", () => { { id: "changed", projectId: "project-1", - filename: "changed.md", + filename: "changed", folderId: null, file: { name: "changed.md", @@ -501,7 +504,7 @@ describe("canonical asset metadata synchronization", () => { { id: "renamed", projectId: "project-1", - filename: "new-name.md", + filename: "new-name", folderId: null, file: { name: "renamed.md", diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.ts b/packages/asset-uploader/src/canonical-metadata-backfill.ts index e7d58c417a59..a1903742acc9 100644 --- a/packages/asset-uploader/src/canonical-metadata-backfill.ts +++ b/packages/asset-uploader/src/canonical-metadata-backfill.ts @@ -7,6 +7,8 @@ import { import { assetResourceLimits, createAssetFolderHierarchy, + formatAssetName, + getFileNameParts, getMimeTypeByFilename, type AssetFileDocument, } from "@webstudio-is/sdk"; @@ -133,19 +135,18 @@ const createCanonicalDocument = ({ properties: Record; excerpt?: string; }) => { - const name = asset.filename ?? asset.file.name; - const extensionSeparator = asset.file.name.lastIndexOf("."); - const extension = - extensionSeparator === -1 - ? undefined - : asset.file.name.slice(extensionSeparator + 1).toLowerCase(); + const name = formatAssetName({ + name: asset.file.name, + filename: asset.filename, + }); + const extension = getFileNameParts(asset.file.name).extension.toLowerCase(); const folderId = hierarchy.resolveFolderId(asset.folderId ?? undefined); const folderNames = hierarchy.getPath(folderId).map((folder) => folder.name); return normalizeAssetFileDocument({ asset: { id: asset.id, name, - ...(extension === undefined ? {} : { extension }), + ...(extension === "" ? {} : { extension }), ...(folderId === undefined ? {} : { folderId, folderNames }), mimeType: getMimeTypeByFilename(asset.file.name), size: asset.file.size, diff --git a/packages/asset-uploader/src/patch.test.ts b/packages/asset-uploader/src/patch.test.ts index 9c5947a02d4f..0daa5e275976 100644 --- a/packages/asset-uploader/src/patch.test.ts +++ b/packages/asset-uploader/src/patch.test.ts @@ -379,7 +379,7 @@ describe("patchAssets (msw)", () => { let localAssetRow = { ...assetRow, projectId, - filename: "post.md", + filename: "post", folderId: "folder" as string | null, file: { ...assetRow.file, diff --git a/packages/asset-uploader/src/resource-index-build.ts b/packages/asset-uploader/src/resource-index-build.ts index 532406e9dc45..1402b502dc92 100644 --- a/packages/asset-uploader/src/resource-index-build.ts +++ b/packages/asset-uploader/src/resource-index-build.ts @@ -40,6 +40,7 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ resourceId, query, entries, + assetRevision, signal, }: { client: Client; @@ -48,6 +49,7 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ resourceId: string; query: string; entries: readonly CanonicalAssetFileEntry[]; + assetRevision?: string; signal?: AbortSignal; }) => { assertNotCancelled(signal); @@ -56,6 +58,7 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ resourceId, query, entries, + assetRevision, }); assertNotCancelled(signal); await beginAssetResourceIndexBuild({ diff --git a/packages/asset-uploader/src/resource-index-lifecycle.ts b/packages/asset-uploader/src/resource-index-lifecycle.ts index 382e6de7afc2..303ddde30c4c 100644 --- a/packages/asset-uploader/src/resource-index-lifecycle.ts +++ b/packages/asset-uploader/src/resource-index-lifecycle.ts @@ -1,3 +1,4 @@ +import { computeCanonicalAssetRevision } from "@webstudio-is/asset-resource"; import { getAssetResourceQuery, type Resource } from "@webstudio-is/sdk"; export { getAssetResourceQuery } from "@webstudio-is/sdk"; import type { Client } from "@webstudio-is/postgrest/index.server"; @@ -62,6 +63,7 @@ export const synchronizeAssetResourceIndexQueries = async ({ } await synchronizeCanonicalAssets({ projectId, client, assetClient }); const entries = await loadCanonicalAssetFileEntries({ client, projectId }); + const assetRevision = await computeCanonicalAssetRevision(entries); const updatedResourceIds: string[] = []; for (const { resourceId, query } of changed) { await buildPersistAndActivateAssetResourceIndex({ @@ -71,6 +73,7 @@ export const synchronizeAssetResourceIndexQueries = async ({ resourceId, query, entries, + assetRevision, }); updatedResourceIds.push(resourceId); } diff --git a/packages/asset-uploader/src/resource-index-maintenance.ts b/packages/asset-uploader/src/resource-index-maintenance.ts index dbdfa77e9661..bcf317366db7 100644 --- a/packages/asset-uploader/src/resource-index-maintenance.ts +++ b/packages/asset-uploader/src/resource-index-maintenance.ts @@ -1,6 +1,7 @@ -import type { - CanonicalAssetFileEntry, - ImmutableAssetResourceIndexStore, +import { + computeCanonicalAssetRevision, + type CanonicalAssetFileEntry, + type ImmutableAssetResourceIndexStore, } from "@webstudio-is/asset-resource"; import type { Client } from "@webstudio-is/postgrest/index.server"; import type { AssetClient } from "./client"; @@ -70,6 +71,7 @@ export const updateAssetResourceIndexesAfterCanonicalChange = async ({ // schema-less field. V1 therefore treats all query resources in this project // as affected, but rebuilds them from compact canonical rows loaded once. const entries = await loadCanonicalAssetFileEntries({ client, projectId }); + const assetRevision = await computeCanonicalAssetRevision(entries); const updatedResourceIds: string[] = []; const errors: unknown[] = []; for (const resource of resources) { @@ -81,6 +83,7 @@ export const updateAssetResourceIndexesAfterCanonicalChange = async ({ resourceId: resource.resourceId, query: resource.query, entries, + assetRevision, }); updatedResourceIds.push(resource.resourceId); } catch (error) { @@ -159,6 +162,7 @@ export const reconcileAssetResourceIndexesForPublication = async ({ resourceId: resource.resourceId, query: resource.query, entries, + assetRevision, }); } catch (error) { // A concurrent build of the same identity may activate first. The diff --git a/packages/asset-uploader/src/revision.test.ts b/packages/asset-uploader/src/revision.test.ts index 35352f6c1102..fe37c40be64c 100644 --- a/packages/asset-uploader/src/revision.test.ts +++ b/packages/asset-uploader/src/revision.test.ts @@ -288,7 +288,7 @@ describe("asset content revisions", () => { expect(canonicalDocument).toMatchObject({ _id: "asset", - name: "post", + name: "post.md", contentRef: revisionName, properties: { title: "Updated post" }, excerpt: "Updated body", diff --git a/packages/asset-uploader/src/upload.test.ts b/packages/asset-uploader/src/upload.test.ts index c3efd6f79bfb..c57c71f54c25 100644 --- a/packages/asset-uploader/src/upload.test.ts +++ b/packages/asset-uploader/src/upload.test.ts @@ -744,7 +744,7 @@ describe("createUploadTicket", () => { }); describe("uploadFile", () => { - test("indexes only the newly uploaded Markdown asset after commit", async () => { + test("leaves asset resource maintenance to the committed asset patch", async () => { const source = "---\ntitle: New post\n---\nPost body"; const sourceBytes = new TextEncoder().encode(source); let persisted: Record | undefined; @@ -815,18 +815,19 @@ describe("uploadFile", () => { }) ); + const storage = { + uploadFile: async () => ({ + format: "md", + size: sourceBytes.byteLength, + meta: {}, + }), + readFile, + }; await expect( uploadFile( "post.md", new Blob([source]).stream(), - { - uploadFile: async () => ({ - format: "md", - size: sourceBytes.byteLength, - meta: {}, - }), - readFile, - }, + storage, createContext(), undefined ) @@ -835,19 +836,11 @@ describe("uploadFile", () => { name: "post.md", type: "file", }); - expect(readFile).toHaveBeenCalledOnce(); - expect(persisted).toMatchObject({ - projectId: "project-1", - assetId: "asset-1", - document: { - name: "post.md", - properties: { title: "New post" }, - excerpt: "Post body", - }, - }); + expect(readFile).not.toHaveBeenCalled(); + expect(persisted).toBeUndefined(); }); - test("keeps a committed upload when resource synchronization fails", async () => { + test("accepts an upload-only storage client", async () => { const source = "Post body"; const uploadedFile = { name: "post.md", @@ -887,8 +880,6 @@ describe("uploadFile", () => { }), db.get("AssetFolder", () => json([])) ); - const logError = vi.spyOn(console, "error").mockImplementation(() => {}); - await expect( uploadFile( "post.md", @@ -899,19 +890,11 @@ describe("uploadFile", () => { size: source.length, meta: {}, }), - readFile: async () => { - throw new Error("Storage read failed"); - }, }, createContext(), undefined ) ).resolves.toMatchObject({ id: "asset-1", name: "post.md" }); - expect(logError).toHaveBeenCalledWith( - "Uploaded asset resource synchronization failed", - expect.any(Error) - ); - logError.mockRestore(); }); test("returns uploaded asset with reserved asset row id", async () => { @@ -991,9 +974,6 @@ describe("uploadFile", () => { size: 5, meta: { width: 10, height: 20 }, }), - readFile: async () => ({ - data: { async *[Symbol.asyncIterator]() {} }, - }), }, createContext(), undefined @@ -1045,9 +1025,6 @@ describe("uploadFile", () => { uploadFile: async () => { throw new Error("Storage upload failed"); }, - readFile: async () => ({ - data: { async *[Symbol.asyncIterator]() {} }, - }), }, createContext(), undefined, @@ -1136,9 +1113,6 @@ describe("uploadFile", () => { new Blob(["orphan"]).stream(), { uploadFile: async () => ({ format: "txt", size: 6, meta: {} }), - readFile: async () => { - throw new Error("read should not run"); - }, }, createContext(), undefined diff --git a/packages/asset-uploader/src/upload.ts b/packages/asset-uploader/src/upload.ts index f78f8a1686d4..2f1687101571 100644 --- a/packages/asset-uploader/src/upload.ts +++ b/packages/asset-uploader/src/upload.ts @@ -12,14 +12,13 @@ import { type Asset, } from "@webstudio-is/sdk"; import type { Database } from "@webstudio-is/postgrest/index.server"; -import type { AssetClient, AssetUploadClient } from "./client"; +import type { AssetUploadClient } from "./client"; import type { AssetDataOverride } from "./utils/get-asset-data"; import { createUniqueAssetFilename } from "./utils/get-unique-filename"; import { sanitizeS3Key } from "./utils/sanitize-s3-key"; import { formatAsset } from "./utils/format-asset"; import { assertPostgrestSuccess } from "./patch-utils"; import type { UploadTicket } from "./types"; -import { synchronizeAssetResourceStateAfterAssetChange } from "./resource-index-maintenance"; type UploadData = { projectId: string; @@ -493,7 +492,7 @@ export const getUploadedAsset = async ({ export const uploadFile = async ( name: string, data: ReadableStream, - client: AssetClient, + client: AssetUploadClient, context: AppContext, assetInfoFallback: | { width: number; height: number; format: string } @@ -510,13 +509,12 @@ export const uploadFile = async ( assetDataOverride, onUploadError ); - let uploadedAsset: Asset; try { const projectId = file.uploaderProjectId; if (typeof projectId !== "string") { throw Error("File uploader project is missing"); } - uploadedAsset = await getUploadedAsset({ + return await getUploadedAsset({ name, projectId, context, @@ -526,15 +524,4 @@ export const uploadFile = async ( await cleanupUploadError(name, context, onUploadError); throw error; } - try { - await synchronizeAssetResourceStateAfterAssetChange({ - client: context.postgrest.client, - assetClient: client, - projectId: uploadedAsset.projectId, - assetId: uploadedAsset.id, - }); - } catch (error) { - console.error("Uploaded asset resource synchronization failed", error); - } - return uploadedAsset; }; diff --git a/packages/sdk/src/asset-resource-config.test.ts b/packages/sdk/src/asset-resource-config.test.ts index 8e56897b4d28..03d3ab61b3c8 100644 --- a/packages/sdk/src/asset-resource-config.test.ts +++ b/packages/sdk/src/asset-resource-config.test.ts @@ -37,6 +37,17 @@ describe("asset query resource configuration", () => { ); }); + test("normalizes parameter names before serialization", () => { + const body = createAssetQueryResourceBody({ + query: "*[properties.slug == $slug]", + parameters: [{ name: " slug ", value: '"post"' }], + }); + + expect(parseAssetQueryResourceBody(body).parameters).toEqual([ + { name: "slug", value: '"post"' }, + ]); + }); + test("only recognizes exact system Assets resource contracts", () => { expect(isStoredAssetQueryResource(createResource())).toBe(true); expect(isAssetsResource(createResource())).toBe(true); diff --git a/packages/sdk/src/asset-resource-config.ts b/packages/sdk/src/asset-resource-config.ts index 61bf25b35d87..b116e16e08df 100644 --- a/packages/sdk/src/asset-resource-config.ts +++ b/packages/sdk/src/asset-resource-config.ts @@ -7,6 +7,8 @@ export type AssetQueryParameterBinding = { value: string; }; +export const normalizeAssetQueryParameterName = (name: string) => name.trim(); + const getStaticStringLiteral = (expression: string) => { try { const value = JSON.parse(expression); @@ -48,7 +50,11 @@ export const createAssetQueryResourceBody = ({ generateObjectExpression( new Map( parameters - .filter(({ name }) => name.trim().length > 0) + .map(({ name, value }) => ({ + name: normalizeAssetQueryParameterName(name), + value, + })) + .filter(({ name }) => name.length > 0) .map(({ name, value }) => [name, value]) ) ), From 37b862d69c68dea332c12a3d3077f0420f1baf09 Mon Sep 17 00:00:00 2001 From: Oleg Isonen Date: Mon, 20 Jul 2026 21:29:48 +0100 Subject: [PATCH 08/14] fix: refresh asset resource fixtures --- .../.template/package.json | 1 + .../$resources.asset-query-manifest.ts | 2 + .../$resources.asset-query-runtime.ts | 7 ++ .../app/routes/[another-page]._index.tsx | 11 ++- .../app/routes/_index.tsx | 11 ++- fixtures/react-router-cloudflare/package.json | 3 +- .../.template/package.json | 1 + .../$resources.asset-query-manifest.ts | 2 + .../$resources.asset-query-runtime.ts | 7 ++ .../app/routes/[another-page]._index.tsx | 11 ++- .../react-router-docker/app/routes/_index.tsx | 11 ++- fixtures/react-router-docker/package.json | 3 +- .../.template/package.json | 1 + .../$resources.asset-query-manifest.ts | 2 + .../$resources.asset-query-runtime.ts | 7 ++ .../app/routes/[another-page]._index.tsx | 11 ++- .../app/routes/_index.tsx | 11 ++- fixtures/react-router-netlify/package.json | 3 +- .../.template/package.json | 1 + .../$resources.asset-query-manifest.ts | 2 + .../$resources.asset-query-runtime.ts | 7 ++ .../app/routes/[another-page]._index.tsx | 11 ++- .../react-router-vercel/app/routes/_index.tsx | 11 ++- fixtures/react-router-vercel/package.json | 3 +- .../$resources.asset-query-manifest.ts | 2 + .../$resources.asset-query-runtime.ts | 7 ++ .../app/asset-resource-fetch.ts | 91 +++++++++++++++++++ fixtures/ssg-cloudflare-pages/package.json | 3 +- .../pages/another-page/+data.ts | 23 ++++- .../ssg-cloudflare-pages/pages/index/+data.ts | 23 ++++- fixtures/ssg-cloudflare-pages/vite.config.ts | 7 +- .../$resources.asset-query-manifest.ts | 2 + .../$resources.asset-query-runtime.ts | 7 ++ .../app/asset-resource-fetch.ts | 91 +++++++++++++++++++ .../ssg-netlify-by-project-id/package.json | 3 +- .../pages/index/+data.ts | 23 ++++- .../pages/redirect/+data.ts | 23 ++++- .../ssg-netlify-by-project-id/vite.config.ts | 7 +- .../$resources.asset-query-manifest.ts | 2 + .../$resources.asset-query-runtime.ts | 7 ++ .../app/routes/[another-page]._index.tsx | 11 ++- .../app/routes/_index.tsx | 11 ++- .../package.json | 3 +- .../webstudio-features/.template/package.json | 1 + .../$resources.asset-query-manifest.ts | 2 + .../$resources.asset-query-runtime.ts | 7 ++ .../routes/[_route_with_symbols_]._index.tsx | 11 ++- .../app/routes/[animations]._index.tsx | 11 ++- .../app/routes/[assets1]._index.tsx | 11 ++- .../app/routes/[class-names]._index.tsx | 11 ++- .../app/routes/[content-block]._index.tsx | 11 ++- .../app/routes/[duration]._index.tsx | 11 ++- .../app/routes/[expressions]._index.tsx | 11 ++- .../app/routes/[form]._index.tsx | 11 ++- .../app/routes/[head-tag]._index.tsx | 11 ++- .../app/routes/[heading-with-id]._index.tsx | 11 ++- .../routes/[nested].[nested-page]._index.tsx | 11 ++- .../app/routes/[radix]._index.tsx | 11 ++- .../app/routes/[resources]._index.tsx | 11 ++- .../app/routes/[sitemap.xml]._index.tsx | 11 ++- .../app/routes/[text-duration]._index.tsx | 11 ++- .../webstudio-features/app/routes/_index.tsx | 11 ++- fixtures/webstudio-features/package.json | 3 +- .../scripts/benchmark-scale.mjs | 17 ++-- .../src/candidate-selection.test.ts | 4 +- packages/cli/templates/internal/package.json | 1 + pnpm-lock.yaml | 24 +++++ 67 files changed, 631 insertions(+), 90 deletions(-) create mode 100644 fixtures/react-router-cloudflare/app/__generated__/$resources.asset-query-manifest.ts create mode 100644 fixtures/react-router-cloudflare/app/__generated__/$resources.asset-query-runtime.ts create mode 100644 fixtures/react-router-docker/app/__generated__/$resources.asset-query-manifest.ts create mode 100644 fixtures/react-router-docker/app/__generated__/$resources.asset-query-runtime.ts create mode 100644 fixtures/react-router-netlify/app/__generated__/$resources.asset-query-manifest.ts create mode 100644 fixtures/react-router-netlify/app/__generated__/$resources.asset-query-runtime.ts create mode 100644 fixtures/react-router-vercel/app/__generated__/$resources.asset-query-manifest.ts create mode 100644 fixtures/react-router-vercel/app/__generated__/$resources.asset-query-runtime.ts create mode 100644 fixtures/ssg-cloudflare-pages/app/__generated__/$resources.asset-query-manifest.ts create mode 100644 fixtures/ssg-cloudflare-pages/app/__generated__/$resources.asset-query-runtime.ts create mode 100644 fixtures/ssg-cloudflare-pages/app/asset-resource-fetch.ts create mode 100644 fixtures/ssg-netlify-by-project-id/app/__generated__/$resources.asset-query-manifest.ts create mode 100644 fixtures/ssg-netlify-by-project-id/app/__generated__/$resources.asset-query-runtime.ts create mode 100644 fixtures/ssg-netlify-by-project-id/app/asset-resource-fetch.ts create mode 100644 fixtures/webstudio-cloudflare-template/app/__generated__/$resources.asset-query-manifest.ts create mode 100644 fixtures/webstudio-cloudflare-template/app/__generated__/$resources.asset-query-runtime.ts create mode 100644 fixtures/webstudio-features/app/__generated__/$resources.asset-query-manifest.ts create mode 100644 fixtures/webstudio-features/app/__generated__/$resources.asset-query-runtime.ts diff --git a/fixtures/react-router-cloudflare/.template/package.json b/fixtures/react-router-cloudflare/.template/package.json index 8b6d3adc7318..1c9aa4ba3a6d 100644 --- a/fixtures/react-router-cloudflare/.template/package.json +++ b/fixtures/react-router-cloudflare/.template/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@webstudio-is/asset-resource": "workspace:*", "@webstudio-is/image": "workspace:*", "@webstudio-is/react-sdk": "workspace:*", "@webstudio-is/sdk": "workspace:*", diff --git a/fixtures/react-router-cloudflare/app/__generated__/$resources.asset-query-manifest.ts b/fixtures/react-router-cloudflare/app/__generated__/$resources.asset-query-manifest.ts new file mode 100644 index 000000000000..d9a4469a0376 --- /dev/null +++ b/fixtures/react-router-cloudflare/app/__generated__/$resources.asset-query-manifest.ts @@ -0,0 +1,2 @@ +export const assetQueryDeploymentId = "f565d527-32e7-4731-bc71-aca9e9574587"; +export const assetQueryManifest = []; diff --git a/fixtures/react-router-cloudflare/app/__generated__/$resources.asset-query-runtime.ts b/fixtures/react-router-cloudflare/app/__generated__/$resources.asset-query-runtime.ts new file mode 100644 index 000000000000..df58a6ea191d --- /dev/null +++ b/fixtures/react-router-cloudflare/app/__generated__/$resources.asset-query-runtime.ts @@ -0,0 +1,7 @@ +export const createGeneratedAssetResourceFetch = async ({ + fallback, +}: { + request: Request; + context: unknown; + fallback: typeof fetch; +}): Promise => fallback; diff --git a/fixtures/react-router-cloudflare/app/routes/[another-page]._index.tsx b/fixtures/react-router-cloudflare/app/routes/[another-page]._index.tsx index b4af7cc10396..12b520dfe062 100644 --- a/fixtures/react-router-cloudflare/app/routes/[another-page]._index.tsx +++ b/fixtures/react-router-cloudflare/app/routes/[another-page]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/react-router-cloudflare/app/routes/_index.tsx b/fixtures/react-router-cloudflare/app/routes/_index.tsx index 867d53e46b71..004867494687 100644 --- a/fixtures/react-router-cloudflare/app/routes/_index.tsx +++ b/fixtures/react-router-cloudflare/app/routes/_index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/react-router-cloudflare/package.json b/fixtures/react-router-cloudflare/package.json index 81f371483186..64dafca0cf12 100644 --- a/fixtures/react-router-cloudflare/package.json +++ b/fixtures/react-router-cloudflare/package.json @@ -34,7 +34,8 @@ "wrangler": "^4.14.1", "@webstudio-is/wsauth": "workspace:*", "@react-router/node": "^7.5.3", - "@react-router/serve": "^7.5.3" + "@react-router/serve": "^7.5.3", + "@webstudio-is/asset-resource": "workspace:*" }, "type": "module", "private": true, diff --git a/fixtures/react-router-docker/.template/package.json b/fixtures/react-router-docker/.template/package.json index 8b6d3adc7318..1c9aa4ba3a6d 100644 --- a/fixtures/react-router-docker/.template/package.json +++ b/fixtures/react-router-docker/.template/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@webstudio-is/asset-resource": "workspace:*", "@webstudio-is/image": "workspace:*", "@webstudio-is/react-sdk": "workspace:*", "@webstudio-is/sdk": "workspace:*", diff --git a/fixtures/react-router-docker/app/__generated__/$resources.asset-query-manifest.ts b/fixtures/react-router-docker/app/__generated__/$resources.asset-query-manifest.ts new file mode 100644 index 000000000000..d9a4469a0376 --- /dev/null +++ b/fixtures/react-router-docker/app/__generated__/$resources.asset-query-manifest.ts @@ -0,0 +1,2 @@ +export const assetQueryDeploymentId = "f565d527-32e7-4731-bc71-aca9e9574587"; +export const assetQueryManifest = []; diff --git a/fixtures/react-router-docker/app/__generated__/$resources.asset-query-runtime.ts b/fixtures/react-router-docker/app/__generated__/$resources.asset-query-runtime.ts new file mode 100644 index 000000000000..df58a6ea191d --- /dev/null +++ b/fixtures/react-router-docker/app/__generated__/$resources.asset-query-runtime.ts @@ -0,0 +1,7 @@ +export const createGeneratedAssetResourceFetch = async ({ + fallback, +}: { + request: Request; + context: unknown; + fallback: typeof fetch; +}): Promise => fallback; diff --git a/fixtures/react-router-docker/app/routes/[another-page]._index.tsx b/fixtures/react-router-docker/app/routes/[another-page]._index.tsx index b4af7cc10396..12b520dfe062 100644 --- a/fixtures/react-router-docker/app/routes/[another-page]._index.tsx +++ b/fixtures/react-router-docker/app/routes/[another-page]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/react-router-docker/app/routes/_index.tsx b/fixtures/react-router-docker/app/routes/_index.tsx index 867d53e46b71..004867494687 100644 --- a/fixtures/react-router-docker/app/routes/_index.tsx +++ b/fixtures/react-router-docker/app/routes/_index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/react-router-docker/package.json b/fixtures/react-router-docker/package.json index 2ebb1dc0033a..dba72a5daab7 100644 --- a/fixtures/react-router-docker/package.json +++ b/fixtures/react-router-docker/package.json @@ -30,7 +30,8 @@ "react-router": "^7.5.3", "vite": "^6.3.4", "webstudio": "workspace:*", - "@webstudio-is/wsauth": "workspace:*" + "@webstudio-is/wsauth": "workspace:*", + "@webstudio-is/asset-resource": "workspace:*" }, "private": true, "sideEffects": false, diff --git a/fixtures/react-router-netlify/.template/package.json b/fixtures/react-router-netlify/.template/package.json index 8b6d3adc7318..1c9aa4ba3a6d 100644 --- a/fixtures/react-router-netlify/.template/package.json +++ b/fixtures/react-router-netlify/.template/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@webstudio-is/asset-resource": "workspace:*", "@webstudio-is/image": "workspace:*", "@webstudio-is/react-sdk": "workspace:*", "@webstudio-is/sdk": "workspace:*", diff --git a/fixtures/react-router-netlify/app/__generated__/$resources.asset-query-manifest.ts b/fixtures/react-router-netlify/app/__generated__/$resources.asset-query-manifest.ts new file mode 100644 index 000000000000..d9a4469a0376 --- /dev/null +++ b/fixtures/react-router-netlify/app/__generated__/$resources.asset-query-manifest.ts @@ -0,0 +1,2 @@ +export const assetQueryDeploymentId = "f565d527-32e7-4731-bc71-aca9e9574587"; +export const assetQueryManifest = []; diff --git a/fixtures/react-router-netlify/app/__generated__/$resources.asset-query-runtime.ts b/fixtures/react-router-netlify/app/__generated__/$resources.asset-query-runtime.ts new file mode 100644 index 000000000000..df58a6ea191d --- /dev/null +++ b/fixtures/react-router-netlify/app/__generated__/$resources.asset-query-runtime.ts @@ -0,0 +1,7 @@ +export const createGeneratedAssetResourceFetch = async ({ + fallback, +}: { + request: Request; + context: unknown; + fallback: typeof fetch; +}): Promise => fallback; diff --git a/fixtures/react-router-netlify/app/routes/[another-page]._index.tsx b/fixtures/react-router-netlify/app/routes/[another-page]._index.tsx index b4af7cc10396..12b520dfe062 100644 --- a/fixtures/react-router-netlify/app/routes/[another-page]._index.tsx +++ b/fixtures/react-router-netlify/app/routes/[another-page]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/react-router-netlify/app/routes/_index.tsx b/fixtures/react-router-netlify/app/routes/_index.tsx index 867d53e46b71..004867494687 100644 --- a/fixtures/react-router-netlify/app/routes/_index.tsx +++ b/fixtures/react-router-netlify/app/routes/_index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/react-router-netlify/package.json b/fixtures/react-router-netlify/package.json index ec470c398f06..f18944c38c2c 100644 --- a/fixtures/react-router-netlify/package.json +++ b/fixtures/react-router-netlify/package.json @@ -31,7 +31,8 @@ "vite": "^6.3.4", "webstudio": "workspace:*", "@webstudio-is/wsauth": "workspace:*", - "@react-router/serve": "^7.5.3" + "@react-router/serve": "^7.5.3", + "@webstudio-is/asset-resource": "workspace:*" }, "type": "module", "private": true, diff --git a/fixtures/react-router-vercel/.template/package.json b/fixtures/react-router-vercel/.template/package.json index 8b6d3adc7318..1c9aa4ba3a6d 100644 --- a/fixtures/react-router-vercel/.template/package.json +++ b/fixtures/react-router-vercel/.template/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@webstudio-is/asset-resource": "workspace:*", "@webstudio-is/image": "workspace:*", "@webstudio-is/react-sdk": "workspace:*", "@webstudio-is/sdk": "workspace:*", diff --git a/fixtures/react-router-vercel/app/__generated__/$resources.asset-query-manifest.ts b/fixtures/react-router-vercel/app/__generated__/$resources.asset-query-manifest.ts new file mode 100644 index 000000000000..d9a4469a0376 --- /dev/null +++ b/fixtures/react-router-vercel/app/__generated__/$resources.asset-query-manifest.ts @@ -0,0 +1,2 @@ +export const assetQueryDeploymentId = "f565d527-32e7-4731-bc71-aca9e9574587"; +export const assetQueryManifest = []; diff --git a/fixtures/react-router-vercel/app/__generated__/$resources.asset-query-runtime.ts b/fixtures/react-router-vercel/app/__generated__/$resources.asset-query-runtime.ts new file mode 100644 index 000000000000..df58a6ea191d --- /dev/null +++ b/fixtures/react-router-vercel/app/__generated__/$resources.asset-query-runtime.ts @@ -0,0 +1,7 @@ +export const createGeneratedAssetResourceFetch = async ({ + fallback, +}: { + request: Request; + context: unknown; + fallback: typeof fetch; +}): Promise => fallback; diff --git a/fixtures/react-router-vercel/app/routes/[another-page]._index.tsx b/fixtures/react-router-vercel/app/routes/[another-page]._index.tsx index b4af7cc10396..12b520dfe062 100644 --- a/fixtures/react-router-vercel/app/routes/[another-page]._index.tsx +++ b/fixtures/react-router-vercel/app/routes/[another-page]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/react-router-vercel/app/routes/_index.tsx b/fixtures/react-router-vercel/app/routes/_index.tsx index 867d53e46b71..004867494687 100644 --- a/fixtures/react-router-vercel/app/routes/_index.tsx +++ b/fixtures/react-router-vercel/app/routes/_index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/react-router-vercel/package.json b/fixtures/react-router-vercel/package.json index 3a3b6cade0a6..137911b1562d 100644 --- a/fixtures/react-router-vercel/package.json +++ b/fixtures/react-router-vercel/package.json @@ -31,7 +31,8 @@ "vite": "^6.3.4", "webstudio": "workspace:*", "@webstudio-is/wsauth": "workspace:*", - "@react-router/serve": "^7.5.3" + "@react-router/serve": "^7.5.3", + "@webstudio-is/asset-resource": "workspace:*" }, "private": true, "type": "module", diff --git a/fixtures/ssg-cloudflare-pages/app/__generated__/$resources.asset-query-manifest.ts b/fixtures/ssg-cloudflare-pages/app/__generated__/$resources.asset-query-manifest.ts new file mode 100644 index 000000000000..e486f6d38f77 --- /dev/null +++ b/fixtures/ssg-cloudflare-pages/app/__generated__/$resources.asset-query-manifest.ts @@ -0,0 +1,2 @@ +export const assetQueryDeploymentId = "a2e8de30-03d5-4514-a3a6-406b3266a3af"; +export const assetQueryManifest = []; diff --git a/fixtures/ssg-cloudflare-pages/app/__generated__/$resources.asset-query-runtime.ts b/fixtures/ssg-cloudflare-pages/app/__generated__/$resources.asset-query-runtime.ts new file mode 100644 index 000000000000..df58a6ea191d --- /dev/null +++ b/fixtures/ssg-cloudflare-pages/app/__generated__/$resources.asset-query-runtime.ts @@ -0,0 +1,7 @@ +export const createGeneratedAssetResourceFetch = async ({ + fallback, +}: { + request: Request; + context: unknown; + fallback: typeof fetch; +}): Promise => fallback; diff --git a/fixtures/ssg-cloudflare-pages/app/asset-resource-fetch.ts b/fixtures/ssg-cloudflare-pages/app/asset-resource-fetch.ts new file mode 100644 index 000000000000..c058910b8184 --- /dev/null +++ b/fixtures/ssg-cloudflare-pages/app/asset-resource-fetch.ts @@ -0,0 +1,91 @@ +import { open } from "node:fs/promises"; +import { resolve, sep } from "node:path"; +import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource"; + +const resolvePublicAsset = (path: string) => { + const publicDirectory = resolve("public"); + const pathname = new URL(path, "https://webstudio.local").pathname; + const filePath = resolve(publicDirectory, `.${pathname}`); + if (filePath.startsWith(`${publicDirectory}${sep}`) === false) { + throw new Error("Static asset path is outside the public directory"); + } + return filePath; +}; + +const getByteRange = (header: string | null, size: number) => { + if (header?.startsWith("bytes=") !== true) { + return { offset: 0, length: size, partial: false }; + } + const values = header.slice("bytes=".length).split("-"); + const offset = Number(values[0]); + const requestedEnd = Number(values[1]); + if ( + values.length !== 2 || + Number.isSafeInteger(offset) === false || + Number.isSafeInteger(requestedEnd) === false || + offset < 0 || + requestedEnd < offset || + offset >= size + ) { + return; + } + const end = Math.min(requestedEnd, size - 1); + return { offset, length: end - offset + 1, partial: true }; +}; + +export const fetchSsgPublicAsset = async (path: string, init?: RequestInit) => { + let file; + try { + file = await open(resolvePublicAsset(path), "r"); + } catch (error) { + if ( + typeof error === "object" && + error !== null && + Reflect.get(error, "code") === "ENOENT" + ) { + return new Response("Not found", { status: 404 }); + } + throw error; + } + try { + const { size } = await file.stat(); + const range = getByteRange(new Headers(init?.headers).get("range"), size); + if (range === undefined) { + return new Response("Range not satisfiable", { + status: 416, + headers: { "content-range": `bytes */${size}` }, + }); + } + const body = range.partial + ? await file + .read(new Uint8Array(range.length), 0, range.length, range.offset) + .then(({ buffer, bytesRead }) => buffer.subarray(0, bytesRead)) + : await file.readFile(); + const headers = new Headers({ "content-length": String(body.length) }); + if (range.partial) { + headers.set( + "content-range", + `bytes ${range.offset}-${range.offset + body.length - 1}/${size}` + ); + } + return new Response(Uint8Array.from(body).buffer, { + status: range.partial ? 206 : 200, + headers, + }); + } finally { + await file.close(); + } +}; + +export const createSsgAssetResourceFetch = ({ + deploymentId, + manifest, +}: { + deploymentId: string; + manifest: Parameters[0]["manifest"]; +}) => + createPublishedAssetResourceFetch({ + deploymentId, + manifest, + fetchAsset: fetchSsgPublicAsset, + }); diff --git a/fixtures/ssg-cloudflare-pages/package.json b/fixtures/ssg-cloudflare-pages/package.json index b87cb403bf04..51230fb1bc55 100644 --- a/fixtures/ssg-cloudflare-pages/package.json +++ b/fixtures/ssg-cloudflare-pages/package.json @@ -42,6 +42,7 @@ "react": "18.3.0-canary-14898b6a9-20240318", "react-dom": "18.3.0-canary-14898b6a9-20240318", "vike": "^0.4.229", - "@webstudio-is/wsauth": "workspace:*" + "@webstudio-is/wsauth": "workspace:*", + "@webstudio-is/asset-resource": "workspace:*" } } diff --git a/fixtures/ssg-cloudflare-pages/pages/another-page/+data.ts b/fixtures/ssg-cloudflare-pages/pages/another-page/+data.ts index 35c8bd18f6a1..86707ed562de 100644 --- a/fixtures/ssg-cloudflare-pages/pages/another-page/+data.ts +++ b/fixtures/ssg-cloudflare-pages/pages/another-page/+data.ts @@ -5,8 +5,22 @@ import { getResources, } from "../../app/__generated__/[another-page]._index.server"; import { assets } from "../../app/__generated__/$resources.assets"; +import { + assetQueryDeploymentId, + assetQueryManifest, +} from "../../app/__generated__/$resources.asset-query-manifest"; +import { createSsgAssetResourceFetch } from "../../app/asset-resource-fetch"; + +const fetchAssetResource = createSsgAssetResourceFetch({ + deploymentId: assetQueryDeploymentId, + manifest: assetQueryManifest, +}); -const customFetch: typeof fetch = (input, init) => { +const customFetch: typeof fetch = async (input, init) => { + const assetResourceResponse = await fetchAssetResource(input, init); + if (assetResourceResponse !== undefined) { + return assetResourceResponse; + } if (typeof input !== "string") { return fetch(input, init); } @@ -26,13 +40,13 @@ const customFetch: typeof fetch = (input, init) => { }; const response = new Response(JSON.stringify(data)); response.headers.set("content-type", "application/json; charset=utf-8"); - return Promise.resolve(response); + return response; } if (isLocalResource(input, "assets")) { const response = new Response(JSON.stringify(assets)); response.headers.set("content-type", "application/json; charset=utf-8"); - return Promise.resolve(response); + return response; } return fetch(input, init); @@ -55,7 +69,8 @@ export const data = async (pageContext: PageContextServer) => { const resources = await loadResources( customFetch, - getResources({ system }).data + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/ssg-cloudflare-pages/pages/index/+data.ts b/fixtures/ssg-cloudflare-pages/pages/index/+data.ts index 8f86daf8a549..2277f7c53a35 100644 --- a/fixtures/ssg-cloudflare-pages/pages/index/+data.ts +++ b/fixtures/ssg-cloudflare-pages/pages/index/+data.ts @@ -5,8 +5,22 @@ import { getResources, } from "../../app/__generated__/_index.server"; import { assets } from "../../app/__generated__/$resources.assets"; +import { + assetQueryDeploymentId, + assetQueryManifest, +} from "../../app/__generated__/$resources.asset-query-manifest"; +import { createSsgAssetResourceFetch } from "../../app/asset-resource-fetch"; + +const fetchAssetResource = createSsgAssetResourceFetch({ + deploymentId: assetQueryDeploymentId, + manifest: assetQueryManifest, +}); -const customFetch: typeof fetch = (input, init) => { +const customFetch: typeof fetch = async (input, init) => { + const assetResourceResponse = await fetchAssetResource(input, init); + if (assetResourceResponse !== undefined) { + return assetResourceResponse; + } if (typeof input !== "string") { return fetch(input, init); } @@ -26,13 +40,13 @@ const customFetch: typeof fetch = (input, init) => { }; const response = new Response(JSON.stringify(data)); response.headers.set("content-type", "application/json; charset=utf-8"); - return Promise.resolve(response); + return response; } if (isLocalResource(input, "assets")) { const response = new Response(JSON.stringify(assets)); response.headers.set("content-type", "application/json; charset=utf-8"); - return Promise.resolve(response); + return response; } return fetch(input, init); @@ -55,7 +69,8 @@ export const data = async (pageContext: PageContextServer) => { const resources = await loadResources( customFetch, - getResources({ system }).data + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/ssg-cloudflare-pages/vite.config.ts b/fixtures/ssg-cloudflare-pages/vite.config.ts index 8e2bf4e3d6d2..83d1624d67ab 100644 --- a/fixtures/ssg-cloudflare-pages/vite.config.ts +++ b/fixtures/ssg-cloudflare-pages/vite.config.ts @@ -2,14 +2,17 @@ import { defineConfig } from "vite"; import react from "@vitejs/plugin-react"; import vike from "vike/plugin"; +const sourceConditions = + process.env.WEBSTUDIO_LOCAL_CLI_BOOTSTRAPPED === "1" ? ["webstudio"] : []; + export default defineConfig({ plugins: [react(), vike()], resolve: { - conditions: ["browser", "development|production"], + conditions: [...sourceConditions, "browser", "development|production"], }, ssr: { resolve: { - conditions: ["node", "development|production"], + conditions: [...sourceConditions, "node", "development|production"], }, }, }); diff --git a/fixtures/ssg-netlify-by-project-id/app/__generated__/$resources.asset-query-manifest.ts b/fixtures/ssg-netlify-by-project-id/app/__generated__/$resources.asset-query-manifest.ts new file mode 100644 index 000000000000..d89d0845fa3a --- /dev/null +++ b/fixtures/ssg-netlify-by-project-id/app/__generated__/$resources.asset-query-manifest.ts @@ -0,0 +1,2 @@ +export const assetQueryDeploymentId = "8f0fdff2-e76b-4f86-9203-ea7df5f1143c"; +export const assetQueryManifest = []; diff --git a/fixtures/ssg-netlify-by-project-id/app/__generated__/$resources.asset-query-runtime.ts b/fixtures/ssg-netlify-by-project-id/app/__generated__/$resources.asset-query-runtime.ts new file mode 100644 index 000000000000..df58a6ea191d --- /dev/null +++ b/fixtures/ssg-netlify-by-project-id/app/__generated__/$resources.asset-query-runtime.ts @@ -0,0 +1,7 @@ +export const createGeneratedAssetResourceFetch = async ({ + fallback, +}: { + request: Request; + context: unknown; + fallback: typeof fetch; +}): Promise => fallback; diff --git a/fixtures/ssg-netlify-by-project-id/app/asset-resource-fetch.ts b/fixtures/ssg-netlify-by-project-id/app/asset-resource-fetch.ts new file mode 100644 index 000000000000..c058910b8184 --- /dev/null +++ b/fixtures/ssg-netlify-by-project-id/app/asset-resource-fetch.ts @@ -0,0 +1,91 @@ +import { open } from "node:fs/promises"; +import { resolve, sep } from "node:path"; +import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource"; + +const resolvePublicAsset = (path: string) => { + const publicDirectory = resolve("public"); + const pathname = new URL(path, "https://webstudio.local").pathname; + const filePath = resolve(publicDirectory, `.${pathname}`); + if (filePath.startsWith(`${publicDirectory}${sep}`) === false) { + throw new Error("Static asset path is outside the public directory"); + } + return filePath; +}; + +const getByteRange = (header: string | null, size: number) => { + if (header?.startsWith("bytes=") !== true) { + return { offset: 0, length: size, partial: false }; + } + const values = header.slice("bytes=".length).split("-"); + const offset = Number(values[0]); + const requestedEnd = Number(values[1]); + if ( + values.length !== 2 || + Number.isSafeInteger(offset) === false || + Number.isSafeInteger(requestedEnd) === false || + offset < 0 || + requestedEnd < offset || + offset >= size + ) { + return; + } + const end = Math.min(requestedEnd, size - 1); + return { offset, length: end - offset + 1, partial: true }; +}; + +export const fetchSsgPublicAsset = async (path: string, init?: RequestInit) => { + let file; + try { + file = await open(resolvePublicAsset(path), "r"); + } catch (error) { + if ( + typeof error === "object" && + error !== null && + Reflect.get(error, "code") === "ENOENT" + ) { + return new Response("Not found", { status: 404 }); + } + throw error; + } + try { + const { size } = await file.stat(); + const range = getByteRange(new Headers(init?.headers).get("range"), size); + if (range === undefined) { + return new Response("Range not satisfiable", { + status: 416, + headers: { "content-range": `bytes */${size}` }, + }); + } + const body = range.partial + ? await file + .read(new Uint8Array(range.length), 0, range.length, range.offset) + .then(({ buffer, bytesRead }) => buffer.subarray(0, bytesRead)) + : await file.readFile(); + const headers = new Headers({ "content-length": String(body.length) }); + if (range.partial) { + headers.set( + "content-range", + `bytes ${range.offset}-${range.offset + body.length - 1}/${size}` + ); + } + return new Response(Uint8Array.from(body).buffer, { + status: range.partial ? 206 : 200, + headers, + }); + } finally { + await file.close(); + } +}; + +export const createSsgAssetResourceFetch = ({ + deploymentId, + manifest, +}: { + deploymentId: string; + manifest: Parameters[0]["manifest"]; +}) => + createPublishedAssetResourceFetch({ + deploymentId, + manifest, + fetchAsset: fetchSsgPublicAsset, + }); diff --git a/fixtures/ssg-netlify-by-project-id/package.json b/fixtures/ssg-netlify-by-project-id/package.json index ea76cd0ba852..a339f7d1abbb 100644 --- a/fixtures/ssg-netlify-by-project-id/package.json +++ b/fixtures/ssg-netlify-by-project-id/package.json @@ -42,6 +42,7 @@ "react": "18.3.0-canary-14898b6a9-20240318", "react-dom": "18.3.0-canary-14898b6a9-20240318", "vike": "^0.4.229", - "@webstudio-is/wsauth": "workspace:*" + "@webstudio-is/wsauth": "workspace:*", + "@webstudio-is/asset-resource": "workspace:*" } } diff --git a/fixtures/ssg-netlify-by-project-id/pages/index/+data.ts b/fixtures/ssg-netlify-by-project-id/pages/index/+data.ts index 8f86daf8a549..2277f7c53a35 100644 --- a/fixtures/ssg-netlify-by-project-id/pages/index/+data.ts +++ b/fixtures/ssg-netlify-by-project-id/pages/index/+data.ts @@ -5,8 +5,22 @@ import { getResources, } from "../../app/__generated__/_index.server"; import { assets } from "../../app/__generated__/$resources.assets"; +import { + assetQueryDeploymentId, + assetQueryManifest, +} from "../../app/__generated__/$resources.asset-query-manifest"; +import { createSsgAssetResourceFetch } from "../../app/asset-resource-fetch"; + +const fetchAssetResource = createSsgAssetResourceFetch({ + deploymentId: assetQueryDeploymentId, + manifest: assetQueryManifest, +}); -const customFetch: typeof fetch = (input, init) => { +const customFetch: typeof fetch = async (input, init) => { + const assetResourceResponse = await fetchAssetResource(input, init); + if (assetResourceResponse !== undefined) { + return assetResourceResponse; + } if (typeof input !== "string") { return fetch(input, init); } @@ -26,13 +40,13 @@ const customFetch: typeof fetch = (input, init) => { }; const response = new Response(JSON.stringify(data)); response.headers.set("content-type", "application/json; charset=utf-8"); - return Promise.resolve(response); + return response; } if (isLocalResource(input, "assets")) { const response = new Response(JSON.stringify(assets)); response.headers.set("content-type", "application/json; charset=utf-8"); - return Promise.resolve(response); + return response; } return fetch(input, init); @@ -55,7 +69,8 @@ export const data = async (pageContext: PageContextServer) => { const resources = await loadResources( customFetch, - getResources({ system }).data + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/ssg-netlify-by-project-id/pages/redirect/+data.ts b/fixtures/ssg-netlify-by-project-id/pages/redirect/+data.ts index 31c7efd090b7..2944e6e22655 100644 --- a/fixtures/ssg-netlify-by-project-id/pages/redirect/+data.ts +++ b/fixtures/ssg-netlify-by-project-id/pages/redirect/+data.ts @@ -5,8 +5,22 @@ import { getResources, } from "../../app/__generated__/[redirect]._index.server"; import { assets } from "../../app/__generated__/$resources.assets"; +import { + assetQueryDeploymentId, + assetQueryManifest, +} from "../../app/__generated__/$resources.asset-query-manifest"; +import { createSsgAssetResourceFetch } from "../../app/asset-resource-fetch"; + +const fetchAssetResource = createSsgAssetResourceFetch({ + deploymentId: assetQueryDeploymentId, + manifest: assetQueryManifest, +}); -const customFetch: typeof fetch = (input, init) => { +const customFetch: typeof fetch = async (input, init) => { + const assetResourceResponse = await fetchAssetResource(input, init); + if (assetResourceResponse !== undefined) { + return assetResourceResponse; + } if (typeof input !== "string") { return fetch(input, init); } @@ -26,13 +40,13 @@ const customFetch: typeof fetch = (input, init) => { }; const response = new Response(JSON.stringify(data)); response.headers.set("content-type", "application/json; charset=utf-8"); - return Promise.resolve(response); + return response; } if (isLocalResource(input, "assets")) { const response = new Response(JSON.stringify(assets)); response.headers.set("content-type", "application/json; charset=utf-8"); - return Promise.resolve(response); + return response; } return fetch(input, init); @@ -55,7 +69,8 @@ export const data = async (pageContext: PageContextServer) => { const resources = await loadResources( customFetch, - getResources({ system }).data + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/ssg-netlify-by-project-id/vite.config.ts b/fixtures/ssg-netlify-by-project-id/vite.config.ts index 8e2bf4e3d6d2..83d1624d67ab 100644 --- a/fixtures/ssg-netlify-by-project-id/vite.config.ts +++ b/fixtures/ssg-netlify-by-project-id/vite.config.ts @@ -2,14 +2,17 @@ import { defineConfig } from "vite"; import react from "@vitejs/plugin-react"; import vike from "vike/plugin"; +const sourceConditions = + process.env.WEBSTUDIO_LOCAL_CLI_BOOTSTRAPPED === "1" ? ["webstudio"] : []; + export default defineConfig({ plugins: [react(), vike()], resolve: { - conditions: ["browser", "development|production"], + conditions: [...sourceConditions, "browser", "development|production"], }, ssr: { resolve: { - conditions: ["node", "development|production"], + conditions: [...sourceConditions, "node", "development|production"], }, }, }); diff --git a/fixtures/webstudio-cloudflare-template/app/__generated__/$resources.asset-query-manifest.ts b/fixtures/webstudio-cloudflare-template/app/__generated__/$resources.asset-query-manifest.ts new file mode 100644 index 000000000000..e486f6d38f77 --- /dev/null +++ b/fixtures/webstudio-cloudflare-template/app/__generated__/$resources.asset-query-manifest.ts @@ -0,0 +1,2 @@ +export const assetQueryDeploymentId = "a2e8de30-03d5-4514-a3a6-406b3266a3af"; +export const assetQueryManifest = []; diff --git a/fixtures/webstudio-cloudflare-template/app/__generated__/$resources.asset-query-runtime.ts b/fixtures/webstudio-cloudflare-template/app/__generated__/$resources.asset-query-runtime.ts new file mode 100644 index 000000000000..df58a6ea191d --- /dev/null +++ b/fixtures/webstudio-cloudflare-template/app/__generated__/$resources.asset-query-runtime.ts @@ -0,0 +1,7 @@ +export const createGeneratedAssetResourceFetch = async ({ + fallback, +}: { + request: Request; + context: unknown; + fallback: typeof fetch; +}): Promise => fallback; diff --git a/fixtures/webstudio-cloudflare-template/app/routes/[another-page]._index.tsx b/fixtures/webstudio-cloudflare-template/app/routes/[another-page]._index.tsx index a0ca5169c720..e63c611375e9 100644 --- a/fixtures/webstudio-cloudflare-template/app/routes/[another-page]._index.tsx +++ b/fixtures/webstudio-cloudflare-template/app/routes/[another-page]._index.tsx @@ -46,6 +46,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -123,9 +124,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-cloudflare-template/app/routes/_index.tsx b/fixtures/webstudio-cloudflare-template/app/routes/_index.tsx index cb5d8b908e2c..53ecfa89b515 100644 --- a/fixtures/webstudio-cloudflare-template/app/routes/_index.tsx +++ b/fixtures/webstudio-cloudflare-template/app/routes/_index.tsx @@ -46,6 +46,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -123,9 +124,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-cloudflare-template/package.json b/fixtures/webstudio-cloudflare-template/package.json index f8a39ec1ca2a..b131567e7fc6 100644 --- a/fixtures/webstudio-cloudflare-template/package.json +++ b/fixtures/webstudio-cloudflare-template/package.json @@ -46,7 +46,8 @@ "react-dom": "18.3.0-canary-14898b6a9-20240318", "webstudio": "workspace:*", "worktop": "0.8.0-next.18", - "zod": "^4.4.3" + "zod": "^4.4.3", + "@webstudio-is/asset-resource": "workspace:*" }, "devDependencies": { "@cloudflare/workers-types": "^4.20240620.0", diff --git a/fixtures/webstudio-features/.template/package.json b/fixtures/webstudio-features/.template/package.json index fddf1bd5d9c3..2dd9ca28c304 100644 --- a/fixtures/webstudio-features/.template/package.json +++ b/fixtures/webstudio-features/.template/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@webstudio-is/asset-resource": "workspace:*", "@webstudio-is/image": "workspace:*", "@webstudio-is/react-sdk": "workspace:*", "@webstudio-is/sdk": "workspace:*", diff --git a/fixtures/webstudio-features/app/__generated__/$resources.asset-query-manifest.ts b/fixtures/webstudio-features/app/__generated__/$resources.asset-query-manifest.ts new file mode 100644 index 000000000000..7400fd8a417e --- /dev/null +++ b/fixtures/webstudio-features/app/__generated__/$resources.asset-query-manifest.ts @@ -0,0 +1,2 @@ +export const assetQueryDeploymentId = "1b66ee06-8ea5-4420-9a0e-e0d3a67aca32"; +export const assetQueryManifest = []; diff --git a/fixtures/webstudio-features/app/__generated__/$resources.asset-query-runtime.ts b/fixtures/webstudio-features/app/__generated__/$resources.asset-query-runtime.ts new file mode 100644 index 000000000000..df58a6ea191d --- /dev/null +++ b/fixtures/webstudio-features/app/__generated__/$resources.asset-query-runtime.ts @@ -0,0 +1,7 @@ +export const createGeneratedAssetResourceFetch = async ({ + fallback, +}: { + request: Request; + context: unknown; + fallback: typeof fetch; +}): Promise => fallback; diff --git a/fixtures/webstudio-features/app/routes/[_route_with_symbols_]._index.tsx b/fixtures/webstudio-features/app/routes/[_route_with_symbols_]._index.tsx index a23fe4a9fae7..691c6b21c499 100644 --- a/fixtures/webstudio-features/app/routes/[_route_with_symbols_]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[_route_with_symbols_]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[animations]._index.tsx b/fixtures/webstudio-features/app/routes/[animations]._index.tsx index ac5591b6e98a..f4366e95c381 100644 --- a/fixtures/webstudio-features/app/routes/[animations]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[animations]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[assets1]._index.tsx b/fixtures/webstudio-features/app/routes/[assets1]._index.tsx index 357aadb289c9..c81a9f6d2293 100644 --- a/fixtures/webstudio-features/app/routes/[assets1]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[assets1]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[class-names]._index.tsx b/fixtures/webstudio-features/app/routes/[class-names]._index.tsx index b56fc4e83f59..c8211d351e2c 100644 --- a/fixtures/webstudio-features/app/routes/[class-names]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[class-names]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[content-block]._index.tsx b/fixtures/webstudio-features/app/routes/[content-block]._index.tsx index 9888b1b046af..f1cb2b20e525 100644 --- a/fixtures/webstudio-features/app/routes/[content-block]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[content-block]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[duration]._index.tsx b/fixtures/webstudio-features/app/routes/[duration]._index.tsx index be87b8ace54b..9de93512e60c 100644 --- a/fixtures/webstudio-features/app/routes/[duration]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[duration]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[expressions]._index.tsx b/fixtures/webstudio-features/app/routes/[expressions]._index.tsx index e32498f9d7e4..7222a0fa046d 100644 --- a/fixtures/webstudio-features/app/routes/[expressions]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[expressions]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[form]._index.tsx b/fixtures/webstudio-features/app/routes/[form]._index.tsx index 16684fdc4f9d..a132ca4e7d03 100644 --- a/fixtures/webstudio-features/app/routes/[form]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[form]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[head-tag]._index.tsx b/fixtures/webstudio-features/app/routes/[head-tag]._index.tsx index f2c9620fff25..c753a0fdca33 100644 --- a/fixtures/webstudio-features/app/routes/[head-tag]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[head-tag]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[heading-with-id]._index.tsx b/fixtures/webstudio-features/app/routes/[heading-with-id]._index.tsx index cc76334f286a..81a0c7dbad29 100644 --- a/fixtures/webstudio-features/app/routes/[heading-with-id]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[heading-with-id]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[nested].[nested-page]._index.tsx b/fixtures/webstudio-features/app/routes/[nested].[nested-page]._index.tsx index e747d8b61789..a8188d29910a 100644 --- a/fixtures/webstudio-features/app/routes/[nested].[nested-page]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[nested].[nested-page]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[radix]._index.tsx b/fixtures/webstudio-features/app/routes/[radix]._index.tsx index f4070bb89eab..76c413ddb778 100644 --- a/fixtures/webstudio-features/app/routes/[radix]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[radix]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[resources]._index.tsx b/fixtures/webstudio-features/app/routes/[resources]._index.tsx index 31d4962a8140..323add31b59c 100644 --- a/fixtures/webstudio-features/app/routes/[resources]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[resources]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[sitemap.xml]._index.tsx b/fixtures/webstudio-features/app/routes/[sitemap.xml]._index.tsx index 14ef8fba335c..5eb53ed83d48 100644 --- a/fixtures/webstudio-features/app/routes/[sitemap.xml]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[sitemap.xml]._index.tsx @@ -20,6 +20,7 @@ import { assetBaseUrl, imageLoader } from "../constants.mjs"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -98,9 +99,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/[text-duration]._index.tsx b/fixtures/webstudio-features/app/routes/[text-duration]._index.tsx index d842f7f7d879..a3fbc353936b 100644 --- a/fixtures/webstudio-features/app/routes/[text-duration]._index.tsx +++ b/fixtures/webstudio-features/app/routes/[text-duration]._index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/app/routes/_index.tsx b/fixtures/webstudio-features/app/routes/_index.tsx index 867d53e46b71..004867494687 100644 --- a/fixtures/webstudio-features/app/routes/_index.tsx +++ b/fixtures/webstudio-features/app/routes/_index.tsx @@ -45,6 +45,7 @@ import css from "../__generated__/index.css?url"; import { sitemap } from "../__generated__/$resources.sitemap.xml"; import { assets } from "../__generated__/$resources.assets"; import { authRoutes } from "../__generated__/$resources.wsauth.server"; +import { createGeneratedAssetResourceFetch } from "../__generated__/$resources.asset-query-runtime"; const authenticateProductionRequest = (request: Request) => { const host = @@ -122,9 +123,15 @@ export const loader = async (arg: LoaderFunctionArgs) => { pathname: url.pathname, }; + const generatedFetch = await createGeneratedAssetResourceFetch({ + request: arg.request, + context: arg.context, + fallback: customFetch, + }); const resources = await loadResources( - customFetch, - getResources({ system }).data + generatedFetch, + getResources({ system }).data, + url ); const pageMeta = getPageMeta({ system, resources }); diff --git a/fixtures/webstudio-features/package.json b/fixtures/webstudio-features/package.json index e9a01b8d7893..a0fb4c9bdc1b 100644 --- a/fixtures/webstudio-features/package.json +++ b/fixtures/webstudio-features/package.json @@ -32,7 +32,8 @@ "vite": "^6.3.4", "webstudio": "workspace:*", "@webstudio-is/wsauth": "workspace:*", - "@react-router/serve": "^7.5.3" + "@react-router/serve": "^7.5.3", + "@webstudio-is/asset-resource": "workspace:*" }, "devDependencies": { "@types/react": "^18.2.70", diff --git a/packages/asset-resource/scripts/benchmark-scale.mjs b/packages/asset-resource/scripts/benchmark-scale.mjs index 4f6a9d6c69e9..5c9292213ba3 100644 --- a/packages/asset-resource/scripts/benchmark-scale.mjs +++ b/packages/asset-resource/scripts/benchmark-scale.mjs @@ -79,8 +79,7 @@ const listingQuery = `*[ "slug": properties.slug, excerpt }`; -const detailQuery = - `*[properties.slug == $slug][0]{_id, revision, contentRef, "title": properties.title}`; +const detailQuery = `*[properties.slug == $slug][0]{_id, revision, contentRef, "title": properties.title}`; const buildIndex = (query = listingQuery) => buildAssetResourceIndex({ projectId, resourceId, query, entries }); @@ -173,10 +172,12 @@ const detailAndHydration = await measure(50, async () => { if (file === undefined) { throw new Error("Scale fixture content is missing"); } - const bytes = encoder.encode(file.source).subarray( - range?.offset ?? 0, - range === undefined ? undefined : range.offset + range.length - ); + const bytes = encoder + .encode(file.source) + .subarray( + range?.offset ?? 0, + range === undefined ? undefined : range.offset + range.length + ); return { data: { async *[Symbol.asyncIterator]() { @@ -189,7 +190,9 @@ const detailAndHydration = await measure(50, async () => { }); const workerBundle = await bundle({ - entryPoints: [new URL("../src/published-runtime.ts", import.meta.url).pathname], + entryPoints: [ + new URL("../src/published-runtime.ts", import.meta.url).pathname, + ], bundle: true, format: "esm", minify: true, diff --git a/packages/asset-resource/src/candidate-selection.test.ts b/packages/asset-resource/src/candidate-selection.test.ts index aad992ec9c74..7457c1fafbba 100644 --- a/packages/asset-resource/src/candidate-selection.test.ts +++ b/packages/asset-resource/src/candidate-selection.test.ts @@ -68,9 +68,7 @@ describe("asset resource candidate selection", () => { test("does not split a parameter-dependent disjunction", async () => { const selection = await selectAssetResourceCandidates({ - tree: parse( - `*[properties.locale == "en" || properties.slug == $slug]` - ), + tree: parse(`*[properties.locale == "en" || properties.slug == $slug]`), documents, }); diff --git a/packages/cli/templates/internal/package.json b/packages/cli/templates/internal/package.json index 0647aeb8388b..853f67d9c139 100644 --- a/packages/cli/templates/internal/package.json +++ b/packages/cli/templates/internal/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@webstudio-is/asset-resource": "workspace:*", "@webstudio-is/image": "workspace:*", "@webstudio-is/react-sdk": "workspace:*", "@webstudio-is/sdk": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 43d89e9a29a4..93c1a4ee9eb3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -528,6 +528,9 @@ importers: '@react-router/serve': specifier: ^7.5.3 version: 7.5.3(react-router@7.5.3(react-dom@18.3.0-canary-14898b6a9-20240318(react@18.3.0-canary-14898b6a9-20240318))(react@18.3.0-canary-14898b6a9-20240318))(typescript@7.0.2) + '@webstudio-is/asset-resource': + specifier: workspace:* + version: link:../../packages/asset-resource '@webstudio-is/image': specifier: workspace:* version: link:../../packages/image @@ -598,6 +601,9 @@ importers: '@react-router/serve': specifier: ^7.5.3 version: 7.5.3(react-router@7.5.3(react-dom@18.3.0-canary-14898b6a9-20240318(react@18.3.0-canary-14898b6a9-20240318))(react@18.3.0-canary-14898b6a9-20240318))(typescript@7.0.2) + '@webstudio-is/asset-resource': + specifier: workspace:* + version: link:../../packages/asset-resource '@webstudio-is/image': specifier: workspace:* version: link:../../packages/image @@ -674,6 +680,9 @@ importers: '@react-router/serve': specifier: ^7.5.3 version: 7.5.3(react-router@7.5.3(react-dom@18.3.0-canary-14898b6a9-20240318(react@18.3.0-canary-14898b6a9-20240318))(react@18.3.0-canary-14898b6a9-20240318))(typescript@7.0.2) + '@webstudio-is/asset-resource': + specifier: workspace:* + version: link:../../packages/asset-resource '@webstudio-is/image': specifier: workspace:* version: link:../../packages/image @@ -744,6 +753,9 @@ importers: '@vercel/react-router': specifier: ^1.1.0 version: 1.1.0(@react-router/dev@7.5.3(@react-router/serve@7.5.3(react-router@7.5.3(react-dom@18.3.0-canary-14898b6a9-20240318(react@18.3.0-canary-14898b6a9-20240318))(react@18.3.0-canary-14898b6a9-20240318))(typescript@7.0.2))(@types/node@22.13.10)(jiti@2.4.2)(lightningcss@1.31.1)(react-router@7.5.3(react-dom@18.3.0-canary-14898b6a9-20240318(react@18.3.0-canary-14898b6a9-20240318))(react@18.3.0-canary-14898b6a9-20240318))(tsx@4.19.3)(typescript@7.0.2)(vite@6.3.4(@types/node@22.13.10)(jiti@2.4.2)(lightningcss@1.31.1)(tsx@4.19.3)(yaml@2.9.0))(wrangler@4.14.1)(yaml@2.9.0))(@react-router/node@7.5.3(react-router@7.5.3(react-dom@18.3.0-canary-14898b6a9-20240318(react@18.3.0-canary-14898b6a9-20240318))(react@18.3.0-canary-14898b6a9-20240318))(typescript@7.0.2))(isbot@5.1.25)(react-dom@18.3.0-canary-14898b6a9-20240318(react@18.3.0-canary-14898b6a9-20240318))(react@18.3.0-canary-14898b6a9-20240318) + '@webstudio-is/asset-resource': + specifier: workspace:* + version: link:../../packages/asset-resource '@webstudio-is/image': specifier: workspace:* version: link:../../packages/image @@ -799,6 +811,9 @@ importers: fixtures/ssg-cloudflare-pages: dependencies: + '@webstudio-is/asset-resource': + specifier: workspace:* + version: link:../../packages/asset-resource '@webstudio-is/image': specifier: workspace:* version: link:../../packages/image @@ -854,6 +869,9 @@ importers: fixtures/ssg-netlify-by-project-id: dependencies: + '@webstudio-is/asset-resource': + specifier: workspace:* + version: link:../../packages/asset-resource '@webstudio-is/image': specifier: workspace:* version: link:../../packages/image @@ -924,6 +942,9 @@ importers: '@remix-run/server-runtime': specifier: 2.16.5 version: 2.16.5(typescript@7.0.2) + '@webstudio-is/asset-resource': + specifier: workspace:* + version: link:../../packages/asset-resource '@webstudio-is/image': specifier: workspace:* version: link:../../packages/image @@ -1009,6 +1030,9 @@ importers: '@react-router/serve': specifier: ^7.5.3 version: 7.5.3(react-router@7.5.3(react-dom@18.3.0-canary-14898b6a9-20240318(react@18.3.0-canary-14898b6a9-20240318))(react@18.3.0-canary-14898b6a9-20240318))(typescript@7.0.2) + '@webstudio-is/asset-resource': + specifier: workspace:* + version: link:../../packages/asset-resource '@webstudio-is/image': specifier: workspace:* version: link:../../packages/image From 23c8396f6919525544e0155dce830d2288c28e23 Mon Sep 17 00:00:00 2001 From: Oleg Isonen Date: Mon, 20 Jul 2026 21:52:41 +0100 Subject: [PATCH 09/14] fix: synchronize asset resource indexes --- .../app/services/api-build.server.test.ts | 6 +- apps/builder/app/services/api-build.server.ts | 19 ++- .../builder/app/services/api-router.server.ts | 14 +++ .../$resources/assets-field-catalog.server.ts | 7 +- apps/builder/app/shared/db/canvas.server.ts | 34 ++---- .../sync/patch/patch-service.server.test.ts | 2 +- .../shared/sync/patch/patch-service.server.ts | 96 ++------------- ...synchronize-asset-resource-patch.server.ts | 110 ++++++++++++++++++ .../asset-resource/src/candidate-selection.ts | 37 +----- packages/asset-resource/src/groq-ast.ts | 32 +++++ .../asset-resource/src/published-runtime.ts | 11 +- .../asset-resource/src/query-validation.ts | 51 ++------ packages/asset-resource/src/sha256.ts | 9 +- .../src/builder-api-data-boundary.test.ts | 11 +- .../src/canonical-metadata-backfill.ts | 8 +- .../asset-uploader/src/field-catalog.test.ts | 14 ++- packages/asset-uploader/src/field-catalog.ts | 9 ++ .../asset-uploader/src/query-preview.test.ts | 4 + packages/asset-uploader/src/query-preview.ts | 10 +- .../src/resource-index-maintenance.test.ts | 43 +++++++ .../src/resource-index-maintenance.ts | 92 +++++++++++++++ .../src/resource-index-rebuild.test.ts | 50 +------- .../src/resource-index-rebuild.ts | 16 +-- .../src/runtime/asset-resources.ts | 5 +- packages/project-build/src/runtime/cutover.ts | 13 +++ packages/project/src/db/build-patch.test.ts | 13 ++- packages/project/src/db/build-patch.ts | 17 ++- 27 files changed, 454 insertions(+), 279 deletions(-) create mode 100644 apps/builder/app/shared/synchronize-asset-resource-patch.server.ts create mode 100644 packages/asset-resource/src/groq-ast.ts diff --git a/apps/builder/app/services/api-build.server.test.ts b/apps/builder/app/services/api-build.server.test.ts index 16f0ba78af83..6fe350d6ec96 100644 --- a/apps/builder/app/services/api-build.server.test.ts +++ b/apps/builder/app/services/api-build.server.test.ts @@ -247,7 +247,8 @@ describe("api build patch commits", () => { clientVersion: 1, transactions: [transaction], }, - ctx + ctx, + expect.any(Function) ); expect(patchBuild).toHaveBeenNthCalledWith( 2, @@ -262,7 +263,8 @@ describe("api build patch commits", () => { }, ], }, - ctx + ctx, + expect.any(Function) ); }); diff --git a/apps/builder/app/services/api-build.server.ts b/apps/builder/app/services/api-build.server.ts index 7d0ce655be57..962a1f5b674a 100644 --- a/apps/builder/app/services/api-build.server.ts +++ b/apps/builder/app/services/api-build.server.ts @@ -15,6 +15,7 @@ import type { AppContext } from "@webstudio-is/trpc-interface/index.server"; import { serializePages } from "@webstudio-is/project-migrations/pages"; import { assertApiProjectPermit } from "./api-permits.server"; import { throwApiError } from "./api-errors.server"; +import { synchronizeAssetResourcesAfterBuildPatch } from "../shared/synchronize-asset-resource-patch.server"; export const loadBuildByProjectVersion = async ( ctx: AppContext, @@ -163,7 +164,23 @@ export const commitBuildTransactions = async ({ clientVersion, transactions, }, - ctx + ctx, + async ({ previousBuild, build, changes }) => { + try { + await synchronizeAssetResourcesAfterBuildPatch({ + context: ctx, + projectId, + previousResources: previousBuild.resources, + resources: build.resources, + changes, + }); + } catch (error) { + console.error( + "Asset resource post-patch synchronization failed", + error + ); + } + } ); if (result.status === "version_mismatched") { throwApiError("CONFLICT", result.errors); diff --git a/apps/builder/app/services/api-router.server.ts b/apps/builder/app/services/api-router.server.ts index be5c4aa65421..607c9efaa977 100644 --- a/apps/builder/app/services/api-router.server.ts +++ b/apps/builder/app/services/api-router.server.ts @@ -57,6 +57,7 @@ import { import { componentMetas } from "~/shared/component-metas.server"; import { assetResourceQueryRequest, + getAssetResourceQuery, type Asset, type AssetFolder, } from "@webstudio-is/sdk"; @@ -784,6 +785,7 @@ export const apiRouter = router({ await loadBuilderAssetFieldCatalog({ projectId: input.projectId, context: ctx, + assetClient: createAssetClient(), }), { command: "get-asset-field-catalog", @@ -818,11 +820,23 @@ export const apiRouter = router({ "build", async ({ ctx, input }) => { try { + const build = await loadDevBuildByProjectId(ctx, input.projectId); + const resource = build.resources.find( + ({ id }) => id === input.resourceId + ); + const query = + resource === undefined + ? undefined + : getAssetResourceQuery(resource); + if (query === undefined) { + throw new AssetResourceIndexNotFoundError(); + } const result = await rebuildAssetResourceIndex({ client: ctx.postgrest.client, store: createAssetClient().resourceIndexStore, projectId: input.projectId, resourceId: input.resourceId, + query, }); const status = await loadAssetResourceIndexStatus({ projectId: input.projectId, diff --git a/apps/builder/app/shared/$resources/assets-field-catalog.server.ts b/apps/builder/app/shared/$resources/assets-field-catalog.server.ts index f1912cf5bdb1..70972f7553cd 100644 --- a/apps/builder/app/shared/$resources/assets-field-catalog.server.ts +++ b/apps/builder/app/shared/$resources/assets-field-catalog.server.ts @@ -5,6 +5,7 @@ import { AuthorizationError } from "@webstudio-is/trpc-interface/index.server"; import { privateNoStoreResponseHeaders } from "~/services/cache-control.server"; import { createContext } from "../context.server"; import { isBuilder } from "../router-utils"; +import { createAssetClient } from "../asset-client"; export const loader = async ({ request }: { request: Request }) => { if (isBuilder(request) === false) { @@ -20,7 +21,11 @@ export const loader = async ({ request }: { request: Request }) => { try { const context = await createContext(request); - const catalog = await loadBuilderAssetFieldCatalog({ projectId, context }); + const catalog = await loadBuilderAssetFieldCatalog({ + projectId, + context, + assetClient: createAssetClient(), + }); return json(catalog, { headers: privateNoStoreResponseHeaders }); } catch (error) { if (error instanceof AuthorizationError) { diff --git a/apps/builder/app/shared/db/canvas.server.ts b/apps/builder/app/shared/db/canvas.server.ts index 66e02f3c2b51..69a231ef911b 100644 --- a/apps/builder/app/shared/db/canvas.server.ts +++ b/apps/builder/app/shared/db/canvas.server.ts @@ -11,9 +11,8 @@ import { collectFontFamiliesFromStyleDecls } from "@webstudio-is/project-build/r import { loadAssetDataByProject, loadCanonicalAssetFileEntries, - loadAssetResourceIndexSnapshots, + prepareAssetResourceIndexSnapshotsForPublication, getAssetResourceQuery, - reconcileAssetResourceIndexesForPublication, synchronizeCanonicalAssets, } from "@webstudio-is/asset-uploader/index.server"; import type { AppContext } from "@webstudio-is/trpc-interface/index.server"; @@ -192,26 +191,17 @@ const addProjectMetadata = async ( ); const expectedAssetRevision = await computeCanonicalAssetRevision(canonicalEntries); - await reconcileAssetResourceIndexesForPublication({ - client: context.postgrest.client, - store: assetClient.resourceIndexStore, - projectId: project.id, - resources: indexedResources, - entries: canonicalEntries, - assetRevision: expectedAssetRevision, - }); - assetResourceIndexes = await loadAssetResourceIndexSnapshots({ - client: context.postgrest.client, - projectId: project.id, - resources: indexedResources, - read: assetClient.readFile, - referenceId: data.build.id, - garbageCollectionStore: - assetClient.resourceIndexStore.delete === undefined - ? undefined - : { delete: assetClient.resourceIndexStore.delete }, - expectedAssetRevision, - }); + assetResourceIndexes = + await prepareAssetResourceIndexSnapshotsForPublication({ + client: context.postgrest.client, + store: assetClient.resourceIndexStore, + projectId: project.id, + resources: indexedResources, + entries: canonicalEntries, + read: assetClient.readFile, + referenceId: data.build.id, + assetRevision: expectedAssetRevision, + }); } return { diff --git a/apps/builder/app/shared/sync/patch/patch-service.server.test.ts b/apps/builder/app/shared/sync/patch/patch-service.server.test.ts index 3b0465bec17f..7cf5317e2edd 100644 --- a/apps/builder/app/shared/sync/patch/patch-service.server.test.ts +++ b/apps/builder/app/shared/sync/patch/patch-service.server.test.ts @@ -210,7 +210,7 @@ describe("applyPatchRequest", () => { expect(updateAssetResourceIndexesAfterCanonicalChange).toHaveBeenCalledWith( expect.objectContaining({ projectId: "project-1", - changedAssetIds: ["folder-1"], + changedAssetIds: ["project-assets"], }) ); }); diff --git a/apps/builder/app/shared/sync/patch/patch-service.server.ts b/apps/builder/app/shared/sync/patch/patch-service.server.ts index 438ff0afd01e..6e683bb187d8 100644 --- a/apps/builder/app/shared/sync/patch/patch-service.server.ts +++ b/apps/builder/app/shared/sync/patch/patch-service.server.ts @@ -15,14 +15,7 @@ import type { NormalizedPatchRequest, PatchEntry, } from "./patch-normalize.server"; -import { - synchronizeAllCanonicalAssetStandardMetadata, - synchronizeCanonicalAsset, - synchronizeAssetResourceIndexQueries, - updateAssetResourceIndexesAfterCanonicalChange, -} from "@webstudio-is/asset-uploader/index.server"; -import { createAssetClient } from "../../asset-client"; -import type { Resource } from "@webstudio-is/sdk"; +import { synchronizeAssetResourcesAfterBuildPatch } from "../../synchronize-asset-resource-patch.server"; type BuildRow = Database["public"]["Tables"]["Build"]["Row"]; type BuildColumn = keyof BuildRow; @@ -328,8 +321,6 @@ const applyAuthorizedEntries = async ({ }; }; -const parseResources = (value: string) => JSON.parse(value) as Resource[]; - export const applyPatchRequest = async ( context: AppContext, patch: NormalizedPatchRequest @@ -371,84 +362,13 @@ export const applyPatchRequest = async ( return applied; } - if ( - build?.resources !== undefined && - applied.build?.resources !== undefined && - build.resources !== applied.build.resources - ) { - try { - await synchronizeAssetResourceIndexQueries({ - client: context.postgrest.client, - assetClient: createAssetClient(), - projectId: patch.projectId, - previousResources: parseResources(build.resources), - resources: parseResources(applied.build.resources), - }); - } catch (error) { - // The resource patch is already committed. Keep its response successful; - // index status exposes build failures without encouraging a duplicate patch. - console.error("Asset resource index synchronization failed", error); - } - } - - const assetChanges = authorized.flatMap(({ entry }) => - entry.transaction.payload.filter( - ({ namespace }) => namespace === "assets" || namespace === "assetFolders" - ) - ); - if (assetChanges.length > 0) { - try { - const assetClient = createAssetClient(); - const hasFolderChanges = assetChanges.some( - ({ namespace }) => namespace === "assetFolders" - ); - if (hasFolderChanges) { - await synchronizeAllCanonicalAssetStandardMetadata({ - client: context.postgrest.client, - projectId: patch.projectId, - }); - } - const addedAssetIds = [ - ...new Set( - assetChanges - .filter(({ namespace }) => namespace === "assets") - .flatMap(({ patches }) => - patches.flatMap(({ op, path }) => - op === "add" && path.length === 1 && typeof path[0] === "string" - ? [path[0]] - : [] - ) - ) - ), - ]; - for (const assetId of addedAssetIds) { - await synchronizeCanonicalAsset({ - client: context.postgrest.client, - assetClient, - projectId: patch.projectId, - assetId, - }); - } - const changedAssetIds = [ - ...new Set( - assetChanges.flatMap(({ patches }) => - patches - .map(({ path }) => path[0]) - .filter((id): id is string => typeof id === "string") - ) - ), - ]; - await updateAssetResourceIndexesAfterCanonicalChange({ - client: context.postgrest.client, - store: assetClient.resourceIndexStore, - projectId: patch.projectId, - changedAssetIds: - changedAssetIds.length === 0 ? ["project-assets"] : changedAssetIds, - }); - } catch (error) { - console.error("Asset resource index maintenance failed", error); - } - } + await synchronizeAssetResourcesAfterBuildPatch({ + context, + projectId: patch.projectId, + previousResources: build?.resources, + resources: applied.build?.resources, + changes: authorized.flatMap(({ entry }) => entry.transaction.payload), + }); const entriesByTransactionId = new Map(); for (const entryResult of [ diff --git a/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts b/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts new file mode 100644 index 000000000000..0746eccf5080 --- /dev/null +++ b/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts @@ -0,0 +1,110 @@ +import { + synchronizeAllCanonicalAssetStandardMetadata, + synchronizeAssetResourceIndexQueries, + synchronizeCanonicalAsset, + updateAssetResourceIndexesAfterCanonicalChange, +} from "@webstudio-is/asset-uploader/index.server"; +import type { BuildPatchChange } from "@webstudio-is/project/index.server"; +import type { AppContext } from "@webstudio-is/trpc-interface/index.server"; +import type { Resource } from "@webstudio-is/sdk"; +import { createAssetClient } from "./asset-client"; + +const parseResources = (value: string) => JSON.parse(value) as Resource[]; + +export const synchronizeAssetResourcesAfterBuildPatch = async ({ + context, + projectId, + previousResources, + resources, + changes, +}: { + context: AppContext; + projectId: string; + previousResources?: string; + resources?: string; + changes: readonly BuildPatchChange[]; +}) => { + const hasResourceChanges = + previousResources !== undefined && + resources !== undefined && + previousResources !== resources; + const assetChanges = changes.filter( + ({ namespace }) => namespace === "assets" || namespace === "assetFolders" + ); + if (hasResourceChanges === false && assetChanges.length === 0) { + return; + } + let assetClient: ReturnType; + try { + assetClient = createAssetClient(); + } catch (error) { + console.error("Asset resource client initialization failed", error); + return; + } + if (hasResourceChanges) { + try { + await synchronizeAssetResourceIndexQueries({ + client: context.postgrest.client, + assetClient, + projectId, + previousResources: parseResources(previousResources), + resources: parseResources(resources), + }); + } catch (error) { + console.error("Asset resource index synchronization failed", error); + } + } + + if (assetChanges.length === 0) { + return; + } + + try { + if (assetChanges.some(({ namespace }) => namespace === "assetFolders")) { + await synchronizeAllCanonicalAssetStandardMetadata({ + client: context.postgrest.client, + projectId, + }); + } + const directAssetChanges = assetChanges.filter( + ({ namespace }) => namespace === "assets" + ); + const addedAssetIds = [ + ...new Set( + directAssetChanges.flatMap(({ patches }) => + patches.flatMap(({ op, path }) => + op === "add" && path.length === 1 && typeof path[0] === "string" + ? [path[0]] + : [] + ) + ) + ), + ]; + for (const assetId of addedAssetIds) { + await synchronizeCanonicalAsset({ + client: context.postgrest.client, + assetClient, + projectId, + assetId, + }); + } + const changedAssetIds = [ + ...new Set( + directAssetChanges.flatMap(({ patches }) => + patches + .map(({ path }) => path[0]) + .filter((id): id is string => typeof id === "string") + ) + ), + ]; + await updateAssetResourceIndexesAfterCanonicalChange({ + client: context.postgrest.client, + store: assetClient.resourceIndexStore, + projectId, + changedAssetIds: + changedAssetIds.length === 0 ? ["project-assets"] : changedAssetIds, + }); + } catch (error) { + console.error("Asset resource index maintenance failed", error); + } +}; diff --git a/packages/asset-resource/src/candidate-selection.ts b/packages/asset-resource/src/candidate-selection.ts index 1310b87e8bcc..9fcaae0f8ac3 100644 --- a/packages/asset-resource/src/candidate-selection.ts +++ b/packages/asset-resource/src/candidate-selection.ts @@ -1,5 +1,6 @@ import { evaluate, type ExprNode } from "groq-js/1"; import type { AssetFileDocument } from "@webstudio-is/sdk"; +import { isGroqAstNode, visitGroqAst } from "./groq-ast"; const assetResourceCandidatePolicyV1 = { records: "safe-static-filter-superset", @@ -7,16 +8,6 @@ const assetResourceCandidatePolicyV1 = { content: "reference-only", } as const; -type AstNode = { - type: string; -}; - -const isAstNode = (value: unknown): value is AstNode => - typeof value === "object" && - value !== null && - "type" in value && - typeof value.type === "string"; - const compareStrings = (left: string, right: string) => { if (left < right) { return -1; @@ -27,29 +18,9 @@ const compareStrings = (left: string, right: string) => { return 0; }; -const visitAst = (node: AstNode, visit: (node: AstNode) => void) => { - visit(node); - if (node.type === "Value") { - return; - } - for (const value of Object.values(node)) { - if (Array.isArray(value)) { - for (const item of value) { - if (isAstNode(item)) { - visitAst(item, visit); - } - } - continue; - } - if (isAstNode(value)) { - visitAst(value, visit); - } - } -}; - export const getAssetResourceParameterNames = (tree: ExprNode) => { const names = new Set(); - visitAst(tree, (node) => { + visitGroqAst(tree, (node) => { if ( node.type === "Parameter" && "name" in node && @@ -82,7 +53,7 @@ const stableStaticNodeTypes = new Set([ const isStableStaticExpression = (tree: ExprNode) => { let stable = true; - visitAst(tree, (node) => { + visitGroqAst(tree, (node) => { if (stableStaticNodeTypes.has(node.type) === false) { stable = false; } @@ -131,7 +102,7 @@ const collectDatasetFilters = (node: ExprNode): DatasetFilterAnalysis => { return { expressions: [], isUntransformedDataset: true }; } - if ("base" in node && isAstNode(node.base)) { + if ("base" in node && isGroqAstNode(node.base)) { const analysis = collectDatasetFilters(node.base as ExprNode); if (node.type === "Filter" && analysis.isUntransformedDataset) { return { diff --git a/packages/asset-resource/src/groq-ast.ts b/packages/asset-resource/src/groq-ast.ts new file mode 100644 index 000000000000..b8f98942d5cc --- /dev/null +++ b/packages/asset-resource/src/groq-ast.ts @@ -0,0 +1,32 @@ +import type { ExprNode } from "groq-js/1"; + +export type GroqAstNode = { type: string }; + +export const isGroqAstNode = (value: unknown): value is GroqAstNode => + typeof value === "object" && + value !== null && + "type" in value && + typeof value.type === "string"; + +export const visitGroqAst = ( + tree: ExprNode, + visit: (node: GroqAstNode, depth: number) => boolean | void +) => { + const walk = (node: GroqAstNode, depth: number) => { + if (visit(node, depth) === false || node.type === "Value") { + return; + } + for (const value of Object.values(node)) { + if (Array.isArray(value)) { + for (const item of value) { + if (isGroqAstNode(item)) { + walk(item, depth + 1); + } + } + } else if (isGroqAstNode(value)) { + walk(value, depth + 1); + } + } + }; + walk(tree, 1); +}; diff --git a/packages/asset-resource/src/published-runtime.ts b/packages/asset-resource/src/published-runtime.ts index 26a43992b75d..7ab38c663112 100644 --- a/packages/asset-resource/src/published-runtime.ts +++ b/packages/asset-resource/src/published-runtime.ts @@ -14,6 +14,7 @@ import { } from "./index"; import { AssetResourceHydrationError } from "./hydration"; import { verifyAssetResourceIndex } from "./resource-index"; +import { sha256Hex } from "./sha256"; export type PublishedAssetResourceManifestEntry = { resourceId: string; @@ -118,15 +119,9 @@ export const getPublishedAssetResourceCacheKey = async ({ request: Request; }) => { const body = await request.clone().text(); - const digest = await crypto.subtle.digest( - "SHA-256", - new TextEncoder().encode( - `${deploymentId}\n${entry.resourceId}\n${entry.revision}\n${body}` - ) + const hash = await sha256Hex( + `${deploymentId}\n${entry.resourceId}\n${entry.revision}\n${body}` ); - const hash = Array.from(new Uint8Array(digest), (byte) => - byte.toString(16).padStart(2, "0") - ).join(""); const url = new URL(request.url); url.searchParams.set("ws-asset-resource", hash); return new Request(url, { method: "GET" }); diff --git a/packages/asset-resource/src/query-validation.ts b/packages/asset-resource/src/query-validation.ts index dc76dba1fdb0..4b693bf55daf 100644 --- a/packages/asset-resource/src/query-validation.ts +++ b/packages/asset-resource/src/query-validation.ts @@ -1,6 +1,7 @@ import { parse, type ExprNode } from "groq-js/1"; import { assetResourceLimits } from "@webstudio-is/sdk"; import { getAssetResourceParameterNames } from "./candidate-selection"; +import { visitGroqAst } from "./groq-ast"; export class AssetResourceQueryValidationError extends Error { readonly code: "INVALID_QUERY" | "QUERY_COMPLEXITY_EXCEEDED"; @@ -22,19 +23,11 @@ export class AssetResourceQueryValidationError extends Error { } } -type AstNode = { type: string }; - -const isAstNode = (value: unknown): value is AstNode => - typeof value === "object" && - value !== null && - "type" in value && - typeof value.type === "string"; - const measureAst = (tree: ExprNode) => { let nodes = 0; let maxDepth = 0; let datasetScans = 0; - const visit = (node: AstNode, depth: number) => { + visitGroqAst(tree, (node, depth) => { nodes += 1; maxDepth = Math.max(maxDepth, depth); if (node.type === "Everything") { @@ -42,26 +35,11 @@ const measureAst = (tree: ExprNode) => { } if ( nodes > assetResourceLimits.queryAstNodes || - maxDepth > assetResourceLimits.queryAstDepth || - node.type === "Value" + maxDepth > assetResourceLimits.queryAstDepth ) { - return; + return false; } - for (const value of Object.values(node)) { - if (Array.isArray(value)) { - for (const item of value) { - if (isAstNode(item)) { - visit(item, depth + 1); - } - } - continue; - } - if (isAstNode(value)) { - visit(value, depth + 1); - } - } - }; - visit(tree, 1); + }); return { nodes, depth: maxDepth, datasetScans }; }; @@ -171,24 +149,11 @@ const getAttributePath = (node: unknown): string | undefined => { export const getAssetResourceReferencedFieldPaths = (query: string) => { const { tree } = validateAssetResourceQuery(query); const paths = new Set(); - const visited = new Set(); - const visit = (value: unknown) => { - if (typeof value !== "object" || value === null || visited.has(value)) { - return; - } - visited.add(value); - const path = getAttributePath(value); + visitGroqAst(tree, (node) => { + const path = getAttributePath(node); if (path !== undefined) { paths.add(path); } - for (const child of Object.values(value)) { - if (Array.isArray(child)) { - child.forEach(visit); - } else { - visit(child); - } - } - }; - visit(tree); + }); return Array.from(paths).sort(); }; diff --git a/packages/asset-resource/src/sha256.ts b/packages/asset-resource/src/sha256.ts index f27b3d9bfce4..1583a206e68b 100644 --- a/packages/asset-resource/src/sha256.ts +++ b/packages/asset-resource/src/sha256.ts @@ -1,9 +1,12 @@ -export const sha256 = async (value: string | Uint8Array) => { +export const sha256Hex = async (value: string | Uint8Array) => { const bytes = new Uint8Array( typeof value === "string" ? new TextEncoder().encode(value) : value ); const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes); - return `sha256:${Array.from(new Uint8Array(digest), (byte) => + return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0") - ).join("")}`; + ).join(""); }; + +export const sha256 = async (value: string | Uint8Array) => + `sha256:${await sha256Hex(value)}`; diff --git a/packages/asset-uploader/src/builder-api-data-boundary.test.ts b/packages/asset-uploader/src/builder-api-data-boundary.test.ts index 579a4f326e10..0d11a597e97e 100644 --- a/packages/asset-uploader/src/builder-api-data-boundary.test.ts +++ b/packages/asset-uploader/src/builder-api-data-boundary.test.ts @@ -6,6 +6,7 @@ import { import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; import { loadBuilderAssetFieldCatalog } from "./field-catalog"; import { previewAssetResourceQuery } from "./query-preview"; +import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; vi.mock("@webstudio-is/trpc-interface/index.server", () => ({ authorizeProject: { hasProjectPermit: vi.fn() }, @@ -14,6 +15,9 @@ vi.mock("@webstudio-is/trpc-interface/index.server", () => ({ vi.mock("./canonical-metadata-persistence", () => ({ loadCanonicalAssetFileEntries: vi.fn(), })); +vi.mock("./canonical-metadata-backfill", () => ({ + synchronizeCanonicalAssets: vi.fn(), +})); const projectId = "project-1"; const context = { @@ -27,8 +31,9 @@ describe("Builder asset-resource API data boundary", () => { vi.mocked(loadCanonicalAssetFileEntries).mockResolvedValue([]); }); - test("catalog and preview requests use persisted rows without rescanning files", async () => { - await loadBuilderAssetFieldCatalog({ projectId, context }); + test("catalog and preview synchronize before reading persisted rows", async () => { + const assetClient = { readFile: vi.fn() }; + await loadBuilderAssetFieldCatalog({ projectId, context, assetClient }); await previewAssetResourceQuery({ projectId, request: { @@ -38,8 +43,10 @@ describe("Builder asset-resource API data boundary", () => { content: { mode: "none" }, }, context, + assetClient, }); + expect(synchronizeCanonicalAssets).toHaveBeenCalledTimes(2); expect(loadCanonicalAssetFileEntries).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.ts b/packages/asset-uploader/src/canonical-metadata-backfill.ts index a1903742acc9..c548e18d2618 100644 --- a/packages/asset-uploader/src/canonical-metadata-backfill.ts +++ b/packages/asset-uploader/src/canonical-metadata-backfill.ts @@ -24,6 +24,8 @@ import { } from "./canonical-metadata-persistence"; const markdownExtension = /\.md$/i; +type CanonicalAssetClient = Pick & + Partial>; export const createAssetContentRevision = ({ storageName, @@ -182,7 +184,7 @@ const indexCanonicalAsset = async ({ asset: UploadedAssetRow; hierarchy: AssetFolderHierarchy; client: Client; - assetClient: AssetClient; + assetClient: CanonicalAssetClient; }) => { let properties: Record = {}; let excerpt: string | undefined; @@ -233,7 +235,7 @@ export const synchronizeCanonicalAsset = async ({ projectId: string; assetId: string; client: Client; - assetClient: AssetClient; + assetClient: CanonicalAssetClient; }) => { const assets = await loadUploadedAssets(projectId, client, [assetId]); const asset = assets[0]; @@ -267,7 +269,7 @@ export const synchronizeCanonicalAssets = async ({ }: { projectId: string; client: Client; - assetClient: AssetClient; + assetClient: CanonicalAssetClient; concurrency?: number; }) => { if (Number.isInteger(concurrency) === false || concurrency <= 0) { diff --git a/packages/asset-uploader/src/field-catalog.test.ts b/packages/asset-uploader/src/field-catalog.test.ts index af8842fa9d96..c543037ce523 100644 --- a/packages/asset-uploader/src/field-catalog.test.ts +++ b/packages/asset-uploader/src/field-catalog.test.ts @@ -6,6 +6,7 @@ import { } from "@webstudio-is/trpc-interface/index.server"; import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; import { loadBuilderAssetFieldCatalog } from "./field-catalog"; +import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; vi.mock("@webstudio-is/trpc-interface/index.server", () => ({ authorizeProject: { hasProjectPermit: vi.fn() }, @@ -14,12 +15,16 @@ vi.mock("@webstudio-is/trpc-interface/index.server", () => ({ vi.mock("./canonical-metadata-persistence", () => ({ loadCanonicalAssetFileEntries: vi.fn(), })); +vi.mock("./canonical-metadata-backfill", () => ({ + synchronizeCanonicalAssets: vi.fn(), +})); const projectId = "project-1"; const postgrestClient = { from: vi.fn() } as never; const context = { postgrest: { client: postgrestClient }, } as unknown as AppContext; +const assetClient = { readFile: vi.fn() }; describe("loadBuilderAssetFieldCatalog", () => { beforeEach(() => { @@ -51,7 +56,11 @@ describe("loadBuilderAssetFieldCatalog", () => { }, ]); - const result = await loadBuilderAssetFieldCatalog({ projectId, context }); + const result = await loadBuilderAssetFieldCatalog({ + projectId, + context, + assetClient, + }); expect(authorizeProject.hasProjectPermit).toHaveBeenCalledWith( { projectId, permit: "view" }, @@ -61,6 +70,7 @@ describe("loadBuilderAssetFieldCatalog", () => { client: postgrestClient, projectId, }); + expect(synchronizeCanonicalAssets).toHaveBeenCalledOnce(); expect(result.fields["properties.title"]).toEqual({ types: ["string"], occurrences: 1, @@ -71,7 +81,7 @@ describe("loadBuilderAssetFieldCatalog", () => { vi.mocked(authorizeProject.hasProjectPermit).mockResolvedValue(false); await expect( - loadBuilderAssetFieldCatalog({ projectId, context }) + loadBuilderAssetFieldCatalog({ projectId, context, assetClient }) ).rejects.toBeInstanceOf(AuthorizationError); expect(loadCanonicalAssetFileEntries).not.toHaveBeenCalled(); }); diff --git a/packages/asset-uploader/src/field-catalog.ts b/packages/asset-uploader/src/field-catalog.ts index 185615850a5a..8c0c820c1695 100644 --- a/packages/asset-uploader/src/field-catalog.ts +++ b/packages/asset-uploader/src/field-catalog.ts @@ -8,13 +8,17 @@ import { type AppContext, } from "@webstudio-is/trpc-interface/index.server"; import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; +import type { AssetClient } from "./client"; +import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; export const loadBuilderAssetFieldCatalog = async ({ projectId, context, + assetClient, }: { projectId: string; context: AppContext; + assetClient: Pick; }) => { const canView = await authorizeProject.hasProjectPermit( { projectId, permit: "view" }, @@ -26,6 +30,11 @@ export const loadBuilderAssetFieldCatalog = async ({ ); } + await synchronizeCanonicalAssets({ + projectId, + client: context.postgrest.client, + assetClient, + }); const entries = await loadCanonicalAssetFileEntries({ client: context.postgrest.client, projectId, diff --git a/packages/asset-uploader/src/query-preview.test.ts b/packages/asset-uploader/src/query-preview.test.ts index 12bdba9cd061..4234df2240c5 100644 --- a/packages/asset-uploader/src/query-preview.test.ts +++ b/packages/asset-uploader/src/query-preview.test.ts @@ -14,6 +14,9 @@ vi.mock("@webstudio-is/trpc-interface/index.server", () => ({ vi.mock("./canonical-metadata-persistence", () => ({ loadCanonicalAssetFileEntries: vi.fn(), })); +vi.mock("./canonical-metadata-backfill", () => ({ + synchronizeCanonicalAssets: vi.fn(), +})); const projectId = "project-1"; const postgrestClient = { from: vi.fn() } as never; @@ -86,6 +89,7 @@ describe("previewAssetResourceQuery", () => { content: { mode: "none" }, }, context, + assetClient: { readFile: vi.fn() }, }) ).rejects.toBeInstanceOf(AuthorizationError); expect(loadCanonicalAssetFileEntries).not.toHaveBeenCalled(); diff --git a/packages/asset-uploader/src/query-preview.ts b/packages/asset-uploader/src/query-preview.ts index bed31130c317..0c4e9f5d5aa7 100644 --- a/packages/asset-uploader/src/query-preview.ts +++ b/packages/asset-uploader/src/query-preview.ts @@ -11,6 +11,7 @@ import { type AppContext, } from "@webstudio-is/trpc-interface/index.server"; import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; +import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; export const previewAssetResourceQuery = async ({ projectId, @@ -21,7 +22,7 @@ export const previewAssetResourceQuery = async ({ projectId: string; request: AssetResourceQueryRequest; context: AppContext; - assetClient?: Pick; + assetClient: Pick; }) => { const canView = await authorizeProject.hasProjectPermit( { projectId, permit: "view" }, @@ -32,6 +33,11 @@ export const previewAssetResourceQuery = async ({ "You don't have access to preview this project's asset resources" ); } + await synchronizeCanonicalAssets({ + projectId, + client: context.postgrest.client, + assetClient, + }); const entries = await loadCanonicalAssetFileEntries({ client: context.postgrest.client, projectId, @@ -43,6 +49,6 @@ export const previewAssetResourceQuery = async ({ queryHash: await computeAssetResourceQueryHash(request.query), indexRevision: `metadata:${assetRevision}`, assetRevision, - read: assetClient?.readFile, + read: assetClient.readFile, }); }; diff --git a/packages/asset-uploader/src/resource-index-maintenance.test.ts b/packages/asset-uploader/src/resource-index-maintenance.test.ts index 7574c3687b14..818506b910f9 100644 --- a/packages/asset-uploader/src/resource-index-maintenance.test.ts +++ b/packages/asset-uploader/src/resource-index-maintenance.test.ts @@ -12,6 +12,7 @@ import { } from "@webstudio-is/postgrest/testing"; import { reconcileAssetResourceIndexesForPublication, + prepareAssetResourceIndexSnapshotsForPublication, updateAssetResourceIndexesAfterCanonicalChange, } from "./resource-index-maintenance"; @@ -173,4 +174,46 @@ describe("incremental resource index maintenance", () => { ).resolves.toEqual(["posts"]); expect(putIfAbsent).toHaveBeenCalledOnce(); }); + + test("builds a historical query snapshot without replacing current state", async () => { + const historicalQuery = `*[extension == "md"]`; + const historicalQueryHash = + await computeAssetResourceQueryHash(historicalQuery); + const assetRevision = await computeCanonicalAssetRevision([entry]); + server.use( + db.get("AssetResourceIndexState", () => + json([ + { + resourceId: "posts", + queryHash: `sha256:${"1".repeat(64)}`, + deletedAt: null, + }, + ]) + ) + ); + const putIfAbsent = vi.fn(); + const read = vi.fn(); + + const snapshots = await prepareAssetResourceIndexSnapshotsForPublication({ + client: testContext.postgrest.client, + store: { putIfAbsent }, + projectId: "project-1", + resources: [ + { + resourceId: "posts", + query: historicalQuery, + queryHash: historicalQueryHash, + }, + ], + entries: [entry], + assetRevision, + read, + referenceId: "historical-build", + }); + + expect(snapshots).toHaveLength(1); + expect(snapshots[0]?.index.queryHash).toBe(historicalQueryHash); + expect(putIfAbsent).not.toHaveBeenCalled(); + expect(read).not.toHaveBeenCalled(); + }); }); diff --git a/packages/asset-uploader/src/resource-index-maintenance.ts b/packages/asset-uploader/src/resource-index-maintenance.ts index bcf317366db7..b79a2cc52239 100644 --- a/packages/asset-uploader/src/resource-index-maintenance.ts +++ b/packages/asset-uploader/src/resource-index-maintenance.ts @@ -1,4 +1,5 @@ import { + buildAssetResourceIndex, computeCanonicalAssetRevision, type CanonicalAssetFileEntry, type ImmutableAssetResourceIndexStore, @@ -12,6 +13,10 @@ import { AssetResourceIndexBuildSupersededError, buildPersistAndActivateAssetResourceIndex, } from "./resource-index-build"; +import { + loadAssetResourceIndexSnapshots, + type AssetResourceIndexSnapshot, +} from "./resource-index-snapshot"; export const synchronizeAssetResourceStateAfterAssetChange = async ({ client, @@ -176,3 +181,90 @@ export const reconcileAssetResourceIndexesForPublication = async ({ } return rebuilt; }; + +export const prepareAssetResourceIndexSnapshotsForPublication = async ({ + client, + store, + projectId, + resources, + entries, + assetRevision, + read, + referenceId, +}: { + client: Client; + store: ImmutableAssetResourceIndexStore; + projectId: string; + resources: readonly { + resourceId: string; + query: string; + queryHash: string; + }[]; + entries: readonly CanonicalAssetFileEntry[]; + assetRevision: string; + read: (name: string) => Promise<{ data: AsyncIterable }>; + referenceId: string; +}): Promise => { + if (resources.length === 0) { + return []; + } + const statesResult = await client + .from("AssetResourceIndexState") + .select("resourceId, queryHash, deletedAt") + .eq("projectId", projectId) + .in( + "resourceId", + [...new Set(resources.map(({ resourceId }) => resourceId))].sort() + ); + assertPostgrestSuccess(statesResult); + const states = new Map( + (statesResult.data ?? []).map((state) => [state.resourceId, state]) + ); + const currentResources = resources.filter((resource) => { + const state = states.get(resource.resourceId); + return ( + state !== undefined && + state.deletedAt === null && + state.queryHash === resource.queryHash + ); + }); + await reconcileAssetResourceIndexesForPublication({ + client, + store, + projectId, + resources: currentResources, + entries, + assetRevision, + }); + const currentSnapshots = await loadAssetResourceIndexSnapshots({ + client, + projectId, + resources: currentResources, + read, + expectedAssetRevision: assetRevision, + referenceId, + garbageCollectionStore: + store.delete === undefined ? undefined : { delete: store.delete }, + }); + const snapshots = new Map( + currentSnapshots.map((snapshot) => [snapshot.resourceId, snapshot]) + ); + for (const resource of resources) { + if (snapshots.has(resource.resourceId)) { + continue; + } + const index = await buildAssetResourceIndex({ + projectId, + resourceId: resource.resourceId, + query: resource.query, + entries, + assetRevision, + }); + snapshots.set(resource.resourceId, { + resourceId: resource.resourceId, + revision: index.integrity.checksum, + index, + }); + } + return resources.map(({ resourceId }) => snapshots.get(resourceId)!); +}; diff --git a/packages/asset-uploader/src/resource-index-rebuild.test.ts b/packages/asset-uploader/src/resource-index-rebuild.test.ts index 0f02dd0b9911..33a18e492b94 100644 --- a/packages/asset-uploader/src/resource-index-rebuild.test.ts +++ b/packages/asset-uploader/src/resource-index-rebuild.test.ts @@ -6,10 +6,7 @@ import { json, testContext, } from "@webstudio-is/postgrest/testing"; -import { - AssetResourceIndexNotFoundError, - rebuildAssetResourceIndex, -} from "./resource-index-rebuild"; +import { rebuildAssetResourceIndex } from "./resource-index-rebuild"; const server = createTestServer(); const document = { @@ -33,9 +30,6 @@ const entry = createCanonicalAssetFileEntry({ describe("explicit resource index rebuild", () => { test("rebuilds the persisted query from canonical metadata", async () => { server.use( - db.get("AssetResourceIndexState", () => - json({ query: `*[properties.slug == $slug]` }) - ), db.get("AssetFileMetadata", () => json([ { @@ -58,51 +52,11 @@ describe("explicit resource index rebuild", () => { store: { putIfAbsent }, projectId: "project-1", resourceId: "posts", + query: `*[properties.slug == $slug]`, }); expect(result.index.parameterNames).toEqual(["slug"]); expect(result.index.documents).toHaveLength(1); expect(putIfAbsent).toHaveBeenCalledOnce(); }); - - test("fails clearly when the resource has no persisted index state", async () => { - server.use( - db.get("AssetResourceIndexState", ({ request }) => { - expect(new URL(request.url).searchParams.get("deletedAt")).toBe( - "is.null" - ); - return json(null); - }) - ); - await expect( - rebuildAssetResourceIndex({ - client: testContext.postgrest.client, - store: { putIfAbsent: vi.fn() }, - projectId: "project-1", - resourceId: "missing", - }) - ).rejects.toBeInstanceOf(AssetResourceIndexNotFoundError); - }); - - test("treats a deleted resource index state as not found", async () => { - const putIfAbsent = vi.fn(); - server.use( - db.get("AssetResourceIndexState", ({ request }) => { - const search = new URL(request.url).searchParams; - return search.get("deletedAt") === "is.null" - ? json(null) - : json({ query: "*[]" }); - }) - ); - - await expect( - rebuildAssetResourceIndex({ - client: testContext.postgrest.client, - store: { putIfAbsent }, - projectId: "project-1", - resourceId: "deleted", - }) - ).rejects.toBeInstanceOf(AssetResourceIndexNotFoundError); - expect(putIfAbsent).not.toHaveBeenCalled(); - }); }); diff --git a/packages/asset-uploader/src/resource-index-rebuild.ts b/packages/asset-uploader/src/resource-index-rebuild.ts index 9ce7999745d4..203512c73ff7 100644 --- a/packages/asset-uploader/src/resource-index-rebuild.ts +++ b/packages/asset-uploader/src/resource-index-rebuild.ts @@ -1,6 +1,5 @@ import type { ImmutableAssetResourceIndexStore } from "@webstudio-is/asset-resource"; import type { Client } from "@webstudio-is/postgrest/index.server"; -import { assertPostgrestSuccess } from "./patch-utils"; import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; import { buildPersistAndActivateAssetResourceIndex } from "./resource-index-build"; @@ -16,30 +15,21 @@ export const rebuildAssetResourceIndex = async ({ store, projectId, resourceId, + query, }: { client: Client; store: ImmutableAssetResourceIndexStore; projectId: string; resourceId: string; + query: string; }) => { - const state = await client - .from("AssetResourceIndexState") - .select("query") - .eq("projectId", projectId) - .eq("resourceId", resourceId) - .is("deletedAt", null) - .maybeSingle(); - assertPostgrestSuccess(state); - if (state.data === null) { - throw new AssetResourceIndexNotFoundError(); - } const entries = await loadCanonicalAssetFileEntries({ client, projectId }); return await buildPersistAndActivateAssetResourceIndex({ client, store, projectId, resourceId, - query: state.data.query, + query, entries, }); }; diff --git a/packages/project-build/src/runtime/asset-resources.ts b/packages/project-build/src/runtime/asset-resources.ts index 384dc65e61b7..00f177a29a53 100644 --- a/packages/project-build/src/runtime/asset-resources.ts +++ b/packages/project-build/src/runtime/asset-resources.ts @@ -4,7 +4,7 @@ import { assetResourceQueryRequest, createAssetQueryResourceBody, decodeDataSourceVariable, - isAssetsResource as isSdkAssetsResource, + isAssetsResource, isStoredAssetQueryResource, parseAssetQueryResourceBody, SYSTEM_VARIABLE_ID, @@ -147,9 +147,6 @@ const parseJsonExpression = (expression: string | undefined) => { } }; -export const isAssetsResource = (resource: Resource) => - isSdkAssetsResource(resource); - export const parseAssetResourceConfiguration = ( resource: Resource ): DecodedAssetResourceConfiguration | undefined => { diff --git a/packages/project-build/src/runtime/cutover.ts b/packages/project-build/src/runtime/cutover.ts index b11f9b8a6710..0271e18df18b 100644 --- a/packages/project-build/src/runtime/cutover.ts +++ b/packages/project-build/src/runtime/cutover.ts @@ -251,6 +251,19 @@ export const builderRuntimeCutoverManifests = [ operationIds: ["variables.list", "resources.list"] as const, callers: ["appRouter.api.variables", "appRouter.api.resources"] as const, }, + { + family: "assets-resource-operations", + operationIds: [ + "assetsResources.list", + "assetsResources.get", + "assetsResources.create", + "assetsResources.update", + ] as const, + callers: [ + "appRouter.api.assetsResources", + "MCP/CLI Assets resources", + ] as const, + }, { family: "resource-mutations", operationIds: [ diff --git a/packages/project/src/db/build-patch.test.ts b/packages/project/src/db/build-patch.test.ts index 535c43f6d6c3..336336f1247c 100644 --- a/packages/project/src/db/build-patch.test.ts +++ b/packages/project/src/db/build-patch.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import { createTestServer, db, @@ -281,6 +281,7 @@ describe("patchBuild", () => { test("applies transactions, validates data, and updates build with optimistic version guard", async () => { let updatedBuild: unknown; + const onCommitted = vi.fn(); server.use( db.get("Build", () => json([buildRow])), db.patch("Build", async ({ request }) => { @@ -296,7 +297,8 @@ describe("patchBuild", () => { clientVersion: 3, transactions: [transaction()], }, - createContext() + createContext(), + onCommitted ); expect(result).toEqual({ status: "ok", version: 4 }); @@ -304,6 +306,13 @@ describe("patchBuild", () => { version: 4, lastTransactionId: "tx-1", }); + expect(onCommitted).toHaveBeenCalledWith( + expect.objectContaining({ + previousBuild: buildRow, + build: expect.objectContaining({ version: 4 }), + changes: transaction().payload, + }) + ); expect(JSON.parse((updatedBuild as { props: string }).props)).toEqual([ { id: "prop-1", diff --git a/packages/project/src/db/build-patch.ts b/packages/project/src/db/build-patch.ts index 37812bad3bf8..bf61ed1a774c 100644 --- a/packages/project/src/db/build-patch.ts +++ b/packages/project/src/db/build-patch.ts @@ -23,6 +23,12 @@ export type PatchBuildResult = | { status: "version_mismatched"; errors: string } | { status: "error"; errors: string }; +export type BuildPatchCommitted = { + previousBuild: Database["public"]["Tables"]["Build"]["Row"]; + build: Database["public"]["Tables"]["Build"]["Row"]; + changes: BuildPatchChange[]; +}; + type PatchLoadedBuildResult = | { status: "ok"; @@ -131,7 +137,8 @@ export const patchBuild = async ( transactions: BuildPatchTransaction[]; clientVersion: number; }, - context: AppContext + context: AppContext, + onCommitted?: (result: BuildPatchCommitted) => Promise ): Promise => { const build = await loadRawBuildById(context, buildId); const result = await patchLoadedBuild( @@ -143,5 +150,13 @@ export const patchBuild = async ( return result; } + if (result.build !== build) { + await onCommitted?.({ + previousBuild: build, + build: result.build, + changes: transactions.flatMap(({ payload }) => payload), + }); + } + return { status: "ok", version: result.version }; }; From 21db1ddcd890f75835196ea307d45844635e82ff Mon Sep 17 00:00:00 2001 From: Oleg Isonen Date: Mon, 20 Jul 2026 22:14:19 +0100 Subject: [PATCH 10/14] fix: complete asset resource lifecycle --- .../settings-panel/asset-query-form-utils.ts | 7 +- .../settings-panel/asset-query-form.tsx | 5 +- .../sync/patch/patch-service.server.test.ts | 82 ++++ ...synchronize-asset-resource-patch.server.ts | 32 +- .../docs/asset-resource-architecture.md | 9 +- .../scripts/benchmark-index.mjs | 2 +- .../scripts/benchmark-scale.mjs | 2 +- .../asset-resource/src/resource-index.test.ts | 20 +- packages/asset-resource/src/resource-index.ts | 9 +- .../src/builder-api-data-boundary.test.ts | 40 +- .../src/builder-canonical-entries.ts | 48 +++ .../asset-uploader/src/field-catalog.test.ts | 43 +- packages/asset-uploader/src/field-catalog.ts | 35 +- .../asset-uploader/src/query-preview.test.ts | 44 +- packages/asset-uploader/src/query-preview.ts | 32 +- .../src/resource-index-build.test.ts | 47 ++- .../src/resource-index-build.ts | 12 +- .../src/resource-index-lifecycle.test.ts | 29 +- .../src/resource-index-lifecycle.ts | 52 ++- .../src/resource-index-maintenance.ts | 7 + .../src/resource-index-persistence.ts | 12 + .../src/resource-index-rebuild.ts | 10 +- .../postgrest/src/__generated__/db-types.ts | 384 ++++++++++++++++-- .../supabase/tests/asset-resource-indexes.sql | 30 +- .../migration.sql | 3 + .../migration.sql | 8 +- .../migration.sql | 2 + .../migration.sql | 4 +- packages/prisma-client/prisma/schema.prisma | 1 + packages/project-build/src/mcp.test.ts | 31 ++ packages/project-build/src/mcp.ts | 26 +- .../src/runtime/asset-resources.ts | 5 +- .../src/runtime/registry.test.ts | 4 + packages/sdk/src/schema/asset-resource.ts | 8 +- 34 files changed, 865 insertions(+), 220 deletions(-) create mode 100644 packages/asset-uploader/src/builder-canonical-entries.ts diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts index a8d3307e5ff1..cd03791c3869 100644 --- a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts @@ -1,4 +1,5 @@ import { + assetResourceParameterName, assetResourceContentOptions, assetResourceLimits, normalizeAssetQueryParameterName, @@ -63,7 +64,11 @@ export const getAssetQueryConfigurationError = ({ const names = parameters.map(({ name }) => normalizeAssetQueryParameterName(name) ); - if (names.some((name) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) === false)) { + if ( + names.some( + (name) => assetResourceParameterName.safeParse(name).success === false + ) + ) { return "Runtime parameter names must be valid identifiers."; } if (new Set(names).size !== names.length) { diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx b/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx index eaa6ffb4b97c..23391841ba41 100644 --- a/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { useStore } from "@nanostores/react"; import { + assetResourceParameterName, assetResourceContentOptions, assetResourceLimits, assetResourceIndexStatus, @@ -484,7 +485,9 @@ export const AssetQueryForm = ({ new Set( parameters .map(({ name }) => name) - .filter((name) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) + .filter( + (name) => assetResourceParameterName.safeParse(name).success + ) ) ), resourceFieldPaths, diff --git a/apps/builder/app/shared/sync/patch/patch-service.server.test.ts b/apps/builder/app/shared/sync/patch/patch-service.server.test.ts index 7cf5317e2edd..30b3f3f754b9 100644 --- a/apps/builder/app/shared/sync/patch/patch-service.server.test.ts +++ b/apps/builder/app/shared/sync/patch/patch-service.server.test.ts @@ -265,6 +265,88 @@ describe("applyPatchRequest", () => { ); }); + test("reparses a swapped asset revision before rebuilding indexes", async () => { + const assetEntry = { + ...patch.entries[0], + transaction: { + id: "tx-asset-revision", + payload: [ + { + namespace: "assets", + patches: [ + { + op: "replace", + path: ["asset-1", "name"], + value: "post-revision.md", + }, + ], + }, + ], + } as never, + }; + authorizePatchEntries.mockResolvedValue({ + authorized: [{ entry: assetEntry, context: { writer: 1 } }], + rejected: [], + }); + patchLoadedBuild.mockImplementation(async ({ build }) => ({ + status: "ok", + version: 4, + build: { ...build, version: 4 }, + })); + + await applyPatchRequest(createContext(), { + ...patch, + entries: [assetEntry], + }); + + expect(synchronizeCanonicalAsset).toHaveBeenCalledWith( + expect.objectContaining({ assetId: "asset-1" }) + ); + expect(synchronizeCanonicalAsset.mock.invocationCallOrder[0]).toBeLessThan( + updateAssetResourceIndexesAfterCanonicalChange.mock.invocationCallOrder[0] + ); + }); + + test("skips index maintenance for asset descriptions", async () => { + const assetEntry = { + ...patch.entries[0], + transaction: { + id: "tx-asset-description", + payload: [ + { + namespace: "assets", + patches: [ + { + op: "replace", + path: ["asset-1", "description"], + value: "Decorative description", + }, + ], + }, + ], + } as never, + }; + authorizePatchEntries.mockResolvedValue({ + authorized: [{ entry: assetEntry, context: { writer: 1 } }], + rejected: [], + }); + patchLoadedBuild.mockImplementation(async ({ build }) => ({ + status: "ok", + version: 4, + build: { ...build, version: 4 }, + })); + + await applyPatchRequest(createContext(), { + ...patch, + entries: [assetEntry], + }); + + expect(synchronizeCanonicalAsset).not.toHaveBeenCalled(); + expect( + updateAssetResourceIndexesAfterCanonicalChange + ).not.toHaveBeenCalled(); + }); + test("returns per-entry partial results while applying authorized entries", async () => { authorizePatchEntries.mockResolvedValue({ authorized: [{ entry: patch.entries[1], context: { writer: 2 } }], diff --git a/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts b/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts index 0746eccf5080..065cb3322884 100644 --- a/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts +++ b/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts @@ -69,18 +69,22 @@ export const synchronizeAssetResourcesAfterBuildPatch = async ({ const directAssetChanges = assetChanges.filter( ({ namespace }) => namespace === "assets" ); - const addedAssetIds = [ + const canonicalAssetIds = [ ...new Set( directAssetChanges.flatMap(({ patches }) => - patches.flatMap(({ op, path }) => - op === "add" && path.length === 1 && typeof path[0] === "string" + patches.flatMap(({ path }) => + typeof path[0] === "string" && + (path.length === 1 || + path[1] === "name" || + path[1] === "filename" || + path[1] === "folderId") ? [path[0]] : [] ) ) ), ]; - for (const assetId of addedAssetIds) { + for (const assetId of canonicalAssetIds) { await synchronizeCanonicalAsset({ client: context.postgrest.client, assetClient, @@ -88,21 +92,19 @@ export const synchronizeAssetResourcesAfterBuildPatch = async ({ assetId, }); } - const changedAssetIds = [ - ...new Set( - directAssetChanges.flatMap(({ patches }) => - patches - .map(({ path }) => path[0]) - .filter((id): id is string => typeof id === "string") - ) - ), - ]; + const hasFolderChanges = assetChanges.some( + ({ namespace }) => namespace === "assetFolders" + ); + if (hasFolderChanges === false && canonicalAssetIds.length === 0) { + return; + } await updateAssetResourceIndexesAfterCanonicalChange({ client: context.postgrest.client, store: assetClient.resourceIndexStore, projectId, - changedAssetIds: - changedAssetIds.length === 0 ? ["project-assets"] : changedAssetIds, + changedAssetIds: hasFolderChanges + ? ["project-assets"] + : canonicalAssetIds, }); } catch (error) { console.error("Asset resource index maintenance failed", error); diff --git a/apps/builder/docs/asset-resource-architecture.md b/apps/builder/docs/asset-resource-architecture.md index 6a896b94ece3..860e201e8692 100644 --- a/apps/builder/docs/asset-resource-architecture.md +++ b/apps/builder/docs/asset-resource-architecture.md @@ -232,6 +232,7 @@ final runtime evaluation: Index construction accepts only persisted `CanonicalAssetFileEntry` values. It derives the asset revision and candidate documents from those entries and +rejects a caller-provided revision that does not match that exact entry set. It has no asset-storage reader, so creating or changing a resource query cannot reopen Markdown files. Content is read only by the separate post-selection hydration path. @@ -251,6 +252,9 @@ An active revision is usable only when all four values agree: Building a replacement never mutates the active revision. Validation and any static evaluation finish first; activation is one atomic reference change. A +unique attempt ID guards activation, failure, and cancellation, so concurrent +builds with the same query and asset revisions cannot complete each other's +state transitions. A failed or superseded build leaves the previous valid revision active but marks the resource stale until a matching revision can be activated. Publication cannot proceed while it is stale. @@ -267,8 +271,9 @@ shares one object accidentally. The active state protects the current revision, and publication holds a temporary `BUILD` reference while it reads an immutable snapshot. Published deployments contain their own static copy and no longer depend on the private object. After activation, query deletion, and snapshot -release, best-effort cleanup claims unreferenced revisions, deletes their private -objects, and then removes their database rows. Garbage claims are 15-minute +release, best-effort cleanup runs once per mutation or publication batch, +claims unreferenced revisions, deletes their private objects, and then removes +their database rows. Garbage claims are 15-minute leases. A later cleanup worker rotates and resumes an expired claim, so a crash before or after object deletion cannot permanently strand the revision row. diff --git a/packages/asset-resource/scripts/benchmark-index.mjs b/packages/asset-resource/scripts/benchmark-index.mjs index 0520e3d6e275..188c077d93a2 100644 --- a/packages/asset-resource/scripts/benchmark-index.mjs +++ b/packages/asset-resource/scripts/benchmark-index.mjs @@ -85,4 +85,4 @@ const result = { verify: await measure(20, () => verifyAssetResourceIndex(parsed)), }; -console.log(JSON.stringify(result, null, 2)); +console.info(JSON.stringify(result, null, 2)); diff --git a/packages/asset-resource/scripts/benchmark-scale.mjs b/packages/asset-resource/scripts/benchmark-scale.mjs index 5c9292213ba3..c0faa2a48e9c 100644 --- a/packages/asset-resource/scripts/benchmark-scale.mjs +++ b/packages/asset-resource/scripts/benchmark-scale.mjs @@ -207,7 +207,7 @@ const retainedCopies = Array.from({ length: 10 }, () => JSON.parse(serialized)); const memoryAfter = process.memoryUsage().heapUsed; void retainedCopies; -console.log( +console.info( JSON.stringify( { environment: { node: process.version, platform: process.platform }, diff --git a/packages/asset-resource/src/resource-index.test.ts b/packages/asset-resource/src/resource-index.test.ts index 8bb0ba176bb8..a1bc1195585f 100644 --- a/packages/asset-resource/src/resource-index.test.ts +++ b/packages/asset-resource/src/resource-index.test.ts @@ -76,12 +76,12 @@ describe("resource index build", () => { await expect(verifyAssetResourceIndex(index)).resolves.toEqual(index); }); - test("reuses a canonical revision prepared for a batch", async () => { + test("accepts a canonical revision prepared from the same entries", async () => { const entry = createCanonicalAssetFileEntry({ projectId: "project-1", document: createDocument("post", {}), }); - const assetRevision = `sha256:${"a".repeat(64)}`; + const assetRevision = await computeCanonicalAssetRevision([entry]); const index = await buildAssetResourceIndex({ projectId: "project-1", @@ -94,6 +94,22 @@ describe("resource index build", () => { expect(index.assetRevision).toBe(assetRevision); }); + test("rejects an asset revision unrelated to the prepared entries", async () => { + const entry = createCanonicalAssetFileEntry({ + projectId: "project-1", + document: createDocument("post", {}), + }); + await expect( + buildAssetResourceIndex({ + projectId: "project-1", + resourceId: "posts", + query: "*[]", + entries: [entry], + assetRevision: `sha256:${"0".repeat(64)}`, + }) + ).rejects.toThrow("do not match the supplied asset revision"); + }); + test("treats all frontmatter fields as schemaless query data", async () => { const entries = [ createCanonicalAssetFileEntry({ diff --git a/packages/asset-resource/src/resource-index.ts b/packages/asset-resource/src/resource-index.ts index 42e1c5eed9f2..45525727eb2c 100644 --- a/packages/asset-resource/src/resource-index.ts +++ b/packages/asset-resource/src/resource-index.ts @@ -104,6 +104,12 @@ export const buildAssetResourceIndex = async ({ assetRevision?: string; }) => { const validatedQuery = validateAssetResourceQuery(query); + const preparedAssetRevision = await computeCanonicalAssetRevision(entries); + if (assetRevision !== undefined && assetRevision !== preparedAssetRevision) { + throw new Error( + "Prepared canonical entries do not match the supplied asset revision" + ); + } const assetIds = new Set(); for (const entry of entries) { if (entry.projectId !== projectId) { @@ -130,8 +136,7 @@ export const buildAssetResourceIndex = async ({ version: 1, resourceId, query, - assetRevision: - assetRevision ?? (await computeCanonicalAssetRevision(entries)), + assetRevision: preparedAssetRevision, queryMode: selection.queryMode, parameterNames: selection.parameterNames, documents: selection.documents, diff --git a/packages/asset-uploader/src/builder-api-data-boundary.test.ts b/packages/asset-uploader/src/builder-api-data-boundary.test.ts index 0d11a597e97e..36ab90b2b1ca 100644 --- a/packages/asset-uploader/src/builder-api-data-boundary.test.ts +++ b/packages/asset-uploader/src/builder-api-data-boundary.test.ts @@ -1,39 +1,36 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; -import { - authorizeProject, - type AppContext, -} from "@webstudio-is/trpc-interface/index.server"; -import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; +import { type AppContext } from "@webstudio-is/trpc-interface/index.server"; import { loadBuilderAssetFieldCatalog } from "./field-catalog"; import { previewAssetResourceQuery } from "./query-preview"; -import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; - -vi.mock("@webstudio-is/trpc-interface/index.server", () => ({ - authorizeProject: { hasProjectPermit: vi.fn() }, - AuthorizationError: class AuthorizationError extends Error {}, -})); -vi.mock("./canonical-metadata-persistence", () => ({ - loadCanonicalAssetFileEntries: vi.fn(), -})); -vi.mock("./canonical-metadata-backfill", () => ({ - synchronizeCanonicalAssets: vi.fn(), -})); const projectId = "project-1"; const context = { postgrest: { client: {} }, } as unknown as AppContext; +const hasProjectPermit = vi.fn(); +const synchronizeCanonicalAssets = vi.fn(); +const loadCanonicalAssetFileEntries = vi.fn(); +const dependencies = { + hasProjectPermit, + synchronizeCanonicalAssets, + loadCanonicalAssetFileEntries, +}; describe("Builder asset-resource API data boundary", () => { beforeEach(() => { - vi.clearAllMocks(); - vi.mocked(authorizeProject.hasProjectPermit).mockResolvedValue(true); - vi.mocked(loadCanonicalAssetFileEntries).mockResolvedValue([]); + hasProjectPermit.mockReset().mockResolvedValue(true); + synchronizeCanonicalAssets.mockReset(); + loadCanonicalAssetFileEntries.mockReset().mockResolvedValue([]); }); test("catalog and preview synchronize before reading persisted rows", async () => { const assetClient = { readFile: vi.fn() }; - await loadBuilderAssetFieldCatalog({ projectId, context, assetClient }); + await loadBuilderAssetFieldCatalog({ + projectId, + context, + assetClient, + dependencies, + }); await previewAssetResourceQuery({ projectId, request: { @@ -44,6 +41,7 @@ describe("Builder asset-resource API data boundary", () => { }, context, assetClient, + dependencies, }); expect(synchronizeCanonicalAssets).toHaveBeenCalledTimes(2); diff --git a/packages/asset-uploader/src/builder-canonical-entries.ts b/packages/asset-uploader/src/builder-canonical-entries.ts new file mode 100644 index 000000000000..b6e14505f193 --- /dev/null +++ b/packages/asset-uploader/src/builder-canonical-entries.ts @@ -0,0 +1,48 @@ +import { + authorizeProject, + AuthorizationError, + type AppContext, +} from "@webstudio-is/trpc-interface/index.server"; +import type { AssetClient } from "./client"; +import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; +import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; + +export type BuilderCanonicalEntriesDependencies = { + hasProjectPermit?: typeof authorizeProject.hasProjectPermit; + synchronizeCanonicalAssets?: typeof synchronizeCanonicalAssets; + loadCanonicalAssetFileEntries?: typeof loadCanonicalAssetFileEntries; +}; + +export const loadAuthorizedBuilderCanonicalEntries = async ({ + projectId, + context, + assetClient, + authorizationError, + dependencies = {}, +}: { + projectId: string; + context: AppContext; + assetClient: Pick; + authorizationError: string; + dependencies?: BuilderCanonicalEntriesDependencies; +}) => { + const canView = await ( + dependencies.hasProjectPermit ?? authorizeProject.hasProjectPermit + )({ projectId, permit: "view" }, context); + if (canView === false) { + throw new AuthorizationError(authorizationError); + } + await (dependencies.synchronizeCanonicalAssets ?? synchronizeCanonicalAssets)( + { + projectId, + client: context.postgrest.client, + assetClient, + } + ); + return await ( + dependencies.loadCanonicalAssetFileEntries ?? loadCanonicalAssetFileEntries + )({ + client: context.postgrest.client, + projectId, + }); +}; diff --git a/packages/asset-uploader/src/field-catalog.test.ts b/packages/asset-uploader/src/field-catalog.test.ts index c543037ce523..a8b0b17a2218 100644 --- a/packages/asset-uploader/src/field-catalog.test.ts +++ b/packages/asset-uploader/src/field-catalog.test.ts @@ -1,23 +1,9 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; import { - authorizeProject, AuthorizationError, type AppContext, } from "@webstudio-is/trpc-interface/index.server"; -import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; import { loadBuilderAssetFieldCatalog } from "./field-catalog"; -import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; - -vi.mock("@webstudio-is/trpc-interface/index.server", () => ({ - authorizeProject: { hasProjectPermit: vi.fn() }, - AuthorizationError: class AuthorizationError extends Error {}, -})); -vi.mock("./canonical-metadata-persistence", () => ({ - loadCanonicalAssetFileEntries: vi.fn(), -})); -vi.mock("./canonical-metadata-backfill", () => ({ - synchronizeCanonicalAssets: vi.fn(), -})); const projectId = "project-1"; const postgrestClient = { from: vi.fn() } as never; @@ -25,16 +11,25 @@ const context = { postgrest: { client: postgrestClient }, } as unknown as AppContext; const assetClient = { readFile: vi.fn() }; +const hasProjectPermit = vi.fn(); +const synchronizeCanonicalAssets = vi.fn(); +const loadCanonicalAssetFileEntries = vi.fn(); +const dependencies = { + hasProjectPermit, + synchronizeCanonicalAssets, + loadCanonicalAssetFileEntries, +}; describe("loadBuilderAssetFieldCatalog", () => { beforeEach(() => { - vi.mocked(authorizeProject.hasProjectPermit).mockReset(); - vi.mocked(loadCanonicalAssetFileEntries).mockReset(); + hasProjectPermit.mockReset(); + loadCanonicalAssetFileEntries.mockReset(); + synchronizeCanonicalAssets.mockReset(); }); test("authorizes view access and derives fields from persisted metadata", async () => { - vi.mocked(authorizeProject.hasProjectPermit).mockResolvedValue(true); - vi.mocked(loadCanonicalAssetFileEntries).mockResolvedValue([ + hasProjectPermit.mockResolvedValue(true); + loadCanonicalAssetFileEntries.mockResolvedValue([ { projectId, assetId: "post-1", @@ -60,9 +55,10 @@ describe("loadBuilderAssetFieldCatalog", () => { projectId, context, assetClient, + dependencies, }); - expect(authorizeProject.hasProjectPermit).toHaveBeenCalledWith( + expect(hasProjectPermit).toHaveBeenCalledWith( { projectId, permit: "view" }, context ); @@ -78,10 +74,15 @@ describe("loadBuilderAssetFieldCatalog", () => { }); test("does not load metadata without view access", async () => { - vi.mocked(authorizeProject.hasProjectPermit).mockResolvedValue(false); + hasProjectPermit.mockResolvedValue(false); await expect( - loadBuilderAssetFieldCatalog({ projectId, context, assetClient }) + loadBuilderAssetFieldCatalog({ + projectId, + context, + assetClient, + dependencies, + }) ).rejects.toBeInstanceOf(AuthorizationError); expect(loadCanonicalAssetFileEntries).not.toHaveBeenCalled(); }); diff --git a/packages/asset-uploader/src/field-catalog.ts b/packages/asset-uploader/src/field-catalog.ts index 8c0c820c1695..f6997c10cb94 100644 --- a/packages/asset-uploader/src/field-catalog.ts +++ b/packages/asset-uploader/src/field-catalog.ts @@ -2,42 +2,31 @@ import { createAssetFieldCatalog, toBuilderAssetFieldCatalog, } from "@webstudio-is/asset-resource"; -import { - authorizeProject, - AuthorizationError, - type AppContext, -} from "@webstudio-is/trpc-interface/index.server"; -import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; +import type { AppContext } from "@webstudio-is/trpc-interface/index.server"; import type { AssetClient } from "./client"; -import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; +import { + loadAuthorizedBuilderCanonicalEntries, + type BuilderCanonicalEntriesDependencies, +} from "./builder-canonical-entries"; export const loadBuilderAssetFieldCatalog = async ({ projectId, context, assetClient, + dependencies = {}, }: { projectId: string; context: AppContext; assetClient: Pick; + dependencies?: BuilderCanonicalEntriesDependencies; }) => { - const canView = await authorizeProject.hasProjectPermit( - { projectId, permit: "view" }, - context - ); - if (canView === false) { - throw new AuthorizationError( - "You don't have access to this project's asset field catalog" - ); - } - - await synchronizeCanonicalAssets({ + const entries = await loadAuthorizedBuilderCanonicalEntries({ projectId, - client: context.postgrest.client, + context, assetClient, - }); - const entries = await loadCanonicalAssetFileEntries({ - client: context.postgrest.client, - projectId, + authorizationError: + "You don't have access to this project's asset field catalog", + dependencies, }); const catalog = await createAssetFieldCatalog(entries); return toBuilderAssetFieldCatalog(catalog); diff --git a/packages/asset-uploader/src/query-preview.test.ts b/packages/asset-uploader/src/query-preview.test.ts index 4234df2240c5..e6589ba983a1 100644 --- a/packages/asset-uploader/src/query-preview.test.ts +++ b/packages/asset-uploader/src/query-preview.test.ts @@ -1,39 +1,35 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; import { - authorizeProject, AuthorizationError, type AppContext, } from "@webstudio-is/trpc-interface/index.server"; -import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; import { previewAssetResourceQuery } from "./query-preview"; -vi.mock("@webstudio-is/trpc-interface/index.server", () => ({ - authorizeProject: { hasProjectPermit: vi.fn() }, - AuthorizationError: class AuthorizationError extends Error {}, -})); -vi.mock("./canonical-metadata-persistence", () => ({ - loadCanonicalAssetFileEntries: vi.fn(), -})); -vi.mock("./canonical-metadata-backfill", () => ({ - synchronizeCanonicalAssets: vi.fn(), -})); - const projectId = "project-1"; const postgrestClient = { from: vi.fn() } as never; const context = { postgrest: { client: postgrestClient }, } as unknown as AppContext; const revision = `sha256:${"b".repeat(64)}`; +const hasProjectPermit = vi.fn(); +const synchronizeCanonicalAssets = vi.fn(); +const loadCanonicalAssetFileEntries = vi.fn(); +const dependencies = { + hasProjectPermit, + synchronizeCanonicalAssets, + loadCanonicalAssetFileEntries, +}; describe("previewAssetResourceQuery", () => { beforeEach(() => { - vi.mocked(authorizeProject.hasProjectPermit).mockReset(); - vi.mocked(loadCanonicalAssetFileEntries).mockReset(); + hasProjectPermit.mockReset(); + synchronizeCanonicalAssets.mockReset(); + loadCanonicalAssetFileEntries.mockReset(); }); test("evaluates persisted canonical metadata without reading files", async () => { - vi.mocked(authorizeProject.hasProjectPermit).mockResolvedValue(true); - vi.mocked(loadCanonicalAssetFileEntries).mockResolvedValue([ + hasProjectPermit.mockResolvedValue(true); + loadCanonicalAssetFileEntries.mockResolvedValue([ { projectId, assetId: "post-1", @@ -66,6 +62,7 @@ describe("previewAssetResourceQuery", () => { }, context, assetClient: { readFile }, + dependencies, }); expect(result.result).toEqual([{ _id: "post-1" }]); @@ -78,7 +75,7 @@ describe("previewAssetResourceQuery", () => { }); test("does not load metadata without view access", async () => { - vi.mocked(authorizeProject.hasProjectPermit).mockResolvedValue(false); + hasProjectPermit.mockResolvedValue(false); await expect( previewAssetResourceQuery({ projectId, @@ -90,14 +87,15 @@ describe("previewAssetResourceQuery", () => { }, context, assetClient: { readFile: vi.fn() }, + dependencies, }) ).rejects.toBeInstanceOf(AuthorizationError); expect(loadCanonicalAssetFileEntries).not.toHaveBeenCalled(); }); test("hydrates only identities retained by the query result", async () => { - vi.mocked(authorizeProject.hasProjectPermit).mockResolvedValue(true); - vi.mocked(loadCanonicalAssetFileEntries).mockResolvedValue([ + hasProjectPermit.mockResolvedValue(true); + loadCanonicalAssetFileEntries.mockResolvedValue([ { projectId, assetId: "post-1", @@ -137,6 +135,7 @@ describe("previewAssetResourceQuery", () => { }, context, assetClient: { readFile }, + dependencies, }); expect(result.content["post-1"]?.text).toBe("Post"); @@ -148,7 +147,7 @@ describe("previewAssetResourceQuery", () => { }); test("hydrates exactly one complete file selected by a slug detail query", async () => { - vi.mocked(authorizeProject.hasProjectPermit).mockResolvedValue(true); + hasProjectPermit.mockResolvedValue(true); const entries = ["first", "selected"].map((slug) => ({ projectId, assetId: `post-${slug}`, @@ -170,7 +169,7 @@ describe("previewAssetResourceQuery", () => { { path: "properties.slug", type: "string" as const }, ], })); - vi.mocked(loadCanonicalAssetFileEntries).mockResolvedValue(entries); + loadCanonicalAssetFileEntries.mockResolvedValue(entries); const readFile = vi.fn(async (contentRef: string) => ({ data: { async *[Symbol.asyncIterator]() { @@ -192,6 +191,7 @@ describe("previewAssetResourceQuery", () => { }, context, assetClient: { readFile }, + dependencies, }); expect(result.content).toEqual({ diff --git a/packages/asset-uploader/src/query-preview.ts b/packages/asset-uploader/src/query-preview.ts index 0c4e9f5d5aa7..3c2f1bcf6842 100644 --- a/packages/asset-uploader/src/query-preview.ts +++ b/packages/asset-uploader/src/query-preview.ts @@ -5,42 +5,32 @@ import { } from "@webstudio-is/asset-resource"; import type { AssetResourceQueryRequest } from "@webstudio-is/sdk"; import type { AssetClient } from "./client"; +import type { AppContext } from "@webstudio-is/trpc-interface/index.server"; import { - authorizeProject, - AuthorizationError, - type AppContext, -} from "@webstudio-is/trpc-interface/index.server"; -import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; -import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; + loadAuthorizedBuilderCanonicalEntries, + type BuilderCanonicalEntriesDependencies, +} from "./builder-canonical-entries"; export const previewAssetResourceQuery = async ({ projectId, request, context, assetClient, + dependencies = {}, }: { projectId: string; request: AssetResourceQueryRequest; context: AppContext; assetClient: Pick; + dependencies?: BuilderCanonicalEntriesDependencies; }) => { - const canView = await authorizeProject.hasProjectPermit( - { projectId, permit: "view" }, - context - ); - if (canView === false) { - throw new AuthorizationError( - "You don't have access to preview this project's asset resources" - ); - } - await synchronizeCanonicalAssets({ + const entries = await loadAuthorizedBuilderCanonicalEntries({ projectId, - client: context.postgrest.client, + context, assetClient, - }); - const entries = await loadCanonicalAssetFileEntries({ - client: context.postgrest.client, - projectId, + authorizationError: + "You don't have access to preview this project's asset resources", + dependencies, }); const assetRevision = await computeCanonicalAssetRevision(entries); return await executeAndHydrateAssetResourceQuery({ diff --git a/packages/asset-uploader/src/resource-index-build.test.ts b/packages/asset-uploader/src/resource-index-build.test.ts index 10fb94069c96..4816f6b86c5f 100644 --- a/packages/asset-uploader/src/resource-index-build.test.ts +++ b/packages/asset-uploader/src/resource-index-build.test.ts @@ -57,10 +57,6 @@ describe("resource index build and activation", () => { ), }); return json(true); - }), - db.post("rpc/claim_asset_resource_index_garbage", () => { - events.push("cleanup"); - return json([]); }) ); const result = await buildPersistAndActivateAssetResourceIndex({ @@ -78,7 +74,7 @@ describe("resource index build and activation", () => { entries: [entry], }); - expect(events).toEqual(["begin", "persist", "activate", "cleanup"]); + expect(events).toEqual(["begin", "persist", "activate"]); expect(result.index.documents).toHaveLength(1); }); @@ -105,6 +101,47 @@ describe("resource index build and activation", () => { ).rejects.toBeInstanceOf(AssetResourceIndexBuildSupersededError); }); + test("uses a unique identity for each otherwise identical build attempt", async () => { + const begun: string[] = []; + const activated: string[] = []; + server.use( + db.post("rpc/begin_asset_resource_index_build", async ({ request }) => { + const body = (await request.json()) as { + p_build_attempt_id: string; + }; + begun.push(body.p_build_attempt_id); + return json(null); + }), + db.post("rpc/activate_asset_resource_index", async ({ request }) => { + const body = (await request.json()) as { + p_build_attempt_id: string; + }; + activated.push(body.p_build_attempt_id); + return json(true); + }) + ); + const input = { + client: testContext.postgrest.client, + store: { + putIfAbsent: async ({ checksum }: { checksum: string }) => ({ + status: "created" as const, + checksum, + }), + }, + projectId: "project-1", + resourceId: "posts", + query: "*[]", + entries: [entry], + }; + + await buildPersistAndActivateAssetResourceIndex(input); + await buildPersistAndActivateAssetResourceIndex(input); + + expect(begun).toHaveLength(2); + expect(new Set(begun).size).toBe(2); + expect(activated).toEqual(begun); + }); + test("marks a current failed attempt without trying to activate it", async () => { const activate = vi.fn(); server.use( diff --git a/packages/asset-uploader/src/resource-index-build.ts b/packages/asset-uploader/src/resource-index-build.ts index 1402b502dc92..6f90fc33606f 100644 --- a/packages/asset-uploader/src/resource-index-build.ts +++ b/packages/asset-uploader/src/resource-index-build.ts @@ -11,7 +11,6 @@ import { cancelAssetResourceIndexBuild, failAssetResourceIndexBuild, } from "./resource-index-persistence"; -import { collectAssetResourceIndexGarbageBestEffort } from "./resource-index-garbage-collection"; export class AssetResourceIndexBuildSupersededError extends Error { constructor() { @@ -52,6 +51,7 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ assetRevision?: string; signal?: AbortSignal; }) => { + const buildAttemptId = crypto.randomUUID(); assertNotCancelled(signal); const index = await buildAssetResourceIndex({ projectId, @@ -68,6 +68,7 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ query, queryHash: index.queryHash, assetRevision: index.assetRevision, + buildAttemptId, }); try { @@ -84,18 +85,13 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ revision: persisted.revision, queryHash: index.queryHash, assetRevision: index.assetRevision, + buildAttemptId, checksum: index.integrity.checksum, objectKey: persisted.key, }); if (activated === false) { throw new AssetResourceIndexBuildSupersededError(); } - if (store.delete !== undefined) { - await collectAssetResourceIndexGarbageBestEffort({ - client, - store: { delete: store.delete }, - }); - } return { index, persisted }; } catch (error) { if (error instanceof AssetResourceIndexBuildCancelledError) { @@ -105,6 +101,7 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ resourceId, queryHash: index.queryHash, assetRevision: index.assetRevision, + buildAttemptId, }); throw error; } @@ -115,6 +112,7 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ resourceId, queryHash: index.queryHash, assetRevision: index.assetRevision, + buildAttemptId, }); } catch (stateError) { throw new AggregateError( diff --git a/packages/asset-uploader/src/resource-index-lifecycle.test.ts b/packages/asset-uploader/src/resource-index-lifecycle.test.ts index 307150bcbc5a..92111f662d5a 100644 --- a/packages/asset-uploader/src/resource-index-lifecycle.test.ts +++ b/packages/asset-uploader/src/resource-index-lifecycle.test.ts @@ -5,19 +5,7 @@ const loadCanonicalAssetFileEntries = vi.hoisted(() => vi.fn()); const synchronizeCanonicalAssets = vi.hoisted(() => vi.fn()); const buildPersistAndActivateAssetResourceIndex = vi.hoisted(() => vi.fn()); const deleteAssetResourceIndexQuery = vi.hoisted(() => vi.fn()); - -vi.mock("./canonical-metadata-persistence", () => ({ - loadCanonicalAssetFileEntries, -})); -vi.mock("./canonical-metadata-backfill", () => ({ - synchronizeCanonicalAssets, -})); -vi.mock("./resource-index-build", () => ({ - buildPersistAndActivateAssetResourceIndex, -})); -vi.mock("./resource-index-persistence", () => ({ - deleteAssetResourceIndexQuery, -})); +const collectAssetResourceIndexGarbageBestEffort = vi.hoisted(() => vi.fn()); import { getAssetResourceQuery, @@ -40,6 +28,13 @@ const resource = (id: string, query: string): Resource => ({ ]) ), }); +const dependencies = { + loadCanonicalAssetFileEntries, + synchronizeCanonicalAssets, + buildPersistAndActivateAssetResourceIndex, + deleteAssetResourceIndexQuery, + collectAssetResourceIndexGarbageBestEffort, +}; describe("asset resource query lifecycle", () => { beforeEach(() => { @@ -47,6 +42,7 @@ describe("asset resource query lifecycle", () => { synchronizeCanonicalAssets.mockReset().mockResolvedValue({}); buildPersistAndActivateAssetResourceIndex.mockReset().mockResolvedValue({}); deleteAssetResourceIndexQuery.mockReset().mockResolvedValue(true); + collectAssetResourceIndexGarbageBestEffort.mockReset(); }); test("extracts only fixed queryable-asset GROQ definitions", () => { @@ -65,10 +61,13 @@ describe("asset resource query lifecycle", () => { await expect( synchronizeAssetResourceIndexQueries({ client: {} as never, - assetClient: { resourceIndexStore: {} } as never, + assetClient: { + resourceIndexStore: { delete: vi.fn() }, + } as never, projectId: "project-1", previousResources: previous, resources: current, + dependencies, }) ).resolves.toEqual({ deletedResourceIds: [], @@ -77,6 +76,7 @@ describe("asset resource query lifecycle", () => { expect(loadCanonicalAssetFileEntries).toHaveBeenCalledOnce(); expect(synchronizeCanonicalAssets).toHaveBeenCalledOnce(); expect(buildPersistAndActivateAssetResourceIndex).toHaveBeenCalledTimes(2); + expect(collectAssetResourceIndexGarbageBestEffort).toHaveBeenCalledOnce(); }); test("deletes index ownership when a query is deleted or changed away", async () => { @@ -94,6 +94,7 @@ describe("asset resource query lifecycle", () => { resource("changed-away", "*[]"), ], resources: [external], + dependencies, }) ).resolves.toEqual({ deletedResourceIds: ["changed-away", "deleted"], diff --git a/packages/asset-uploader/src/resource-index-lifecycle.ts b/packages/asset-uploader/src/resource-index-lifecycle.ts index 303ddde30c4c..fe737d3ff303 100644 --- a/packages/asset-uploader/src/resource-index-lifecycle.ts +++ b/packages/asset-uploader/src/resource-index-lifecycle.ts @@ -15,13 +15,33 @@ export const synchronizeAssetResourceIndexQueries = async ({ projectId, previousResources, resources, + dependencies = {}, }: { client: Client; assetClient: AssetClientWithResourceIndexStore; projectId: string; previousResources: readonly Resource[]; resources: readonly Resource[]; + dependencies?: { + synchronizeCanonicalAssets?: typeof synchronizeCanonicalAssets; + loadCanonicalAssetFileEntries?: typeof loadCanonicalAssetFileEntries; + buildPersistAndActivateAssetResourceIndex?: typeof buildPersistAndActivateAssetResourceIndex; + deleteAssetResourceIndexQuery?: typeof deleteAssetResourceIndexQuery; + collectAssetResourceIndexGarbageBestEffort?: typeof collectAssetResourceIndexGarbageBestEffort; + }; }) => { + const synchronize = + dependencies.synchronizeCanonicalAssets ?? synchronizeCanonicalAssets; + const loadEntries = + dependencies.loadCanonicalAssetFileEntries ?? loadCanonicalAssetFileEntries; + const buildIndex = + dependencies.buildPersistAndActivateAssetResourceIndex ?? + buildPersistAndActivateAssetResourceIndex; + const deleteQuery = + dependencies.deleteAssetResourceIndexQuery ?? deleteAssetResourceIndexQuery; + const collectGarbage = + dependencies.collectAssetResourceIndexGarbageBestEffort ?? + collectAssetResourceIndexGarbageBestEffort; const previous = new Map( previousResources.map((resource) => [ resource.id, @@ -47,26 +67,26 @@ export const synchronizeAssetResourceIndexQueries = async ({ .sort((left, right) => left.resourceId.localeCompare(right.resourceId)); for (const resourceId of deletedResourceIds) { - await deleteAssetResourceIndexQuery({ client, projectId, resourceId }); - } - if ( - deletedResourceIds.length > 0 && - assetClient.resourceIndexStore.delete !== undefined - ) { - await collectAssetResourceIndexGarbageBestEffort({ - client, - store: { delete: assetClient.resourceIndexStore.delete }, - }); + await deleteQuery({ client, projectId, resourceId }); } if (changed.length === 0) { + if ( + deletedResourceIds.length > 0 && + assetClient.resourceIndexStore.delete !== undefined + ) { + await collectGarbage({ + client, + store: { delete: assetClient.resourceIndexStore.delete }, + }); + } return { deletedResourceIds, updatedResourceIds: [] }; } - await synchronizeCanonicalAssets({ projectId, client, assetClient }); - const entries = await loadCanonicalAssetFileEntries({ client, projectId }); + await synchronize({ projectId, client, assetClient }); + const entries = await loadEntries({ client, projectId }); const assetRevision = await computeCanonicalAssetRevision(entries); const updatedResourceIds: string[] = []; for (const { resourceId, query } of changed) { - await buildPersistAndActivateAssetResourceIndex({ + await buildIndex({ client, store: assetClient.resourceIndexStore, projectId, @@ -77,5 +97,11 @@ export const synchronizeAssetResourceIndexQueries = async ({ }); updatedResourceIds.push(resourceId); } + if (assetClient.resourceIndexStore.delete !== undefined) { + await collectGarbage({ + client, + store: { delete: assetClient.resourceIndexStore.delete }, + }); + } return { deletedResourceIds, updatedResourceIds }; }; diff --git a/packages/asset-uploader/src/resource-index-maintenance.ts b/packages/asset-uploader/src/resource-index-maintenance.ts index b79a2cc52239..c1861a846712 100644 --- a/packages/asset-uploader/src/resource-index-maintenance.ts +++ b/packages/asset-uploader/src/resource-index-maintenance.ts @@ -17,6 +17,7 @@ import { loadAssetResourceIndexSnapshots, type AssetResourceIndexSnapshot, } from "./resource-index-snapshot"; +import { collectAssetResourceIndexGarbageBestEffort } from "./resource-index-garbage-collection"; export const synchronizeAssetResourceStateAfterAssetChange = async ({ client, @@ -95,6 +96,12 @@ export const updateAssetResourceIndexesAfterCanonicalChange = async ({ errors.push(error); } } + if (store.delete !== undefined) { + await collectAssetResourceIndexGarbageBestEffort({ + client, + store: { delete: store.delete }, + }); + } if (errors.length > 0) { throw new AggregateError( errors, diff --git a/packages/asset-uploader/src/resource-index-persistence.ts b/packages/asset-uploader/src/resource-index-persistence.ts index 4dbfc81363ed..bd846e62e410 100644 --- a/packages/asset-uploader/src/resource-index-persistence.ts +++ b/packages/asset-uploader/src/resource-index-persistence.ts @@ -8,6 +8,7 @@ export const beginAssetResourceIndexBuild = async ({ query, queryHash, assetRevision, + buildAttemptId, }: { client: Client; projectId: string; @@ -15,6 +16,7 @@ export const beginAssetResourceIndexBuild = async ({ query: string; queryHash: string; assetRevision: string; + buildAttemptId: string; }) => { const result = await client.rpc("begin_asset_resource_index_build", { p_project_id: projectId, @@ -22,6 +24,7 @@ export const beginAssetResourceIndexBuild = async ({ p_query: query, p_query_hash: queryHash, p_asset_revision: assetRevision, + p_build_attempt_id: buildAttemptId, }); assertPostgrestSuccess(result); }; @@ -33,6 +36,7 @@ export const activateAssetResourceIndex = async ({ revision, queryHash, assetRevision, + buildAttemptId, checksum, objectKey, }: { @@ -42,6 +46,7 @@ export const activateAssetResourceIndex = async ({ revision: string; queryHash: string; assetRevision: string; + buildAttemptId: string; checksum: string; objectKey: string; }) => { @@ -51,6 +56,7 @@ export const activateAssetResourceIndex = async ({ p_revision: revision, p_query_hash: queryHash, p_asset_revision: assetRevision, + p_build_attempt_id: buildAttemptId, p_checksum: checksum, p_object_key: objectKey, }); @@ -64,18 +70,21 @@ export const failAssetResourceIndexBuild = async ({ resourceId, queryHash, assetRevision, + buildAttemptId, }: { client: Client; projectId: string; resourceId: string; queryHash: string; assetRevision: string; + buildAttemptId: string; }) => { const result = await client.rpc("fail_asset_resource_index_build", { p_project_id: projectId, p_resource_id: resourceId, p_query_hash: queryHash, p_asset_revision: assetRevision, + p_build_attempt_id: buildAttemptId, p_build_error: { code: "INDEX_BUILD_FAILED", message: "Resource index build failed", @@ -91,18 +100,21 @@ export const cancelAssetResourceIndexBuild = async ({ resourceId, queryHash, assetRevision, + buildAttemptId, }: { client: Client; projectId: string; resourceId: string; queryHash: string; assetRevision: string; + buildAttemptId: string; }) => { const result = await client.rpc("cancel_asset_resource_index_build", { p_project_id: projectId, p_resource_id: resourceId, p_query_hash: queryHash, p_asset_revision: assetRevision, + p_build_attempt_id: buildAttemptId, }); assertPostgrestSuccess(result); return result.data === true; diff --git a/packages/asset-uploader/src/resource-index-rebuild.ts b/packages/asset-uploader/src/resource-index-rebuild.ts index 203512c73ff7..4ca746f3a075 100644 --- a/packages/asset-uploader/src/resource-index-rebuild.ts +++ b/packages/asset-uploader/src/resource-index-rebuild.ts @@ -2,6 +2,7 @@ import type { ImmutableAssetResourceIndexStore } from "@webstudio-is/asset-resou import type { Client } from "@webstudio-is/postgrest/index.server"; import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; import { buildPersistAndActivateAssetResourceIndex } from "./resource-index-build"; +import { collectAssetResourceIndexGarbageBestEffort } from "./resource-index-garbage-collection"; export class AssetResourceIndexNotFoundError extends Error { constructor() { @@ -24,7 +25,7 @@ export const rebuildAssetResourceIndex = async ({ query: string; }) => { const entries = await loadCanonicalAssetFileEntries({ client, projectId }); - return await buildPersistAndActivateAssetResourceIndex({ + const result = await buildPersistAndActivateAssetResourceIndex({ client, store, projectId, @@ -32,4 +33,11 @@ export const rebuildAssetResourceIndex = async ({ query, entries, }); + if (store.delete !== undefined) { + await collectAssetResourceIndexGarbageBestEffort({ + client, + store: { delete: store.delete }, + }); + } + return result; }; diff --git a/packages/postgrest/src/__generated__/db-types.ts b/packages/postgrest/src/__generated__/db-types.ts index 04bf30c0e9e7..88d90f9abd70 100644 --- a/packages/postgrest/src/__generated__/db-types.ts +++ b/packages/postgrest/src/__generated__/db-types.ts @@ -7,31 +7,6 @@ export type Json = | Json[]; export type Database = { - graphql_public: { - Tables: { - [_ in never]: never; - }; - Views: { - [_ in never]: never; - }; - Functions: { - graphql: { - Args: { - extensions?: Json; - operationName?: string; - query?: string; - variables?: Json; - }; - Returns: Json; - }; - }; - Enums: { - [_ in never]: never; - }; - CompositeTypes: { - [_ in never]: never; - }; - }; public: { Tables: { _prisma_migrations: { @@ -109,6 +84,44 @@ export type Database = { }, ]; }; + AssetFileMetadata: { + Row: { + assetId: string; + createdAt: string; + document: Json; + fieldContributions: Json; + projectId: string; + revision: string; + updatedAt: string; + }; + Insert: { + assetId: string; + createdAt?: string; + document: Json; + fieldContributions?: Json; + projectId: string; + revision: string; + updatedAt?: string; + }; + Update: { + assetId?: string; + createdAt?: string; + document?: Json; + fieldContributions?: Json; + projectId?: string; + revision?: string; + updatedAt?: string; + }; + Relationships: [ + { + foreignKeyName: "AssetFileMetadata_assetId_projectId_fkey"; + columns: ["assetId", "projectId"]; + isOneToOne: false; + referencedRelation: "Asset"; + referencedColumns: ["id", "projectId"]; + }, + ]; + }; AssetFolder: { Row: { createdAt: string; @@ -155,6 +168,172 @@ export type Database = { }, ]; }; + AssetResourceIndexReference: { + Row: { + createdAt: string; + projectId: string; + referenceId: string; + resourceId: string; + revision: string; + type: Database["public"]["Enums"]["AssetResourceIndexReferenceType"]; + }; + Insert: { + createdAt?: string; + projectId: string; + referenceId: string; + resourceId: string; + revision: string; + type: Database["public"]["Enums"]["AssetResourceIndexReferenceType"]; + }; + Update: { + createdAt?: string; + projectId?: string; + referenceId?: string; + resourceId?: string; + revision?: string; + type?: Database["public"]["Enums"]["AssetResourceIndexReferenceType"]; + }; + Relationships: [ + { + foreignKeyName: "AssetResourceIndexReference_index_fkey"; + columns: ["projectId", "resourceId", "revision"]; + isOneToOne: false; + referencedRelation: "AssetResourceIndexRevision"; + referencedColumns: ["projectId", "resourceId", "revision"]; + }, + ]; + }; + AssetResourceIndexRevision: { + Row: { + assetRevision: string; + checksum: string; + createdAt: string; + gcClaimId: string | null; + gcStartedAt: string | null; + objectKey: string; + projectId: string; + queryHash: string; + resourceId: string; + revision: string; + unreferencedAt: string | null; + }; + Insert: { + assetRevision: string; + checksum: string; + createdAt?: string; + gcClaimId?: string | null; + gcStartedAt?: string | null; + objectKey: string; + projectId: string; + queryHash: string; + resourceId: string; + revision: string; + unreferencedAt?: string | null; + }; + Update: { + assetRevision?: string; + checksum?: string; + createdAt?: string; + gcClaimId?: string | null; + gcStartedAt?: string | null; + objectKey?: string; + projectId?: string; + queryHash?: string; + resourceId?: string; + revision?: string; + unreferencedAt?: string | null; + }; + Relationships: [ + { + foreignKeyName: "AssetResourceIndexRevision_projectId_fkey"; + columns: ["projectId"]; + isOneToOne: false; + referencedRelation: "DashboardProject"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "AssetResourceIndexRevision_projectId_fkey"; + columns: ["projectId"]; + isOneToOne: false; + referencedRelation: "Project"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "AssetResourceIndexRevision_resource_fkey"; + columns: ["projectId", "resourceId"]; + isOneToOne: false; + referencedRelation: "AssetResourceIndexState"; + referencedColumns: ["projectId", "resourceId"]; + }, + ]; + }; + AssetResourceIndexState: { + Row: { + activeRevision: string | null; + assetRevision: string; + buildAttemptId: string; + buildError: Json | null; + buildStatus: Database["public"]["Enums"]["AssetResourceIndexBuildStatus"]; + createdAt: string; + deletedAt: string | null; + projectId: string; + query: string; + queryHash: string; + resourceId: string; + updatedAt: string; + }; + Insert: { + activeRevision?: string | null; + assetRevision: string; + buildAttemptId: string; + buildError?: Json | null; + buildStatus?: Database["public"]["Enums"]["AssetResourceIndexBuildStatus"]; + createdAt?: string; + deletedAt?: string | null; + projectId: string; + query: string; + queryHash: string; + resourceId: string; + updatedAt?: string; + }; + Update: { + activeRevision?: string | null; + assetRevision?: string; + buildAttemptId?: string; + buildError?: Json | null; + buildStatus?: Database["public"]["Enums"]["AssetResourceIndexBuildStatus"]; + createdAt?: string; + deletedAt?: string | null; + projectId?: string; + query?: string; + queryHash?: string; + resourceId?: string; + updatedAt?: string; + }; + Relationships: [ + { + foreignKeyName: "AssetResourceIndexState_activeRevision_fkey"; + columns: ["projectId", "resourceId", "activeRevision"]; + isOneToOne: false; + referencedRelation: "AssetResourceIndexRevision"; + referencedColumns: ["projectId", "resourceId", "revision"]; + }, + { + foreignKeyName: "AssetResourceIndexState_projectId_fkey"; + columns: ["projectId"]; + isOneToOne: false; + referencedRelation: "DashboardProject"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "AssetResourceIndexState_projectId_fkey"; + columns: ["projectId"]; + isOneToOne: false; + referencedRelation: "Project"; + referencedColumns: ["id"]; + }, + ]; + }; AuthorizationToken: { Row: { canClone: boolean; @@ -418,7 +597,6 @@ export type Database = { }; File: { Row: { - contentHash: string | null; createdAt: string; description: string | null; format: string; @@ -431,7 +609,6 @@ export type Database = { uploaderProjectId: string | null; }; Insert: { - contentHash?: string | null; createdAt?: string; description?: string | null; format: string; @@ -444,7 +621,6 @@ export type Database = { uploaderProjectId?: string | null; }; Update: { - contentHash?: string | null; createdAt?: string; description?: string | null; format?: string; @@ -1070,6 +1246,60 @@ export type Database = { }; }; Functions: { + activate_asset_resource_index: { + Args: { + p_asset_revision: string; + p_build_attempt_id: string; + p_checksum: string; + p_object_key: string; + p_project_id: string; + p_query_hash: string; + p_resource_id: string; + p_revision: string; + }; + Returns: boolean; + }; + add_asset_resource_index_reference: { + Args: { + p_project_id: string; + p_reference_id: string; + p_resource_id: string; + p_revision: string; + p_type: Database["public"]["Enums"]["AssetResourceIndexReferenceType"]; + }; + Returns: undefined; + }; + begin_asset_resource_index_build: { + Args: { + p_asset_revision: string; + p_build_attempt_id: string; + p_project_id: string; + p_query: string; + p_query_hash: string; + p_resource_id: string; + }; + Returns: undefined; + }; + cancel_asset_resource_index_build: { + Args: { + p_asset_revision: string; + p_build_attempt_id: string; + p_project_id: string; + p_query_hash: string; + p_resource_id: string; + }; + Returns: boolean; + }; + claim_asset_resource_index_garbage: { + Args: { p_before: string; p_limit: number }; + Returns: { + gcClaimId: string; + objectKey: string; + projectId: string; + resourceId: string; + revision: string; + }[]; + }; clone_project: { Args: { domain: string; @@ -1104,6 +1334,14 @@ export type Database = { Args: { from_date?: string; to_date?: string }; Returns: undefined; }; + delete_asset_resource_index_query: { + Args: { p_project_id: string; p_resource_id: string }; + Returns: boolean; + }; + delete_stale_asset_file_metadata: { + Args: { p_asset_ids: string[]; p_project_id: string }; + Returns: number; + }; domainsVirtual: { Args: { "": Database["public"]["Tables"]["Project"]["Row"] }; Returns: { @@ -1127,6 +1365,34 @@ export type Database = { isSetofReturn: true; }; }; + fail_asset_resource_index_build: { + Args: { + p_asset_revision: string; + p_build_attempt_id: string; + p_build_error: Json; + p_project_id: string; + p_query_hash: string; + p_resource_id: string; + }; + Returns: boolean; + }; + finalize_asset_resource_index_garbage: { + Args: { + p_gc_claim_id: string; + p_project_id: string; + p_resource_id: string; + p_revision: string; + }; + Returns: boolean; + }; + get_unpublishable_asset_resource_indexes: { + Args: { + p_asset_revision: string; + p_project_id: string; + p_resources: Json; + }; + Returns: Json; + }; latestBuildVirtual: | { Args: { @@ -1202,6 +1468,34 @@ export type Database = { isSetofReturn: true; }; }; + release_asset_resource_index_garbage_claim: { + Args: { + p_gc_claim_id: string; + p_project_id: string; + p_resource_id: string; + p_revision: string; + }; + Returns: boolean; + }; + remove_asset_resource_index_reference: { + Args: { + p_project_id: string; + p_reference_id: string; + p_type: Database["public"]["Enums"]["AssetResourceIndexReferenceType"]; + }; + Returns: number; + }; + replace_asset_file_metadata: { + Args: { + p_asset_id: string; + p_document: Json; + p_field_contributions: Json; + p_project_id: string; + p_revision: string; + p_source: Json; + }; + Returns: boolean; + }; restore_development_build: { Args: { from_build_id: string; project_id: string }; Returns: string; @@ -1215,8 +1509,31 @@ export type Database = { }; Returns: string; }; + uuid_generate_v1: { Args: never; Returns: string }; + uuid_generate_v1mc: { Args: never; Returns: string }; + uuid_generate_v3: { + Args: { name: string; namespace: string }; + Returns: string; + }; + uuid_generate_v4: { Args: never; Returns: string }; + uuid_generate_v5: { + Args: { name: string; namespace: string }; + Returns: string; + }; + uuid_nil: { Args: never; Returns: string }; + uuid_ns_dns: { Args: never; Returns: string }; + uuid_ns_oid: { Args: never; Returns: string }; + uuid_ns_url: { Args: never; Returns: string }; + uuid_ns_x500: { Args: never; Returns: string }; }; Enums: { + AssetResourceIndexBuildStatus: + | "PENDING" + | "BUILDING" + | "ACTIVE" + | "STALE" + | "FAILED"; + AssetResourceIndexReferenceType: "RESOURCE" | "BUILD" | "DEPLOYMENT"; AuthorizationRelation: | "viewers" | "editors" @@ -1358,11 +1675,16 @@ export type CompositeTypes< : never; export const Constants = { - graphql_public: { - Enums: {}, - }, public: { Enums: { + AssetResourceIndexBuildStatus: [ + "PENDING", + "BUILDING", + "ACTIVE", + "STALE", + "FAILED", + ], + AssetResourceIndexReferenceType: ["RESOURCE", "BUILD", "DEPLOYMENT"], AuthorizationRelation: [ "viewers", "editors", diff --git a/packages/postgrest/supabase/tests/asset-resource-indexes.sql b/packages/postgrest/supabase/tests/asset-resource-indexes.sql index 8c884496b193..20261e7d115f 100644 --- a/packages/postgrest/supabase/tests/asset-resource-indexes.sql +++ b/packages/postgrest/supabase/tests/asset-resource-indexes.sql @@ -12,7 +12,8 @@ SELECT lives_ok( 'posts', '*[]', 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', - 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' + 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'attempt-active' ) $$, 'A valid query and asset revision begin an index build' @@ -25,6 +26,7 @@ SELECT is( 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'attempt-active', 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', 'projects/test/resources/posts/index.json' ), @@ -53,7 +55,30 @@ SELECT begin_asset_resource_index_build( 'posts', '*[]', 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', - 'sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' + 'sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', + 'attempt-old' +); + +SELECT begin_asset_resource_index_build( + 'resource-index-test-project', + 'posts', + '*[]', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', + 'attempt-current' +); + +SELECT is( + fail_asset_resource_index_build( + 'resource-index-test-project', + 'posts', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', + 'attempt-old', + '{"code":"INDEX_BUILD_FAILED","message":"Old attempt failed"}'::JSONB + ), + FALSE, + 'An older identical attempt cannot fail the current build' ); SELECT is( @@ -62,6 +87,7 @@ SELECT is( 'posts', 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', 'sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', + 'attempt-current', '{"code":"INDEX_BUILD_FAILED","message":"Resource index build failed"}'::JSONB ), TRUE, diff --git a/packages/prisma-client/prisma/migrations/20260718122000_asset_resource_indexes/migration.sql b/packages/prisma-client/prisma/migrations/20260718122000_asset_resource_indexes/migration.sql index 30e94a1bfeb0..7531811d211b 100644 --- a/packages/prisma-client/prisma/migrations/20260718122000_asset_resource_indexes/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718122000_asset_resource_indexes/migration.sql @@ -12,6 +12,7 @@ CREATE TABLE "AssetResourceIndexState" ( "query" TEXT NOT NULL, "queryHash" TEXT NOT NULL, "assetRevision" TEXT NOT NULL, + "buildAttemptId" TEXT NOT NULL, "buildStatus" "AssetResourceIndexBuildStatus" NOT NULL DEFAULT 'PENDING', "activeRevision" TEXT, "buildError" JSONB, @@ -30,6 +31,8 @@ CREATE TABLE "AssetResourceIndexState" ( CHECK (BTRIM("query") <> '' AND OCTET_LENGTH("query") <= 32768), CONSTRAINT "AssetResourceIndexState_assetRevision_check" CHECK ("assetRevision" ~ '^sha256:[0-9a-f]{64}$'), + CONSTRAINT "AssetResourceIndexState_buildAttemptId_check" + CHECK (BTRIM("buildAttemptId") <> ''), CONSTRAINT "AssetResourceIndexState_activeRevision_check" CHECK ( "activeRevision" IS NULL diff --git a/packages/prisma-client/prisma/migrations/20260718123000_activate_asset_resource_index/migration.sql b/packages/prisma-client/prisma/migrations/20260718123000_activate_asset_resource_index/migration.sql index 92ae24707a06..0f425f730f22 100644 --- a/packages/prisma-client/prisma/migrations/20260718123000_activate_asset_resource_index/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718123000_activate_asset_resource_index/migration.sql @@ -3,7 +3,8 @@ CREATE FUNCTION begin_asset_resource_index_build( p_resource_id TEXT, p_query TEXT, p_query_hash TEXT, - p_asset_revision TEXT + p_asset_revision TEXT, + p_build_attempt_id TEXT ) RETURNS VOID LANGUAGE plpgsql @@ -22,6 +23,7 @@ BEGIN "query", "queryHash", "assetRevision", + "buildAttemptId", "buildStatus", "buildError", "deletedAt", @@ -32,6 +34,7 @@ BEGIN p_query, p_query_hash, p_asset_revision, + p_build_attempt_id, 'BUILDING', NULL, NULL, @@ -42,6 +45,7 @@ BEGIN "query" = EXCLUDED."query", "queryHash" = EXCLUDED."queryHash", "assetRevision" = EXCLUDED."assetRevision", + "buildAttemptId" = EXCLUDED."buildAttemptId", "buildStatus" = 'BUILDING', "buildError" = NULL, "deletedAt" = NULL, @@ -55,6 +59,7 @@ CREATE FUNCTION activate_asset_resource_index( p_revision TEXT, p_query_hash TEXT, p_asset_revision TEXT, + p_build_attempt_id TEXT, p_checksum TEXT, p_object_key TEXT ) @@ -77,6 +82,7 @@ BEGIN AND "resourceId" = p_resource_id AND "queryHash" = p_query_hash AND "assetRevision" = p_asset_revision + AND "buildAttemptId" = p_build_attempt_id AND "buildStatus" = 'BUILDING' FOR UPDATE; diff --git a/packages/prisma-client/prisma/migrations/20260718124000_fail_asset_resource_index_build/migration.sql b/packages/prisma-client/prisma/migrations/20260718124000_fail_asset_resource_index_build/migration.sql index 5e96836a2799..8ca978d9398e 100644 --- a/packages/prisma-client/prisma/migrations/20260718124000_fail_asset_resource_index_build/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718124000_fail_asset_resource_index_build/migration.sql @@ -3,6 +3,7 @@ CREATE FUNCTION fail_asset_resource_index_build( p_resource_id TEXT, p_query_hash TEXT, p_asset_revision TEXT, + p_build_attempt_id TEXT, p_build_error JSONB ) RETURNS BOOLEAN @@ -28,6 +29,7 @@ BEGIN AND "resourceId" = p_resource_id AND "queryHash" = p_query_hash AND "assetRevision" = p_asset_revision + AND "buildAttemptId" = p_build_attempt_id AND "buildStatus" = 'BUILDING'; RETURN FOUND; diff --git a/packages/prisma-client/prisma/migrations/20260718125000_cancel_asset_resource_index_build/migration.sql b/packages/prisma-client/prisma/migrations/20260718125000_cancel_asset_resource_index_build/migration.sql index 2cf1894fe20e..ae3a41f1c23f 100644 --- a/packages/prisma-client/prisma/migrations/20260718125000_cancel_asset_resource_index_build/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718125000_cancel_asset_resource_index_build/migration.sql @@ -2,7 +2,8 @@ CREATE FUNCTION cancel_asset_resource_index_build( p_project_id TEXT, p_resource_id TEXT, p_query_hash TEXT, - p_asset_revision TEXT + p_asset_revision TEXT, + p_build_attempt_id TEXT ) RETURNS BOOLEAN LANGUAGE plpgsql @@ -23,6 +24,7 @@ BEGIN AND "resourceId" = p_resource_id AND "queryHash" = p_query_hash AND "assetRevision" = p_asset_revision + AND "buildAttemptId" = p_build_attempt_id AND "buildStatus" = 'BUILDING'; RETURN FOUND; diff --git a/packages/prisma-client/prisma/schema.prisma b/packages/prisma-client/prisma/schema.prisma index 33f6d7c38dd1..d6f32224aea5 100644 --- a/packages/prisma-client/prisma/schema.prisma +++ b/packages/prisma-client/prisma/schema.prisma @@ -219,6 +219,7 @@ model AssetResourceIndexState { query String queryHash String assetRevision String + buildAttemptId String buildStatus AssetResourceIndexBuildStatus @default(PENDING) activeRevision String? buildError Json? diff --git a/packages/project-build/src/mcp.test.ts b/packages/project-build/src/mcp.test.ts index 04ceebea49a0..7d791d150649 100644 --- a/packages/project-build/src/mcp.test.ts +++ b/packages/project-build/src/mcp.test.ts @@ -7169,6 +7169,37 @@ describe("project session mcp adapter", () => { } }); + test("returns actionable structured screenshot failures", async () => { + const server = await createProjectSessionMcpServer({ + operations: publicMcpOperations, + createProjectSession: createSessionFactory(), + executeOperation: createExecuteOperation(), + captureScreenshot: vi.fn(async () => { + throw new Error("Chromium executable was not found"); + }), + }); + const { client, close } = await createConnectedClient(server); + + try { + const result = await client.callTool({ + name: "screenshot", + arguments: { url: "https://example.com" }, + }); + expect(result).toMatchObject({ + isError: true, + structuredContent: { + ok: false, + error: { + code: "SCREENSHOT_CAPTURE_FAILED", + message: expect.stringContaining("preview.status"), + }, + }, + }); + } finally { + await close(); + } + }); + test("returns stable SDK tool errors from resolver", async () => { const error = new Error( "Project session snapshot changed on disk." diff --git a/packages/project-build/src/mcp.ts b/packages/project-build/src/mcp.ts index af1ef2130c17..7bdc7bbb3a97 100644 --- a/packages/project-build/src/mcp.ts +++ b/packages/project-build/src/mcp.ts @@ -6556,13 +6556,25 @@ export const createProjectSessionMcpCore = ({ return toMetaResult(await downloadAsset(getDownloadAssetInput(input))); } if (name === "screenshot" && captureScreenshot !== undefined) { - return toMetaResult( - await captureScreenshot(getScreenshotInput(input), { - report: (message) => { - reportToolProgress?.(message); - }, - }) - ); + const screenshotInput = getScreenshotInput(input); + try { + return toMetaResult( + await captureScreenshot(screenshotInput, { + report: (message) => { + reportToolProgress?.(message); + }, + }) + ); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + throw Object.assign( + new Error( + `Screenshot capture failed: ${message}. Check preview.status, verify that the route loads, and retry with an installed browser or explicit browserPath.` + ), + { code: "SCREENSHOT_CAPTURE_FAILED", cause: error } + ); + } } if (name === "screenshot.diff" && diffScreenshots !== undefined) { return toMetaResult( diff --git a/packages/project-build/src/runtime/asset-resources.ts b/packages/project-build/src/runtime/asset-resources.ts index 00f177a29a53..b42ef6b952be 100644 --- a/packages/project-build/src/runtime/asset-resources.ts +++ b/packages/project-build/src/runtime/asset-resources.ts @@ -1,6 +1,7 @@ import { assetResourceContentOptions, assetResourceLimits, + assetResourceParameterName, assetResourceQueryRequest, createAssetQueryResourceBody, decodeDataSourceVariable, @@ -26,10 +27,8 @@ import { import { throwBuilderRuntimeError } from "./errors"; import { paginateOutput, paginatedOutputInputSchema } from "./output"; -const parameterName = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/); - export const assetsQueryParameterBindingInput = z.object({ - name: parameterName, + name: assetResourceParameterName, value: resourceExpressionInput, }); diff --git a/packages/project-build/src/runtime/registry.test.ts b/packages/project-build/src/runtime/registry.test.ts index 76c43805957a..f43c334d7cb2 100644 --- a/packages/project-build/src/runtime/registry.test.ts +++ b/packages/project-build/src/runtime/registry.test.ts @@ -2618,6 +2618,10 @@ describe("builder runtime registry", () => { }, }, ], + [ + "assetsResources.create", + { name: "All assets", scopeInstanceId: "heading" }, + ], ["resources.delete", { resourceId: "resource", force: true }], [ "assetFolders.update", diff --git a/packages/sdk/src/schema/asset-resource.ts b/packages/sdk/src/schema/asset-resource.ts index 93580a4002e0..87c9f1c86ae3 100644 --- a/packages/sdk/src/schema/asset-resource.ts +++ b/packages/sdk/src/schema/asset-resource.ts @@ -152,6 +152,10 @@ export const assetResourceLimits = { const sha256Revision = z.string().regex(/^sha256:[a-f0-9]{64}$/); +export const assetResourceParameterName = z + .string() + .regex(/^[A-Za-z_][A-Za-z0-9_]*$/); + export const assetResourceIndexV1 = z .strictObject({ format: z.literal("webstudio-resource-index"), @@ -161,7 +165,7 @@ export const assetResourceIndexV1 = z assetRevision: sha256Revision, queryMode: z.enum(["static", "parameterized"]), parameterNames: z - .array(z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/)) + .array(assetResourceParameterName) .max(assetResourceLimits.parameterCount), documents: z.array(assetFileDocument), integrity: z.strictObject({ @@ -261,7 +265,7 @@ export const assetResourceQueryRequest = z.object({ { error: "Asset resource query exceeds the UTF-8 byte limit" } ), parameters: z - .record(z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/), z.json()) + .record(assetResourceParameterName, z.json()) .refine( (parameters) => Object.keys(parameters).length <= assetResourceLimits.parameterCount, From a4ac720ac7d5d5e9d0ca2faa5993c455d9f38793 Mon Sep 17 00:00:00 2001 From: Oleg Isonen Date: Tue, 21 Jul 2026 18:59:13 +0100 Subject: [PATCH 11/14] fix: harden queryable asset resources --- apps/builder/.env | 1 + .../asset-query-form-utils.test.ts | 9 + .../settings-panel/asset-query-form-utils.ts | 14 +- .../settings-panel/asset-query-form.tsx | 33 ++- .../settings-panel/resource-panel.tsx | 3 +- apps/builder/app/env/env.server.ts | 2 + .../rest.resources-loader-utils.test.ts | 18 ++ .../app/routes/rest.resources-loader-utils.ts | 5 + .../app/routes/rest.resources-loader.ts | 5 +- apps/builder/app/services/api-build.server.ts | 1 + .../app/services/api-router.server.test.ts | 17 +- .../builder/app/services/api-router.server.ts | 18 +- .../assets-field-catalog.server.test.ts | 50 ++-- .../$resources/assets-field-catalog.server.ts | 26 +- .../assets-index-status.server.test.ts | 51 ++-- .../$resources/assets-index-status.server.ts | 23 +- .../$resources/assets-query.server.test.ts | 123 +++++---- .../shared/$resources/assets-query.server.ts | 38 ++- .../shared/$resources/assets.server.test.ts | 32 +-- .../app/shared/$resources/assets.server.ts | 11 +- apps/builder/app/shared/asset-client.ts | 19 ++ apps/builder/app/shared/db/canvas.server.ts | 18 +- .../app/shared/groq-completion.test.ts | 57 ++++ apps/builder/app/shared/groq-completion.ts | 33 ++- .../sync/patch/patch-service.server.test.ts | 207 +++++++++++--- .../shared/sync/patch/patch-service.server.ts | 6 +- ...synchronize-asset-resource-patch.server.ts | 143 ++++++---- .../docs/asset-resource-architecture.md | 129 +++++---- .../app/asset-resource-fetch.ts | 1 + .../app/asset-resource-fetch.ts | 1 + packages/asset-resource/src/blog-e2e.test.ts | 1 + .../asset-resource/src/candidate-selection.ts | 11 +- packages/asset-resource/src/canonical.ts | 17 +- packages/asset-resource/src/field-catalog.ts | 12 +- packages/asset-resource/src/hydration.test.ts | 35 +++ packages/asset-resource/src/hydration.ts | 9 +- .../asset-resource/src/index-storage.test.ts | 4 + packages/asset-resource/src/index-storage.ts | 7 +- packages/asset-resource/src/index.test.ts | 2 +- packages/asset-resource/src/markdown.test.ts | 12 + packages/asset-resource/src/markdown.ts | 6 + .../src/published-runtime.test.ts | 52 +++- .../asset-resource/src/published-runtime.ts | 22 +- .../src/query-validation.test.ts | 11 + .../asset-resource/src/query-validation.ts | 13 +- packages/asset-resource/src/resource-index.ts | 12 +- packages/asset-resource/src/stable-json.ts | 2 +- .../src/transport-parity.test.ts | 2 +- .../asset-uploader/src/async-utils.test.ts | 59 ++++ packages/asset-uploader/src/async-utils.ts | 41 +++ .../src/canonical-metadata-backfill.test.ts | 9 +- .../src/canonical-metadata-backfill.ts | 41 ++- .../canonical-metadata-persistence.test.ts | 18 ++ .../src/canonical-metadata-persistence.ts | 29 +- packages/asset-uploader/src/client.ts | 6 + packages/asset-uploader/src/clients/fs/fs.ts | 17 +- .../src/clients/fs/immutable-object.test.ts | 7 + .../src/clients/fs/immutable-object.ts | 8 +- packages/asset-uploader/src/clients/s3/s3.ts | 57 ++-- .../src/clients/storage-separation.test.ts | 44 +++ packages/asset-uploader/src/index.server.ts | 5 + packages/asset-uploader/src/patch.test.ts | 9 +- .../src/resource-index-build.test.ts | 58 +++- .../src/resource-index-build.ts | 52 +++- .../resource-index-garbage-collection.test.ts | 37 ++- .../src/resource-index-garbage-collection.ts | 98 ++++--- .../src/resource-index-lifecycle.test.ts | 59 +++- .../src/resource-index-lifecycle.ts | 70 +++-- .../src/resource-index-maintenance.test.ts | 40 ++- .../src/resource-index-maintenance.ts | 27 +- .../src/resource-index-persistence.ts | 34 +++ .../src/resource-index-rebuild.test.ts | 20 +- .../src/resource-index-rebuild.ts | 33 ++- .../src/resource-index-snapshot.test.ts | 58 ++++ .../src/resource-index-snapshot.ts | 35 ++- ...esource-index-status-authorization.test.ts | 23 +- .../src/resource-index-status.ts | 27 +- .../cli/src/commands/api-command-metadata.ts | 5 + packages/cli/src/commands/api-command.ts | 13 + packages/cli/src/prebuild.test.ts | 87 +++++- packages/cli/src/prebuild.ts | 72 ++++- .../templates/ssg/app/asset-resource-fetch.ts | 1 + .../postgrest/src/__generated__/db-types.ts | 36 ++- packages/postgrest/src/index.server.ts | 20 +- .../supabase/tests/asset-file-metadata.sql | 30 ++ .../supabase/tests/asset-resource-indexes.sql | 261 +++++++++++++++++- .../migration.sql | 1 + .../migration.sql | 5 +- .../migration.sql | 260 +++++++++++++++-- .../migration.sql | 12 +- .../migration.sql | 11 + packages/prisma-client/prisma/schema.prisma | 1 + .../src/runtime/asset-resources.ts | 7 +- .../src/runtime/registry.test.ts | 32 +++ packages/sdk/src/resource-loader.test.ts | 34 ++- packages/sdk/src/resource-loader.ts | 27 +- 96 files changed, 2611 insertions(+), 649 deletions(-) create mode 100644 apps/builder/app/routes/rest.resources-loader-utils.test.ts create mode 100644 apps/builder/app/routes/rest.resources-loader-utils.ts create mode 100644 packages/asset-uploader/src/async-utils.test.ts create mode 100644 packages/asset-uploader/src/async-utils.ts create mode 100644 packages/asset-uploader/src/clients/storage-separation.test.ts diff --git a/apps/builder/.env b/apps/builder/.env index 0a25a02949ce..40713faacf83 100644 --- a/apps/builder/.env +++ b/apps/builder/.env @@ -24,6 +24,7 @@ POSTGREST_API_KEY= # S3 envs # S3_BUCKET= +# S3_RESOURCE_INDEX_BUCKET= # S3_REGION= # S3_ACL="public-read" # S3_ENDPOINT= diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts index 06517b0a742b..f67956d7bfb9 100644 --- a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts @@ -6,6 +6,7 @@ import { getAssetFileTypeGroqPredicate, getAssetQueryConfigurationError, isEmptyAssetQueryResult, + normalizeAssetQueryParameterBindings, parseAssetQueryResourceBody, } from "./asset-query-form-utils"; @@ -96,6 +97,14 @@ describe("asset query resource body", () => { expect(getAssetFileTypeGroqPredicate("md")).toBe('extension == "md"'); }); + test("normalizes runtime parameter names without changing expressions", () => { + expect( + normalizeAssetQueryParameterBindings([ + { name: " slug ", value: "system.params.slug" }, + ]) + ).toEqual([{ name: "slug", value: "system.params.slug" }]); + }); + test("rejects invalid and duplicate query options before saving", () => { expect( getAssetQueryConfigurationError({ diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts index cd03791c3869..1b4368ea8687 100644 --- a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts @@ -13,6 +13,16 @@ export { } from "@webstudio-is/sdk"; export type { AssetQueryParameterBinding } from "@webstudio-is/sdk"; +export const normalizeAssetQueryParameterBindings = < + Binding extends { name: string }, +>( + parameters: readonly Binding[] +) => + parameters.map((parameter) => ({ + ...parameter, + name: normalizeAssetQueryParameterName(parameter.name), + })); + export const getAssetIndexStatusLabel = ( status: AssetResourceIndexStatus | undefined ) => { @@ -61,8 +71,8 @@ export const getAssetQueryConfigurationError = ({ if (parameters.length > assetResourceLimits.parameterCount) { return `Use at most ${assetResourceLimits.parameterCount} runtime parameters.`; } - const names = parameters.map(({ name }) => - normalizeAssetQueryParameterName(name) + const names = normalizeAssetQueryParameterBindings(parameters).map( + ({ name }) => name ); if ( names.some( diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx b/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx index 23391841ba41..3a9700f313d7 100644 --- a/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx @@ -10,6 +10,7 @@ import { isLiteralExpression, type AssetResourceContentOptions, type AssetResourceIndexStatus, + type AssetResourceQuerySuccess, type BuilderAssetFieldCatalog, type Resource, } from "@webstudio-is/sdk"; @@ -46,6 +47,7 @@ import { getAssetIndexStatusLabel, getAssetQueryConfigurationError, isEmptyAssetQueryResult, + normalizeAssetQueryParameterBindings, parseAssetQueryResourceBody, type AssetQueryParameterBinding, } from "./asset-query-form-utils"; @@ -282,8 +284,20 @@ const AssetQueryPreview = ({ | { type: "idle" } | { type: "loading" } | { type: "error"; message: string } - | { type: "success"; result: unknown } + | { type: "success"; response: AssetResourceQuerySuccess } >({ type: "idle" }); + const previewInputKey = JSON.stringify({ + query, + parameters, + resultLimit, + content, + }); + const previewInputKeyRef = useRef(previewInputKey); + previewInputKeyRef.current = previewInputKey; + + useEffect(() => { + setPreviewState({ type: "idle" }); + }, [previewInputKey]); useEffect(() => { if (resourceId === undefined) { @@ -321,12 +335,13 @@ const AssetQueryPreview = ({ }, [onIndexStatusChange, resourceId, resourceRevision]); const preview = async () => { + const requestedInputKey = previewInputKey; setPreviewState({ type: "loading" }); try { const response = await previewBuilderAssetQuery({ query, parameters: Object.fromEntries( - parameters + normalizeAssetQueryParameterBindings(parameters) .filter(({ name }) => name.trim().length > 0) .map(({ name, value }) => [ name, @@ -336,6 +351,9 @@ const AssetQueryPreview = ({ resultLimit, content, }); + if (previewInputKeyRef.current !== requestedInputKey) { + return; + } const parsed = assetResourceQueryResponse.safeParse(response.data); if (parsed.success === false) { setPreviewState({ @@ -348,14 +366,17 @@ const AssetQueryPreview = ({ setPreviewState({ type: "error", message: parsed.data.error.message }); return; } - setPreviewState({ type: "success", result: parsed.data.result }); + setPreviewState({ type: "success", response: parsed.data }); } catch { + if (previewInputKeyRef.current !== requestedInputKey) { + return; + } setPreviewState({ type: "error", message: "The query preview failed." }); } }; const isEmpty = previewState.type === "success" && - isEmptyAssetQueryResult(previewState.result); + isEmptyAssetQueryResult(previewState.response.result); return ( @@ -383,7 +404,7 @@ const AssetQueryPreview = ({ title="Query preview" size="small" readOnly={true} - value={JSON.stringify(previewState.result, null, 2)} + value={JSON.stringify(previewState.response, null, 2)} onChange={() => {}} onChangeComplete={() => {}} /> @@ -483,7 +504,7 @@ export const AssetQueryForm = ({ catalog: fieldCatalog, parameterNames: Array.from( new Set( - parameters + normalizeAssetQueryParameterBindings(parameters) .map(({ name }) => name) .filter( (name) => assetResourceParameterName.safeParse(name).success diff --git a/apps/builder/app/builder/features/settings-panel/resource-panel.tsx b/apps/builder/app/builder/features/settings-panel/resource-panel.tsx index 917d7d4d1fb8..a725b5b07b2c 100644 --- a/apps/builder/app/builder/features/settings-panel/resource-panel.tsx +++ b/apps/builder/app/builder/features/settings-panel/resource-panel.tsx @@ -21,6 +21,7 @@ import { encodeDataVariableId, generateObjectExpression, isLiteralExpression, + isStoredAssetQueryResource, parseObjectExpression, SYSTEM_VARIABLE_ID, systemParameter, @@ -923,7 +924,7 @@ export const SystemResourceForm = forwardRef< ? resources.get(variable.resourceId) : undefined; const isStoredAssetQuery = - resource?.url === JSON.stringify(assetsQueryResourceUrl); + resource !== undefined && isStoredAssetQueryResource(resource); const assetsLocalResource = { label: "Assets", diff --git a/apps/builder/app/env/env.server.ts b/apps/builder/app/env/env.server.ts index 2154f5926463..bc960881d29b 100644 --- a/apps/builder/app/env/env.server.ts +++ b/apps/builder/app/env/env.server.ts @@ -35,6 +35,7 @@ const environment = z.object({ S3_ACCESS_KEY_ID: z.string().optional(), S3_SECRET_ACCESS_KEY: z.string().optional(), S3_BUCKET: z.string().optional(), + S3_RESOURCE_INDEX_BUCKET: z.string().optional(), S3_ACL: z.string().optional(), // Origin for /cdn-cgi/image/ cloudflare endpoint (without ending slash) @@ -99,6 +100,7 @@ const rawEnv = { S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY, S3_BUCKET: process.env.S3_BUCKET, + S3_RESOURCE_INDEX_BUCKET: process.env.S3_RESOURCE_INDEX_BUCKET, S3_ACL: process.env.S3_ACL, RESIZE_ORIGIN: process.env.RESIZE_ORIGIN, ENTRI_APPLICATION_ID: process.env.ENTRI_APPLICATION_ID, diff --git a/apps/builder/app/routes/rest.resources-loader-utils.test.ts b/apps/builder/app/routes/rest.resources-loader-utils.test.ts new file mode 100644 index 000000000000..e1e692b91ef8 --- /dev/null +++ b/apps/builder/app/routes/rest.resources-loader-utils.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, test } from "vitest"; +import { createLocalResourceRequest } from "./rest.resources-loader-utils"; + +describe("resources loader local dispatch", () => { + test("creates nested requests from relative local resource URLs", async () => { + const request = createLocalResourceRequest( + new Request("https://p-project.builder.example/rest/resources-loader"), + "/$resources/assets/query?resourceId=posts", + { method: "POST", body: '{"query":"*[]"}' } + ); + + expect(request.url).toBe( + "https://p-project.builder.example/$resources/assets/query?resourceId=posts" + ); + expect(request.method).toBe("POST"); + expect(await request.text()).toBe('{"query":"*[]"}'); + }); +}); diff --git a/apps/builder/app/routes/rest.resources-loader-utils.ts b/apps/builder/app/routes/rest.resources-loader-utils.ts new file mode 100644 index 000000000000..605b7120556b --- /dev/null +++ b/apps/builder/app/routes/rest.resources-loader-utils.ts @@ -0,0 +1,5 @@ +export const createLocalResourceRequest = ( + outerRequest: Request, + input: string, + init?: RequestInit +) => new Request(new URL(input, outerRequest.url), init); diff --git a/apps/builder/app/routes/rest.resources-loader.ts b/apps/builder/app/routes/rest.resources-loader.ts index f9c1a13e1682..b581dea0a790 100644 --- a/apps/builder/app/routes/rest.resources-loader.ts +++ b/apps/builder/app/routes/rest.resources-loader.ts @@ -13,6 +13,7 @@ import { preventCrossOriginCookie } from "~/services/no-cross-origin-cookie"; import { checkCsrf } from "~/services/csrf-session.server"; import { getResourceKey } from "~/shared/resources"; import { privateNoStoreResponseHeaders } from "~/services/cache-control.server"; +import { createLocalResourceRequest } from "./rest.resources-loader-utils"; export const action = async ({ request }: ActionFunctionArgs) => { preventCrossOriginCookie(request); @@ -43,14 +44,14 @@ export const action = async ({ request }: ActionFunctionArgs) => { if (isLocalResource(input, "assets/query")) { return assetsQueryLoader({ request, - resourceRequest: new Request(input, init), + resourceRequest: createLocalResourceRequest(request, input, init), }); } if (isLocalResource(input, "assets/index-status")) { return assetsIndexStatusLoader({ request, - resourceRequest: new Request(input, init), + resourceRequest: createLocalResourceRequest(request, input, init), }); } diff --git a/apps/builder/app/services/api-build.server.ts b/apps/builder/app/services/api-build.server.ts index 962a1f5b674a..dbe057f6b27d 100644 --- a/apps/builder/app/services/api-build.server.ts +++ b/apps/builder/app/services/api-build.server.ts @@ -169,6 +169,7 @@ export const commitBuildTransactions = async ({ try { await synchronizeAssetResourcesAfterBuildPatch({ context: ctx, + buildId, projectId, previousResources: previousBuild.resources, resources: build.resources, diff --git a/apps/builder/app/services/api-router.server.test.ts b/apps/builder/app/services/api-router.server.test.ts index f9e1b04f37ec..c9c03a19d7c3 100644 --- a/apps/builder/app/services/api-router.server.test.ts +++ b/apps/builder/app/services/api-router.server.test.ts @@ -167,6 +167,20 @@ describe("api router build operation adapters", () => { test("rebuilds a query index through the authenticated recovery operation", async () => { vi.spyOn(authDb, "getTokenInfo").mockResolvedValue(createToken()); vi.spyOn(authorizeProject, "hasProjectPermit").mockResolvedValue(true); + vi.spyOn(projectBuild, "loadDevBuildByProjectId").mockResolvedValue({ + id: "build-1", + resources: [ + { + id: "resource-1", + name: "Posts", + control: "system", + method: "post", + url: JSON.stringify("/$resources/assets/query"), + headers: [], + body: '{query:"*[]"}', + }, + ], + } as never); vi.spyOn(assetUploader, "rebuildAssetResourceIndex").mockResolvedValue({ index: {}, persisted: { revision: "revision-2", key: "indexes/revision-2.json" }, @@ -782,7 +796,8 @@ describe("api router permits", () => { projectId: "project-1", clientVersion: 3, }), - expect.anything() + expect.anything(), + expect.any(Function) ); }); diff --git a/apps/builder/app/services/api-router.server.ts b/apps/builder/app/services/api-router.server.ts index 607c9efaa977..e71091454992 100644 --- a/apps/builder/app/services/api-router.server.ts +++ b/apps/builder/app/services/api-router.server.ts @@ -31,7 +31,7 @@ import { AssetResourceHydrationError, AssetResourceQueryExecutionError, AssetResourceQueryValidationError, - getAssetResourceReferencedFieldPaths, + getAssetResourceReferencedFieldPathsFromTree, validateAssetResourceQuery, } from "@webstudio-is/asset-resource"; import { @@ -89,7 +89,10 @@ import { executeApiRuntimeMutation, executeApiRuntimeOperation, } from "./api-runtime.server"; -import { createAssetClient } from "../shared/asset-client"; +import { + createAssetClient, + createAssetClientWithResourceIndexStore, +} from "../shared/asset-client"; const assertApiPublishDomains = ({ auth, @@ -741,8 +744,8 @@ export const apiRouter = router({ valid: true as const, queryMode: validated.queryMode, parameterNames: validated.parameterNames, - referencedFieldPaths: getAssetResourceReferencedFieldPaths( - input.query + referencedFieldPaths: getAssetResourceReferencedFieldPathsFromTree( + validated.tree ), astNodes: validated.astNodes, astDepth: validated.astDepth, @@ -831,12 +834,17 @@ export const apiRouter = router({ if (query === undefined) { throw new AssetResourceIndexNotFoundError(); } + const assetClient = createAssetClientWithResourceIndexStore(); const result = await rebuildAssetResourceIndex({ client: ctx.postgrest.client, - store: createAssetClient().resourceIndexStore, + assetClient, projectId: input.projectId, resourceId: input.resourceId, query, + source: { + buildId: build.id, + resources: JSON.stringify(build.resources), + }, }); const status = await loadAssetResourceIndexStatus({ projectId: input.projectId, diff --git a/apps/builder/app/shared/$resources/assets-field-catalog.server.test.ts b/apps/builder/app/shared/$resources/assets-field-catalog.server.test.ts index 79657af75afb..227e67d20539 100644 --- a/apps/builder/app/shared/$resources/assets-field-catalog.server.test.ts +++ b/apps/builder/app/shared/$resources/assets-field-catalog.server.test.ts @@ -1,17 +1,14 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; -import { loadBuilderAssetFieldCatalog } from "@webstudio-is/asset-uploader/index.server"; import { AuthorizationError } from "@webstudio-is/trpc-interface/index.server"; -import { createContext } from "../context.server"; import { loader } from "./assets-field-catalog.server"; -vi.mock("@webstudio-is/asset-uploader/index.server", () => ({ - loadBuilderAssetFieldCatalog: vi.fn(), -})); -vi.mock("../context.server", () => ({ - createContext: vi.fn(), -})); - const projectId = "090e6e14-ae50-4b2e-bd22-71733cec05bb"; +const assetClient = { readFile: vi.fn() }; +const dependencies = { + createContext: vi.fn(), + createAssetClient: vi.fn(() => assetClient), + loadBuilderAssetFieldCatalog: vi.fn(), +}; const catalog = { format: "webstudio-builder-asset-field-catalog" as const, version: 1 as const, @@ -27,36 +24,43 @@ const catalog = { describe("asset field catalog system resource", () => { beforeEach(() => { - vi.mocked(createContext).mockResolvedValue({} as never); - vi.mocked(loadBuilderAssetFieldCatalog).mockResolvedValue(catalog); + dependencies.createContext.mockResolvedValue({} as never); + dependencies.loadBuilderAssetFieldCatalog.mockResolvedValue(catalog); }); test("returns the authenticated project's compact field catalog", async () => { - const response = await loader({ - request: new Request( - `https://p-${projectId}.localhost/$resources/assets/field-catalog` - ), - }); + const response = await loader( + { + request: new Request( + `https://p-${projectId}.localhost/$resources/assets/field-catalog` + ), + }, + dependencies + ); expect(response.status).toBe(200); expect(response.headers.get("cache-control")).toContain("private"); await expect(response.json()).resolves.toEqual(catalog); - expect(loadBuilderAssetFieldCatalog).toHaveBeenCalledWith({ + expect(dependencies.loadBuilderAssetFieldCatalog).toHaveBeenCalledWith({ projectId, context: expect.anything(), + assetClient, }); }); test("returns a structured forbidden response", async () => { - vi.mocked(loadBuilderAssetFieldCatalog).mockRejectedValue( + dependencies.loadBuilderAssetFieldCatalog.mockRejectedValue( new AuthorizationError("denied") ); - const response = await loader({ - request: new Request( - `https://p-${projectId}.localhost/$resources/assets/field-catalog` - ), - }); + const response = await loader( + { + request: new Request( + `https://p-${projectId}.localhost/$resources/assets/field-catalog` + ), + }, + dependencies + ); expect(response.status).toBe(403); await expect(response.json()).resolves.toMatchObject({ diff --git a/apps/builder/app/shared/$resources/assets-field-catalog.server.ts b/apps/builder/app/shared/$resources/assets-field-catalog.server.ts index 70972f7553cd..6e40e2403850 100644 --- a/apps/builder/app/shared/$resources/assets-field-catalog.server.ts +++ b/apps/builder/app/shared/$resources/assets-field-catalog.server.ts @@ -7,7 +7,25 @@ import { createContext } from "../context.server"; import { isBuilder } from "../router-utils"; import { createAssetClient } from "../asset-client"; -export const loader = async ({ request }: { request: Request }) => { +type Dependencies = { + createContext: typeof createContext; + createAssetClient: () => Pick< + ReturnType, + "readFile" + >; + loadBuilderAssetFieldCatalog: typeof loadBuilderAssetFieldCatalog; +}; + +const defaultDependencies: Dependencies = { + createContext, + createAssetClient, + loadBuilderAssetFieldCatalog, +}; + +export const loader = async ( + { request }: { request: Request }, + dependencies = defaultDependencies +) => { if (isBuilder(request) === false) { throw new Error( "Asset field catalog can only be accessed from the builder interface" @@ -20,11 +38,11 @@ export const loader = async ({ request }: { request: Request }) => { } try { - const context = await createContext(request); - const catalog = await loadBuilderAssetFieldCatalog({ + const context = await dependencies.createContext(request); + const catalog = await dependencies.loadBuilderAssetFieldCatalog({ projectId, context, - assetClient: createAssetClient(), + assetClient: dependencies.createAssetClient(), }); return json(catalog, { headers: privateNoStoreResponseHeaders }); } catch (error) { diff --git a/apps/builder/app/shared/$resources/assets-index-status.server.test.ts b/apps/builder/app/shared/$resources/assets-index-status.server.test.ts index 2b034c59d7d9..882ff524fc40 100644 --- a/apps/builder/app/shared/$resources/assets-index-status.server.test.ts +++ b/apps/builder/app/shared/$resources/assets-index-status.server.test.ts @@ -1,14 +1,11 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; -import { loadAssetResourceIndexStatus } from "@webstudio-is/asset-uploader/index.server"; -import { createContext } from "../context.server"; import { loader } from "./assets-index-status.server"; -vi.mock("@webstudio-is/asset-uploader/index.server", () => ({ - loadAssetResourceIndexStatus: vi.fn(), -})); -vi.mock("../context.server", () => ({ createContext: vi.fn() })); - const projectId = "090e6e14-ae50-4b2e-bd22-71733cec05bb"; +const dependencies = { + createContext: vi.fn(), + loadAssetResourceIndexStatus: vi.fn(), +}; const outerRequest = () => new Request(`https://p-${projectId}.localhost/rest/resources-loader`); const innerRequest = (resourceId?: string) => { @@ -23,8 +20,8 @@ const innerRequest = (resourceId?: string) => { describe("asset index status system resource", () => { beforeEach(() => { - vi.mocked(createContext).mockResolvedValue({} as never); - vi.mocked(loadAssetResourceIndexStatus).mockReset(); + dependencies.createContext.mockResolvedValue({} as never); + dependencies.loadAssetResourceIndexStatus.mockReset(); }); test("returns the authenticated resource status", async () => { @@ -36,16 +33,19 @@ describe("asset index status system resource", () => { activeRevision: "active-revision", updatedAt: "2026-07-18T12:00:00.000Z", }; - vi.mocked(loadAssetResourceIndexStatus).mockResolvedValue(status); + dependencies.loadAssetResourceIndexStatus.mockResolvedValue(status); - const response = await loader({ - request: outerRequest(), - resourceRequest: innerRequest("posts"), - }); + const response = await loader( + { + request: outerRequest(), + resourceRequest: innerRequest("posts"), + }, + dependencies + ); expect(response.status).toBe(200); await expect(response.json()).resolves.toEqual({ ok: true, status }); - expect(loadAssetResourceIndexStatus).toHaveBeenCalledWith({ + expect(dependencies.loadAssetResourceIndexStatus).toHaveBeenCalledWith({ projectId, resourceId: "posts", context: expect.anything(), @@ -53,17 +53,20 @@ describe("asset index status system resource", () => { }); test("rejects a missing resource ID and reports missing state", async () => { - const invalid = await loader({ - request: outerRequest(), - resourceRequest: innerRequest(), - }); + const invalid = await loader( + { request: outerRequest(), resourceRequest: innerRequest() }, + dependencies + ); expect(invalid.status).toBe(400); - vi.mocked(loadAssetResourceIndexStatus).mockResolvedValue(undefined); - const missing = await loader({ - request: outerRequest(), - resourceRequest: innerRequest("missing"), - }); + dependencies.loadAssetResourceIndexStatus.mockResolvedValue(undefined); + const missing = await loader( + { + request: outerRequest(), + resourceRequest: innerRequest("missing"), + }, + dependencies + ); expect(missing.status).toBe(404); await expect(missing.json()).resolves.toMatchObject({ ok: false, diff --git a/apps/builder/app/shared/$resources/assets-index-status.server.ts b/apps/builder/app/shared/$resources/assets-index-status.server.ts index 8141c2b29f7d..5fd60c62e583 100644 --- a/apps/builder/app/shared/$resources/assets-index-status.server.ts +++ b/apps/builder/app/shared/$resources/assets-index-status.server.ts @@ -6,6 +6,8 @@ import { privateNoStoreResponseHeaders } from "~/services/cache-control.server"; import { createContext } from "../context.server"; import { isBuilder } from "../router-utils"; +const defaultDependencies = { createContext, loadAssetResourceIndexStatus }; + const failure = ({ code, message, @@ -22,13 +24,16 @@ const failure = ({ { status, headers: privateNoStoreResponseHeaders } ); -export const loader = async ({ - request, - resourceRequest, -}: { - request: Request; - resourceRequest: Request; -}) => { +export const loader = async ( + { + request, + resourceRequest, + }: { + request: Request; + resourceRequest: Request; + }, + dependencies = defaultDependencies +) => { if (isBuilder(request) === false) { return failure({ code: "FORBIDDEN", @@ -53,8 +58,8 @@ export const loader = async ({ } try { - const context = await createContext(request); - const status = await loadAssetResourceIndexStatus({ + const context = await dependencies.createContext(request); + const status = await dependencies.loadAssetResourceIndexStatus({ projectId, resourceId, context, diff --git a/apps/builder/app/shared/$resources/assets-query.server.test.ts b/apps/builder/app/shared/$resources/assets-query.server.test.ts index b20dd36101cb..c8d666e68bdd 100644 --- a/apps/builder/app/shared/$resources/assets-query.server.test.ts +++ b/apps/builder/app/shared/$resources/assets-query.server.test.ts @@ -1,22 +1,17 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; -import { previewAssetResourceQuery } from "@webstudio-is/asset-uploader/index.server"; import { AssetResourceQueryExecutionError, AssetResourceQueryValidationError, } from "@webstudio-is/asset-resource"; import { AuthorizationError } from "@webstudio-is/trpc-interface/index.server"; -import { createContext } from "../context.server"; import { loader } from "./assets-query.server"; -vi.mock("@webstudio-is/asset-uploader/index.server", () => ({ - previewAssetResourceQuery: vi.fn(), -})); -vi.mock("../context.server", () => ({ createContext: vi.fn() })); -vi.mock("../asset-client", () => ({ - createAssetClient: vi.fn(() => ({ readFile: vi.fn() })), -})); - const projectId = "090e6e14-ae50-4b2e-bd22-71733cec05bb"; +const dependencies = { + createContext: vi.fn(), + createAssetClient: vi.fn(() => ({ readFile: vi.fn() })), + previewAssetResourceQuery: vi.fn(), +}; const outerRequest = () => new Request(`https://p-${projectId}.localhost/rest/resources-loader`); const innerRequest = (body: unknown) => @@ -27,8 +22,8 @@ const innerRequest = (body: unknown) => describe("asset query preview system resource", () => { beforeEach(() => { - vi.mocked(createContext).mockResolvedValue({} as never); - vi.mocked(previewAssetResourceQuery).mockReset(); + dependencies.createContext.mockResolvedValue({} as never); + dependencies.previewAssetResourceQuery.mockReset(); }); test("uses outer authentication context and the inner query body", async () => { @@ -45,17 +40,22 @@ describe("asset query preview system resource", () => { hydratedBytes: 0, }, }; - vi.mocked(previewAssetResourceQuery).mockResolvedValue(responseBody); + dependencies.previewAssetResourceQuery.mockResolvedValue(responseBody); - const response = await loader({ - request: outerRequest(), - resourceRequest: innerRequest({ query: "*[]" }), - }); + const response = await loader( + { + request: outerRequest(), + resourceRequest: innerRequest({ query: "*[]" }), + }, + dependencies + ); expect(response.status).toBe(200); await expect(response.json()).resolves.toEqual(responseBody); - expect(createContext).toHaveBeenCalledWith(expect.any(Request)); - expect(previewAssetResourceQuery).toHaveBeenCalledWith({ + expect(dependencies.createContext).toHaveBeenCalledWith( + expect.any(Request) + ); + expect(dependencies.previewAssetResourceQuery).toHaveBeenCalledWith({ projectId, request: { query: "*[]", @@ -69,23 +69,26 @@ describe("asset query preview system resource", () => { }); test("returns structured invalid-request and forbidden failures", async () => { - const invalid = await loader({ - request: outerRequest(), - resourceRequest: innerRequest({ query: "" }), - }); + const invalid = await loader( + { request: outerRequest(), resourceRequest: innerRequest({ query: "" }) }, + dependencies + ); expect(invalid.status).toBe(400); await expect(invalid.json()).resolves.toMatchObject({ ok: false, error: { code: "INVALID_REQUEST" }, }); - vi.mocked(previewAssetResourceQuery).mockRejectedValue( + dependencies.previewAssetResourceQuery.mockRejectedValue( new AuthorizationError("denied") ); - const forbidden = await loader({ - request: outerRequest(), - resourceRequest: innerRequest({ query: "*[]" }), - }); + const forbidden = await loader( + { + request: outerRequest(), + resourceRequest: innerRequest({ query: "*[]" }), + }, + dependencies + ); expect(forbidden.status).toBe(403); await expect(forbidden.json()).resolves.toMatchObject({ ok: false, @@ -94,16 +97,19 @@ describe("asset query preview system resource", () => { }); test("rejects malformed JSON and accepts hydration options", async () => { - const malformed = await loader({ - request: outerRequest(), - resourceRequest: new Request( - `https://p-${projectId}.localhost/$resources/assets/query`, - { method: "POST", body: "{" } - ), - }); + const malformed = await loader( + { + request: outerRequest(), + resourceRequest: new Request( + `https://p-${projectId}.localhost/$resources/assets/query`, + { method: "POST", body: "{" } + ), + }, + dependencies + ); expect(malformed.status).toBe(400); - vi.mocked(previewAssetResourceQuery).mockResolvedValue({ + dependencies.previewAssetResourceQuery.mockResolvedValue({ ok: true, result: null, content: {}, @@ -116,15 +122,18 @@ describe("asset query preview system resource", () => { hydratedBytes: 0, }, }); - const hydration = await loader({ - request: outerRequest(), - resourceRequest: innerRequest({ - query: "*[0]", - content: { mode: "full" }, - }), - }); + const hydration = await loader( + { + request: outerRequest(), + resourceRequest: innerRequest({ + query: "*[0]", + content: { mode: "full" }, + }), + }, + dependencies + ); expect(hydration.status).toBe(200); - expect(previewAssetResourceQuery).toHaveBeenCalledWith({ + expect(dependencies.previewAssetResourceQuery).toHaveBeenCalledWith({ projectId, request: expect.objectContaining({ content: { mode: "full" } }), context: expect.anything(), @@ -133,31 +142,37 @@ describe("asset query preview system resource", () => { }); test("maps query validation and execution errors", async () => { - vi.mocked(previewAssetResourceQuery).mockRejectedValueOnce( + dependencies.previewAssetResourceQuery.mockRejectedValueOnce( new AssetResourceQueryValidationError({ code: "INVALID_QUERY", message: "Invalid GROQ", }) ); - const invalid = await loader({ - request: outerRequest(), - resourceRequest: innerRequest({ query: "*[invalid ==" }), - }); + const invalid = await loader( + { + request: outerRequest(), + resourceRequest: innerRequest({ query: "*[invalid ==" }), + }, + dependencies + ); expect(invalid.status).toBe(400); await expect(invalid.json()).resolves.toMatchObject({ error: { code: "INVALID_QUERY" }, }); - vi.mocked(previewAssetResourceQuery).mockRejectedValueOnce( + dependencies.previewAssetResourceQuery.mockRejectedValueOnce( new AssetResourceQueryExecutionError({ code: "RESULT_LIMIT_EXCEEDED", message: "Too many results", }) ); - const excessive = await loader({ - request: outerRequest(), - resourceRequest: innerRequest({ query: "*[]" }), - }); + const excessive = await loader( + { + request: outerRequest(), + resourceRequest: innerRequest({ query: "*[]" }), + }, + dependencies + ); expect(excessive.status).toBe(400); await expect(excessive.json()).resolves.toMatchObject({ error: { code: "RESULT_LIMIT_EXCEEDED" }, diff --git a/apps/builder/app/shared/$resources/assets-query.server.ts b/apps/builder/app/shared/$resources/assets-query.server.ts index b1494f8f11c7..43679cafcb13 100644 --- a/apps/builder/app/shared/$resources/assets-query.server.ts +++ b/apps/builder/app/shared/$resources/assets-query.server.ts @@ -17,6 +17,21 @@ import { createContext } from "../context.server"; import { isBuilder } from "../router-utils"; import { createAssetClient } from "../asset-client"; +type Dependencies = { + createContext: typeof createContext; + createAssetClient: () => Pick< + ReturnType, + "readFile" + >; + previewAssetResourceQuery: typeof previewAssetResourceQuery; +}; + +const defaultDependencies: Dependencies = { + createContext, + createAssetClient, + previewAssetResourceQuery, +}; + const failure = ({ code, message, @@ -38,13 +53,16 @@ const failure = ({ { status, headers: privateNoStoreResponseHeaders } ); -export const loader = async ({ - request, - resourceRequest, -}: { - request: Request; - resourceRequest: Request; -}) => { +export const loader = async ( + { + request, + resourceRequest, + }: { + request: Request; + resourceRequest: Request; + }, + dependencies = defaultDependencies +) => { if (isBuilder(request) === false) { return failure({ code: "FORBIDDEN", @@ -80,12 +98,12 @@ export const loader = async ({ }); } try { - const context = await createContext(request); - const result = await previewAssetResourceQuery({ + const context = await dependencies.createContext(request); + const result = await dependencies.previewAssetResourceQuery({ projectId, request: parsed.data, context, - assetClient: createAssetClient(), + assetClient: dependencies.createAssetClient(), }); return json(result, { headers: privateNoStoreResponseHeaders }); } catch (error) { diff --git a/apps/builder/app/shared/$resources/assets.server.test.ts b/apps/builder/app/shared/$resources/assets.server.test.ts index 5534ed99d660..c9950f665d45 100644 --- a/apps/builder/app/shared/$resources/assets.server.test.ts +++ b/apps/builder/app/shared/$resources/assets.server.test.ts @@ -1,21 +1,16 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; -import { loadAssetsByProject } from "@webstudio-is/asset-uploader/index.server"; -import { createContext } from "../context.server"; import { loader } from "./assets.server"; -vi.mock("@webstudio-is/asset-uploader/index.server", () => ({ - loadAssetsByProject: vi.fn(), -})); -vi.mock("../context.server", () => ({ - createContext: vi.fn(), -})); - const projectId = "090e6e14-ae50-4b2e-bd22-71733cec05bb"; +const dependencies = { + createContext: vi.fn(), + loadAssetsByProject: vi.fn(), +}; describe("legacy Assets system resource", () => { beforeEach(() => { - vi.mocked(createContext).mockResolvedValue({} as never); - vi.mocked(loadAssetsByProject).mockResolvedValue([ + dependencies.createContext.mockResolvedValue({} as never); + dependencies.loadAssetsByProject.mockResolvedValue([ { id: "asset-1", projectId, @@ -31,17 +26,20 @@ describe("legacy Assets system resource", () => { }); test("keeps returning every runtime asset keyed by asset ID", async () => { - const response = await loader({ - request: new Request( - `https://p-${projectId}.localhost/$resources/assets` - ), - }); + const response = await loader( + { + request: new Request( + `https://p-${projectId}.localhost/$resources/assets` + ), + }, + dependencies + ); expect(response.status).toBe(200); await expect(response.json()).resolves.toEqual({ "asset-1": { url: "/cgi/asset/document.pdf?format=raw" }, }); - expect(loadAssetsByProject).toHaveBeenCalledWith( + expect(dependencies.loadAssetsByProject).toHaveBeenCalledWith( projectId, expect.anything() ); diff --git a/apps/builder/app/shared/$resources/assets.server.ts b/apps/builder/app/shared/$resources/assets.server.ts index bffdd290b240..696cf8498df5 100644 --- a/apps/builder/app/shared/$resources/assets.server.ts +++ b/apps/builder/app/shared/$resources/assets.server.ts @@ -5,11 +5,16 @@ import { toRuntimeAsset } from "@webstudio-is/sdk"; import { isBuilder } from "../router-utils"; import { createContext } from "../context.server"; +const defaultDependencies = { createContext, loadAssetsByProject }; + /** * System Resource that provides the list of assets for the current project. * This allows assets to be dynamically referenced in the builder using the expression editor. */ -export const loader = async ({ request }: { request: Request }) => { +export const loader = async ( + { request }: { request: Request }, + dependencies = defaultDependencies +) => { if (isBuilder(request) === false) { throw new Error( "Asset resource loader can only be accessed from the builder interface" @@ -24,9 +29,9 @@ export const loader = async ({ request }: { request: Request }) => { ); } - const context = await createContext(request); + const context = await dependencies.createContext(request); - const assets = await loadAssetsByProject(projectId, context); + const assets = await dependencies.loadAssetsByProject(projectId, context); const requestUrl = new URL(request.url); const origin = `${requestUrl.protocol}//${requestUrl.host}`; diff --git a/apps/builder/app/shared/asset-client.ts b/apps/builder/app/shared/asset-client.ts index c16b02001155..2dae28161e35 100644 --- a/apps/builder/app/shared/asset-client.ts +++ b/apps/builder/app/shared/asset-client.ts @@ -3,6 +3,7 @@ import { maxSize } from "@webstudio-is/asset-uploader"; import { createFsClient, createS3Client, + type AssetClientWithReadableResourceIndexStore, } from "@webstudio-is/asset-uploader/index.server"; import env from "~/env/env.server"; @@ -25,6 +26,7 @@ export const createAssetClient = () => { accessKeyId: env.S3_ACCESS_KEY_ID, secretAccessKey: env.S3_SECRET_ACCESS_KEY, bucket: env.S3_BUCKET, + resourceIndexBucket: env.S3_RESOURCE_INDEX_BUCKET, acl: env.S3_ACL, maxUploadSize, }); @@ -32,6 +34,23 @@ export const createAssetClient = () => { return createFsClient({ maxUploadSize, fileDirectory: path.join(process.cwd(), fileUploadPath), + resourceIndexDirectory: path.join( + process.cwd(), + "private/asset-resource-indexes" + ), }); } }; + +export const createAssetClientWithResourceIndexStore = () => { + const client = createAssetClient(); + if ( + client.resourceIndexStore === undefined || + client.resourceIndexStore.read === undefined + ) { + throw new Error( + "Private asset resource index storage is not configured. Set S3_RESOURCE_INDEX_BUCKET." + ); + } + return client as AssetClientWithReadableResourceIndexStore; +}; diff --git a/apps/builder/app/shared/db/canvas.server.ts b/apps/builder/app/shared/db/canvas.server.ts index 69a231ef911b..679f6ca8150b 100644 --- a/apps/builder/app/shared/db/canvas.server.ts +++ b/apps/builder/app/shared/db/canvas.server.ts @@ -10,7 +10,7 @@ import { import { collectFontFamiliesFromStyleDecls } from "@webstudio-is/project-build/runtime"; import { loadAssetDataByProject, - loadCanonicalAssetFileEntries, + loadCanonicalAssetFileSnapshot, prepareAssetResourceIndexSnapshotsForPublication, getAssetResourceQuery, synchronizeCanonicalAssets, @@ -30,7 +30,7 @@ import { import { serializePages } from "@webstudio-is/project-migrations/pages"; import { loadById } from "@webstudio-is/project/index.server"; import { getUserById } from "./user.server"; -import { createAssetClient } from "../asset-client"; +import { createAssetClientWithResourceIndexStore } from "../asset-client"; const getPair = (item: Item): [string, Item] => [ item.id, @@ -172,16 +172,17 @@ const addProjectMetadata = async ( }); let assetResourceIndexes: PublishedProjectBundle["assetResourceIndexes"]; if (assetQueryResources.length > 0) { - const assetClient = createAssetClient(); + const assetClient = createAssetClientWithResourceIndexStore(); await synchronizeCanonicalAssets({ client: context.postgrest.client, projectId: project.id, assetClient, }); - const canonicalEntries = await loadCanonicalAssetFileEntries({ - client: context.postgrest.client, - projectId: project.id, - }); + const { entries: canonicalEntries, metadataSnapshot } = + await loadCanonicalAssetFileSnapshot({ + client: context.postgrest.client, + projectId: project.id, + }); const indexedResources = await Promise.all( assetQueryResources.map(async ({ resourceId, query }) => ({ resourceId, @@ -198,9 +199,10 @@ const addProjectMetadata = async ( projectId: project.id, resources: indexedResources, entries: canonicalEntries, - read: assetClient.readFile, + read: assetClient.resourceIndexStore.read, referenceId: data.build.id, assetRevision: expectedAssetRevision, + metadataSnapshot, }); } diff --git a/apps/builder/app/shared/groq-completion.test.ts b/apps/builder/app/shared/groq-completion.test.ts index c8977b705ec9..fb432ff843dd 100644 --- a/apps/builder/app/shared/groq-completion.test.ts +++ b/apps/builder/app/shared/groq-completion.test.ts @@ -82,6 +82,60 @@ describe("GROQ completion", () => { ); }); + test.each([ + "*[properties[", + '*[properties["seo', + '*[properties["seo-title"]', + ])("replaces an incomplete bracket path from its root in %s", async (doc) => { + const result = await complete( + createGroqCompletionSource({ + catalog: { + format: "webstudio-builder-asset-field-catalog", + version: 1, + canonicalRevision: `sha256:${"2".repeat(64)}`, + documentCount: 1, + fields: { + 'properties["seo-title"]': { + types: ["string"], + occurrences: 1, + }, + }, + }, + }), + doc + ); + expect(result?.from).toBe(doc.indexOf("properties")); + expect(result?.options).toContainEqual( + expect.objectContaining({ label: 'properties["seo-title"]' }) + ); + expect(result?.options.map(({ label }) => label)).not.toContain("match"); + }); + + test("completes nested array traversal paths", async () => { + const doc = "*[properties.authors[]."; + const result = await complete( + createGroqCompletionSource({ + catalog: { + format: "webstudio-builder-asset-field-catalog", + version: 1, + canonicalRevision: `sha256:${"3".repeat(64)}`, + documentCount: 1, + fields: { + "properties.authors[].name": { + types: ["string"], + occurrences: 1, + }, + }, + }, + }), + doc + ); + expect(result?.from).toBe(doc.indexOf("properties")); + expect(result?.options).toContainEqual( + expect.objectContaining({ label: "properties.authors[].name" }) + ); + }); + test("displays observed types, optionality, and mixed-type warnings", async () => { const result = await complete( createGroqCompletionSource({ @@ -114,6 +168,9 @@ describe("GROQ completion", () => { test("uses the syntax tree to avoid suggestions inside strings", async () => { const source = createGroqCompletionSource(); await expect(complete(source, '*[name == "partial')).resolves.toBeNull(); + await expect( + complete(source, `*[name == 'properties["partial`) + ).resolves.toBeNull(); const filterResult = await complete(source, "*[name == "); expect(filterResult?.options.map(({ label }) => label)).toContain("match"); diff --git a/apps/builder/app/shared/groq-completion.ts b/apps/builder/app/shared/groq-completion.ts index c94697027a65..0f4799b6b107 100644 --- a/apps/builder/app/shared/groq-completion.ts +++ b/apps/builder/app/shared/groq-completion.ts @@ -48,6 +48,12 @@ const groqVocabulary: Completion[] = [ { label: "upper()", type: "function", apply: "upper()" }, ]; +// Match dotted paths and JSON-quoted bracket segments, including incomplete +// segments while the user is typing (for example `properties["seo`). +const groqFieldPath = + /(?:[A-Za-z_][A-Za-z0-9_]*)(?:(?:\.[A-Za-z_][A-Za-z0-9_]*)|(?:\[\])|(?:\["(?:[^"\\]|\\.)*(?:"\])?))*[.\[]?$/; +const validGroqFieldPath = new RegExp(`^${groqFieldPath.source}`); + const getSyntaxContext = (context: CompletionContext) => { const names: string[] = []; let node: SyntaxNode | null = syntaxTree(context.state).resolveInner( @@ -94,14 +100,14 @@ const getFieldCompletions = ({ export const createGroqCompletionSource = (configuration: GroqCompletionConfiguration = {}) => (context: CompletionContext): CompletionResult | null => { - const token = context.matchBefore( - /(?:\$[A-Za-z0-9_]*|[A-Za-z_][A-Za-z0-9_.$]*)/ - ); + const parameterToken = context.matchBefore(/\$[A-Za-z0-9_]*/); + const fieldToken = context.matchBefore(groqFieldPath); + const token = parameterToken ?? fieldToken; if (token === null && context.explicit === false) { return null; } const text = token?.text ?? ""; - if (text.startsWith("$")) { + if (parameterToken !== null) { return { from: token?.from ?? context.pos, options: (configuration.parameterNames ?? []).map((name) => ({ @@ -114,19 +120,20 @@ export const createGroqCompletionSource = } const syntaxContext = getSyntaxContext(context); - if ( - syntaxContext.some((name) => - ["String", "DoubleString", "SingleString"].includes(name) - ) - ) { + const insideString = syntaxContext.includes("String"); + const insideBracketAttribute = + text.includes('["') && syntaxContext.includes("DoubleString"); + if (insideString && insideBracketAttribute === false) { return null; } - const inDottedPath = - text.includes(".") || syntaxContext.includes("DotAccess"); + const inFieldPath = + text.includes(".") || + text.includes("[") || + syntaxContext.includes("DotAccess"); const fields = getFieldCompletions(configuration); return { from: token?.from ?? context.pos, - options: inDottedPath ? fields : [...fields, ...groqVocabulary], - validFor: /^[A-Za-z_][A-Za-z0-9_.$]*$/, + options: inFieldPath ? fields : [...fields, ...groqVocabulary], + validFor: validGroqFieldPath, }; }; diff --git a/apps/builder/app/shared/sync/patch/patch-service.server.test.ts b/apps/builder/app/shared/sync/patch/patch-service.server.test.ts index 30b3f3f754b9..9a3c8e9a6a78 100644 --- a/apps/builder/app/shared/sync/patch/patch-service.server.test.ts +++ b/apps/builder/app/shared/sync/patch/patch-service.server.test.ts @@ -5,15 +5,6 @@ const authorizePatchEntries = vi.hoisted(() => vi.fn()); const createContentModeCapabilities = vi.hoisted(() => vi.fn(() => ({ capabilities: true })) ); -const synchronizeAssetResourceIndexQueries = vi.hoisted(() => vi.fn()); -const synchronizeCanonicalAsset = vi.hoisted(() => vi.fn()); -const synchronizeAllCanonicalAssetStandardMetadata = vi.hoisted(() => vi.fn()); -const updateAssetResourceIndexesAfterCanonicalChange = vi.hoisted(() => - vi.fn() -); -const createAssetClient = vi.hoisted(() => - vi.fn(() => ({ resourceIndexStore: {} })) -); vi.mock("@webstudio-is/project/index.server", () => ({ patchLoadedBuild, @@ -25,23 +16,62 @@ vi.mock("./patch-auth.server", () => ({ createContentModeCapabilities, createWriterContext: vi.fn(), })); -vi.mock("@webstudio-is/asset-uploader/index.server", () => ({ - synchronizeAssetResourceIndexQueries, - synchronizeCanonicalAsset, - synchronizeAllCanonicalAssetStandardMetadata, - updateAssetResourceIndexesAfterCanonicalChange, -})); -vi.mock("../../asset-client", () => ({ - createAssetClient, -})); - import { applyPatchRequest, loadAuthorizedPatchState, } from "./patch-service.server"; +import { synchronizeAssetResourcesAfterBuildPatch } from "../../synchronize-asset-resource-patch.server"; import type { AppContext } from "@webstudio-is/trpc-interface/index.server"; import type { NormalizedPatchRequest } from "./patch-normalize.server"; +const synchronizeAssetResourceIndexQueries = vi.fn(); +const synchronizeCanonicalAsset = vi.fn(); +const synchronizeAllCanonicalAssetStandardMetadata = vi.fn(); +const updateAssetResourceIndexesAfterCanonicalChange = vi.fn(); +const createAssetClient = vi.fn(() => ({ + resourceIndexStore: { + putIfAbsent: vi.fn(async ({ checksum }: { checksum: string }) => ({ + status: "created" as const, + checksum, + })), + read: vi.fn(async () => ({ + data: { + async *[Symbol.asyncIterator]() {}, + }, + })), + }, + readFile: vi.fn(async () => ({ + data: { + async *[Symbol.asyncIterator]() {}, + }, + })), + uploadFile: vi.fn(async () => ({ format: "file", size: 0, meta: {} })), +})); +const synchronizationDependencies = { + synchronizeAssetResourceIndexQueries, + synchronizeCanonicalAsset, + synchronizeAllCanonicalAssetStandardMetadata, + updateAssetResourceIndexesAfterCanonicalChange, + createAssetClient, +} satisfies Exclude< + Parameters[1], + undefined +>; +const patchDependencies = { + synchronizeAssetResourcesAfterBuildPatch: ( + input: Parameters[0] + ) => + synchronizeAssetResourcesAfterBuildPatch( + input, + synchronizationDependencies + ), +}; + +const applyPatchRequestForTest = ( + context: AppContext, + patch: NormalizedPatchRequest +) => applyPatchRequest(context, patch, patchDependencies); + const buildRow = { projectId: "project-1", version: 3, @@ -161,7 +191,7 @@ describe("applyPatchRequest", () => { })); await expect( - applyPatchRequest(createContext(), { + applyPatchRequestForTest(createContext(), { ...patch, entries: [resourceEntry], }) @@ -175,6 +205,66 @@ describe("applyPatchRequest", () => { ); }); + test("does not repeat canonical reads or changed-query builds for a combined patch", async () => { + const combinedEntry = { + ...patch.entries[0], + transaction: { + id: "tx-combined", + payload: [ + { namespace: "resources", patches: [] }, + { + namespace: "assets", + patches: [ + { + op: "add", + path: ["asset-1"], + value: { id: "asset-1", name: "post.md" }, + }, + ], + }, + ], + } as never, + }; + authorizePatchEntries.mockResolvedValue({ + authorized: [{ entry: combinedEntry, context: { writer: 1 } }], + rejected: [], + }); + const nextResources = JSON.stringify([ + { + id: "posts", + name: "Posts", + control: "system", + method: "post", + url: JSON.stringify("/$resources/assets/query"), + headers: [], + body: '{query:"*[]"}', + }, + ]); + patchLoadedBuild.mockImplementation(async ({ build }) => ({ + status: "ok", + version: 4, + build: { ...build, version: 4, resources: nextResources }, + })); + synchronizeAssetResourceIndexQueries.mockResolvedValueOnce({ + deletedResourceIds: [], + updatedResourceIds: ["posts"], + }); + + await applyPatchRequestForTest(createContext(), { + ...patch, + entries: [combinedEntry], + }); + + expect(synchronizeCanonicalAsset).not.toHaveBeenCalled(); + expect(synchronizeAllCanonicalAssetStandardMetadata).not.toHaveBeenCalled(); + expect(updateAssetResourceIndexesAfterCanonicalChange).toHaveBeenCalledWith( + expect.objectContaining({ + changedAssetIds: ["asset-1"], + excludedResourceIds: ["posts"], + }) + ); + }); + test("maintains canonical paths and indexes after asset-folder patches", async () => { const folderEntry = { ...patch.entries[0], @@ -201,7 +291,10 @@ describe("applyPatchRequest", () => { })); await expect( - applyPatchRequest(createContext(), { ...patch, entries: [folderEntry] }) + applyPatchRequestForTest(createContext(), { + ...patch, + entries: [folderEntry], + }) ).resolves.toMatchObject({ status: "ok" }); expect(synchronizeAllCanonicalAssetStandardMetadata).toHaveBeenCalledWith({ client: expect.anything(), @@ -245,7 +338,10 @@ describe("applyPatchRequest", () => { })); await expect( - applyPatchRequest(createContext(), { ...patch, entries: [assetEntry] }) + applyPatchRequestForTest(createContext(), { + ...patch, + entries: [assetEntry], + }) ).resolves.toMatchObject({ status: "ok" }); expect(synchronizeCanonicalAsset).toHaveBeenCalledWith({ @@ -294,7 +390,7 @@ describe("applyPatchRequest", () => { build: { ...build, version: 4 }, })); - await applyPatchRequest(createContext(), { + await applyPatchRequestForTest(createContext(), { ...patch, entries: [assetEntry], }); @@ -307,6 +403,46 @@ describe("applyPatchRequest", () => { ); }); + test("does not reparse files after standard asset metadata changes", async () => { + const assetEntry = { + ...patch.entries[0], + transaction: { + id: "tx-asset-filename", + payload: [ + { + namespace: "assets", + patches: [ + { + op: "replace", + path: ["asset-1", "filename"], + value: "Public post title.md", + }, + ], + }, + ], + } as never, + }; + authorizePatchEntries.mockResolvedValue({ + authorized: [{ entry: assetEntry, context: { writer: 1 } }], + rejected: [], + }); + patchLoadedBuild.mockImplementation(async ({ build }) => ({ + status: "ok", + version: 4, + build: { ...build, version: 4 }, + })); + + await applyPatchRequestForTest(createContext(), { + ...patch, + entries: [assetEntry], + }); + + expect(synchronizeCanonicalAsset).not.toHaveBeenCalled(); + expect(updateAssetResourceIndexesAfterCanonicalChange).toHaveBeenCalledWith( + expect.objectContaining({ changedAssetIds: ["asset-1"] }) + ); + }); + test("skips index maintenance for asset descriptions", async () => { const assetEntry = { ...patch.entries[0], @@ -336,7 +472,7 @@ describe("applyPatchRequest", () => { build: { ...build, version: 4 }, })); - await applyPatchRequest(createContext(), { + await applyPatchRequestForTest(createContext(), { ...patch, entries: [assetEntry], }); @@ -365,7 +501,7 @@ describe("applyPatchRequest", () => { const context = createContext(); - const result = await applyPatchRequest(context, patch); + const result = await applyPatchRequestForTest(context, patch); expect(createContentModeCapabilities).not.toHaveBeenCalled(); expect(context.selectedColumns).toEqual([ @@ -416,7 +552,7 @@ describe("applyPatchRequest", () => { })); const context = createContext(); - const result = await applyPatchRequest(context, patch); + const result = await applyPatchRequestForTest(context, patch); expect(createContentModeCapabilities).toHaveBeenCalledWith({ breakpoints: JSON.stringify([]), @@ -450,7 +586,7 @@ describe("applyPatchRequest", () => { }); const context = createContext(); - const result = await applyPatchRequest(context, patch); + const result = await applyPatchRequestForTest(context, patch); expect(context.selectedColumns).toEqual(["projectId, version"]); expect(patchLoadedBuild).not.toHaveBeenCalled(); @@ -490,7 +626,7 @@ describe("applyPatchRequest", () => { })); const context = createContext(); - const result = await applyPatchRequest(context, assetsPatch); + const result = await applyPatchRequestForTest(context, assetsPatch); expect(context.selectedColumns).toEqual([ "projectId, version", @@ -532,7 +668,7 @@ describe("applyPatchRequest", () => { })) .mockResolvedValueOnce({ status: "error", errors: "entry failed" }); - const result = await applyPatchRequest(createContext(), patch); + const result = await applyPatchRequestForTest(createContext(), patch); expect(result).toEqual({ status: "partial", @@ -569,7 +705,7 @@ describe("applyPatchRequest", () => { build: { ...build, version: 5 }, })); - const result = await applyPatchRequest(createContext(), patch); + const result = await applyPatchRequestForTest(createContext(), patch); expect(patchLoadedBuild).toHaveBeenCalledTimes(2); expect(patchLoadedBuild).toHaveBeenNthCalledWith( @@ -633,7 +769,10 @@ describe("applyPatchRequest", () => { build: { ...build, version: 4 }, })); - const result = await applyPatchRequest(createContext(), duplicatePatch); + const result = await applyPatchRequestForTest( + createContext(), + duplicatePatch + ); expect(result).toEqual({ status: "partial", @@ -668,9 +807,9 @@ describe("applyPatchRequest", () => { })) .mockRejectedValueOnce(new Error("PostgREST unavailable")); - await expect(applyPatchRequest(createContext(), patch)).rejects.toThrow( - "PostgREST unavailable" - ); + await expect( + applyPatchRequestForTest(createContext(), patch) + ).rejects.toThrow("PostgREST unavailable"); }); }); diff --git a/apps/builder/app/shared/sync/patch/patch-service.server.ts b/apps/builder/app/shared/sync/patch/patch-service.server.ts index 6e683bb187d8..5f39d46b2e37 100644 --- a/apps/builder/app/shared/sync/patch/patch-service.server.ts +++ b/apps/builder/app/shared/sync/patch/patch-service.server.ts @@ -323,7 +323,8 @@ const applyAuthorizedEntries = async ({ export const applyPatchRequest = async ( context: AppContext, - patch: NormalizedPatchRequest + patch: NormalizedPatchRequest, + dependencies = { synchronizeAssetResourcesAfterBuildPatch } ): Promise => { const state = await loadBuildState(context, patch.buildId); assertBuildProject(state, patch); @@ -362,8 +363,9 @@ export const applyPatchRequest = async ( return applied; } - await synchronizeAssetResourcesAfterBuildPatch({ + await dependencies.synchronizeAssetResourcesAfterBuildPatch({ context, + buildId: patch.buildId, projectId: patch.projectId, previousResources: build?.resources, resources: applied.build?.resources, diff --git a/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts b/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts index 065cb3322884..9ae80a361ec1 100644 --- a/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts +++ b/apps/builder/app/shared/synchronize-asset-resource-patch.server.ts @@ -7,23 +7,72 @@ import { import type { BuildPatchChange } from "@webstudio-is/project/index.server"; import type { AppContext } from "@webstudio-is/trpc-interface/index.server"; import type { Resource } from "@webstudio-is/sdk"; -import { createAssetClient } from "./asset-client"; +import { createAssetClientWithResourceIndexStore } from "./asset-client"; + +const defaultDependencies = { + createAssetClient: createAssetClientWithResourceIndexStore, + synchronizeAllCanonicalAssetStandardMetadata, + synchronizeAssetResourceIndexQueries, + synchronizeCanonicalAsset, + updateAssetResourceIndexesAfterCanonicalChange, +}; const parseResources = (value: string) => JSON.parse(value) as Resource[]; -export const synchronizeAssetResourcesAfterBuildPatch = async ({ - context, - projectId, - previousResources, - resources, - changes, -}: { - context: AppContext; - projectId: string; - previousResources?: string; - resources?: string; - changes: readonly BuildPatchChange[]; -}) => { +const getCanonicalAssetChanges = (changes: readonly BuildPatchChange[]) => { + const changedAssetIds = new Set(); + const contentAssetIds = new Set(); + for (const { namespace, patches } of changes) { + if (namespace !== "assets") { + continue; + } + for (const { path } of patches) { + const assetId = path[0]; + if (typeof assetId !== "string") { + continue; + } + const field = path[1]; + if ( + path.length !== 1 && + field !== "name" && + field !== "filename" && + field !== "folderId" + ) { + continue; + } + changedAssetIds.add(assetId); + // Asset patching already synchronizes filename and folder metadata. + // Adds, deletes, and file swaps still require reading or removing the + // canonical content entry. + if (path.length === 1 || field === "name") { + contentAssetIds.add(assetId); + } + } + } + return { + changedAssetIds: [...changedAssetIds], + contentAssetIds: [...contentAssetIds], + }; +}; + +export const synchronizeAssetResourcesAfterBuildPatch = async ( + { + context, + buildId, + projectId, + previousResources, + resources, + changes, + }: { + context: AppContext; + buildId: string; + projectId: string; + previousResources?: string; + resources?: string; + changes: readonly BuildPatchChange[]; + }, + dependencies = defaultDependencies +) => { const hasResourceChanges = previousResources !== undefined && resources !== undefined && @@ -34,22 +83,27 @@ export const synchronizeAssetResourcesAfterBuildPatch = async ({ if (hasResourceChanges === false && assetChanges.length === 0) { return; } - let assetClient: ReturnType; + let assetClient: ReturnType; try { - assetClient = createAssetClient(); + assetClient = dependencies.createAssetClient(); } catch (error) { console.error("Asset resource client initialization failed", error); return; } + let canonicalAlreadySynchronized = false; + let updatedResourceIds: string[] = []; if (hasResourceChanges) { try { - await synchronizeAssetResourceIndexQueries({ + const result = await dependencies.synchronizeAssetResourceIndexQueries({ client: context.postgrest.client, assetClient, projectId, previousResources: parseResources(previousResources), resources: parseResources(resources), + source: { buildId, resources }, }); + updatedResourceIds = result.updatedResourceIds; + canonicalAlreadySynchronized = updatedResourceIds.length > 0; } catch (error) { console.error("Asset resource index synchronization failed", error); } @@ -60,51 +114,36 @@ export const synchronizeAssetResourcesAfterBuildPatch = async ({ } try { - if (assetChanges.some(({ namespace }) => namespace === "assetFolders")) { - await synchronizeAllCanonicalAssetStandardMetadata({ - client: context.postgrest.client, - projectId, - }); - } - const directAssetChanges = assetChanges.filter( - ({ namespace }) => namespace === "assets" + const hasFolderChanges = assetChanges.some( + ({ namespace }) => namespace === "assetFolders" ); - const canonicalAssetIds = [ - ...new Set( - directAssetChanges.flatMap(({ patches }) => - patches.flatMap(({ path }) => - typeof path[0] === "string" && - (path.length === 1 || - path[1] === "name" || - path[1] === "filename" || - path[1] === "folderId") - ? [path[0]] - : [] - ) - ) - ), - ]; - for (const assetId of canonicalAssetIds) { - await synchronizeCanonicalAsset({ + const { changedAssetIds, contentAssetIds } = + getCanonicalAssetChanges(assetChanges); + if (canonicalAlreadySynchronized === false && hasFolderChanges) { + await dependencies.synchronizeAllCanonicalAssetStandardMetadata({ client: context.postgrest.client, - assetClient, projectId, - assetId, }); } - const hasFolderChanges = assetChanges.some( - ({ namespace }) => namespace === "assetFolders" - ); - if (hasFolderChanges === false && canonicalAssetIds.length === 0) { + if (canonicalAlreadySynchronized === false) { + for (const assetId of contentAssetIds) { + await dependencies.synchronizeCanonicalAsset({ + client: context.postgrest.client, + assetClient, + projectId, + assetId, + }); + } + } + if (hasFolderChanges === false && changedAssetIds.length === 0) { return; } - await updateAssetResourceIndexesAfterCanonicalChange({ + await dependencies.updateAssetResourceIndexesAfterCanonicalChange({ client: context.postgrest.client, store: assetClient.resourceIndexStore, projectId, - changedAssetIds: hasFolderChanges - ? ["project-assets"] - : canonicalAssetIds, + changedAssetIds: hasFolderChanges ? ["project-assets"] : changedAssetIds, + excludedResourceIds: updatedResourceIds, }); } catch (error) { console.error("Asset resource index maintenance failed", error); diff --git a/apps/builder/docs/asset-resource-architecture.md b/apps/builder/docs/asset-resource-architecture.md index 860e201e8692..005a71ffc5c5 100644 --- a/apps/builder/docs/asset-resource-architecture.md +++ b/apps/builder/docs/asset-resource-architecture.md @@ -84,13 +84,13 @@ only when their corresponding UI needs them. Opening the editor may fetch the compact field catalog and one resource's index status; neither response contains Markdown bodies or public storage URLs. -The authenticated field-catalog and preview APIs read persisted -`AssetFileMetadata` rows. The status API reads one -`AssetResourceIndexState` row. None of these request paths invokes canonical -metadata backfill, synchronization, recovery, rebuild, or an asset-storage -reader. Full or partial file bytes are available only through the explicit -post-selection hydration contract, never through Builder startup or editor -initialization. +The authenticated field-catalog and preview APIs first synchronize canonical +metadata, then read persisted `AssetFileMetadata` rows. Synchronization reuses +unchanged rows and reads a bounded prefix only for new, changed, or inconsistent +Markdown assets. The status API reads one `AssetResourceIndexState` row and does +not read private index objects. Neither the catalog nor status response contains +file bodies. Preview reads full or partial bytes only when its explicit +post-selection hydration option requests them. The browser has no direct credentials for private index objects in R2. The Builder resource bridge supplies the outer authenticated request context while @@ -202,12 +202,11 @@ Every saved resource query is parsed before indexing. The parsed GROQ tree, rather than string matching, determines whether it references runtime parameters. -A **static query** has no parameter nodes. The index build evaluates it against -the matching canonical asset revision, validates its limits and JSON result, -and stores the materialized result plus the selected hydration identities. At -runtime the pinned result is returned without evaluating GROQ again. Changing -the query, publication policy, relevant asset metadata, or selected file -revision invalidates that materialization. +A **static query** has no parameter nodes. Like a parameterized query, its index +stores a deterministic candidate dataset and the validated query. The runtime +evaluates GROQ against that pinned dataset, then applies the result limit and +hydrates selected files. `queryMode` records whether parameters are required; +it does not select a separate materialized-result format in V1. A **parameterized query** contains at least one `$parameter`. Its index stores a deterministically ordered candidate dataset and the validated query needed for @@ -232,16 +231,16 @@ final runtime evaluation: Index construction accepts only persisted `CanonicalAssetFileEntry` values. It derives the asset revision and candidate documents from those entries and -rejects a caller-provided revision that does not match that exact entry set. It -has no asset-storage reader, so creating or changing a resource query cannot -reopen Markdown files. Content is read only by the separate post-selection -hydration path. +rejects a caller-provided revision that does not match that exact entry set. +Resource synchronization may first read bounded prefixes for canonical rows +that are new, changed, or in need of recovery; after that synchronization, the +pure index builder uses only persisted lightweight documents. Complete content +is read only by the separate post-selection hydration path. -The query hash is SHA-256 over the index-format version, exact UTF-8 query, and -static resource options that affect candidates or results. Runtime parameter -values and request-specific lower limits are not part of the query hash. The -immutable index revision identifies the resource ID, query hash, canonical -asset revision, and index checksum. +The query hash is SHA-256 over the exact UTF-8 query. Runtime parameter values +and request-specific limits are not part of it. The immutable index checksum +covers the complete versioned artifact, including the resource ID, query hash, +canonical asset revision, candidate documents, and format version. An active revision is usable only when all four values agree: @@ -251,19 +250,27 @@ An active revision is usable only when all four values agree: 4. The stored bytes match the index checksum. Building a replacement never mutates the active revision. Validation and any -static evaluation finish first; activation is one atomic reference change. A -unique attempt ID guards activation, failure, and cancellation, so concurrent +static evaluation finish first. The build transaction pre-registers or renews +the content-addressed revision with a 24-hour unreferenced lease before +uploading the immutable object, so garbage collection cannot race a reused +object and a crash between upload and activation remains discoverable. +Activation is one atomic reference change. A unique +attempt ID guards activation, failure, and cancellation, so concurrent builds with the same query and asset revisions cannot complete each other's state transitions. A failed or superseded build leaves the previous valid revision active but marks the resource stale until a matching revision can be activated. Publication -cannot proceed while it is stale. +never consumes a stale revision: it first reconciles the current query against +the publication snapshot, or builds a historical query snapshot in memory. Private index artifacts use a dedicated S3-compatible R2 store. The adapter writes with `If-None-Match: *`, records the index checksum as object metadata, and verifies that checksum with `HEAD` when an idempotent write finds an existing object. It sets no public ACL and must use a bucket with no public -object access, separate from public asset-delivery storage. +object access, separate from public asset-delivery storage. Builder configures +that bucket with `S3_RESOURCE_INDEX_BUCKET`; query-index operations fail closed +when remote asset storage is enabled without it. Filesystem development stores +indexes below `private/asset-resource-indexes`, outside the public asset tree. V1 does not deduplicate index objects across resources: `resourceId` is part of both object identity and the revision primary key. Cleanup therefore never @@ -273,9 +280,11 @@ snapshot. Published deployments contain their own static copy and no longer depend on the private object. After activation, query deletion, and snapshot release, best-effort cleanup runs once per mutation or publication batch, claims unreferenced revisions, deletes their private objects, and then removes -their database rows. Garbage claims are 15-minute -leases. A later cleanup worker rotates and resumes an expired claim, so a crash -before or after object deletion cannot permanently strand the revision row. +their database rows. Temporary `BUILD` references stop protecting revisions +after 24 hours, so a failed release cannot retain an object forever. Garbage +claims are 15-minute leases. A later +cleanup worker rotates and resumes an expired claim, so a crash before or after +object deletion cannot permanently strand the revision row. ## Schemaless frontmatter and publication @@ -457,9 +466,10 @@ resource definition call different endpoints from `/` and `/blog/post`. Generated route templates pass the normalized public request URL. Builder preview passes the source origin decoded from the project Builder URL rather than the internal `/rest/resources-loader` URL or project subdomain. Absolute -URLs remain unchanged. Local `/$resources/*` detection accepts either relative -or resolved absolute URLs, so system resources continue to dispatch internally -in both environments. +URLs remain unchanged. Local `/$resources/*` URLs remain relative through the +transport so generated adapters can distinguish them from an identical path on +an external origin. Generated dynamic adapters also require the request origin +to match the deployment origin before dispatching a local asset query. `createAssetResourceRequest` adapts the typed query contract to the existing resource transport. It produces `POST /$resources/assets/query`, marks it as a @@ -476,11 +486,11 @@ entry shadowing the other. Requests without an identity remain compatible when the published manifest contains only one matching query. Both cache layers hash the complete normalized POST body. Builder -`getResourceKey` and generated-runtime `getResourceCacheKey` have collision -tests that independently vary the GROQ query, runtime parameters, pinned index -revision, and content options. Cache-control remains part of the generated -Cache API key, while the request method prevents a POST query from colliding -with a legacy GET resource at the same URL. +`getResourceKey` and the published-runtime cache key have collision tests that +independently vary the GROQ query, runtime parameters, pinned index revision, +and content options. Published keys additionally include deployment, resource, +and immutable index revision identity. The published adapter handles only POST +asset-query requests, so they cannot collide with the legacy GET resource. Resource loading accepts an optional `AbortSignal` and timeout. Either aborts the underlying fetch and always removes its listener and timer. Caller @@ -655,22 +665,22 @@ frontmatter and does not add 1,000 source files to the repository. Baseline captured on 2026-07-18 with Node 22.22.1 on Darwin: -| Measurement | Result | -| ----------------------------------------------------------- | ----------------------------------------: | -| Initial frontmatter backfill | 282.890 ms, 1,000 reads | -| One-file metadata plus index update | 33.634 ms, 1 read and 999 unchanged reads | -| Query index cold build | 59.920 ms | -| Warm rebuild median / p95 | 32.816 / 35.148 ms | -| Changed-query rebuild median / p95 | 32.444 / 32.706 ms | -| Public candidates | 900 | -| Index JSON / gzip | 433,249 / 31,485 bytes | -| Cold JSON parse plus integrity verification median / p95 | 16.752 / 22.900 ms | -| Runtime minified / gzip bundle | 449,344 / 102,176 bytes | -| Warm listing median / p95 | 4.258 / 4.871 ms | -| `$slug` selection plus complete-file hydration median / p95 | 1.111 / 1.575 ms | -| Estimated parsed-index heap per copy | 569,809 bytes | -| Published static files | 900 Markdown + 1 index | -| Generated TypeScript index bytes | 0 | +| Measurement | Result | +| ----------------------------------------------------------- | ------------------------------------------------: | +| Initial frontmatter backfill | 282.890 ms, 1,000 reads | +| One-file metadata plus index update | 33.634 ms, 1 read and 999 unchanged files skipped | +| Query index cold build | 59.920 ms | +| Warm rebuild median / p95 | 32.816 / 35.148 ms | +| Changed-query rebuild median / p95 | 32.444 / 32.706 ms | +| Public candidates | 900 | +| Index JSON / gzip | 433,249 / 31,485 bytes | +| Cold JSON parse plus integrity verification median / p95 | 16.752 / 22.900 ms | +| Runtime minified / gzip bundle | 449,344 / 102,176 bytes | +| Warm listing median / p95 | 4.258 / 4.871 ms | +| `$slug` selection plus complete-file hydration median / p95 | 1.111 / 1.575 ms | +| Estimated parsed-index heap per copy | 569,809 bytes | +| Published static files | 900 Markdown + 1 index | +| Generated TypeScript index bytes | 0 | Storage-operation timings in this local benchmark cover serialization, integrity checks, and store calls with an in-memory immutable-store adapter; @@ -680,7 +690,7 @@ baselines for this machine, not universal latency guarantees. ## Operations and rollout -Operational metrics are defined by `AssetResourceOperationalMetrics`: +Before production rollout, the hosting telemetry adapter must expose: - Active, indexing, failed, and stale index counts. - Oldest stale-index age. @@ -688,11 +698,12 @@ Operational metrics are defined by `AssetResourceOperationalMetrics`: - Orphaned immutable objects. - Garbage-collection failures. -Alerts fire for any failed build, oversized attempt, orphan, or GC failure, and -when an index remains stale for more than 15 minutes. Build duration, immutable -store latency, index bytes, query latency, hydration bytes, and cache hit rate -should additionally be emitted as tagged histograms/counters by the hosting -telemetry adapter using project-safe identifiers. +Configure alerts for any failed build, oversized attempt, orphan, or GC failure, +and when an index remains stale for more than 15 minutes. The adapter must also +emit build duration, immutable-store latency, index bytes, query latency, +hydration bytes, and cache hit rate as tagged histograms or counters using +project-safe identifiers. This instrumentation is a rollout requirement, not +part of the asset-resource package itself. `assetResource` defaults to disabled. Keep rollout experimental until production canaries confirm the 1,000-file memory, CPU, publish-time, static-file-count, diff --git a/fixtures/ssg-cloudflare-pages/app/asset-resource-fetch.ts b/fixtures/ssg-cloudflare-pages/app/asset-resource-fetch.ts index c058910b8184..e77ae6a279cd 100644 --- a/fixtures/ssg-cloudflare-pages/app/asset-resource-fetch.ts +++ b/fixtures/ssg-cloudflare-pages/app/asset-resource-fetch.ts @@ -85,6 +85,7 @@ export const createSsgAssetResourceFetch = ({ manifest: Parameters[0]["manifest"]; }) => createPublishedAssetResourceFetch({ + baseUrl: "https://webstudio.local", deploymentId, manifest, fetchAsset: fetchSsgPublicAsset, diff --git a/fixtures/ssg-netlify-by-project-id/app/asset-resource-fetch.ts b/fixtures/ssg-netlify-by-project-id/app/asset-resource-fetch.ts index c058910b8184..e77ae6a279cd 100644 --- a/fixtures/ssg-netlify-by-project-id/app/asset-resource-fetch.ts +++ b/fixtures/ssg-netlify-by-project-id/app/asset-resource-fetch.ts @@ -85,6 +85,7 @@ export const createSsgAssetResourceFetch = ({ manifest: Parameters[0]["manifest"]; }) => createPublishedAssetResourceFetch({ + baseUrl: "https://webstudio.local", deploymentId, manifest, fetchAsset: fetchSsgPublicAsset, diff --git a/packages/asset-resource/src/blog-e2e.test.ts b/packages/asset-resource/src/blog-e2e.test.ts index d732400eefd9..4de6288d6b21 100644 --- a/packages/asset-resource/src/blog-e2e.test.ts +++ b/packages/asset-resource/src/blog-e2e.test.ts @@ -79,6 +79,7 @@ describe("published Markdown blog end-to-end", () => { }; }); const runtimeFetch = createPublishedAssetResourceFetch({ + baseUrl: "https://blog.example", deploymentId: "blog-deployment", manifest, fetchAsset, diff --git a/packages/asset-resource/src/candidate-selection.ts b/packages/asset-resource/src/candidate-selection.ts index 9fcaae0f8ac3..d2c1e0308c8b 100644 --- a/packages/asset-resource/src/candidate-selection.ts +++ b/packages/asset-resource/src/candidate-selection.ts @@ -1,6 +1,7 @@ import { evaluate, type ExprNode } from "groq-js/1"; import type { AssetFileDocument } from "@webstudio-is/sdk"; import { isGroqAstNode, visitGroqAst } from "./groq-ast"; +import { compareStrings } from "./stable-json"; const assetResourceCandidatePolicyV1 = { records: "safe-static-filter-superset", @@ -8,16 +9,6 @@ const assetResourceCandidatePolicyV1 = { content: "reference-only", } as const; -const compareStrings = (left: string, right: string) => { - if (left < right) { - return -1; - } - if (left > right) { - return 1; - } - return 0; -}; - export const getAssetResourceParameterNames = (tree: ExprNode) => { const names = new Set(); visitGroqAst(tree, (node) => { diff --git a/packages/asset-resource/src/canonical.ts b/packages/asset-resource/src/canonical.ts index a943fcb198ca..cd1560d0efa6 100644 --- a/packages/asset-resource/src/canonical.ts +++ b/packages/asset-resource/src/canonical.ts @@ -1,4 +1,5 @@ import { assetFileDocument, type AssetFileDocument } from "@webstudio-is/sdk"; +import { compareStrings } from "./stable-json"; export type AssetFileMetadataInput = { id: string; @@ -53,19 +54,9 @@ const getObservedType = (value: unknown): ObservedFieldType => { }; const identifier = /^[A-Za-z_][A-Za-z0-9_]*$/; -const appendFieldPath = (path: string, key: string) => +export const appendAssetFieldPath = (path: string, key: string) => identifier.test(key) ? `${path}.${key}` : `${path}[${JSON.stringify(key)}]`; -const compareStrings = (left: string, right: string) => { - if (left < right) { - return -1; - } - if (left > right) { - return 1; - } - return 0; -}; - export const getFieldContributions = ( properties: AssetFileDocument["properties"] ): FieldContribution[] => { @@ -82,7 +73,7 @@ export const getFieldContributions = ( if (value !== null && typeof value === "object") { for (const key of Object.keys(value).sort()) { add( - appendFieldPath(path, key), + appendAssetFieldPath(path, key), (value as Record)[key] ); } @@ -90,7 +81,7 @@ export const getFieldContributions = ( }; for (const key of Object.keys(properties).sort()) { - add(appendFieldPath("properties", key), properties[key]); + add(appendAssetFieldPath("properties", key), properties[key]); } return Array.from(contributions.values()).sort( (left, right) => diff --git a/packages/asset-resource/src/field-catalog.ts b/packages/asset-resource/src/field-catalog.ts index 5a24e7945808..83b210d48de8 100644 --- a/packages/asset-resource/src/field-catalog.ts +++ b/packages/asset-resource/src/field-catalog.ts @@ -9,7 +9,7 @@ import { type FieldContribution, type ObservedFieldType, } from "./canonical"; -import { serializeJsonDeterministically } from "./stable-json"; +import { compareStrings, serializeJsonDeterministically } from "./stable-json"; import { sha256 } from "./sha256"; export type AggregatedFieldType = { @@ -37,16 +37,6 @@ export type AssetFieldCatalog = AggregatedAssetFieldCatalog & { canonicalRevision: string; }; -const compareStrings = (left: string, right: string) => { - if (left < right) { - return -1; - } - if (left > right) { - return 1; - } - return 0; -}; - export const computeCanonicalAssetRevision = async ( entries: readonly CanonicalAssetFileEntry[] ) => { diff --git a/packages/asset-resource/src/hydration.test.ts b/packages/asset-resource/src/hydration.test.ts index 6d67fe2e5ab2..5b7b889607a0 100644 --- a/packages/asset-resource/src/hydration.test.ts +++ b/packages/asset-resource/src/hydration.test.ts @@ -248,4 +248,39 @@ describe("selected asset content hydration", () => { content: { private: { text: "secret" } }, }); }); + + test("waits for every started content read before propagating a failure", async () => { + const failed = createDocument("failed", "x"); + const slow = createDocument("slow", "y"); + let release: (() => void) | undefined; + let slowReadCompleted = false; + const read: AssetResourceContentReader = async (contentRef) => { + if (contentRef === failed.contentRef) { + throw new Error("read failed"); + } + await new Promise((resolve) => { + release = resolve; + }); + slowReadCompleted = true; + return { + data: { + async *[Symbol.asyncIterator]() { + yield encoder.encode("y"); + }, + }, + }; + }; + const pending = hydrateAssetResourceResult({ + result: [identity(failed), identity(slow)], + documents: [failed, slow], + options: { mode: "full" }, + read, + }); + + await expect.poll(() => release).toBeTypeOf("function"); + release?.(); + + await expect(pending).rejects.toThrow("read failed"); + expect(slowReadCompleted).toBe(true); + }); }); diff --git a/packages/asset-resource/src/hydration.ts b/packages/asset-resource/src/hydration.ts index 33842d375618..c5bf17ca527d 100644 --- a/packages/asset-resource/src/hydration.ts +++ b/packages/asset-resource/src/hydration.ts @@ -291,7 +291,7 @@ export const hydrateAssetResourceResult = async ({ }); } }; - await Promise.all( + const settlements = await Promise.allSettled( Array.from( { length: Math.min( @@ -302,6 +302,13 @@ export const hydrateAssetResourceResult = async ({ worker ) ); + const failure = settlements.find( + (settlement): settlement is PromiseRejectedResult => + settlement.status === "rejected" + ); + if (failure !== undefined) { + throw failure.reason; + } return { content, hydratedFileCount: selected.length, diff --git a/packages/asset-resource/src/index-storage.test.ts b/packages/asset-resource/src/index-storage.test.ts index de8850e72c4e..3beaad69d85d 100644 --- a/packages/asset-resource/src/index-storage.test.ts +++ b/packages/asset-resource/src/index-storage.test.ts @@ -40,6 +40,10 @@ describe("resource index persistence", () => { status: "created", }); expect(result.key).toContain("project%2F%2E%2E%2Fprivate"); + expect(result.key).toContain(encodeURIComponent(index.assetRevision)); + expect(result.key).toContain( + `${encodeURIComponent(index.integrity.checksum)}.json` + ); expect(putIfAbsent).toHaveBeenCalledWith( expect.objectContaining({ key: result.key, diff --git a/packages/asset-resource/src/index-storage.ts b/packages/asset-resource/src/index-storage.ts index 2aefbdcea97e..f634ccc98fe4 100644 --- a/packages/asset-resource/src/index-storage.ts +++ b/packages/asset-resource/src/index-storage.ts @@ -14,6 +14,10 @@ export type ImmutableAssetResourceIndexStore = { status: "created" | "exists"; checksum: string; }>; + read?: (key: string) => Promise<{ + data: AsyncIterable; + contentLength?: number; + }>; delete?: (key: string) => Promise<"deleted" | "missing">; }; @@ -38,7 +42,8 @@ export const getAssetResourceIndexObjectKey = ({ encodeKeySegment(index.resourceId), "indexes", encodeKeySegment(index.queryHash), - `${encodeKeySegment(index.assetRevision)}.json`, + encodeKeySegment(index.assetRevision), + `${encodeKeySegment(index.integrity.checksum)}.json`, ].join("/"); export const persistAssetResourceIndex = async ({ diff --git a/packages/asset-resource/src/index.test.ts b/packages/asset-resource/src/index.test.ts index 2ac4de23c87c..c39ba4abd8f9 100644 --- a/packages/asset-resource/src/index.test.ts +++ b/packages/asset-resource/src/index.test.ts @@ -78,7 +78,7 @@ describe("asset resource request transport", () => { expect(fetch).toHaveBeenCalledOnce(); const [url, init] = fetch.mock.calls[0]; - expect(url).toBe("https://example.com/$resources/assets/query"); + expect(url).toBe("/$resources/assets/query"); expect(init).toEqual({ method: "post", headers: new Headers([["content-type", "application/json"]]), diff --git a/packages/asset-resource/src/markdown.test.ts b/packages/asset-resource/src/markdown.test.ts index e3f552216350..42eea4f0626b 100644 --- a/packages/asset-resource/src/markdown.test.ts +++ b/packages/asset-resource/src/markdown.test.ts @@ -128,6 +128,18 @@ tags: [web, studio] extractMarkdownFrontmatter("---\ntitle: first\ntitle: second\n---\n") ).rejects.toMatchObject({ code: "FRONTMATTER_INVALID" }); }); + + test("rejects tagged YAML values that are not JSON", async () => { + for (const value of [ + "!!timestamp 2020-01-01", + "!!set\n item: null", + "!!omap\n - item: value", + ]) { + await expect( + extractMarkdownFrontmatter(`---\nvalue: ${value}\n---\n`) + ).rejects.toMatchObject({ code: "FRONTMATTER_INVALID" }); + } + }); }); describe("Markdown body and excerpt extraction", () => { diff --git a/packages/asset-resource/src/markdown.ts b/packages/asset-resource/src/markdown.ts index 2bfc796e1b28..c78ea9a24766 100644 --- a/packages/asset-resource/src/markdown.ts +++ b/packages/asset-resource/src/markdown.ts @@ -280,6 +280,12 @@ const parseYamlProperties = ( return input.map((item) => normalize(item, depth + 1)); } if (typeof input === "object") { + if (Object.getPrototypeOf(input) !== Object.prototype) { + throw new MarkdownMetadataError( + "FRONTMATTER_INVALID", + "Markdown frontmatter contains a non-JSON object" + ); + } const result: Record = {}; for (const [key, child] of Object.entries(input)) { fields += 1; diff --git a/packages/asset-resource/src/published-runtime.test.ts b/packages/asset-resource/src/published-runtime.test.ts index 1ac075f265f0..f5684ce5fb65 100644 --- a/packages/asset-resource/src/published-runtime.test.ts +++ b/packages/asset-resource/src/published-runtime.test.ts @@ -60,6 +60,7 @@ const createRuntime = async () => { manifest, fetchAsset, runtimeFetch: createPublishedAssetResourceFetch({ + baseUrl: "https://site.example", deploymentId: "build-1", manifest, fetchAsset, @@ -88,6 +89,27 @@ const withResourceId = (request: Request, resourceId: string) => { describe("published asset resource runtime", () => { beforeEach(() => __testing.clearParsedIndexCache()); + test("accepts generated local resource URLs as relative strings", async () => { + const { manifest, fetchAsset } = await createRuntime(); + const fetchResource = createPublishedAssetResourceFetch({ + baseUrl: "https://site.example", + deploymentId: "deployment-1", + manifest, + fetchAsset, + }); + + const response = await fetchResource("/$resources/assets/query", { + method: "POST", + headers: { + "content-type": "application/json", + "x-webstudio-resource-id": "posts", + }, + body: JSON.stringify({ query, parameters: { slug: "hello" } }), + }); + + expect(response?.status).toBe(200); + }); + test("evaluates runtime parameters and caches one parsed immutable index per isolate", async () => { const { runtimeFetch, fetchAsset } = await createRuntime(); const first = await runtimeFetch(createQueryRequest()); @@ -153,7 +175,17 @@ describe("published asset resource runtime", () => { "https://external.example/data" ); expect(await fallbackResponse.text()).toBe("fallback"); - expect(fallback).toHaveBeenCalledOnce(); + + const externalQueryResponse = await generatedFetch( + "https://external.example/$resources/assets/query", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ query, parameters: { slug: "post" } }), + } + ); + expect(await externalQueryResponse.text()).toBe("fallback"); + expect(fallback).toHaveBeenCalledTimes(2); }); test("isolates parsed indexes for deployments that use the same public path", async () => { @@ -183,6 +215,7 @@ describe("published asset resource runtime", () => { ]; const secondFetchAsset = vi.fn(async () => Response.json(secondIndex)); const secondRuntime = createPublishedAssetResourceFetch({ + baseUrl: "https://site.example", deploymentId: "build-2", manifest: secondManifest, fetchAsset: secondFetchAsset, @@ -227,6 +260,7 @@ describe("published asset resource runtime", () => { ) ); const runtimeFetch = createPublishedAssetResourceFetch({ + baseUrl: "https://site.example", deploymentId: "build-1", manifest, fetchAsset, @@ -257,6 +291,7 @@ describe("published asset resource runtime", () => { }), }; const runtimeFetch = createPublishedAssetResourceFetch({ + baseUrl: "https://site.example", deploymentId: "build-1", manifest, fetchAsset, @@ -276,7 +311,7 @@ describe("published asset resource runtime", () => { expect(cache.put).toHaveBeenCalledOnce(); }); - test("rejects stale revisions and keys caches by deployment, revision, parameters, and hydration", async () => { + test("rejects stale revisions and keys caches by deployment, revision, parameters, hydration, and policy", async () => { const { runtimeFetch, manifest } = await createRuntime(); const staleRequest = createQueryRequest(); const body = await staleRequest.json(); @@ -325,6 +360,16 @@ describe("published asset resource runtime", () => { }), }) ); + const changedCachePolicyRequest = createQueryRequest(); + changedCachePolicyRequest.headers.set( + "cache-control", + "public, max-age=60" + ); + const changedCachePolicy = await getKey( + "build-1", + manifest[0], + changedCachePolicyRequest + ); expect( new Set([ first.url, @@ -332,7 +377,8 @@ describe("published asset resource runtime", () => { changedRevision.url, changedHydration.url, changedParameters.url, + changedCachePolicy.url, ]) - ).toHaveLength(5); + ).toHaveLength(6); }); }); diff --git a/packages/asset-resource/src/published-runtime.ts b/packages/asset-resource/src/published-runtime.ts index 7ab38c663112..bdd6ca9fc368 100644 --- a/packages/asset-resource/src/published-runtime.ts +++ b/packages/asset-resource/src/published-runtime.ts @@ -106,8 +106,14 @@ const failure = ({ status ); -const getRequest = (input: RequestInfo | URL, init?: RequestInit) => - new Request(input, init); +const getRequest = ( + input: RequestInfo | URL, + baseUrl: string | URL, + init?: RequestInit +) => + typeof input === "string" || input instanceof URL + ? new Request(new URL(input, baseUrl), init) + : new Request(input, init); export const getPublishedAssetResourceCacheKey = async ({ deploymentId, @@ -119,8 +125,9 @@ export const getPublishedAssetResourceCacheKey = async ({ request: Request; }) => { const body = await request.clone().text(); + const cacheControl = request.headers.get("cache-control"); const hash = await sha256Hex( - `${deploymentId}\n${entry.resourceId}\n${entry.revision}\n${body}` + `${deploymentId}\n${entry.resourceId}\n${entry.revision}\n${body}\n${cacheControl}` ); const url = new URL(request.url); url.searchParams.set("ws-asset-resource", hash); @@ -132,12 +139,15 @@ export const createPublishedAssetResourceFetch = ({ manifest, fetchAsset, cache, + baseUrl, }: { deploymentId: string; manifest: readonly PublishedAssetResourceManifestEntry[]; fetchAsset: PublishedAssetFetch; cache?: Pick; + baseUrl: string | URL; }) => { + const baseOrigin = new URL(baseUrl).origin; const entriesByQueryHash = new Map< string, PublishedAssetResourceManifestEntry[] @@ -151,7 +161,10 @@ export const createPublishedAssetResourceFetch = ({ input: RequestInfo | URL, init?: RequestInit ): Promise => { - const request = getRequest(input, init); + const request = getRequest(input, baseUrl, init); + if (new URL(request.url).origin !== baseOrigin) { + return; + } if ( new URL(request.url).pathname !== assetsQueryResourceUrl || request.method.toUpperCase() !== "POST" @@ -332,6 +345,7 @@ export const createGeneratedAssetResourceFetch = async ({ manifest, fetchAsset, cache, + baseUrl: request.url, }); return async (input, init) => (await fetchResource(input, init)) ?? fallback(input, init); diff --git a/packages/asset-resource/src/query-validation.test.ts b/packages/asset-resource/src/query-validation.test.ts index 1b61bc674be0..12fdd3227263 100644 --- a/packages/asset-resource/src/query-validation.test.ts +++ b/packages/asset-resource/src/query-validation.test.ts @@ -3,6 +3,7 @@ import { assetResourceLimits } from "@webstudio-is/sdk"; import { AssetResourceQueryValidationError, getAssetResourceReferencedFieldPaths, + getAssetResourceReferencedFieldPathsFromTree, validateAssetResourceQuery, } from "./query-validation"; @@ -105,4 +106,14 @@ describe("asset resource query validation", () => { "properties.title", ]); }); + + test("formats dynamic field names with valid bracket notation", () => { + const validated = validateAssetResourceQuery( + '*[properties["seo-title"] == "Post"]' + ); + + expect( + getAssetResourceReferencedFieldPathsFromTree(validated.tree) + ).toEqual(["properties", 'properties["seo-title"]']); + }); }); diff --git a/packages/asset-resource/src/query-validation.ts b/packages/asset-resource/src/query-validation.ts index 4b693bf55daf..5c7bb5b303ae 100644 --- a/packages/asset-resource/src/query-validation.ts +++ b/packages/asset-resource/src/query-validation.ts @@ -2,6 +2,7 @@ import { parse, type ExprNode } from "groq-js/1"; import { assetResourceLimits } from "@webstudio-is/sdk"; import { getAssetResourceParameterNames } from "./candidate-selection"; import { visitGroqAst } from "./groq-ast"; +import { appendAssetFieldPath } from "./canonical"; export class AssetResourceQueryValidationError extends Error { readonly code: "INVALID_QUERY" | "QUERY_COMPLEXITY_EXCEEDED"; @@ -143,11 +144,12 @@ const getAttributePath = (node: unknown): string | undefined => { const basePath = getAttributePath(attribute.base); return basePath === undefined ? attribute.name - : `${basePath}.${attribute.name}`; + : appendAssetFieldPath(basePath, attribute.name); }; -export const getAssetResourceReferencedFieldPaths = (query: string) => { - const { tree } = validateAssetResourceQuery(query); +export const getAssetResourceReferencedFieldPathsFromTree = ( + tree: ExprNode +) => { const paths = new Set(); visitGroqAst(tree, (node) => { const path = getAttributePath(node); @@ -157,3 +159,8 @@ export const getAssetResourceReferencedFieldPaths = (query: string) => { }); return Array.from(paths).sort(); }; + +export const getAssetResourceReferencedFieldPaths = (query: string) => + getAssetResourceReferencedFieldPathsFromTree( + validateAssetResourceQuery(query).tree + ); diff --git a/packages/asset-resource/src/resource-index.ts b/packages/asset-resource/src/resource-index.ts index 45525727eb2c..1fd3fc3b830b 100644 --- a/packages/asset-resource/src/resource-index.ts +++ b/packages/asset-resource/src/resource-index.ts @@ -7,19 +7,9 @@ import type { CanonicalAssetFileEntry } from "./canonical"; import { computeCanonicalAssetRevision } from "./field-catalog"; import { selectAssetResourceCandidates } from "./candidate-selection"; import { validateAssetResourceQuery } from "./query-validation"; -import { serializeJsonDeterministically } from "./stable-json"; +import { compareStrings, serializeJsonDeterministically } from "./stable-json"; import { sha256 } from "./sha256"; -const compareStrings = (left: string, right: string) => { - if (left < right) { - return -1; - } - if (left > right) { - return 1; - } - return 0; -}; - export type AssetResourceIndexInput = Omit< AssetResourceIndexV1, "parameterNames" | "documents" diff --git a/packages/asset-resource/src/stable-json.ts b/packages/asset-resource/src/stable-json.ts index f6d24e4dfd72..450257ad53cd 100644 --- a/packages/asset-resource/src/stable-json.ts +++ b/packages/asset-resource/src/stable-json.ts @@ -1,4 +1,4 @@ -const compareStrings = (left: string, right: string) => { +export const compareStrings = (left: string, right: string) => { if (left < right) { return -1; } diff --git a/packages/asset-resource/src/transport-parity.test.ts b/packages/asset-resource/src/transport-parity.test.ts index 5e3c9640efcd..83992ffd9c93 100644 --- a/packages/asset-resource/src/transport-parity.test.ts +++ b/packages/asset-resource/src/transport-parity.test.ts @@ -51,7 +51,7 @@ describe("Builder and generated resource transport parity", () => { expect(builderFetch.mock.calls).toEqual(generatedFetch.mock.calls); expect(builderFetch).toHaveBeenCalledWith( - "https://site.example/$resources/assets/query", + "/$resources/assets/query", expect.objectContaining({ method: "post", headers: new Headers([["content-type", "application/json"]]), diff --git a/packages/asset-uploader/src/async-utils.test.ts b/packages/asset-uploader/src/async-utils.test.ts new file mode 100644 index 000000000000..e35b6246bf2d --- /dev/null +++ b/packages/asset-uploader/src/async-utils.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, test } from "vitest"; +import { mapBounded, runBounded } from "./async-utils"; + +describe("bounded async work", () => { + test("limits concurrency and preserves result order", async () => { + let active = 0; + let maximumActive = 0; + const releases: Array<() => void> = []; + const pending = mapBounded([3, 2, 1], 2, async (value) => { + active += 1; + maximumActive = Math.max(maximumActive, active); + await new Promise((resolve) => releases.push(resolve)); + active -= 1; + return value * 2; + }); + + await expect.poll(() => releases.length).toBe(2); + releases.shift()?.(); + await expect.poll(() => releases.length).toBe(2); + releases.shift()?.(); + releases.shift()?.(); + + await expect(pending).resolves.toEqual([6, 4, 2]); + expect(maximumActive).toBe(2); + }); + + test("rejects invalid concurrency instead of silently skipping work", async () => { + await expect(runBounded([1], 0, async () => {})).rejects.toThrow( + "Concurrency must be a positive safe integer" + ); + }); + + test("waits for every started worker before propagating a failure", async () => { + let release: (() => void) | undefined; + let completed = false; + let rejected = false; + const pending = mapBounded(["fail", "slow"], 2, async (value) => { + if (value === "fail") { + throw new Error("failed"); + } + await new Promise((resolve) => { + release = resolve; + }); + completed = true; + return value; + }); + const observed = pending.catch((error) => { + rejected = true; + throw error; + }); + + await expect.poll(() => release).toBeTypeOf("function"); + expect(rejected).toBe(false); + release?.(); + + await expect(observed).rejects.toThrow("failed"); + expect(completed).toBe(true); + }); +}); diff --git a/packages/asset-uploader/src/async-utils.ts b/packages/asset-uploader/src/async-utils.ts new file mode 100644 index 000000000000..f31d4e6fe1b5 --- /dev/null +++ b/packages/asset-uploader/src/async-utils.ts @@ -0,0 +1,41 @@ +export const mapBounded = async ( + values: readonly Value[], + concurrency: number, + run: (value: Value) => Promise +) => { + if (Number.isSafeInteger(concurrency) === false || concurrency <= 0) { + throw new Error("Concurrency must be a positive safe integer"); + } + let cursor = 0; + const results = new Array(values.length); + const worker = async () => { + while (cursor < values.length) { + const index = cursor; + cursor += 1; + results[index] = await run(values[index]); + } + }; + const workers = Array.from( + { length: Math.min(concurrency, values.length) }, + worker + ); + const settlements = await Promise.allSettled(workers); + const failures = settlements.flatMap((settlement) => + settlement.status === "rejected" ? [settlement.reason] : [] + ); + if (failures.length === 1) { + throw failures[0]; + } + if (failures.length > 1) { + throw new AggregateError(failures, "Multiple bounded workers failed"); + } + return results; +}; + +export const runBounded = async ( + values: readonly Value[], + concurrency: number, + run: (value: Value) => Promise +) => { + await mapBounded(values, concurrency, run); +}; diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.test.ts b/packages/asset-uploader/src/canonical-metadata-backfill.test.ts index 6378b934047e..a2509438dd02 100644 --- a/packages/asset-uploader/src/canonical-metadata-backfill.test.ts +++ b/packages/asset-uploader/src/canonical-metadata-backfill.test.ts @@ -457,14 +457,19 @@ describe("canonical asset metadata synchronization", () => { client: testContext.postgrest.client, }) ).resolves.toBe(1); + const revision = createAssetContentRevision({ + storageName: "stored.md", + updatedAt: "2026-07-18T05:00:00.000Z", + size: 20, + }); expect(persisted).toMatchObject({ assetId: "asset-1", - revision: previousDocument.revision, + revision, document: { name: "new.md", path: "Blog/new.md", contentRef: "stored.md", - revision: previousDocument.revision, + revision, properties: { title: "Post" }, excerpt: "Body", }, diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.ts b/packages/asset-uploader/src/canonical-metadata-backfill.ts index c548e18d2618..fc2b92eb8c4d 100644 --- a/packages/asset-uploader/src/canonical-metadata-backfill.ts +++ b/packages/asset-uploader/src/canonical-metadata-backfill.ts @@ -22,6 +22,7 @@ import { loadCanonicalAssetFileEntriesForRecovery, replaceCanonicalAssetFileEntry, } from "./canonical-metadata-persistence"; +import { runBounded } from "./async-utils"; const markdownExtension = /\.md$/i; type CanonicalAssetClient = Pick & @@ -96,24 +97,6 @@ const loadUploadedAssets = async ( return (result.data ?? []) as UploadedAssetRow[]; }; -const runBounded = async ( - values: readonly Value[], - concurrency: number, - run: (value: Value) => Promise -) => { - let cursor = 0; - const worker = async () => { - while (cursor < values.length) { - const value = values[cursor]; - cursor += 1; - await run(value); - } - }; - await Promise.all( - Array.from({ length: Math.min(concurrency, values.length) }, worker) - ); -}; - type AssetFolderHierarchy = ReturnType; const getCanonicalMetadataSource = (asset: UploadedAssetRow) => ({ @@ -397,18 +380,32 @@ export const synchronizeCanonicalAssetStandardMetadata = async ({ loadAssetFoldersByProjectWithClient(projectId, client), ]); const assetsById = new Map(assets.map((asset) => [asset.id, asset])); + const entriesByAssetId = new Map(); + for (const entry of entries) { + const assetEntries = entriesByAssetId.get(entry.assetId) ?? []; + assetEntries.push(entry); + entriesByAssetId.set(entry.assetId, assetEntries); + } const folderMap = new Map(folders.map((folder) => [folder.id, folder])); const hierarchy = createAssetFolderHierarchy(folderMap); let updated = 0; - for (const entry of entries) { - const asset = assetsById.get(entry.assetId); - if (asset === undefined) { + for (const asset of assetsById.values()) { + const revision = createAssetContentRevision({ + storageName: asset.file.name, + updatedAt: asset.file.updatedAt, + size: asset.file.size, + }); + const assetEntries = entriesByAssetId.get(asset.id) ?? []; + const entry = + assetEntries.find((candidate) => candidate.revision === revision) ?? + (assetEntries.length === 1 ? assetEntries[0] : undefined); + if (entry === undefined) { continue; } const document = createCanonicalDocument({ asset, hierarchy, - revision: entry.revision, + revision, properties: entry.document.properties, excerpt: entry.document.excerpt, }); diff --git a/packages/asset-uploader/src/canonical-metadata-persistence.test.ts b/packages/asset-uploader/src/canonical-metadata-persistence.test.ts index d77a34da5386..b3eac7893aa3 100644 --- a/packages/asset-uploader/src/canonical-metadata-persistence.test.ts +++ b/packages/asset-uploader/src/canonical-metadata-persistence.test.ts @@ -10,6 +10,7 @@ import { deleteStaleCanonicalAssetFileEntries, loadCanonicalAssetFileEntries, loadCanonicalAssetFileEntry, + loadCanonicalAssetFileSnapshot, replaceCanonicalAssetFileEntry, } from "./canonical-metadata-persistence"; @@ -35,6 +36,7 @@ const entry = createCanonicalAssetFileEntry({ }); const row = { ...entry, + metadataToken: "metadata-token-1", document, fieldContributions: entry.fieldContributions, createdAt: "2026-07-18T00:00:00.000Z", @@ -63,6 +65,22 @@ describe("canonical asset metadata persistence", () => { ).resolves.toEqual(entry); }); + test("loads metadata tokens atomically with canonical entries", async () => { + server.use(db.get("AssetFileMetadata", () => json([row]))); + + await expect( + loadCanonicalAssetFileSnapshot({ + client: testContext.postgrest.client, + projectId: "project-1", + }) + ).resolves.toEqual({ + entries: [entry], + metadataSnapshot: [ + { assetId: "asset-1", metadataToken: "metadata-token-1" }, + ], + }); + }); + test("replaces older revisions for one canonical asset", async () => { server.use( db.post("rpc/replace_asset_file_metadata", async ({ request }) => { diff --git a/packages/asset-uploader/src/canonical-metadata-persistence.ts b/packages/asset-uploader/src/canonical-metadata-persistence.ts index 76267bd24b2d..d58dd5c54682 100644 --- a/packages/asset-uploader/src/canonical-metadata-persistence.ts +++ b/packages/asset-uploader/src/canonical-metadata-persistence.ts @@ -6,6 +6,14 @@ import type { Client, Database } from "@webstudio-is/postgrest/index.server"; import { assertPostgrestSuccess } from "./patch-utils"; type MetadataRow = Database["public"]["Tables"]["AssetFileMetadata"]["Row"]; +type StoredMetadataRow = MetadataRow & { + metadataToken: string; +}; + +export type CanonicalAssetMetadataSnapshot = { + assetId: string; + metadataToken: string; +}[]; export type CanonicalAssetMetadataSource = { storageName: string; @@ -118,7 +126,7 @@ const loadCanonicalAssetFileMetadataRows = async ({ client: Client; projectId: string; assetIds?: string[]; -}): Promise => { +}): Promise => { let query = client .from("AssetFileMetadata") .select() @@ -131,7 +139,7 @@ const loadCanonicalAssetFileMetadataRows = async ({ } const result = await query.order("assetId").order("revision"); assertPostgrestSuccess(result); - return result.data ?? []; + return (result.data ?? []) as StoredMetadataRow[]; }; export const loadCanonicalAssetFileEntries = async ({ @@ -151,6 +159,23 @@ export const loadCanonicalAssetFileEntries = async ({ return rows.map(parseMetadataRow); }; +export const loadCanonicalAssetFileSnapshot = async ({ + client, + projectId, +}: { + client: Client; + projectId: string; +}) => { + const rows = await loadCanonicalAssetFileMetadataRows({ client, projectId }); + return { + entries: rows.map(parseMetadataRow), + metadataSnapshot: rows.map(({ assetId, metadataToken }) => ({ + assetId, + metadataToken, + })), + }; +}; + export const loadCanonicalAssetFileEntriesForRecovery = async ({ client, projectId, diff --git a/packages/asset-uploader/src/client.ts b/packages/asset-uploader/src/client.ts index dfdc8456a54c..f392ce91ae5b 100644 --- a/packages/asset-uploader/src/client.ts +++ b/packages/asset-uploader/src/client.ts @@ -41,3 +41,9 @@ export type AssetClient = AssetUploadClient & { export type AssetClientWithResourceIndexStore = AssetClient & { resourceIndexStore: ImmutableAssetResourceIndexStore; }; + +export type AssetClientWithReadableResourceIndexStore = AssetClient & { + resourceIndexStore: ImmutableAssetResourceIndexStore & { + read: NonNullable; + }; +}; diff --git a/packages/asset-uploader/src/clients/fs/fs.ts b/packages/asset-uploader/src/clients/fs/fs.ts index cc1173ab1bac..1df1c3a57c49 100644 --- a/packages/asset-uploader/src/clients/fs/fs.ts +++ b/packages/asset-uploader/src/clients/fs/fs.ts @@ -1,20 +1,23 @@ -import type { AssetClientWithResourceIndexStore } from "../../client"; +import type { AssetClient } from "../../client"; import { uploadToFs } from "./upload"; import { readFromFs } from "./read"; import { createFsImmutableResourceIndexStore } from "./immutable-object"; type FsClientOptions = { fileDirectory: string; + resourceIndexDirectory?: string; maxUploadSize: number; }; -export const createFsClient = ( - options: FsClientOptions -): AssetClientWithResourceIndexStore => { +export const createFsClient = (options: FsClientOptions): AssetClient => { return { - resourceIndexStore: createFsImmutableResourceIndexStore( - options.fileDirectory - ), + ...(options.resourceIndexDirectory === undefined + ? {} + : { + resourceIndexStore: createFsImmutableResourceIndexStore( + options.resourceIndexDirectory + ), + }), uploadFile: (name, type, data, _assetInfoFallback, assetDataOverride) => uploadToFs({ name, diff --git a/packages/asset-uploader/src/clients/fs/immutable-object.test.ts b/packages/asset-uploader/src/clients/fs/immutable-object.test.ts index 3a24e67ee2c1..6badf3ba6df2 100644 --- a/packages/asset-uploader/src/clients/fs/immutable-object.test.ts +++ b/packages/asset-uploader/src/clients/fs/immutable-object.test.ts @@ -36,6 +36,13 @@ describe("filesystem immutable resource index storage", () => { await expect( readFile(join(directory, "projects/one/index.json"), "utf8") ).resolves.toBe("one"); + const stored = await store.read?.(object.key); + const chunks: Uint8Array[] = []; + for await (const chunk of stored?.data ?? []) { + chunks.push(chunk); + } + expect(Buffer.concat(chunks).toString("utf8")).toBe("one"); + expect(stored?.contentLength).toBe(3); await expect(store.delete?.(object.key)).resolves.toBe("deleted"); await expect(store.delete?.(object.key)).resolves.toBe("missing"); }); diff --git a/packages/asset-uploader/src/clients/fs/immutable-object.ts b/packages/asset-uploader/src/clients/fs/immutable-object.ts index 3af87d466d9d..9fcaf694c592 100644 --- a/packages/asset-uploader/src/clients/fs/immutable-object.ts +++ b/packages/asset-uploader/src/clients/fs/immutable-object.ts @@ -1,4 +1,5 @@ -import { mkdir, readFile, unlink, writeFile } from "node:fs/promises"; +import { createReadStream } from "node:fs"; +import { mkdir, readFile, stat, unlink, writeFile } from "node:fs/promises"; import { dirname, resolve, sep } from "node:path"; import type { ImmutableAssetResourceIndexStore } from "@webstudio-is/asset-resource"; @@ -35,6 +36,11 @@ export const createFsImmutableResourceIndexStore = ( throw error; } }, + read: async (key) => { + const path = resolvePath(key); + const file = await stat(path); + return { data: createReadStream(path), contentLength: file.size }; + }, delete: async (key) => { try { await unlink(resolvePath(key)); diff --git a/packages/asset-uploader/src/clients/s3/s3.ts b/packages/asset-uploader/src/clients/s3/s3.ts index e6e31f63c8e0..2c2a879bad03 100644 --- a/packages/asset-uploader/src/clients/s3/s3.ts +++ b/packages/asset-uploader/src/clients/s3/s3.ts @@ -1,9 +1,6 @@ import { Sha256 } from "@aws-crypto/sha256-js"; import { SignatureV4 } from "@smithy/signature-v4"; -import type { - AssetClient, - AssetClientWithResourceIndexStore, -} from "../../client"; +import type { AssetClient } from "../../client"; import { uploadToS3 } from "./upload"; import { readFromS3 } from "./read"; import { @@ -17,13 +14,18 @@ type S3ClientOptions = { accessKeyId: string; secretAccessKey: string; bucket: string; + resourceIndexBucket?: string; acl?: string; maxUploadSize: number; }; -export const createS3Client = ( - options: S3ClientOptions -): AssetClientWithResourceIndexStore => { +export const createS3Client = (options: S3ClientOptions): AssetClient => { + const resourceIndexBucket = options.resourceIndexBucket; + if (resourceIndexBucket === options.bucket) { + throw new Error( + "Resource indexes require a bucket distinct from the public asset bucket" + ); + } const signer = new SignatureV4({ credentials: { accessKeyId: options.accessKeyId, @@ -58,22 +60,33 @@ export const createS3Client = ( }; return { - resourceIndexStore: { - putIfAbsent: (object) => - putImmutableObjectToS3({ - signer, - endpoint: options.endpoint, - bucket: options.bucket, - object, + ...(resourceIndexBucket === undefined + ? {} + : { + resourceIndexStore: { + putIfAbsent: (object) => + putImmutableObjectToS3({ + signer, + endpoint: options.endpoint, + bucket: resourceIndexBucket, + object, + }), + read: (key) => + readFromS3({ + signer, + name: key, + endpoint: options.endpoint, + bucket: resourceIndexBucket, + }), + delete: (key) => + deleteImmutableObjectFromS3({ + signer, + endpoint: options.endpoint, + bucket: resourceIndexBucket, + key, + }), + }, }), - delete: (key) => - deleteImmutableObjectFromS3({ - signer, - endpoint: options.endpoint, - bucket: options.bucket, - key, - }), - }, uploadFile, readFile: (name, range) => readFromS3({ diff --git a/packages/asset-uploader/src/clients/storage-separation.test.ts b/packages/asset-uploader/src/clients/storage-separation.test.ts new file mode 100644 index 000000000000..3fff7579c4b3 --- /dev/null +++ b/packages/asset-uploader/src/clients/storage-separation.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from "vitest"; +import { createFsClient } from "./fs/fs"; +import { createS3Client } from "./s3/s3"; + +const s3Options = { + endpoint: "https://storage.example", + region: "auto", + accessKeyId: "access", + secretAccessKey: "secret", + bucket: "public-assets", + maxUploadSize: 1, +}; + +describe("resource index storage separation", () => { + test("does not put filesystem indexes in the asset directory by default", () => { + expect( + createFsClient({ fileDirectory: "public/assets", maxUploadSize: 1 }) + .resourceIndexStore + ).toBeUndefined(); + expect( + createFsClient({ + fileDirectory: "public/assets", + resourceIndexDirectory: "private/indexes", + maxUploadSize: 1, + }).resourceIndexStore + ).toBeDefined(); + }); + + test("requires a distinct S3 bucket before exposing an index store", () => { + expect(createS3Client(s3Options).resourceIndexStore).toBeUndefined(); + expect(() => + createS3Client({ + ...s3Options, + resourceIndexBucket: s3Options.bucket, + }) + ).toThrow("distinct from the public asset bucket"); + expect( + createS3Client({ + ...s3Options, + resourceIndexBucket: "private-indexes", + }).resourceIndexStore + ).toBeDefined(); + }); +}); diff --git a/packages/asset-uploader/src/index.server.ts b/packages/asset-uploader/src/index.server.ts index b588f23fa4eb..3f73a11d8fe5 100644 --- a/packages/asset-uploader/src/index.server.ts +++ b/packages/asset-uploader/src/index.server.ts @@ -21,3 +21,8 @@ export * from "./resource-index-rebuild"; export * from "./resource-index-snapshot"; export * from "./resource-index-lifecycle"; export * from "./resource-index-garbage-collection"; +export type { + AssetClient, + AssetClientWithResourceIndexStore, + AssetClientWithReadableResourceIndexStore, +} from "./client"; diff --git a/packages/asset-uploader/src/patch.test.ts b/packages/asset-uploader/src/patch.test.ts index 0daa5e275976..2623acc6c06d 100644 --- a/packages/asset-uploader/src/patch.test.ts +++ b/packages/asset-uploader/src/patch.test.ts @@ -18,6 +18,7 @@ import { patchAssetFoldersWithClient, } from "./folder-persistence"; import { patchAssets } from "./patch"; +import { createAssetContentRevision } from "./canonical-metadata-backfill"; import type { Patch } from "immer"; const server = createTestServer(); @@ -376,6 +377,7 @@ describe("patchAssets (msw)", () => { test("persists moving an asset to root as null", async () => { const projectId = uid(); + const updatedAt = "2026-07-18T00:00:00.000Z"; let localAssetRow = { ...assetRow, projectId, @@ -386,6 +388,7 @@ describe("patchAssets (msw)", () => { name: "stored.md", format: "md", size: 100, + updatedAt, meta: "{}", }, }; @@ -399,7 +402,11 @@ describe("patchAssets (msw)", () => { extension: "md", mimeType: "text/markdown", size: 100, - revision: "file:stored.md:revision:100", + revision: createAssetContentRevision({ + storageName: "stored.md", + updatedAt, + size: 100, + }), contentRef: "stored.md", properties: { title: "Post" }, excerpt: "Body", diff --git a/packages/asset-uploader/src/resource-index-build.test.ts b/packages/asset-uploader/src/resource-index-build.test.ts index 4816f6b86c5f..95bed2621e57 100644 --- a/packages/asset-uploader/src/resource-index-build.test.ts +++ b/packages/asset-uploader/src/resource-index-build.test.ts @@ -42,8 +42,18 @@ describe("resource index build and activation", () => { p_query: `*[properties.slug == $slug]`, p_query_hash: expect.stringMatching(/^sha256:/), p_asset_revision: expect.stringMatching(/^sha256:/), + p_revision: expect.stringMatching(/^sha256:/), + p_checksum: expect.stringMatching(/^sha256:/), + p_object_key: expect.stringContaining( + "projects/project-1/resources/posts" + ), + p_metadata_snapshot: [ + { assetId: "post-1", metadataToken: "metadata-token-1" }, + ], + p_build_id: "build-1", + p_resources: "[]", }); - return json(null); + return json(true); }), db.post("rpc/activate_asset_resource_index", async ({ request }) => { events.push("activate"); @@ -72,6 +82,10 @@ describe("resource index build and activation", () => { resourceId: "posts", query: `*[properties.slug == $slug]`, entries: [entry], + metadataSnapshot: [ + { assetId: "post-1", metadataToken: "metadata-token-1" }, + ], + source: { buildId: "build-1", resources: "[]" }, }); expect(events).toEqual(["begin", "persist", "activate"]); @@ -80,7 +94,7 @@ describe("resource index build and activation", () => { test("reports a superseded build when compare-and-swap activation loses", async () => { server.use( - db.post("rpc/begin_asset_resource_index_build", () => json(null)), + db.post("rpc/begin_asset_resource_index_build", () => json(true)), db.post("rpc/activate_asset_resource_index", () => json(false)), db.post("rpc/fail_asset_resource_index_build", () => json(false)) ); @@ -97,10 +111,35 @@ describe("resource index build and activation", () => { resourceId: "posts", query: "*[]", entries: [entry], + metadataSnapshot: [], }) ).rejects.toBeInstanceOf(AssetResourceIndexBuildSupersededError); }); + test("does not persist when its source snapshot is already superseded", async () => { + const putIfAbsent = vi.fn(); + const activate = vi.fn(); + server.use( + db.post("rpc/begin_asset_resource_index_build", () => json(false)), + db.post("rpc/activate_asset_resource_index", activate) + ); + + await expect( + buildPersistAndActivateAssetResourceIndex({ + client: testContext.postgrest.client, + store: { putIfAbsent }, + projectId: "project-1", + resourceId: "posts", + query: "*[]", + entries: [entry], + metadataSnapshot: [], + }) + ).rejects.toBeInstanceOf(AssetResourceIndexBuildSupersededError); + + expect(putIfAbsent).not.toHaveBeenCalled(); + expect(activate).not.toHaveBeenCalled(); + }); + test("uses a unique identity for each otherwise identical build attempt", async () => { const begun: string[] = []; const activated: string[] = []; @@ -110,7 +149,7 @@ describe("resource index build and activation", () => { p_build_attempt_id: string; }; begun.push(body.p_build_attempt_id); - return json(null); + return json(true); }), db.post("rpc/activate_asset_resource_index", async ({ request }) => { const body = (await request.json()) as { @@ -132,6 +171,7 @@ describe("resource index build and activation", () => { resourceId: "posts", query: "*[]", entries: [entry], + metadataSnapshot: [], }; await buildPersistAndActivateAssetResourceIndex(input); @@ -145,7 +185,7 @@ describe("resource index build and activation", () => { test("marks a current failed attempt without trying to activate it", async () => { const activate = vi.fn(); server.use( - db.post("rpc/begin_asset_resource_index_build", () => json(null)), + db.post("rpc/begin_asset_resource_index_build", () => json(true)), db.post("rpc/activate_asset_resource_index", activate), db.post("rpc/fail_asset_resource_index_build", async ({ request }) => { expect(await request.json()).toMatchObject({ @@ -171,23 +211,26 @@ describe("resource index build and activation", () => { resourceId: "posts", query: "*[]", entries: [entry], + metadataSnapshot: [], }) ).rejects.toThrow("Storage unavailable"); expect(activate).not.toHaveBeenCalled(); }); - test("marks an in-flight cancelled build stale and never activates it", async () => { + test("marks an in-flight cancelled build stale without deleting its potentially shared object", async () => { const controller = new AbortController(); const activate = vi.fn(); + const deleteObject = vi.fn(async () => "deleted" as const); let releaseStorage: (() => void) | undefined; server.use( - db.post("rpc/begin_asset_resource_index_build", () => json(null)), + db.post("rpc/begin_asset_resource_index_build", () => json(true)), db.post("rpc/activate_asset_resource_index", activate), db.post("rpc/cancel_asset_resource_index_build", () => json(true)) ); const build = buildPersistAndActivateAssetResourceIndex({ client: testContext.postgrest.client, store: { + delete: deleteObject, putIfAbsent: async ({ checksum }) => { await new Promise((resolve) => { releaseStorage = resolve; @@ -199,6 +242,7 @@ describe("resource index build and activation", () => { resourceId: "posts", query: "*[]", entries: [entry], + metadataSnapshot: [], signal: controller.signal, }); await vi.waitFor(() => expect(releaseStorage).toBeTypeOf("function")); @@ -209,6 +253,7 @@ describe("resource index build and activation", () => { AssetResourceIndexBuildCancelledError ); expect(activate).not.toHaveBeenCalled(); + expect(deleteObject).not.toHaveBeenCalled(); }); test("rejects an invalid query before build state or storage changes", async () => { @@ -221,6 +266,7 @@ describe("resource index build and activation", () => { resourceId: "posts", query: "*[invalid ==", entries: [entry], + metadataSnapshot: [], }) ).rejects.toMatchObject({ code: "INVALID_QUERY" }); expect(putIfAbsent).not.toHaveBeenCalled(); diff --git a/packages/asset-uploader/src/resource-index-build.ts b/packages/asset-uploader/src/resource-index-build.ts index 6f90fc33606f..8973bbb72585 100644 --- a/packages/asset-uploader/src/resource-index-build.ts +++ b/packages/asset-uploader/src/resource-index-build.ts @@ -1,5 +1,6 @@ import { buildAssetResourceIndex, + getAssetResourceIndexObjectKey, persistAssetResourceIndex, type CanonicalAssetFileEntry, type ImmutableAssetResourceIndexStore, @@ -10,7 +11,9 @@ import { beginAssetResourceIndexBuild, cancelAssetResourceIndexBuild, failAssetResourceIndexBuild, + type AssetResourceIndexBuildSource, } from "./resource-index-persistence"; +import type { CanonicalAssetMetadataSnapshot } from "./canonical-metadata-persistence"; export class AssetResourceIndexBuildSupersededError extends Error { constructor() { @@ -40,6 +43,8 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ query, entries, assetRevision, + metadataSnapshot, + source, signal, }: { client: Client; @@ -49,6 +54,8 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ query: string; entries: readonly CanonicalAssetFileEntry[]; assetRevision?: string; + metadataSnapshot: CanonicalAssetMetadataSnapshot; + source?: AssetResourceIndexBuildSource; signal?: AbortSignal; }) => { const buildAttemptId = crypto.randomUUID(); @@ -60,8 +67,9 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ entries, assetRevision, }); + const objectKey = getAssetResourceIndexObjectKey({ projectId, index }); assertNotCancelled(signal); - await beginAssetResourceIndexBuild({ + const begun = await beginAssetResourceIndexBuild({ client, projectId, resourceId, @@ -69,10 +77,21 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ queryHash: index.queryHash, assetRevision: index.assetRevision, buildAttemptId, + revision: index.integrity.checksum, + checksum: index.integrity.checksum, + objectKey, + metadataSnapshot, + source, }); + if (begun === false) { + throw new AssetResourceIndexBuildSupersededError(); + } + let persisted: + | Awaited> + | undefined; try { - const persisted = await persistAssetResourceIndex({ + persisted = await persistAssetResourceIndex({ store, projectId, index, @@ -88,6 +107,8 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ buildAttemptId, checksum: index.integrity.checksum, objectKey: persisted.key, + metadataSnapshot, + source, }); if (activated === false) { throw new AssetResourceIndexBuildSupersededError(); @@ -95,14 +116,25 @@ export const buildPersistAndActivateAssetResourceIndex = async ({ return { index, persisted }; } catch (error) { if (error instanceof AssetResourceIndexBuildCancelledError) { - await cancelAssetResourceIndexBuild({ - client, - projectId, - resourceId, - queryHash: index.queryHash, - assetRevision: index.assetRevision, - buildAttemptId, - }); + try { + await cancelAssetResourceIndexBuild({ + client, + projectId, + resourceId, + queryHash: index.queryHash, + assetRevision: index.assetRevision, + buildAttemptId, + }); + } catch (stateError) { + throw new AggregateError( + [error, stateError], + "Resource index build was cancelled but its state could not be updated", + { cause: error } + ); + } + // The immutable object is content-addressed and may already be shared by + // another concurrent attempt. Deleting it here could break an activated + // revision. The pre-registered revision remains discoverable by GC. throw error; } try { diff --git a/packages/asset-uploader/src/resource-index-garbage-collection.test.ts b/packages/asset-uploader/src/resource-index-garbage-collection.test.ts index 5ce3991e8b89..deac3df87067 100644 --- a/packages/asset-uploader/src/resource-index-garbage-collection.test.ts +++ b/packages/asset-uploader/src/resource-index-garbage-collection.test.ts @@ -5,7 +5,10 @@ import { json, testContext, } from "@webstudio-is/postgrest/testing"; -import { collectAssetResourceIndexGarbage } from "./resource-index-garbage-collection"; +import { + collectAssetResourceIndexGarbage, + collectAssetResourceIndexGarbageBestEffort, +} from "./resource-index-garbage-collection"; const server = createTestServer(); const candidate = { @@ -101,4 +104,36 @@ describe("resource index garbage collection", () => { }) ).resolves.toEqual({ claimed: 1, deleted: 0, missing: 1 }); }); + + test("drains every available batch", async () => { + let claims = 0; + server.use( + db.post("rpc/claim_asset_resource_index_garbage", () => { + claims += 1; + if (claims === 1) { + return json([ + candidate, + { + ...candidate, + revision: `sha256:${"b".repeat(64)}`, + objectKey: "projects/project-1/resources/posts/older.json", + gcClaimId: "claim-2", + }, + ]); + } + return json([]); + }), + db.post("rpc/finalize_asset_resource_index_garbage", () => json(true)) + ); + const remove = vi.fn(async () => "deleted" as const); + + await collectAssetResourceIndexGarbageBestEffort({ + client: testContext.postgrest.client, + store: { delete: remove }, + limit: 2, + }); + + expect(claims).toBe(2); + expect(remove).toHaveBeenCalledTimes(2); + }); }); diff --git a/packages/asset-uploader/src/resource-index-garbage-collection.ts b/packages/asset-uploader/src/resource-index-garbage-collection.ts index ba4c7119b603..d7a28723519b 100644 --- a/packages/asset-uploader/src/resource-index-garbage-collection.ts +++ b/packages/asset-uploader/src/resource-index-garbage-collection.ts @@ -1,6 +1,9 @@ import type { AssetResourceIndexGarbageCollectionStore } from "@webstudio-is/asset-resource"; import type { Client } from "@webstudio-is/postgrest/index.server"; import { assertPostgrestSuccess } from "./patch-utils"; +import { runBounded } from "./async-utils"; + +const garbageCollectionConcurrency = 8; export const collectAssetResourceIndexGarbage = async ({ client, @@ -25,36 +28,16 @@ export const collectAssetResourceIndexGarbage = async ({ let deleted = 0; let missing = 0; const failures: unknown[] = []; - for (const candidate of candidates.data ?? []) { - let objectDeleted = false; - try { - const status = await store.delete(candidate.objectKey); - objectDeleted = true; - const finalized = await client.rpc( - "finalize_asset_resource_index_garbage", - { - p_project_id: candidate.projectId, - p_resource_id: candidate.resourceId, - p_revision: candidate.revision, - p_gc_claim_id: candidate.gcClaimId, - } - ); - assertPostgrestSuccess(finalized); - if (finalized.data !== true) { - throw new Error("Resource index garbage claim could not be finalized"); - } - if (status === "deleted") { - deleted += 1; - } else { - missing += 1; - } - } catch (error) { - // Once storage deletion succeeds, keep the claim so a later worker can - // retry the idempotent delete and database finalization. Releasing it - // would expose a revision row whose object no longer exists. - if (objectDeleted === false) { - const released = await client.rpc( - "release_asset_resource_index_garbage_claim", + await runBounded( + candidates.data ?? [], + garbageCollectionConcurrency, + async (candidate) => { + let objectDeleted = false; + try { + const status = await store.delete(candidate.objectKey); + objectDeleted = true; + const finalized = await client.rpc( + "finalize_asset_resource_index_garbage", { p_project_id: candidate.projectId, p_resource_id: candidate.resourceId, @@ -62,11 +45,47 @@ export const collectAssetResourceIndexGarbage = async ({ p_gc_claim_id: candidate.gcClaimId, } ); - assertPostgrestSuccess(released); + assertPostgrestSuccess(finalized); + if (finalized.data !== true) { + throw new Error( + "Resource index garbage claim could not be finalized" + ); + } + if (status === "deleted") { + deleted += 1; + } else { + missing += 1; + } + } catch (error) { + // Once storage deletion succeeds, keep the claim so a later worker can + // retry the idempotent delete and database finalization. Releasing it + // would expose a revision row whose object no longer exists. + if (objectDeleted === false) { + try { + const released = await client.rpc( + "release_asset_resource_index_garbage_claim", + { + p_project_id: candidate.projectId, + p_resource_id: candidate.resourceId, + p_revision: candidate.revision, + p_gc_claim_id: candidate.gcClaimId, + } + ); + assertPostgrestSuccess(released); + } catch (releaseError) { + failures.push( + new AggregateError( + [error, releaseError], + "Resource index object deletion and claim release failed" + ) + ); + return; + } + } + failures.push(error); } - failures.push(error); } - } + ); if (failures.length > 0) { throw new AggregateError( failures, @@ -83,12 +102,23 @@ export const collectAssetResourceIndexGarbage = async ({ export const collectAssetResourceIndexGarbageBestEffort = async ({ client, store, + limit = 100, }: { client: Client; store: AssetResourceIndexGarbageCollectionStore; + limit?: number; }) => { try { - await collectAssetResourceIndexGarbage({ client, store }); + while (true) { + const result = await collectAssetResourceIndexGarbage({ + client, + store, + limit, + }); + if (result.claimed < limit) { + break; + } + } } catch (error) { console.error("Asset resource index cleanup failed", error); } diff --git a/packages/asset-uploader/src/resource-index-lifecycle.test.ts b/packages/asset-uploader/src/resource-index-lifecycle.test.ts index 92111f662d5a..cd739a2127f2 100644 --- a/packages/asset-uploader/src/resource-index-lifecycle.test.ts +++ b/packages/asset-uploader/src/resource-index-lifecycle.test.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; import { generateObjectExpression, type Resource } from "@webstudio-is/sdk"; -const loadCanonicalAssetFileEntries = vi.hoisted(() => vi.fn()); +const loadCanonicalAssetFileSnapshot = vi.hoisted(() => vi.fn()); const synchronizeCanonicalAssets = vi.hoisted(() => vi.fn()); const buildPersistAndActivateAssetResourceIndex = vi.hoisted(() => vi.fn()); const deleteAssetResourceIndexQuery = vi.hoisted(() => vi.fn()); @@ -29,16 +29,20 @@ const resource = (id: string, query: string): Resource => ({ ), }); const dependencies = { - loadCanonicalAssetFileEntries, + loadCanonicalAssetFileSnapshot, synchronizeCanonicalAssets, buildPersistAndActivateAssetResourceIndex, deleteAssetResourceIndexQuery, collectAssetResourceIndexGarbageBestEffort, }; +const source = { buildId: "build-1", resources: "[]" }; describe("asset resource query lifecycle", () => { beforeEach(() => { - loadCanonicalAssetFileEntries.mockReset().mockResolvedValue([]); + loadCanonicalAssetFileSnapshot.mockReset().mockResolvedValue({ + entries: [], + metadataSnapshot: [], + }); synchronizeCanonicalAssets.mockReset().mockResolvedValue({}); buildPersistAndActivateAssetResourceIndex.mockReset().mockResolvedValue({}); deleteAssetResourceIndexQuery.mockReset().mockResolvedValue(true); @@ -55,9 +59,16 @@ describe("asset resource query lifecycle", () => { ).toBeUndefined(); }); - test("builds created and changed queries from one canonical load", async () => { - const previous = [resource("changed", "*[false]")]; - const current = [resource("changed", "*[]"), resource("created", "*[]")]; + test("rebuilds all current queries from one canonical load", async () => { + const previous = [ + resource("changed", "*[false]"), + resource("unchanged", "*[]"), + ]; + const current = [ + resource("changed", "*[]"), + resource("created", "*[]"), + resource("unchanged", "*[]"), + ]; await expect( synchronizeAssetResourceIndexQueries({ client: {} as never, @@ -67,15 +78,16 @@ describe("asset resource query lifecycle", () => { projectId: "project-1", previousResources: previous, resources: current, + source, dependencies, }) ).resolves.toEqual({ deletedResourceIds: [], - updatedResourceIds: ["changed", "created"], + updatedResourceIds: ["changed", "created", "unchanged"], }); - expect(loadCanonicalAssetFileEntries).toHaveBeenCalledOnce(); + expect(loadCanonicalAssetFileSnapshot).toHaveBeenCalledOnce(); expect(synchronizeCanonicalAssets).toHaveBeenCalledOnce(); - expect(buildPersistAndActivateAssetResourceIndex).toHaveBeenCalledTimes(2); + expect(buildPersistAndActivateAssetResourceIndex).toHaveBeenCalledTimes(3); expect(collectAssetResourceIndexGarbageBestEffort).toHaveBeenCalledOnce(); }); @@ -94,6 +106,7 @@ describe("asset resource query lifecycle", () => { resource("changed-away", "*[]"), ], resources: [external], + source, dependencies, }) ).resolves.toEqual({ @@ -101,7 +114,33 @@ describe("asset resource query lifecycle", () => { updatedResourceIds: [], }); expect(deleteAssetResourceIndexQuery).toHaveBeenCalledTimes(2); - expect(loadCanonicalAssetFileEntries).not.toHaveBeenCalled(); + expect(deleteAssetResourceIndexQuery).toHaveBeenCalledWith( + expect.objectContaining({ source }) + ); + expect(loadCanonicalAssetFileSnapshot).not.toHaveBeenCalled(); expect(synchronizeCanonicalAssets).not.toHaveBeenCalled(); }); + + test("continues independent query work and cleanup after a failure", async () => { + buildPersistAndActivateAssetResourceIndex + .mockRejectedValueOnce(new Error("first build failed")) + .mockResolvedValueOnce({}); + + await expect( + synchronizeAssetResourceIndexQueries({ + client: {} as never, + assetClient: { + resourceIndexStore: { delete: vi.fn() }, + } as never, + projectId: "project-1", + previousResources: [], + resources: [resource("first", "*[]"), resource("second", "*[]")], + source, + dependencies, + }) + ).rejects.toThrow("Failed to synchronize 1 asset resource index queries"); + + expect(buildPersistAndActivateAssetResourceIndex).toHaveBeenCalledTimes(2); + expect(collectAssetResourceIndexGarbageBestEffort).toHaveBeenCalledOnce(); + }); }); diff --git a/packages/asset-uploader/src/resource-index-lifecycle.ts b/packages/asset-uploader/src/resource-index-lifecycle.ts index fe737d3ff303..d521af7c8d81 100644 --- a/packages/asset-uploader/src/resource-index-lifecycle.ts +++ b/packages/asset-uploader/src/resource-index-lifecycle.ts @@ -1,13 +1,13 @@ -import { computeCanonicalAssetRevision } from "@webstudio-is/asset-resource"; import { getAssetResourceQuery, type Resource } from "@webstudio-is/sdk"; export { getAssetResourceQuery } from "@webstudio-is/sdk"; import type { Client } from "@webstudio-is/postgrest/index.server"; import type { AssetClientWithResourceIndexStore } from "./client"; import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; -import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; +import { loadCanonicalAssetFileSnapshot } from "./canonical-metadata-persistence"; import { buildPersistAndActivateAssetResourceIndex } from "./resource-index-build"; import { deleteAssetResourceIndexQuery } from "./resource-index-persistence"; import { collectAssetResourceIndexGarbageBestEffort } from "./resource-index-garbage-collection"; +import type { AssetResourceIndexBuildSource } from "./resource-index-persistence"; export const synchronizeAssetResourceIndexQueries = async ({ client, @@ -15,6 +15,7 @@ export const synchronizeAssetResourceIndexQueries = async ({ projectId, previousResources, resources, + source, dependencies = {}, }: { client: Client; @@ -22,9 +23,10 @@ export const synchronizeAssetResourceIndexQueries = async ({ projectId: string; previousResources: readonly Resource[]; resources: readonly Resource[]; + source: AssetResourceIndexBuildSource; dependencies?: { synchronizeCanonicalAssets?: typeof synchronizeCanonicalAssets; - loadCanonicalAssetFileEntries?: typeof loadCanonicalAssetFileEntries; + loadCanonicalAssetFileSnapshot?: typeof loadCanonicalAssetFileSnapshot; buildPersistAndActivateAssetResourceIndex?: typeof buildPersistAndActivateAssetResourceIndex; deleteAssetResourceIndexQuery?: typeof deleteAssetResourceIndexQuery; collectAssetResourceIndexGarbageBestEffort?: typeof collectAssetResourceIndexGarbageBestEffort; @@ -32,8 +34,9 @@ export const synchronizeAssetResourceIndexQueries = async ({ }) => { const synchronize = dependencies.synchronizeCanonicalAssets ?? synchronizeCanonicalAssets; - const loadEntries = - dependencies.loadCanonicalAssetFileEntries ?? loadCanonicalAssetFileEntries; + const loadSnapshot = + dependencies.loadCanonicalAssetFileSnapshot ?? + loadCanonicalAssetFileSnapshot; const buildIndex = dependencies.buildPersistAndActivateAssetResourceIndex ?? buildPersistAndActivateAssetResourceIndex; @@ -59,15 +62,17 @@ export const synchronizeAssetResourceIndexQueries = async ({ .map(([resourceId]) => resourceId) .sort(); const changed = [...current] - .filter( - ([resourceId, query]) => - query !== undefined && query !== previous.get(resourceId) - ) + .filter(([, query]) => query !== undefined) .map(([resourceId, query]) => ({ resourceId, query: query as string })) .sort((left, right) => left.resourceId.localeCompare(right.resourceId)); + const failures: unknown[] = []; for (const resourceId of deletedResourceIds) { - await deleteQuery({ client, projectId, resourceId }); + try { + await deleteQuery({ client, projectId, resourceId, source }); + } catch (error) { + failures.push(error); + } } if (changed.length === 0) { if ( @@ -79,23 +84,40 @@ export const synchronizeAssetResourceIndexQueries = async ({ store: { delete: assetClient.resourceIndexStore.delete }, }); } + if (failures.length > 0) { + throw new AggregateError( + failures, + `Failed to synchronize ${failures.length} asset resource index queries` + ); + } return { deletedResourceIds, updatedResourceIds: [] }; } - await synchronize({ projectId, client, assetClient }); - const entries = await loadEntries({ client, projectId }); - const assetRevision = await computeCanonicalAssetRevision(entries); const updatedResourceIds: string[] = []; - for (const { resourceId, query } of changed) { - await buildIndex({ + try { + await synchronize({ projectId, client, assetClient }); + const { entries, metadataSnapshot } = await loadSnapshot({ client, - store: assetClient.resourceIndexStore, projectId, - resourceId, - query, - entries, - assetRevision, }); - updatedResourceIds.push(resourceId); + for (const { resourceId, query } of changed) { + try { + await buildIndex({ + client, + store: assetClient.resourceIndexStore, + projectId, + resourceId, + query, + entries, + metadataSnapshot, + source, + }); + updatedResourceIds.push(resourceId); + } catch (error) { + failures.push(error); + } + } + } catch (error) { + failures.push(error); } if (assetClient.resourceIndexStore.delete !== undefined) { await collectGarbage({ @@ -103,5 +125,11 @@ export const synchronizeAssetResourceIndexQueries = async ({ store: { delete: assetClient.resourceIndexStore.delete }, }); } + if (failures.length > 0) { + throw new AggregateError( + failures, + `Failed to synchronize ${failures.length} asset resource index queries` + ); + } return { deletedResourceIds, updatedResourceIds }; }; diff --git a/packages/asset-uploader/src/resource-index-maintenance.test.ts b/packages/asset-uploader/src/resource-index-maintenance.test.ts index 818506b910f9..17ac08d15eaa 100644 --- a/packages/asset-uploader/src/resource-index-maintenance.test.ts +++ b/packages/asset-uploader/src/resource-index-maintenance.test.ts @@ -36,6 +36,7 @@ const entry = createCanonicalAssetFileEntry({ }); const metadataRow = { ...entry, + metadataToken: "metadata-token-1", document, createdAt: "2026-07-18T00:00:00.000Z", updatedAt: "2026-07-18T00:00:00.000Z", @@ -63,7 +64,7 @@ describe("incremental resource index maintenance", () => { db.post("rpc/begin_asset_resource_index_build", async ({ request }) => { const body = (await request.json()) as Record; begun.push(String(body.p_resource_id)); - return json(null); + return json(true); }), db.post("rpc/activate_asset_resource_index", async ({ request }) => { const body = (await request.json()) as Record; @@ -137,6 +138,35 @@ describe("incremental resource index maintenance", () => { expect(putIfAbsent).not.toHaveBeenCalled(); }); + test("does not rebuild resources already updated by query synchronization", async () => { + let metadataLoaded = false; + server.use( + db.get("AssetResourceIndexState", () => + json([{ resourceId: "posts", query: "*[]" }]) + ), + db.get("AssetFileMetadata", () => { + metadataLoaded = true; + return json([metadataRow]); + }) + ); + const putIfAbsent = vi.fn(); + + await expect( + updateAssetResourceIndexesAfterCanonicalChange({ + client: testContext.postgrest.client, + store: { putIfAbsent }, + projectId: "project-1", + changedAssetIds: ["post-1"], + excludedResourceIds: ["posts"], + }) + ).resolves.toEqual({ + changedAssetIds: ["post-1"], + updatedResourceIds: [], + }); + expect(metadataLoaded).toBe(false); + expect(putIfAbsent).not.toHaveBeenCalled(); + }); + test("repairs a stale current-query index before publication", async () => { const query = `*[extension == "md"]`; const queryHash = await computeAssetResourceQueryHash(query); @@ -154,7 +184,7 @@ describe("incremental resource index maintenance", () => { }, ]) ), - db.post("rpc/begin_asset_resource_index_build", () => json(null)), + db.post("rpc/begin_asset_resource_index_build", () => json(true)), db.post("rpc/activate_asset_resource_index", () => json(true)) ); const putIfAbsent = vi.fn(async ({ checksum }) => ({ @@ -170,6 +200,9 @@ describe("incremental resource index maintenance", () => { resources: [{ resourceId: "posts", query, queryHash }], entries: [entry], assetRevision, + metadataSnapshot: [ + { assetId: "post-1", metadataToken: "metadata-token-1" }, + ], }) ).resolves.toEqual(["posts"]); expect(putIfAbsent).toHaveBeenCalledOnce(); @@ -207,6 +240,9 @@ describe("incremental resource index maintenance", () => { ], entries: [entry], assetRevision, + metadataSnapshot: [ + { assetId: "post-1", metadataToken: "metadata-token-1" }, + ], read, referenceId: "historical-build", }); diff --git a/packages/asset-uploader/src/resource-index-maintenance.ts b/packages/asset-uploader/src/resource-index-maintenance.ts index c1861a846712..54a4635fd82a 100644 --- a/packages/asset-uploader/src/resource-index-maintenance.ts +++ b/packages/asset-uploader/src/resource-index-maintenance.ts @@ -1,6 +1,5 @@ import { buildAssetResourceIndex, - computeCanonicalAssetRevision, type CanonicalAssetFileEntry, type ImmutableAssetResourceIndexStore, } from "@webstudio-is/asset-resource"; @@ -8,7 +7,10 @@ import type { Client } from "@webstudio-is/postgrest/index.server"; import type { AssetClient } from "./client"; import { assertPostgrestSuccess } from "./patch-utils"; import { synchronizeCanonicalAsset } from "./canonical-metadata-backfill"; -import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; +import { + loadCanonicalAssetFileSnapshot, + type CanonicalAssetMetadataSnapshot, +} from "./canonical-metadata-persistence"; import { AssetResourceIndexBuildSupersededError, buildPersistAndActivateAssetResourceIndex, @@ -51,11 +53,13 @@ export const updateAssetResourceIndexesAfterCanonicalChange = async ({ store, projectId, changedAssetIds, + excludedResourceIds = [], }: { client: Client; store: ImmutableAssetResourceIndexStore; projectId: string; changedAssetIds: readonly string[]; + excludedResourceIds?: readonly string[]; }) => { const changedIds = [...new Set(changedAssetIds)].sort(); if (changedIds.length === 0) { @@ -68,7 +72,10 @@ export const updateAssetResourceIndexesAfterCanonicalChange = async ({ .is("deletedAt", null) .order("resourceId"); assertPostgrestSuccess(stateResult); - const resources = stateResult.data ?? []; + const excluded = new Set(excludedResourceIds); + const resources = (stateResult.data ?? []).filter( + ({ resourceId }) => excluded.has(resourceId) === false + ); if (resources.length === 0) { return { changedAssetIds: changedIds, updatedResourceIds: [] }; } @@ -76,8 +83,10 @@ export const updateAssetResourceIndexesAfterCanonicalChange = async ({ // Every GROQ query may depend on collection cardinality, ordering, or any // schema-less field. V1 therefore treats all query resources in this project // as affected, but rebuilds them from compact canonical rows loaded once. - const entries = await loadCanonicalAssetFileEntries({ client, projectId }); - const assetRevision = await computeCanonicalAssetRevision(entries); + const { entries, metadataSnapshot } = await loadCanonicalAssetFileSnapshot({ + client, + projectId, + }); const updatedResourceIds: string[] = []; const errors: unknown[] = []; for (const resource of resources) { @@ -89,7 +98,7 @@ export const updateAssetResourceIndexesAfterCanonicalChange = async ({ resourceId: resource.resourceId, query: resource.query, entries, - assetRevision, + metadataSnapshot, }); updatedResourceIds.push(resource.resourceId); } catch (error) { @@ -118,6 +127,7 @@ export const reconcileAssetResourceIndexesForPublication = async ({ resources, entries, assetRevision, + metadataSnapshot, }: { client: Client; store: ImmutableAssetResourceIndexStore; @@ -129,6 +139,7 @@ export const reconcileAssetResourceIndexesForPublication = async ({ }[]; entries: readonly CanonicalAssetFileEntry[]; assetRevision: string; + metadataSnapshot: CanonicalAssetMetadataSnapshot; }) => { if (resources.length === 0) { return []; @@ -175,6 +186,7 @@ export const reconcileAssetResourceIndexesForPublication = async ({ query: resource.query, entries, assetRevision, + metadataSnapshot, }); } catch (error) { // A concurrent build of the same identity may activate first. The @@ -196,6 +208,7 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({ resources, entries, assetRevision, + metadataSnapshot, read, referenceId, }: { @@ -209,6 +222,7 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({ }[]; entries: readonly CanonicalAssetFileEntry[]; assetRevision: string; + metadataSnapshot: CanonicalAssetMetadataSnapshot; read: (name: string) => Promise<{ data: AsyncIterable }>; referenceId: string; }): Promise => { @@ -242,6 +256,7 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({ resources: currentResources, entries, assetRevision, + metadataSnapshot, }); const currentSnapshots = await loadAssetResourceIndexSnapshots({ client, diff --git a/packages/asset-uploader/src/resource-index-persistence.ts b/packages/asset-uploader/src/resource-index-persistence.ts index bd846e62e410..4dbf3f2d0b86 100644 --- a/packages/asset-uploader/src/resource-index-persistence.ts +++ b/packages/asset-uploader/src/resource-index-persistence.ts @@ -1,5 +1,11 @@ import type { Client } from "@webstudio-is/postgrest/index.server"; import { assertPostgrestSuccess } from "./patch-utils"; +import type { CanonicalAssetMetadataSnapshot } from "./canonical-metadata-persistence"; + +export type AssetResourceIndexBuildSource = { + buildId: string; + resources: string; +}; export const beginAssetResourceIndexBuild = async ({ client, @@ -9,6 +15,11 @@ export const beginAssetResourceIndexBuild = async ({ queryHash, assetRevision, buildAttemptId, + revision, + checksum, + objectKey, + metadataSnapshot, + source, }: { client: Client; projectId: string; @@ -17,6 +28,11 @@ export const beginAssetResourceIndexBuild = async ({ queryHash: string; assetRevision: string; buildAttemptId: string; + revision: string; + checksum: string; + objectKey: string; + metadataSnapshot: CanonicalAssetMetadataSnapshot; + source?: AssetResourceIndexBuildSource; }) => { const result = await client.rpc("begin_asset_resource_index_build", { p_project_id: projectId, @@ -25,8 +41,15 @@ export const beginAssetResourceIndexBuild = async ({ p_query_hash: queryHash, p_asset_revision: assetRevision, p_build_attempt_id: buildAttemptId, + p_revision: revision, + p_checksum: checksum, + p_object_key: objectKey, + p_metadata_snapshot: metadataSnapshot, + p_build_id: source?.buildId, + p_resources: source?.resources, }); assertPostgrestSuccess(result); + return result.data === true; }; export const activateAssetResourceIndex = async ({ @@ -39,6 +62,8 @@ export const activateAssetResourceIndex = async ({ buildAttemptId, checksum, objectKey, + metadataSnapshot, + source, }: { client: Client; projectId: string; @@ -49,6 +74,8 @@ export const activateAssetResourceIndex = async ({ buildAttemptId: string; checksum: string; objectKey: string; + metadataSnapshot: CanonicalAssetMetadataSnapshot; + source?: AssetResourceIndexBuildSource; }) => { const result = await client.rpc("activate_asset_resource_index", { p_project_id: projectId, @@ -59,6 +86,9 @@ export const activateAssetResourceIndex = async ({ p_build_attempt_id: buildAttemptId, p_checksum: checksum, p_object_key: objectKey, + p_metadata_snapshot: metadataSnapshot, + p_build_id: source?.buildId, + p_resources: source?.resources, }); assertPostgrestSuccess(result); return result.data === true; @@ -124,14 +154,18 @@ export const deleteAssetResourceIndexQuery = async ({ client, projectId, resourceId, + source, }: { client: Client; projectId: string; resourceId: string; + source?: AssetResourceIndexBuildSource; }) => { const result = await client.rpc("delete_asset_resource_index_query", { p_project_id: projectId, p_resource_id: resourceId, + p_build_id: source?.buildId, + p_resources: source?.resources, }); assertPostgrestSuccess(result); return result.data === true; diff --git a/packages/asset-uploader/src/resource-index-rebuild.test.ts b/packages/asset-uploader/src/resource-index-rebuild.test.ts index 33a18e492b94..567a3151595b 100644 --- a/packages/asset-uploader/src/resource-index-rebuild.test.ts +++ b/packages/asset-uploader/src/resource-index-rebuild.test.ts @@ -34,27 +34,43 @@ describe("explicit resource index rebuild", () => { json([ { ...entry, + metadataToken: "metadata-token-1", document, createdAt: "2026-07-18T00:00:00.000Z", updatedAt: "2026-07-18T00:00:00.000Z", }, ]) ), - db.post("rpc/begin_asset_resource_index_build", () => json(null)), + db.post("rpc/begin_asset_resource_index_build", () => json(true)), db.post("rpc/activate_asset_resource_index", () => json(true)) ); const putIfAbsent = vi.fn(async ({ checksum }) => ({ status: "exists" as const, checksum, })); + const synchronizeCanonicalAssets = vi.fn(async () => ({ + scanned: 1, + indexed: 0, + metadataUpdated: 0, + unchanged: 1, + removed: 0, + skipped: 0, + inconsistent: 0, + })); const result = await rebuildAssetResourceIndex({ client: testContext.postgrest.client, - store: { putIfAbsent }, + assetClient: { resourceIndexStore: { putIfAbsent } } as never, projectId: "project-1", resourceId: "posts", query: `*[properties.slug == $slug]`, + source: { buildId: "build-1", resources: "[]" }, + dependencies: { synchronizeCanonicalAssets }, }); + expect(synchronizeCanonicalAssets).toHaveBeenCalledOnce(); + expect(synchronizeCanonicalAssets.mock.invocationCallOrder[0]).toBeLessThan( + putIfAbsent.mock.invocationCallOrder[0] + ); expect(result.index.parameterNames).toEqual(["slug"]); expect(result.index.documents).toHaveLength(1); expect(putIfAbsent).toHaveBeenCalledOnce(); diff --git a/packages/asset-uploader/src/resource-index-rebuild.ts b/packages/asset-uploader/src/resource-index-rebuild.ts index 4ca746f3a075..d4cbd94d6e3e 100644 --- a/packages/asset-uploader/src/resource-index-rebuild.ts +++ b/packages/asset-uploader/src/resource-index-rebuild.ts @@ -1,8 +1,10 @@ -import type { ImmutableAssetResourceIndexStore } from "@webstudio-is/asset-resource"; import type { Client } from "@webstudio-is/postgrest/index.server"; -import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence"; +import type { AssetClientWithResourceIndexStore } from "./client"; +import { synchronizeCanonicalAssets } from "./canonical-metadata-backfill"; +import { loadCanonicalAssetFileSnapshot } from "./canonical-metadata-persistence"; import { buildPersistAndActivateAssetResourceIndex } from "./resource-index-build"; import { collectAssetResourceIndexGarbageBestEffort } from "./resource-index-garbage-collection"; +import type { AssetResourceIndexBuildSource } from "./resource-index-persistence"; export class AssetResourceIndexNotFoundError extends Error { constructor() { @@ -13,30 +15,45 @@ export class AssetResourceIndexNotFoundError extends Error { export const rebuildAssetResourceIndex = async ({ client, - store, + assetClient, projectId, resourceId, query, + source, + dependencies = {}, }: { client: Client; - store: ImmutableAssetResourceIndexStore; + assetClient: AssetClientWithResourceIndexStore; projectId: string; resourceId: string; query: string; + source: AssetResourceIndexBuildSource; + dependencies?: { + synchronizeCanonicalAssets?: typeof synchronizeCanonicalAssets; + loadCanonicalAssetFileSnapshot?: typeof loadCanonicalAssetFileSnapshot; + }; }) => { - const entries = await loadCanonicalAssetFileEntries({ client, projectId }); + await (dependencies.synchronizeCanonicalAssets ?? synchronizeCanonicalAssets)( + { client, assetClient, projectId } + ); + const { entries, metadataSnapshot } = await ( + dependencies.loadCanonicalAssetFileSnapshot ?? + loadCanonicalAssetFileSnapshot + )({ client, projectId }); const result = await buildPersistAndActivateAssetResourceIndex({ client, - store, + store: assetClient.resourceIndexStore, projectId, resourceId, query, entries, + metadataSnapshot, + source, }); - if (store.delete !== undefined) { + if (assetClient.resourceIndexStore.delete !== undefined) { await collectAssetResourceIndexGarbageBestEffort({ client, - store: { delete: store.delete }, + store: { delete: assetClient.resourceIndexStore.delete }, }); } return result; diff --git a/packages/asset-uploader/src/resource-index-snapshot.test.ts b/packages/asset-uploader/src/resource-index-snapshot.test.ts index 56bb476fe6a2..63c6f15edb11 100644 --- a/packages/asset-uploader/src/resource-index-snapshot.test.ts +++ b/packages/asset-uploader/src/resource-index-snapshot.test.ts @@ -110,4 +110,62 @@ describe("publication resource index snapshots", () => { }) ).rejects.toMatchObject({ resourceId: "posts" }); }); + + test("preserves both snapshot and reference cleanup failures", async () => { + const index = await createAssetResourceIndex({ + format: "webstudio-resource-index", + version: 1, + resourceId: "posts", + query: "*[]", + assetRevision: `sha256:${"a".repeat(64)}`, + queryMode: "static", + parameterNames: [], + documents: [], + }); + const states = createQuery([ + { + resourceId: "posts", + queryHash: index.queryHash, + assetRevision: index.assetRevision, + activeRevision: index.integrity.checksum, + buildStatus: "ACTIVE", + deletedAt: null, + }, + ]); + const revisions = createQuery([ + { + resourceId: "posts", + revision: index.integrity.checksum, + queryHash: index.queryHash, + assetRevision: index.assetRevision, + objectKey: "private/posts.json", + }, + ]); + const client = { + from: vi.fn().mockReturnValueOnce(states).mockReturnValueOnce(revisions), + rpc: vi.fn(async (name: string) => + name === "remove_asset_resource_index_reference" + ? { data: null, error: new Error("cleanup failed") } + : { data: null, error: null } + ), + }; + + const result = loadAssetResourceIndexSnapshots({ + client: client as never, + projectId: "project-1", + resources: [{ resourceId: "posts", queryHash: index.queryHash }], + read: async () => { + throw new Error("read failed"); + }, + referenceId: "build-1", + expectedAssetRevision: index.assetRevision, + }); + + const error = await result.catch((error: unknown) => error); + expect(error).toBeInstanceOf(AggregateError); + expect((error as AggregateError).errors).toEqual([ + expect.objectContaining({ message: "read failed" }), + expect.objectContaining({ message: "cleanup failed" }), + ]); + }); }); diff --git a/packages/asset-uploader/src/resource-index-snapshot.ts b/packages/asset-uploader/src/resource-index-snapshot.ts index 38c1698fbf36..e713bd26ff84 100644 --- a/packages/asset-uploader/src/resource-index-snapshot.ts +++ b/packages/asset-uploader/src/resource-index-snapshot.ts @@ -13,6 +13,7 @@ import { removeAssetResourceIndexReferences, } from "./resource-index-persistence"; import { collectAssetResourceIndexGarbageBestEffort } from "./resource-index-garbage-collection"; +import { mapBounded } from "./async-utils"; type StateRow = Pick< Database["public"]["Tables"]["AssetResourceIndexState"]["Row"], @@ -138,6 +139,8 @@ export const loadAssetResourceIndexSnapshots = async ({ revisions.map((row) => [`${row.resourceId}:${row.revision}`, row]) ); + let snapshots: AssetResourceIndexSnapshot[] = []; + let operationFailure: { error: unknown } | undefined; try { for (const resourceId of uniqueResourceIds) { const state = statesByResource.get(resourceId) as StateRow & { @@ -152,8 +155,10 @@ export const loadAssetResourceIndexSnapshots = async ({ referenceId: operationReferenceId, }); } - return await Promise.all( - uniqueResourceIds.map(async (resourceId) => { + snapshots = await mapBounded( + uniqueResourceIds, + assetResourceLimits.concurrentContentReads, + async (resourceId) => { const state = statesByResource.get(resourceId) as StateRow & { activeRevision: string; }; @@ -182,9 +187,14 @@ export const loadAssetResourceIndexSnapshots = async ({ ); } return { resourceId, revision: state.activeRevision, index }; - }) + } ); - } finally { + } catch (error) { + operationFailure = { error }; + } + + let cleanupFailure: { error: unknown } | undefined; + try { await removeAssetResourceIndexReferences({ client, projectId, @@ -197,5 +207,22 @@ export const loadAssetResourceIndexSnapshots = async ({ store: garbageCollectionStore, }); } + } catch (error) { + cleanupFailure = { error }; + } + + if (operationFailure !== undefined && cleanupFailure !== undefined) { + throw new AggregateError( + [operationFailure.error, cleanupFailure.error], + "Asset resource index snapshot loading and reference cleanup failed", + { cause: operationFailure.error } + ); + } + if (operationFailure !== undefined) { + throw operationFailure.error; + } + if (cleanupFailure !== undefined) { + throw cleanupFailure.error; } + return snapshots; }; diff --git a/packages/asset-uploader/src/resource-index-status-authorization.test.ts b/packages/asset-uploader/src/resource-index-status-authorization.test.ts index cf275fe1e8aa..a591e7671d9d 100644 --- a/packages/asset-uploader/src/resource-index-status-authorization.test.ts +++ b/packages/asset-uploader/src/resource-index-status-authorization.test.ts @@ -1,32 +1,29 @@ import { describe, expect, test, vi } from "vitest"; import { - authorizeProject, AuthorizationError, type AppContext, } from "@webstudio-is/trpc-interface/index.server"; import { loadAssetResourceIndexStatus } from "./resource-index-status"; -vi.mock("@webstudio-is/trpc-interface/index.server", () => ({ - authorizeProject: { hasProjectPermit: vi.fn() }, - AuthorizationError: class AuthorizationError extends Error {}, -})); - describe("loadAssetResourceIndexStatus authorization", () => { test("checks view permission before querying state", async () => { - vi.mocked(authorizeProject.hasProjectPermit).mockResolvedValue(false); + const hasProjectPermit = vi.fn().mockResolvedValue(false); const from = vi.fn(); const context = { postgrest: { client: { from } }, } as unknown as AppContext; await expect( - loadAssetResourceIndexStatus({ - projectId: "project-1", - resourceId: "posts", - context, - }) + loadAssetResourceIndexStatus( + { + projectId: "project-1", + resourceId: "posts", + context, + }, + { hasProjectPermit } + ) ).rejects.toBeInstanceOf(AuthorizationError); - expect(authorizeProject.hasProjectPermit).toHaveBeenCalledWith( + expect(hasProjectPermit).toHaveBeenCalledWith( { projectId: "project-1", permit: "view" }, context ); diff --git a/packages/asset-uploader/src/resource-index-status.ts b/packages/asset-uploader/src/resource-index-status.ts index d9473a78f7d4..d175e436684b 100644 --- a/packages/asset-uploader/src/resource-index-status.ts +++ b/packages/asset-uploader/src/resource-index-status.ts @@ -26,6 +26,10 @@ const stateByBuildStatus = { FAILED: "failed", } as const satisfies Record; +const defaultDependencies = { + hasProjectPermit: authorizeProject.hasProjectPermit, +}; + export const toAssetResourceIndexStatus = (row: StatusRow) => assetResourceIndexStatus.parse({ resourceId: row.resourceId, @@ -43,16 +47,19 @@ export const toAssetResourceIndexStatus = (row: StatusRow) => updatedAt: row.updatedAt, }); -export const loadAssetResourceIndexStatus = async ({ - projectId, - resourceId, - context, -}: { - projectId: string; - resourceId: string; - context: AppContext; -}) => { - const canView = await authorizeProject.hasProjectPermit( +export const loadAssetResourceIndexStatus = async ( + { + projectId, + resourceId, + context, + }: { + projectId: string; + resourceId: string; + context: AppContext; + }, + dependencies = defaultDependencies +) => { + const canView = await dependencies.hasProjectPermit( { projectId, permit: "view" }, context ); diff --git a/packages/cli/src/commands/api-command-metadata.ts b/packages/cli/src/commands/api-command-metadata.ts index b5b2b0d5f484..55f7fbc3e550 100644 --- a/packages/cli/src/commands/api-command-metadata.ts +++ b/packages/cli/src/commands/api-command-metadata.ts @@ -200,6 +200,11 @@ const apiCommandOptionsByCommand: Partial< "update-variable": apiCommand.updateVariableCommandOptions, "delete-variable": apiCommand.deleteVariableCommandOptions, "list-resources": apiCommand.scopedCommandOptions, + "get-assets-resource": apiCommand.assetResourceCommandOptions, + "create-assets-resource": apiCommand.inputCommandOptions, + "update-assets-resource": apiCommand.inputCommandOptions, + "get-asset-resource-index-status": apiCommand.assetResourceCommandOptions, + "rebuild-asset-resource-index": apiCommand.assetResourceCommandOptions, "list-publishes": apiCommand.paginatedListCommandOptions, "create-resource": apiCommand.createResourceCommandOptions, "update-resource": apiCommand.updateResourceCommandOptions, diff --git a/packages/cli/src/commands/api-command.ts b/packages/cli/src/commands/api-command.ts index a2a0c82b4a4d..53b3bf0cd246 100644 --- a/packages/cli/src/commands/api-command.ts +++ b/packages/cli/src/commands/api-command.ts @@ -106,6 +106,19 @@ const requiredInputOption = (yargs: CommonYargsArgv, describe: string) => demandOption: true, }); +export const inputCommandOptions = (yargs: CommonYargsArgv) => + requiredInputOption( + apiCommandOptions(yargs), + "Required JSON file containing the command input." + ); + +export const assetResourceCommandOptions = (yargs: CommonYargsArgv) => + apiCommandOptions(yargs).option("resource", { + type: "string", + describe: "Required Assets resource id", + demandOption: true, + }); + const confirmOption = (yargs: CommonYargsArgv, describe: string) => yargs.option("confirm", { type: "boolean", diff --git a/packages/cli/src/prebuild.test.ts b/packages/cli/src/prebuild.test.ts index de29b1a37d98..382fc41c437c 100644 --- a/packages/cli/src/prebuild.test.ts +++ b/packages/cli/src/prebuild.test.ts @@ -15,9 +15,10 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { bundleVersion } from "@webstudio-is/protocol"; import { createAssetResourceIndex } from "@webstudio-is/asset-resource"; -import { createAssetQueryResourceBody } from "@webstudio-is/sdk"; +import { createAssetQueryResourceBody, type Resource } from "@webstudio-is/sdk"; import { generateRedirectsModule, + getRequiredAssetResourceContentRefs, materializeAssetResourceIndexes, prebuild, } from "./prebuild"; @@ -319,6 +320,63 @@ describe("generateRedirectsModule", () => { }); }); +test("requires candidate files only when an asset query can hydrate content", async () => { + const query = "*[]"; + const index = await createAssetResourceIndex({ + format: "webstudio-resource-index", + version: 1, + resourceId: "posts", + query, + assetRevision: `sha256:${"a".repeat(64)}`, + queryMode: "static", + parameterNames: [], + documents: [ + { + _id: "post-1", + _type: "asset.file", + name: "post.md", + path: "post.md", + key: "post", + extension: "md", + mimeType: "text/markdown", + size: 10, + revision: "post-revision", + contentRef: "post.md", + properties: {}, + }, + ], + }); + const createResource = (contentExpression: string): Resource => ({ + id: "posts", + name: "Posts", + control: "system", + method: "post", + url: '"/$resources/assets/query"', + headers: [], + body: createAssetQueryResourceBody({ + query, + parameters: [], + contentExpression, + }), + }); + const snapshots = [ + { resourceId: "posts", revision: index.integrity.checksum, index }, + ]; + + expect( + getRequiredAssetResourceContentRefs({ + snapshots, + resources: [["posts", createResource('{"mode":"none"}')]], + }) + ).toEqual(new Set()); + expect( + getRequiredAssetResourceContentRefs({ + snapshots, + resources: [["posts", createResource('{"mode":"markdown-body"}')]], + }) + ).toEqual(new Set(["post.md"])); +}); + test("materializes immutable resource indexes as public JSON with a reference-only module", async () => { const index = await createAssetResourceIndex({ format: "webstudio-resource-index", @@ -332,6 +390,12 @@ test("materializes immutable resource indexes as public JSON with a reference-on }); await mkdir("public", { recursive: true }); await mkdir("app/__generated__", { recursive: true }); + await mkdir("public/resource-indexes", { recursive: true }); + await writeFile( + "public/resource-indexes/obsolete-index.json", + "obsolete", + "utf8" + ); await materializeAssetResourceIndexes({ snapshots: [ @@ -348,6 +412,7 @@ test("materializes immutable resource indexes as public JSON with a reference-on const files = await getFilePaths("public/resource-indexes"); expect(files).toHaveLength(1); + expect(files[0]).not.toContain("obsolete-index.json"); expect(JSON.parse(await readFile(files[0], "utf8"))).toEqual(index); const manifestModule = await readFile( "app/__generated__/$resources.asset-query-manifest.ts", @@ -434,17 +499,15 @@ test("executes and hydrates an asset query from SSG public files", async () => { ], }); - const response = await runtimeFetch( - new Request("https://site.example/$resources/assets/query", { - method: "POST", - body: JSON.stringify({ - query, - parameters: { slug: "post" }, - resultLimit: 1, - content: { mode: "full" }, - }), - }) - ); + const response = await runtimeFetch("/$resources/assets/query", { + method: "POST", + body: JSON.stringify({ + query, + parameters: { slug: "post" }, + resultLimit: 1, + content: { mode: "full" }, + }), + }); expect({ status: response?.status, diff --git a/packages/cli/src/prebuild.ts b/packages/cli/src/prebuild.ts index e3851756a9fa..ae43e2ad24d1 100644 --- a/packages/cli/src/prebuild.ts +++ b/packages/cli/src/prebuild.ts @@ -41,6 +41,8 @@ import { ROOT_INSTANCE_ID, elementComponent, toRuntimeAsset, + assetResourceContentOptions, + parseAssetQueryResourceBody, } from "@webstudio-is/sdk"; import { migratePages } from "@webstudio-is/project-migrations/pages"; import { collectFontFamiliesFromStyleDecls } from "@webstudio-is/project-build/runtime"; @@ -139,6 +141,41 @@ const getResourceIndexPublicPath = (resourceId: string, revision: string) => revision )}.json`; +export const getRequiredAssetResourceContentRefs = ({ + snapshots, + resources, +}: { + snapshots: NonNullable; + resources: PublishedProjectBundle["build"]["resources"]; +}) => { + const resourcesById = new Map(resources); + const required = new Set(); + for (const snapshot of snapshots) { + const resource = resourcesById.get(snapshot.resourceId); + if (resource === undefined) { + continue; + } + const { contentExpression } = parseAssetQueryResourceBody(resource.body); + let content: unknown = { mode: "none" }; + if (contentExpression !== undefined) { + try { + content = JSON.parse(contentExpression); + } catch { + // A dynamic or malformed mode cannot prove that hydration is disabled. + content = undefined; + } + } + const parsedContent = assetResourceContentOptions.safeParse(content); + if (parsedContent.success && parsedContent.data.mode === "none") { + continue; + } + for (const document of snapshot.index.documents) { + required.add(document.contentRef); + } + } + return required; +}; + export const generateAssetQueryRuntimeModule = ({ deploymentId, manifest, @@ -182,6 +219,7 @@ export const materializeAssetResourceIndexes = async ({ deploymentId: string; }) => { const targetDirectory = join(publicDirectory, "resource-indexes"); + await rm(targetDirectory, { recursive: true, force: true }); await createFolderIfNotExists(targetDirectory); const manifest = snapshots.map(({ resourceId, revision, index }) => { if ( @@ -950,15 +988,43 @@ export const prebuild = async (options: { if (options.assets === true && siteData.assets.length > 0) { const downloading = createProgress(); - downloading.start("Downloading fonts and images"); + downloading.start("Downloading assets"); + const requiredContentRefs = getRequiredAssetResourceContentRefs({ + snapshots: siteData.assetResourceIndexes ?? [], + resources: siteData.build.resources, + }); + const requiredAssets: Asset[] = []; + const bestEffortAssets: Asset[] = []; + for (const asset of siteData.assets) { + if (requiredContentRefs.delete(asset.name)) { + requiredAssets.push(asset); + } else { + bestEffortAssets.push(asset); + } + } + if (requiredContentRefs.size > 0) { + throw new Error( + `Published asset query indexes reference missing assets: ${[ + ...requiredContentRefs, + ] + .sort() + .join(", ")}` + ); + } await materializeAssetFiles({ - assets: siteData.assets, + assets: requiredAssets, + origin: siteData.origin || "", + sourceAssetsDirectory: join(buildRoot, LOCAL_ASSETS_DIR), + targetAssetsDirectory: join(buildRoot, "public", assetBaseUrl), + }); + await materializeAssetFiles({ + assets: bestEffortAssets, continueOnError: true, origin: siteData.origin || "", sourceAssetsDirectory: join(buildRoot, LOCAL_ASSETS_DIR), targetAssetsDirectory: join(buildRoot, "public", assetBaseUrl), }); - downloading.stop("Downloaded fonts and images"); + downloading.stop("Downloaded assets"); } feedback.step("Build finished"); diff --git a/packages/cli/templates/ssg/app/asset-resource-fetch.ts b/packages/cli/templates/ssg/app/asset-resource-fetch.ts index 5cbceb67cd60..a4f36f229530 100644 --- a/packages/cli/templates/ssg/app/asset-resource-fetch.ts +++ b/packages/cli/templates/ssg/app/asset-resource-fetch.ts @@ -90,6 +90,7 @@ export const createSsgAssetResourceFetch = ({ >[0]["manifest"]; }) => createPublishedAssetResourceFetch({ + baseUrl: "https://webstudio.local", deploymentId, manifest, fetchAsset: fetchSsgPublicAsset, diff --git a/packages/postgrest/src/__generated__/db-types.ts b/packages/postgrest/src/__generated__/db-types.ts index 88d90f9abd70..1745eabb5915 100644 --- a/packages/postgrest/src/__generated__/db-types.ts +++ b/packages/postgrest/src/__generated__/db-types.ts @@ -90,6 +90,7 @@ export type Database = { createdAt: string; document: Json; fieldContributions: Json; + metadataToken: string; projectId: string; revision: string; updatedAt: string; @@ -99,6 +100,7 @@ export type Database = { createdAt?: string; document: Json; fieldContributions?: Json; + metadataToken?: string; projectId: string; revision: string; updatedAt?: string; @@ -108,6 +110,7 @@ export type Database = { createdAt?: string; document?: Json; fieldContributions?: Json; + metadataToken?: string; projectId?: string; revision?: string; updatedAt?: string; @@ -597,6 +600,7 @@ export type Database = { }; File: { Row: { + contentHash: string | null; createdAt: string; description: string | null; format: string; @@ -609,6 +613,7 @@ export type Database = { uploaderProjectId: string | null; }; Insert: { + contentHash?: string | null; createdAt?: string; description?: string | null; format: string; @@ -621,6 +626,7 @@ export type Database = { uploaderProjectId?: string | null; }; Update: { + contentHash?: string | null; createdAt?: string; description?: string | null; format?: string; @@ -1250,11 +1256,14 @@ export type Database = { Args: { p_asset_revision: string; p_build_attempt_id: string; + p_build_id?: string; p_checksum: string; + p_metadata_snapshot?: Json; p_object_key: string; p_project_id: string; p_query_hash: string; p_resource_id: string; + p_resources?: string; p_revision: string; }; Returns: boolean; @@ -1269,16 +1278,30 @@ export type Database = { }; Returns: undefined; }; + asset_file_metadata_snapshot_matches: { + Args: { p_metadata_snapshot: Json; p_project_id: string }; + Returns: boolean; + }; + asset_resource_index_source_matches: { + Args: { p_build_id: string; p_project_id: string; p_resources: string }; + Returns: boolean; + }; begin_asset_resource_index_build: { Args: { p_asset_revision: string; p_build_attempt_id: string; + p_build_id?: string; + p_checksum?: string; + p_metadata_snapshot?: Json; + p_object_key?: string; p_project_id: string; p_query: string; p_query_hash: string; p_resource_id: string; + p_resources?: string; + p_revision?: string; }; - Returns: undefined; + Returns: boolean; }; cancel_asset_resource_index_build: { Args: { @@ -1335,7 +1358,12 @@ export type Database = { Returns: undefined; }; delete_asset_resource_index_query: { - Args: { p_project_id: string; p_resource_id: string }; + Args: { + p_build_id?: string; + p_project_id: string; + p_resource_id: string; + p_resources?: string; + }; Returns: boolean; }; delete_stale_asset_file_metadata: { @@ -1393,6 +1421,10 @@ export type Database = { }; Returns: Json; }; + invalidate_asset_resource_indexes: { + Args: { p_project_id: string }; + Returns: undefined; + }; latestBuildVirtual: | { Args: { diff --git a/packages/postgrest/src/index.server.ts b/packages/postgrest/src/index.server.ts index a5ad11233bbe..476d79c1f3dd 100644 --- a/packages/postgrest/src/index.server.ts +++ b/packages/postgrest/src/index.server.ts @@ -2,26 +2,10 @@ import type { Database } from "./__generated__/db-types"; import { PostgrestClient } from "@supabase/postgrest-js"; export type { Database } from "./__generated__/db-types"; -type DatabaseWithAssetRevision = Omit & { - public: Omit & { - Functions: Database["public"]["Functions"] & { - swap_asset_file: { - Args: { - asset_id: string; - expected_name: string; - project_id: string; - replacement_name: string; - }; - Returns: string; - }; - }; - }; -}; - -export type Client = PostgrestClient; +export type Client = PostgrestClient; export const createClient = (url: string, apiKey: string): Client => { - const client = new PostgrestClient(url, { + const client = new PostgrestClient(url, { headers: { apikey: apiKey, Authorization: `Bearer ${apiKey}`, diff --git a/packages/postgrest/supabase/tests/asset-file-metadata.sql b/packages/postgrest/supabase/tests/asset-file-metadata.sql index f548b9ee9767..6726d5ae69fe 100644 --- a/packages/postgrest/supabase/tests/asset-file-metadata.sql +++ b/packages/postgrest/supabase/tests/asset-file-metadata.sql @@ -54,6 +54,36 @@ SELECT is( 'One asset has one active canonical revision' ); +CREATE TEMP TABLE initial_metadata_token AS +SELECT "metadataToken" +FROM "AssetFileMetadata" +WHERE "projectId" = 'metadata-test-project' + AND "assetId" = 'metadata-test-asset'; + +SELECT is( + replace_asset_file_metadata( + 'metadata-test-project', + 'metadata-test-asset', + 'revision-1', + '{"_id":"metadata-test-asset","revision":"revision-1"}'::JSONB, + '[]'::JSONB, + '{"storageName":"metadata-test.md","fileUpdatedAt":"2026-07-18T10:00:00Z","fileSize":10,"filename":"post.md","folderId":null}'::JSONB + ), + TRUE, + 'Repeated replacement of identical canonical metadata is accepted' +); + +SELECT is( + ( + SELECT "metadataToken" + FROM "AssetFileMetadata" + WHERE "projectId" = 'metadata-test-project' + AND "assetId" = 'metadata-test-asset' + ), + (SELECT "metadataToken" FROM initial_metadata_token), + 'Identical canonical metadata preserves its snapshot token' +); + UPDATE "File" SET "size" = 11, "updatedAt" = '2026-07-18T11:00:00Z' diff --git a/packages/postgrest/supabase/tests/asset-resource-indexes.sql b/packages/postgrest/supabase/tests/asset-resource-indexes.sql index 20261e7d115f..4d4ae53edc57 100644 --- a/packages/postgrest/supabase/tests/asset-resource-indexes.sql +++ b/packages/postgrest/supabase/tests/asset-resource-indexes.sql @@ -5,6 +5,9 @@ SELECT no_plan(); INSERT INTO "Project" ("id", "title", "tags", "domain") VALUES ('resource-index-test-project', 'Resource index test', '{}', 'resource-index-test.example'); +INSERT INTO "Build" ("id", "projectId", pages, resources) +VALUES ('resource-index-test-build', 'resource-index-test-project', '[]', '[]'); + SELECT lives_ok( $$ SELECT begin_asset_resource_index_build( @@ -13,10 +16,28 @@ SELECT lives_ok( '*[]', 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', - 'attempt-active' + 'attempt-active', + '[]'::JSONB, + NULL, + NULL, + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'projects/test/resources/posts/index.json' ) $$, - 'A valid query and asset revision begin an index build' + 'A valid build pre-registers its immutable revision before storage writes' +); + +SELECT is( + ( + SELECT count(*)::INTEGER + FROM "AssetResourceIndexRevision" + WHERE "projectId" = 'resource-index-test-project' + AND "resourceId" = 'posts' + AND revision = 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc' + ), + 1, + 'A crash after storage persistence leaves a revision discoverable by GC' ); SELECT is( @@ -34,6 +55,89 @@ SELECT is( 'A validated revision is recorded and activated atomically' ); +UPDATE "Build" +SET resources = '[{"id":"newer-resource-snapshot"}]' +WHERE id = 'resource-index-test-build' + AND "projectId" = 'resource-index-test-project'; + +SELECT is( + ( + SELECT "buildStatus" + FROM "AssetResourceIndexState" + WHERE "projectId" = 'resource-index-test-project' + AND "resourceId" = 'posts' + ), + 'STALE'::"AssetResourceIndexBuildStatus", + 'A saved resource change invalidates active indexes synchronously' +); + +SELECT is( + begin_asset_resource_index_build( + 'resource-index-test-project', + 'posts', + '*[false]', + 'sha256:1111111111111111111111111111111111111111111111111111111111111111', + 'sha256:2222222222222222222222222222222222222222222222222222222222222222', + 'attempt-stale-resources', + '[]'::JSONB, + 'resource-index-test-build', + '[]' + ), + FALSE, + 'A build from a stale resource snapshot cannot replace current state' +); + +SELECT is( + begin_asset_resource_index_build( + 'resource-index-test-project', + 'posts', + '*[false]', + 'sha256:1111111111111111111111111111111111111111111111111111111111111111', + 'sha256:2222222222222222222222222222222222222222222222222222222222222222', + 'attempt-stale-metadata', + '[{"assetId":"missing","metadataToken":"old"}]'::JSONB, + 'resource-index-test-build', + '[{"id":"newer-resource-snapshot"}]' + ), + FALSE, + 'A build from a stale metadata snapshot cannot replace current state' +); + +UPDATE "Build" +SET resources = '[]' +WHERE id = 'resource-index-test-build' + AND "projectId" = 'resource-index-test-project'; + +SELECT begin_asset_resource_index_build( + 'resource-index-test-project', + 'posts', + '*[]', + 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'attempt-reactivated', + '[]'::JSONB, + 'resource-index-test-build', + '[]' +); + +SELECT is( + activate_asset_resource_index( + 'resource-index-test-project', + 'posts', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'attempt-reactivated', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'projects/test/resources/posts/index.json', + '[]'::JSONB, + 'resource-index-test-build', + '[]' + ), + TRUE, + 'The current resource snapshot can reactivate its matching revision' +); + SELECT is( ( SELECT revision."checksum" @@ -50,13 +154,47 @@ SELECT is( 'Project data resolves an active revision to its checksum and object reference' ); +INSERT INTO "File" (name, format, size, status) +VALUES ('resource-index-test.md', 'file', 10, 'UPLOADED'); + +INSERT INTO "Asset" (id, "projectId", name) +VALUES ('resource-index-test-asset', 'resource-index-test-project', 'resource-index-test.md'); + +INSERT INTO "AssetFileMetadata" ( + "projectId", "assetId", revision, document, "fieldContributions" +) VALUES ( + 'resource-index-test-project', + 'resource-index-test-asset', + 'file:resource-index-test.md:1', + '{}'::JSONB, + '[]'::JSONB +); + +SELECT is( + ( + SELECT "buildStatus" + FROM "AssetResourceIndexState" + WHERE "projectId" = 'resource-index-test-project' + AND "resourceId" = 'posts' + ), + 'STALE'::"AssetResourceIndexBuildStatus", + 'A canonical metadata change invalidates an active index synchronously' +); + +DELETE FROM "AssetFileMetadata" +WHERE "projectId" = 'resource-index-test-project' + AND "assetId" = 'resource-index-test-asset'; + SELECT begin_asset_resource_index_build( 'resource-index-test-project', 'posts', '*[]', 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', 'sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', - 'attempt-old' + 'attempt-old', + '[]'::JSONB, + 'resource-index-test-build', + '[]' ); SELECT begin_asset_resource_index_build( @@ -65,7 +203,36 @@ SELECT begin_asset_resource_index_build( '*[]', 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', 'sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', - 'attempt-current' + 'attempt-current', + '[]'::JSONB, + 'resource-index-test-build', + '[]' +); + +SELECT is( + activate_asset_resource_index( + 'resource-index-test-project', + 'posts', + 'sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', + 'attempt-old', + 'sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', + 'projects/test/resources/posts/superseded.json' + ), + FALSE, + 'A superseded attempt cannot activate its persisted revision' +); + +SELECT ok( + ( + SELECT "unreferencedAt" IS NOT NULL + FROM "AssetResourceIndexRevision" + WHERE "projectId" = 'resource-index-test-project' + AND "resourceId" = 'posts' + AND revision = 'sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' + ), + 'A superseded persisted revision remains discoverable for garbage collection' ); SELECT is( @@ -106,6 +273,71 @@ SELECT is( 'A failed replacement preserves the previous active revision' ); +UPDATE "AssetResourceIndexRevision" +SET "unreferencedAt" = CURRENT_TIMESTAMP - INTERVAL '1 minute' +WHERE "projectId" = 'resource-index-test-project' + AND "resourceId" = 'posts' + AND revision = 'sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; + +SELECT is( + begin_asset_resource_index_build( + 'resource-index-test-project', + 'posts', + '*[]', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', + 'attempt-reserved-revision', + '[]'::JSONB, + 'resource-index-test-build', + '[]', + 'sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', + 'sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', + 'projects/test/resources/posts/superseded.json' + ), + TRUE, + 'Reusing an immutable revision reserves it before the storage write' +); + +SELECT ok( + ( + SELECT "unreferencedAt" > CURRENT_TIMESTAMP + FROM "AssetResourceIndexRevision" + WHERE "projectId" = 'resource-index-test-project' + AND "resourceId" = 'posts' + AND revision = 'sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' + ), + 'A reused unreferenced revision receives a fresh garbage-collection grace period' +); + +SELECT is( + ( + SELECT count(*)::INTEGER + FROM claim_asset_resource_index_garbage(CURRENT_TIMESTAMP, 10) + ), + 0, + 'Garbage collection cannot claim a revision reserved by a current build' +); + +CREATE TEMP TABLE superseded_revision_garbage AS +SELECT * FROM claim_asset_resource_index_garbage('2100-01-01', 10); + +SELECT is( + (SELECT count(*)::INTEGER FROM superseded_revision_garbage), + 1, + 'A superseded persisted revision remains collectable after its grace period' +); + +SELECT is( + finalize_asset_resource_index_garbage( + (SELECT "projectId" FROM superseded_revision_garbage), + (SELECT "resourceId" FROM superseded_revision_garbage), + (SELECT revision FROM superseded_revision_garbage), + (SELECT "gcClaimId" FROM superseded_revision_garbage) + ), + TRUE, + 'The superseded persisted revision can be finalized independently' +); + SELECT throws_ok( $$ UPDATE "AssetResourceIndexState" @@ -214,13 +446,32 @@ SELECT is( 'Removing a deployment releases its revision reference' ); +SELECT lives_ok( + $$ + SELECT add_asset_resource_index_reference( + 'resource-index-test-project', + 'posts', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'BUILD', + 'abandoned-build' + ) + $$, + 'An in-flight build can temporarily retain a revision' +); + +UPDATE "AssetResourceIndexReference" +SET "createdAt" = CURRENT_TIMESTAMP - INTERVAL '25 hours' +WHERE "projectId" = 'resource-index-test-project' + AND type = 'BUILD' + AND "referenceId" = 'abandoned-build'; + CREATE TEMP TABLE claimed_resource_index_garbage AS SELECT * FROM claim_asset_resource_index_garbage('2100-01-01', 10); SELECT is( (SELECT count(*)::INTEGER FROM claimed_resource_index_garbage), 1, - 'The revision becomes collectable after its last reference is removed' + 'An expired transient build reference does not retain a revision forever' ); SELECT is( diff --git a/packages/prisma-client/prisma/migrations/20260718120000_asset_file_metadata/migration.sql b/packages/prisma-client/prisma/migrations/20260718120000_asset_file_metadata/migration.sql index e9f2a2867f7d..d4a2556b475f 100644 --- a/packages/prisma-client/prisma/migrations/20260718120000_asset_file_metadata/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718120000_asset_file_metadata/migration.sql @@ -2,6 +2,7 @@ CREATE TABLE "AssetFileMetadata" ( "projectId" TEXT NOT NULL, "assetId" TEXT NOT NULL, "revision" TEXT NOT NULL, + "metadataToken" TEXT NOT NULL DEFAULT gen_random_uuid()::TEXT, "document" JSONB NOT NULL, "fieldContributions" JSONB NOT NULL DEFAULT '[]', "createdAt" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/packages/prisma-client/prisma/migrations/20260718121000_serialize_asset_file_metadata/migration.sql b/packages/prisma-client/prisma/migrations/20260718121000_serialize_asset_file_metadata/migration.sql index f099a3a8122c..79f50cb935f8 100644 --- a/packages/prisma-client/prisma/migrations/20260718121000_serialize_asset_file_metadata/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718121000_serialize_asset_file_metadata/migration.sql @@ -51,9 +51,12 @@ BEGIN ) ON CONFLICT ("projectId", "assetId", "revision") DO UPDATE SET + "metadataToken" = gen_random_uuid()::TEXT, "document" = EXCLUDED."document", "fieldContributions" = EXCLUDED."fieldContributions", - "updatedAt" = EXCLUDED."updatedAt"; + "updatedAt" = EXCLUDED."updatedAt" + WHERE "AssetFileMetadata"."document" IS DISTINCT FROM EXCLUDED."document" + OR "AssetFileMetadata"."fieldContributions" IS DISTINCT FROM EXCLUDED."fieldContributions"; DELETE FROM public."AssetFileMetadata" WHERE "projectId" = p_project_id diff --git a/packages/prisma-client/prisma/migrations/20260718123000_activate_asset_resource_index/migration.sql b/packages/prisma-client/prisma/migrations/20260718123000_activate_asset_resource_index/migration.sql index 0f425f730f22..f20f40a82edd 100644 --- a/packages/prisma-client/prisma/migrations/20260718123000_activate_asset_resource_index/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718123000_activate_asset_resource_index/migration.sql @@ -1,15 +1,144 @@ +CREATE FUNCTION invalidate_asset_resource_indexes(p_project_id TEXT) +RETURNS VOID +LANGUAGE plpgsql +AS $$ +BEGIN + UPDATE public."AssetResourceIndexState" + SET "buildAttemptId" = gen_random_uuid()::TEXT, + "buildStatus" = (CASE + WHEN "activeRevision" IS NULL THEN 'PENDING' + ELSE 'STALE' + END)::"AssetResourceIndexBuildStatus", + "buildError" = NULL, + "updatedAt" = CURRENT_TIMESTAMP + WHERE "projectId" = p_project_id + AND "deletedAt" IS NULL; +END; +$$; + +CREATE FUNCTION invalidate_asset_resource_indexes_after_metadata_change() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM invalidate_asset_resource_indexes( + COALESCE(NEW."projectId", OLD."projectId") + ); + RETURN NULL; +END; +$$; + +CREATE TRIGGER invalidate_asset_resource_indexes_after_metadata_change +AFTER INSERT OR UPDATE OR DELETE ON public."AssetFileMetadata" +FOR EACH ROW +EXECUTE FUNCTION invalidate_asset_resource_indexes_after_metadata_change(); + +CREATE FUNCTION invalidate_asset_resource_indexes_after_resource_change() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.resources IS DISTINCT FROM OLD.resources THEN + PERFORM invalidate_asset_resource_indexes(NEW."projectId"); + END IF; + RETURN NULL; +END; +$$; + +CREATE TRIGGER invalidate_asset_resource_indexes_after_resource_change +AFTER UPDATE OF resources ON public."Build" +FOR EACH ROW +EXECUTE FUNCTION invalidate_asset_resource_indexes_after_resource_change(); + +CREATE FUNCTION asset_file_metadata_snapshot_matches( + p_project_id TEXT, + p_metadata_snapshot JSONB +) +RETURNS BOOLEAN +LANGUAGE sql +STABLE +AS $$ + SELECT NOT EXISTS ( + ( + SELECT metadata."assetId", metadata."metadataToken" + FROM public."AssetFileMetadata" AS metadata + WHERE metadata."projectId" = p_project_id + EXCEPT + SELECT snapshot."assetId", snapshot."metadataToken" + FROM jsonb_to_recordset(p_metadata_snapshot) AS snapshot( + "assetId" TEXT, + "metadataToken" TEXT + ) + ) + UNION ALL + ( + SELECT snapshot."assetId", snapshot."metadataToken" + FROM jsonb_to_recordset(p_metadata_snapshot) AS snapshot( + "assetId" TEXT, + "metadataToken" TEXT + ) + EXCEPT + SELECT metadata."assetId", metadata."metadataToken" + FROM public."AssetFileMetadata" AS metadata + WHERE metadata."projectId" = p_project_id + ) + ); +$$; + +CREATE FUNCTION asset_resource_index_source_matches( + p_project_id TEXT, + p_build_id TEXT, + p_resources TEXT +) +RETURNS BOOLEAN +LANGUAGE plpgsql +AS $$ +BEGIN + IF (p_build_id IS NULL) <> (p_resources IS NULL) THEN + RAISE EXCEPTION 'Resource index source is incomplete'; + END IF; + + IF p_build_id IS NULL THEN + RETURN TRUE; + END IF; + + PERFORM 1 + FROM public."Build" + WHERE "projectId" = p_project_id + AND "id" = p_build_id + AND resources::JSONB = p_resources::JSONB + FOR SHARE; + + RETURN FOUND; +END; +$$; + CREATE FUNCTION begin_asset_resource_index_build( p_project_id TEXT, p_resource_id TEXT, p_query TEXT, p_query_hash TEXT, p_asset_revision TEXT, - p_build_attempt_id TEXT + p_build_attempt_id TEXT, + p_metadata_snapshot JSONB DEFAULT '[]'::JSONB, + p_build_id TEXT DEFAULT NULL, + p_resources TEXT DEFAULT NULL, + p_revision TEXT DEFAULT NULL, + p_checksum TEXT DEFAULT NULL, + p_object_key TEXT DEFAULT NULL ) -RETURNS VOID +RETURNS BOOLEAN LANGUAGE plpgsql AS $$ BEGIN + IF NOT ( + (p_revision IS NULL AND p_checksum IS NULL AND p_object_key IS NULL) + OR + (p_revision IS NOT NULL AND p_checksum IS NOT NULL AND p_object_key IS NOT NULL) + ) THEN + RAISE EXCEPTION 'Resource index revision registration is incomplete'; + END IF; + PERFORM pg_catalog.pg_advisory_xact_lock( pg_catalog.hashtextextended( pg_catalog.jsonb_build_array(p_project_id, p_resource_id)::TEXT, @@ -17,6 +146,31 @@ BEGIN ) ); + IF asset_file_metadata_snapshot_matches( + p_project_id, + p_metadata_snapshot + ) = FALSE THEN + RETURN FALSE; + END IF; + + IF asset_resource_index_source_matches( + p_project_id, + p_build_id, + p_resources + ) = FALSE THEN + RETURN FALSE; + END IF; + + IF p_build_id IS NULL AND EXISTS ( + SELECT 1 + FROM public."AssetResourceIndexState" + WHERE "projectId" = p_project_id + AND "resourceId" = p_resource_id + AND ("queryHash" <> p_query_hash OR "deletedAt" IS NOT NULL) + ) THEN + RETURN FALSE; + END IF; + INSERT INTO public."AssetResourceIndexState" ( "projectId", "resourceId", @@ -50,6 +204,54 @@ BEGIN "buildError" = NULL, "deletedAt" = NULL, "updatedAt" = CURRENT_TIMESTAMP; + + IF p_revision IS NOT NULL THEN + INSERT INTO public."AssetResourceIndexRevision" ( + "projectId", + "resourceId", + "revision", + "queryHash", + "assetRevision", + "checksum", + "objectKey", + "unreferencedAt" + ) VALUES ( + p_project_id, + p_resource_id, + p_revision, + p_query_hash, + p_asset_revision, + p_checksum, + p_object_key, + CURRENT_TIMESTAMP + INTERVAL '24 hours' + ) + ON CONFLICT ("projectId", "resourceId", "revision") + DO UPDATE SET "unreferencedAt" = CASE + WHEN "AssetResourceIndexRevision"."unreferencedAt" IS NULL THEN NULL + ELSE GREATEST( + "AssetResourceIndexRevision"."unreferencedAt", + CURRENT_TIMESTAMP + INTERVAL '24 hours' + ) + END + WHERE "AssetResourceIndexRevision"."gcClaimId" IS NULL; + + PERFORM 1 + FROM public."AssetResourceIndexRevision" + WHERE "projectId" = p_project_id + AND "resourceId" = p_resource_id + AND revision = p_revision + AND "queryHash" = p_query_hash + AND "assetRevision" = p_asset_revision + AND checksum = p_checksum + AND "objectKey" = p_object_key + AND "gcClaimId" IS NULL + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'Immutable resource index revision metadata conflicts'; + END IF; + END IF; + + RETURN TRUE; END; $$; @@ -61,7 +263,10 @@ CREATE FUNCTION activate_asset_resource_index( p_asset_revision TEXT, p_build_attempt_id TEXT, p_checksum TEXT, - p_object_key TEXT + p_object_key TEXT, + p_metadata_snapshot JSONB DEFAULT '[]'::JSONB, + p_build_id TEXT DEFAULT NULL, + p_resources TEXT DEFAULT NULL ) RETURNS BOOLEAN LANGUAGE plpgsql @@ -76,20 +281,6 @@ BEGIN ) ); - PERFORM 1 - FROM public."AssetResourceIndexState" - WHERE "projectId" = p_project_id - AND "resourceId" = p_resource_id - AND "queryHash" = p_query_hash - AND "assetRevision" = p_asset_revision - AND "buildAttemptId" = p_build_attempt_id - AND "buildStatus" = 'BUILDING' - FOR UPDATE; - - IF NOT FOUND THEN - RETURN FALSE; - END IF; - INSERT INTO public."AssetResourceIndexRevision" ( "projectId", "resourceId", @@ -97,7 +288,8 @@ BEGIN "queryHash", "assetRevision", "checksum", - "objectKey" + "objectKey", + "unreferencedAt" ) VALUES ( p_project_id, p_resource_id, @@ -105,7 +297,8 @@ BEGIN p_query_hash, p_asset_revision, p_checksum, - p_object_key + p_object_key, + CURRENT_TIMESTAMP ) ON CONFLICT ("projectId", "resourceId", "revision") DO NOTHING; @@ -124,6 +317,35 @@ BEGIN RAISE EXCEPTION 'Immutable resource index revision metadata conflicts'; END IF; + IF asset_file_metadata_snapshot_matches( + p_project_id, + p_metadata_snapshot + ) = FALSE THEN + RETURN FALSE; + END IF; + + IF asset_resource_index_source_matches( + p_project_id, + p_build_id, + p_resources + ) = FALSE THEN + RETURN FALSE; + END IF; + + PERFORM 1 + FROM public."AssetResourceIndexState" + WHERE "projectId" = p_project_id + AND "resourceId" = p_resource_id + AND "queryHash" = p_query_hash + AND "assetRevision" = p_asset_revision + AND "buildAttemptId" = p_build_attempt_id + AND "buildStatus" = 'BUILDING' + FOR UPDATE; + + IF NOT FOUND THEN + RETURN FALSE; + END IF; + UPDATE public."AssetResourceIndexState" SET "activeRevision" = p_revision, "buildStatus" = 'ACTIVE', diff --git a/packages/prisma-client/prisma/migrations/20260718127000_delete_asset_resource_index_query/migration.sql b/packages/prisma-client/prisma/migrations/20260718127000_delete_asset_resource_index_query/migration.sql index 6ff26b8f2754..ba69cdeafa87 100644 --- a/packages/prisma-client/prisma/migrations/20260718127000_delete_asset_resource_index_query/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718127000_delete_asset_resource_index_query/migration.sql @@ -1,11 +1,21 @@ CREATE FUNCTION delete_asset_resource_index_query( p_project_id TEXT, - p_resource_id TEXT + p_resource_id TEXT, + p_build_id TEXT DEFAULT NULL, + p_resources TEXT DEFAULT NULL ) RETURNS BOOLEAN LANGUAGE plpgsql AS $$ BEGIN + IF asset_resource_index_source_matches( + p_project_id, + p_build_id, + p_resources + ) = FALSE THEN + RETURN FALSE; + END IF; + PERFORM pg_catalog.pg_advisory_xact_lock( pg_catalog.hashtextextended( pg_catalog.jsonb_build_array(p_project_id, p_resource_id)::TEXT, diff --git a/packages/prisma-client/prisma/migrations/20260718129000_asset_resource_index_gc/migration.sql b/packages/prisma-client/prisma/migrations/20260718129000_asset_resource_index_gc/migration.sql index 4f95925d75dd..c810c0f8405b 100644 --- a/packages/prisma-client/prisma/migrations/20260718129000_asset_resource_index_gc/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718129000_asset_resource_index_gc/migration.sql @@ -31,6 +31,13 @@ BEGIN WHERE reference."projectId" = candidate."projectId" AND reference."resourceId" = candidate."resourceId" AND reference.revision = candidate.revision + -- BUILD references protect only an in-flight snapshot operation. + -- Expiring abandoned references prevents a failed cleanup request + -- from retaining an immutable object forever. + AND ( + reference.type <> 'BUILD' + OR reference."createdAt" > CURRENT_TIMESTAMP - INTERVAL '24 hours' + ) ) AND NOT EXISTS ( SELECT 1 FROM public."AssetResourceIndexState" AS state @@ -73,6 +80,10 @@ BEGIN WHERE reference."projectId" = candidate."projectId" AND reference."resourceId" = candidate."resourceId" AND reference.revision = candidate.revision + AND ( + reference.type <> 'BUILD' + OR reference."createdAt" > CURRENT_TIMESTAMP - INTERVAL '24 hours' + ) ) AND NOT EXISTS ( SELECT 1 FROM public."AssetResourceIndexState" AS state diff --git a/packages/prisma-client/prisma/schema.prisma b/packages/prisma-client/prisma/schema.prisma index d6f32224aea5..469a737e5f4b 100644 --- a/packages/prisma-client/prisma/schema.prisma +++ b/packages/prisma-client/prisma/schema.prisma @@ -59,6 +59,7 @@ model AssetFileMetadata { projectId String assetId String revision String + metadataToken String @default(uuid()) document Json fieldContributions Json @default("[]") createdAt DateTime @default(now()) @db.Timestamptz(3) diff --git a/packages/project-build/src/runtime/asset-resources.ts b/packages/project-build/src/runtime/asset-resources.ts index b42ef6b952be..35c8e4e6f367 100644 --- a/packages/project-build/src/runtime/asset-resources.ts +++ b/packages/project-build/src/runtime/asset-resources.ts @@ -341,17 +341,18 @@ export const updateAssetsResource = ( if (resource === undefined || isAssetsResource(resource) === false) { return throwBuilderRuntimeError("NOT_FOUND", "Assets resource not found"); } + const { name, query: queryUpdate } = input.values; const storedConfiguration = parseAssetResourceConfiguration(resource); if ( isStoredAssetQueryResource(resource) && - storedConfiguration === undefined + storedConfiguration === undefined && + queryUpdate === undefined ) { return throwBuilderRuntimeError( "BAD_REQUEST", - "Stored Assets query configuration could not be decoded" + "Stored Assets query configuration could not be decoded; replace or remove the query to repair it" ); } - const { name, query: queryUpdate } = input.values; const currentQuery = storedConfiguration === undefined ? undefined diff --git a/packages/project-build/src/runtime/registry.test.ts b/packages/project-build/src/runtime/registry.test.ts index f43c334d7cb2..9a3ffaf4dc73 100644 --- a/packages/project-build/src/runtime/registry.test.ts +++ b/packages/project-build/src/runtime/registry.test.ts @@ -1864,6 +1864,38 @@ describe("builder runtime read families", () => { expect( JSON.stringify((disabled as { payload: unknown }).payload) ).not.toContain("/$resources/assets/query"); + + const storedQueryResource = queryState.resources?.get("asset-resource"); + if (storedQueryResource === undefined) { + throw new Error("Expected queried Assets resource"); + } + const invalidQueryState = { + ...queryState, + resources: new Map([ + ...(queryState.resources ?? []), + [ + "invalid-asset-resource", + { + ...storedQueryResource, + id: "invalid-asset-resource", + body: "invalid query configuration", + }, + ], + ]), + } as BuilderState; + const repaired = executeBuilderRuntimeOperation({ + id: "assetsResources.update", + state: invalidQueryState, + input: { + resourceId: "invalid-asset-resource", + values: { query: null }, + scopeInstanceId: "heading", + }, + context, + }); + expect( + JSON.stringify((repaired as { payload: unknown }).payload) + ).toContain('\\"/$resources/assets\\"'); }); test("replaces bounded fixed resource URLs without changing expressions", () => { diff --git a/packages/sdk/src/resource-loader.test.ts b/packages/sdk/src/resource-loader.test.ts index 195f4b5e60b3..1556d1690351 100644 --- a/packages/sdk/src/resource-loader.test.ts +++ b/packages/sdk/src/resource-loader.test.ts @@ -168,15 +168,45 @@ describe("loadResource", () => { }); }); - test("recognizes absolute local resource URLs", () => { + test("does not intercept an external URL that resembles a local resource", () => { expect( isLocalResource("https://example.com/$resources/assets", "assets") - ).toBe(true); + ).toBe(false); + expect(isLocalResource("//$resources/assets", "assets")).toBe(false); + expect(isLocalResource("//example.com/$resources/assets", "assets")).toBe( + false + ); }); test("keeps the legacy Assets path distinct from the query path", () => { expect(isLocalResource("/$resources/assets", "assets")).toBe(true); expect(isLocalResource("/$resources/assets/query", "assets")).toBe(false); + expect( + isLocalResource( + "/$resources/assets/index-status?resourceId=posts", + "assets/index-status" + ) + ).toBe(true); + }); + + test("keeps a local resource relative while resolving ordinary URLs", async () => { + mockFetch.mockResolvedValue(new Response("ok")); + await loadResource( + mockFetch, + { + name: "assets", + url: "/$resources/assets", + searchParams: [{ name: "page", value: 1 }], + method: "get", + headers: [], + }, + "https://example.com/blog/post" + ); + + expect(mockFetch).toHaveBeenCalledWith( + "/$resources/assets?page=1", + expect.any(Object) + ); }); test("returns a structured cancellation failure", async () => { diff --git a/packages/sdk/src/resource-loader.ts b/packages/sdk/src/resource-loader.ts index 8aff44061901..de456703246c 100644 --- a/packages/sdk/src/resource-loader.ts +++ b/packages/sdk/src/resource-loader.ts @@ -8,13 +8,15 @@ const LOCAL_RESOURCE_PREFIX = "$resources"; * Prevents fetch cycles by prefixing local resources. */ export const isLocalResource = (pathname: string, resourceName?: string) => { - let resolvedPathname = pathname; - try { - resolvedPathname = new URL(pathname).pathname; - } catch { - // Keep relative paths unchanged. + const pathEnd = [pathname.indexOf("?"), pathname.indexOf("#")] + .filter((index) => index !== -1) + .reduce((first, index) => Math.min(first, index), pathname.length); + const path = pathname.slice(0, pathEnd); + if (path.startsWith("//")) { + return false; } - const segments = resolvedPathname.split("/").filter(Boolean); + const normalizedPath = path.startsWith("/") ? path.slice(1) : path; + const segments = normalizedPath.split("/"); if (resourceName === undefined) { return segments[0] === LOCAL_RESOURCE_PREFIX; @@ -81,15 +83,20 @@ export const loadResource = async ( try { // cloudflare workers fail when fetching url contains spaces // even though new URL suppose to trim them on parsing by spec - const resolutionBase = - baseUrl === undefined ? undefined : new URL("/", baseUrl); - const url = new URL(resourceRequest.url.trim(), resolutionBase); + const sourceUrl = resourceRequest.url.trim(); + const local = isLocalResource(sourceUrl); + const resolutionBase = local + ? new URL("https://webstudio.local") + : baseUrl === undefined + ? undefined + : new URL("/", baseUrl); + const url = new URL(sourceUrl, resolutionBase); if (searchParams) { for (const { name, value } of searchParams) { url.searchParams.append(name, serializeValue(value)); } } - href = url.href; + href = local ? `${url.pathname}${url.search}` : url.href; } catch { // empty block } From ad22ca5e79b39724ba34208d92864e4ffe261df1 Mon Sep 17 00:00:00 2001 From: Oleg Isonen Date: Tue, 21 Jul 2026 19:27:27 +0100 Subject: [PATCH 12/14] fix: keep tests out of Builder routes --- apps/builder/app/routes/rest.resources-loader.ts | 2 +- .../$resources/create-local-resource-request.test.ts} | 2 +- .../$resources/create-local-resource-request.ts} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename apps/builder/app/{routes/rest.resources-loader-utils.test.ts => shared/$resources/create-local-resource-request.test.ts} (89%) rename apps/builder/app/{routes/rest.resources-loader-utils.ts => shared/$resources/create-local-resource-request.ts} (100%) diff --git a/apps/builder/app/routes/rest.resources-loader.ts b/apps/builder/app/routes/rest.resources-loader.ts index b581dea0a790..698a3c9e22b0 100644 --- a/apps/builder/app/routes/rest.resources-loader.ts +++ b/apps/builder/app/routes/rest.resources-loader.ts @@ -13,7 +13,7 @@ import { preventCrossOriginCookie } from "~/services/no-cross-origin-cookie"; import { checkCsrf } from "~/services/csrf-session.server"; import { getResourceKey } from "~/shared/resources"; import { privateNoStoreResponseHeaders } from "~/services/cache-control.server"; -import { createLocalResourceRequest } from "./rest.resources-loader-utils"; +import { createLocalResourceRequest } from "../shared/$resources/create-local-resource-request"; export const action = async ({ request }: ActionFunctionArgs) => { preventCrossOriginCookie(request); diff --git a/apps/builder/app/routes/rest.resources-loader-utils.test.ts b/apps/builder/app/shared/$resources/create-local-resource-request.test.ts similarity index 89% rename from apps/builder/app/routes/rest.resources-loader-utils.test.ts rename to apps/builder/app/shared/$resources/create-local-resource-request.test.ts index e1e692b91ef8..1c29bd9fd426 100644 --- a/apps/builder/app/routes/rest.resources-loader-utils.test.ts +++ b/apps/builder/app/shared/$resources/create-local-resource-request.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { createLocalResourceRequest } from "./rest.resources-loader-utils"; +import { createLocalResourceRequest } from "./create-local-resource-request"; describe("resources loader local dispatch", () => { test("creates nested requests from relative local resource URLs", async () => { diff --git a/apps/builder/app/routes/rest.resources-loader-utils.ts b/apps/builder/app/shared/$resources/create-local-resource-request.ts similarity index 100% rename from apps/builder/app/routes/rest.resources-loader-utils.ts rename to apps/builder/app/shared/$resources/create-local-resource-request.ts From beeedc15bff2437f1e9dc35f0554c934660222cb Mon Sep 17 00:00:00 2001 From: Oleg Isonen Date: Wed, 22 Jul 2026 18:15:38 +0100 Subject: [PATCH 13/14] docs: regenerate CLI guidance --- packages/cli/src/docs.generated.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/docs.generated.ts b/packages/cli/src/docs.generated.ts index bc3528abec25..2bb8093b78f3 100644 --- a/packages/cli/src/docs.generated.ts +++ b/packages/cli/src/docs.generated.ts @@ -2,11 +2,11 @@ // Edit markdown files in src/docs instead. export const cliDocs = { "api-use-cases": - '# CLI API Use Cases\n\n## Link/configure one project\n\nCommands:\n\n- webstudio init --link --json\n\nNotes:\n\n- Writes local project id and global origin/token config.\n\n## Import synced project bundle into another project\n\nCommands:\n\n- webstudio sync\n- webstudio import --to \n- MCP tool: import {"to":""}\n\nNotes:\n\n- Imports local `.webstudio/data.json` into the destination project.\n- Destination share link must allow build/import access.\n- Use `--skip-assets` only when asset rows and files should not be imported.\n\n## Identify current token\n\nCommands:\n\n- MCP tool: whoami {}\n\n## Check token permissions\n\nCommands:\n\n- webstudio permissions --json\n\n## Inspect project/build/version\n\nCommands:\n\n- MCP tool: inspect {}\n- MCP tool: snapshot {"include":["pages","instances","styles"]}\n\n## Discover CLI/API capabilities\n\nCommands:\n\n- webstudio schema api\n- webstudio schema mcp\n- webstudio man --json\n- webstudio man llm --json\n- MCP tool: meta.index {}\n- MCP tool: meta.guide {"brief":"Create a pricing page"}\n- MCP tool: meta.get_more_tools {"brief":"update-styles"}\n- webstudio mcp list-resources\n- webstudio mcp read-resource webstudio://project/guide\n- webstudio mcp read-resource webstudio://project/expressions\n\nNotes:\n\n- Use `webstudio schema mcp` for a compact machine-readable MCP tool overview. Add `--verbose` or use focused `meta.get_more_tools` calls only when exact input schemas are needed.\n- Use focused MCP tools for discovery first: `meta.index`, `meta.guide`, `meta.get_more_tools`, `components.list`, `components.summary`, `components.search`, `components.get`, `templates.list`, and `templates.get`. Protocol clients can use `resources/list` and `resources/read`; shell agents can use `webstudio mcp list-resources` and `webstudio mcp read-resource `. Read longer resources such as `webstudio://project/tools` and `webstudio://project/components` only when focused tools are insufficient.\n- `components.summary` returns counts by default; request `{"detail":"components","limit":20}` for paginated entries. Registry list tools return compact metadata, while `components.get` and `templates.get` return focused full details.\n- Read `webstudio://project/expressions` before authoring unfamiliar computed text, prop bindings, resource expressions, actions, or Collection item bindings.\n- From a shell, call one MCP tool with the shortcut form `webstudio \'\'`, for example `webstudio components.summary`. The explicit equivalent is `webstudio mcp single-op-call \'\'`. Use `--input-file` for large payloads.\n\n## Inspect external shadcn registry items\n\nCommands:\n\n- webstudio registry inspect --source https://example.com/r/registry.json --item button --json\n- webstudio registry inspect --source ./registry.json --item dialog --json\n- webstudio registry inspect --source https://example.com/r/button.json --json\n\nNotes:\n\n- Reads a local or remote registry item without installing files or changing the configured Webstudio project.\n- Returns the item name, description, package and registry dependencies, file paths/targets, available docs, and a read-only compatibility report.\n- The report explicitly says whether installation or editable-component conversion is supported, lists declared requirements and manual steps, and says when arbitrary source code was not analyzed.\n- This is an inspection step only. It does not install files or change the configured project.\n\n## Understand what MCP can do\n\nCommands:\n\n- webstudio man mcp\n- MCP tool: meta.index {}\n- MCP tool: meta.guide {"brief":"What can Webstudio MCP do?"}\n\nMCP lets agents work on one configured Webstudio project. Agents can:\n\n- Inspect the linked project, token permissions, and latest editable build.\n- Read selected project data for audits, migrations, and repair.\n- Search labels, text, props, resource URLs, asset metadata, and styles.\n- Audit accessibility, security, SEO, performance settings, unused assets, ineffective Collection styles, and unused or duplicate style data.\n- Create and edit pages, folders, redirects, breakpoints, and page templates.\n- Create pages from reusable templates.\n- Update page metadata, SEO fields, auth settings, and marketplace metadata.\n- Insert components and styled JSX sections.\n- Create data-driven lists, grids, cards, and similar repeated UI from array or object data in one Collection operation.\n- Move, copy, wrap, unwrap, convert, rename, retag, and delete elements.\n- Update text, rich text, props, bindings, and actions.\n- Create and update local styles, design tokens, style sources, and CSS variables.\n- Create static data variables and JSON variables.\n- Create HTTP, GraphQL, and system resources.\n- Use system resources for sitemap, current date, and assets.\n- Bind resources to rendered data or form/action props.\n- Manage nested asset folders and upload, inspect, move, duplicate, download, replace, delete, and inspect usage for assets.\n- Publish, unpublish, inspect publish jobs, and manage custom domains.\n- Start preview, capture screenshots, compare screenshot diffs, and use OCR when installed.\n\n## Inspect and refresh MCP session cache\n\nCommands:\n\n- MCP tool: status {}\n- MCP tool: status {"verbose":true}\n- MCP tool: refresh {"namespaces":["pages","instances","styles"]}\n- MCP tool: reset-session {}\n\nNotes:\n\n- Use status before a task to understand the cached ProjectSession state.\n- Use status with `{"verbose":true}` only when debugging full namespaces, freshness, compatibility, or diagnostics.\n- Use refresh when project data may have changed outside the current MCP session.\n- Use reset-session when local cached state is corrupt or incompatible.\n\n## Visually verify rendered work with AI vision\n\nCommands:\n\n- MCP tool: preview.start {"host":"127.0.0.1","port":5173}\n- MCP tool: preview.status {}\n- MCP tool: preview.stop {}\n- MCP tool: screenshot {"path":"/","output":".webstudio/screenshots/home-current.png","viewport":{"width":1440,"height":900},"waitUntil":"load","waitForTimeout":250}\n- MCP tool: screenshot {"path":"/pricing","output":".webstudio/screenshots/pricing-current.png","viewport":{"width":1440,"height":900},"waitUntil":"load","waitForTimeout":250}\n- MCP tool: screenshot {"baseUrl":"http://127.0.0.1:5177","path":"/pricing","output":".webstudio/screenshots/pricing-current.png","viewport":{"width":1440,"height":900},"waitUntil":"load","waitForTimeout":250}\n- MCP tool: screenshot.diff {"baselinePath":".webstudio/screenshots/home-before.png","currentPath":".webstudio/screenshots/home-current.png","outputDir":".webstudio/screenshots/diff"}\n- MCP tool: screenshot.diff {"baselinePath":".webstudio/screenshots/home-before.png","currentPath":".webstudio/screenshots/home-current.png","outputDir":".webstudio/screenshots/diff","expectedText":["Pricing","Start free"]}\n- MCP tool: screenshot.diff {"baselinePath":".webstudio/screenshots/home-before.png","currentPath":".webstudio/screenshots/home-current.png","outputDir":".webstudio/screenshots/diff","expectedVisual":{"maxMismatchPercentage":2,"maxChangedRegions":3,"dominantColorChange":{"channel":"luminance","direction":"increase","minMagnitude":10}}}\n- MCP tool: screenshot.diff {"baselinePath":".webstudio/screenshots/pricing-before.png","currentPath":".webstudio/screenshots/pricing-current.png","outputDir":".webstudio/screenshots/diff"}\n- MCP tool: vision.install-ocr {"confirm":true}\n\nNotes:\n\n- Use this after page/content/style mutations and after generated project files are current so a vision-capable AI can see the production-like generated site.\n- For multi-page work, capture every changed page by `path` through the same preview server; no click navigation is required.\n- After MCP mutations, path screenshots regenerate/restart preview as needed before capture; when preview is fresh, repeated path screenshots reuse the running server.\n- Do not call `preview.start` through one-shot `webstudio mcp single-op-call`: it is long-lived. From a shell, use `webstudio mcp run` with preview.start, screenshot, and preview.stop in one shared process, or use a real long-running MCP client.\n- From one-shot shell calls or another process, pass `baseUrl` with `path` to capture an already-running preview/site without generating, building, starting, or restarting preview.\n- Use preview.stop only in the same long-running MCP server or `webstudio mcp run` process that started preview. A separate one-shot `single-op-call` process does not own another process\'s preview controller.\n- Use waitForSelector when the rendered app has a reliable ready marker, waitUntil:"networkidle" for network-heavy pages, and waitForTimeout only for final visual settling.\n- Preview installs generated app dependencies under `.webstudio/preview` and reuses them across regenerations.\n- Do not add generated-preview dependencies to the repository root `package.json` or `pnpm-lock.yaml`.\n- If dependency installation fails, check npm and network configuration, then reinstall or update the Webstudio CLI if the problem persists.\n- When a baseline exists, use screenshot.diff once per baseline/current page or viewport pair to get changed regions, OCR textAnalysis, and diff artifact paths before deciding whether the result matches. Pass expectedText for explicit pass/fail current-screen text assertions with found and missing text. Pass expectedVisual for pass/fail limits on pixel mismatch percentage, changed-region count, or the overall dominant color/brightness direction.\n- If screenshot.diff reports OCR unavailable and the user agrees to install it, call vision.install-ocr {"confirm":true}; otherwise continue with pixel diff and visual inspection.\n- Compare the PNG, OCR text evidence, and diff artifacts against the user\'s intent for layout, typography, colors, spacing, imagery, and responsive framing; then iterate with focused mutations.\n- Root CLI equivalent: `webstudio screenshot --path /pricing --output pricing.png` generates a temporary production preview, captures that route, and stops the server. For repeated captures, keep `webstudio preview` running and pass its absolute URL to `webstudio screenshot`.\n\n## List pages\n\nCommands:\n\n- MCP tool: list-pages {}\n- MCP tool: list-folders {}\n\n## Read page by id\n\nCommands:\n\n- MCP tool: get-page {"pageId":""}\n\n## Read page by path\n\nCommands:\n\n- MCP tool: get-page-by-path {"path":"/pricing"}\n\n## Create page\n\nCommands:\n\n- MCP tool: create-page {"name":"Pricing","path":"/pricing"}\n- MCP tool: create-page {"name":"Pricing","path":"/pricing","title":"Pricing","meta":{"description":"Plans for teams"}}\n\nNotes:\n\n- `name`, `path`, page `title`, and metadata text fields accept plain fixed values.\n- For computed page titles or metadata, send JavaScript expression code such as `pageTitle ?? "Pricing"`.\n\n## Update page settings/metadata\n\nCommands:\n\n- MCP tool: update-page {"pageId":"","values":{"title":"Pricing","meta":{"description":"Plans","status":200}}}\n- MCP tool: update-page {"pageId":"","values":{"meta":{"auth":{"method":"basic","login":"","password":""}}}}\n\nNotes:\n\n- Page `title` and metadata text fields accept plain fixed values.\n- For computed page titles or metadata, send JavaScript expression code such as `pageTitle ?? "Pricing"`.\n- Page `status` accepts a fixed HTTP status code as a number from 200 through 599 or a JavaScript expression string for a dynamic status.\n\n## Read project settings\n\nCommands:\n\n- MCP tool: get-project-settings {}\n\nNotes:\n\n- Read `meta.agentInstructions` before making project changes. It contains the project\'s own guidance for AI agents.\n- Agent instructions are shared project guidance. Do not store credentials or other secrets there.\n\n## Update project settings\n\nCommands:\n\n- MCP tool: update-project-settings {"meta":{"siteName":"Acme"}}\n- MCP tool: update-project-settings {"meta":{"agentInstructions":"Use existing design tokens and keep product copy concise."}}\n\n## Read marketplace product\n\nCommands:\n\n- MCP tool: get-marketplace-product {}\n\n## Update marketplace product\n\nCommands:\n\n- MCP tool: update-marketplace-product {"category":"pageTemplates","name":"Acme Template","thumbnailAssetId":"asset-id","author":"Acme Studio","email":"hello@example.com","website":"https://example.com","issues":"","description":"Reusable template project for Acme landing pages."}\n\n## List redirects\n\nCommands:\n\n- MCP tool: list-redirects {}\n\n## Create redirect\n\nCommands:\n\n- MCP tool: create-redirect {"old":"/old","new":"/new","status":301}\n\n## Update redirect\n\nCommands:\n\n- MCP tool: update-redirect {"old":"/old","values":{"new":"/newer","status":302}}\n- MCP tool: update-redirect {"old":"/old","values":{"status":null}}\n\n## Delete redirect\n\nCommands:\n\n- MCP tool: delete-redirect {"old":"/old"}\n\n## Set redirects\n\nCommands:\n\n- MCP tool: set-redirects {"redirects":[{"old":"/old","new":"/new","status":"301"}]}\n\n## List breakpoints\n\nCommands:\n\n- MCP tool: list-breakpoints {}\n\n## Create breakpoint\n\nCommands:\n\n- MCP tool: create-breakpoint {"label":"Tablet","maxWidth":991}\n\n## Update breakpoint\n\nCommands:\n\n- MCP tool: update-breakpoint {"breakpointId":"tablet","values":{"label":"Tablet","maxWidth":1023}}\n- MCP tool: update-breakpoint {"breakpointId":"tablet","values":{"condition":null,"minWidth":768}}\n- MCP tool: update-breakpoint {"breakpointId":"tablet","values":{"minWidth":null,"maxWidth":null,"condition":"(hover: hover)"}}\n\n## Delete breakpoint\n\nCommands:\n\n- MCP tool: delete-breakpoint {"breakpointId":"tablet"}\n\n## Duplicate page\n\nCommands:\n\n- MCP tool: duplicate-page {"pageId":"","name":"Pricing Copy","path":"/pricing-copy"}\n- MCP tool: duplicate-page {"pageId":"","name":"Paris","path":"/paris","substitutions":{"text":{"London":"Paris"},"variables":{"city":{"type":"string","value":"Paris"}}}}\n- webstudio duplicate-page --page --name Paris --path /paris --substitutions \'{"text":{"London":"Paris"},"variables":{"city":{"type":"string","value":"Paris"}}}\' --json\n\nNotes:\n\n- Text substitutions replace exact fixed text only in the duplicated page\'s text children, string props, title, and metadata.\n- Variable substitutions are keyed by copied source-variable name and use typed variable values. The operation rejects missing or ambiguous names without committing a partial duplicate.\n- Existing expressions and cloned variable/resource references keep their remapped ids.\n\n## List page templates\n\nCommands:\n\n- MCP tool: list-page-templates {}\n\n## Create page template\n\nCommands:\n\n- MCP tool: create-page-template {"name":"Landing Template","title":"Landing"}\n\n## Update page template\n\nCommands:\n\n- MCP tool: update-page-template {"templateId":"","values":{"name":"Article Template","meta":{"description":"Reusable article layout"}}}\n\n## Delete page template\n\nCommands:\n\n- MCP tool: delete-page-template {"templateId":""}\n\n## Duplicate page template\n\nCommands:\n\n- MCP tool: duplicate-page-template {"templateId":""}\n\n## Reorder page template\n\nCommands:\n\n- MCP tool: reorder-page-template {"sourceTemplateId":"","targetTemplateId":"","position":"before"}\n\n## Create page from template\n\nCommands:\n\n- MCP tool: create-page-from-template {"templateId":"","name":"Landing","path":"/landing"}\n\n## Delete page\n\nCommands:\n\n- MCP tool: delete-page {"pageId":""}\n\n## List folders\n\nCommands:\n\n- MCP tool: list-folders {}\n- MCP tool: list-pages {}\n\n## Create folder\n\nCommands:\n\n- MCP tool: create-folder {"name":"Blog","slug":"blog"}\n\n## Update folder\n\nCommands:\n\n- MCP tool: update-folder {"folderId":"","values":{"name":"Blog","slug":"blog"}}\n\n## Delete folder\n\nCommands:\n\n- MCP tool: delete-folder {"folderId":""}\n\n## List element instances\n\nCommands:\n\n- MCP tool: list-instances {"pagePath":"/","maxDepth":3}\n\n## Inspect one element instance\n\nCommands:\n\n- MCP tool: inspect-instance {"instanceId":"","include":["props","styles","children"]}\n\n## Insert authored JSX or one component template\n\nCommands:\n\n- MCP tool: insert-fragment {"parentInstanceId":"","fragment":"Product OS"}\n- MCP tool: insert-component {"parentInstanceId":"","component":"@webstudio-is/sdk-components-react-radix:Switch"}\n- MCP tool: insert-component {"parentInstanceId":"","component":"Form"}\n\nNotes:\n\n- Use MCP `insert-fragment` as the default way to author styled component trees. It converts JSX to a structured fragment before mutation.\n- Use only exact component ids returned by `components.search`, `components.get`, or `templates.get`. Never derive or guess component ids.\n- The `ws:` namespace contains specific Webstudio core components; it is not HTML-tag shorthand. Use `` for a native `div` and `` for a native form, never `` or ``.\n- For Webstudio\'s complete form structure, discover the Form component and insert its automatic template with `insert-component` using component `"Form"`.\n- MCP receives JSX as a JSON string because MCP arguments are JSON. The CLI converts it locally before the runtime mutation, so the project session receives structured Webstudio data, not JSX source.\n- In `insert-fragment` JSX, use `ws:style={css\\`...\\`}`for Webstudio-native CSS, or use React-style object syntax such as`style={{ padding: 24 }}` when that is simpler. Both forms create editable Webstudio style data.\n- Do not access host globals or dynamic code APIs in JSX fragments, including `process`, `globalThis`, `eval`, `Function`, or `constructor`.\n- Use Webstudio prop names such as `class` and `for`; do not use React aliases `className` or `htmlFor`.\n- Use Webstudio actions for event/action props, for example `onClick={new ActionValue(["event"], expression\\`console.log(event)\\`)}`. Do not pass JavaScript functions such as `onClick={() => ...}`.\n- Plain prop values must be JSON-compatible: `null`, strings, booleans, finite numbers, arrays, and plain objects. Do not pass `undefined`, `Symbol`, `BigInt`, `NaN`, `Infinity`, `Date`, `Map`, `Set`, class instances, or circular objects; omit the prop, use plain data, or use `expression`/`ActionValue` when the value is dynamic.\n- Template-backed components used in JSX must include required child/part components explicitly under the same parent structure as the template, for example ``.\n- Webstudio applies a registered template automatically when using `insert-component`, so composed components such as Switch include required child parts and styles.\n- Use `components.list`, `components.summary`, `components.search`, `components.get`, `templates.list`, and `templates.get` to discover known registry items, component ids, props, templates, insertability, and content model. Read `webstudio://project/components` only when those focused tools are insufficient.\n- Component/template registry items use a shadcn-compatible top-level shape plus Webstudio-specific superset metadata in `meta`. They are for Builder/MCP discovery, not a published shadcn install registry yet.\n- Known components with `contentModel.category: "none"` are not standalone-insertable; insert their root component template instead so required providers/parents are included.\n- Unknown custom component ids are a low-level extension mechanism, not a discovery fallback. Agents must not synthesize them.\n\n## Make a region editable in Content mode\n\nCommands:\n\n- MCP tool: insert-component {"parentInstanceId":"","component":"ws:block"}\n- MCP tool: inspect-instance {"instanceId":"","include":["children"]}\n\nNotes:\n\n- When a page will be handed to a Content-mode editor, wrap every region they should be able to edit in a Content Block (`ws:block`). Content-mode editors can edit text and supported props only in Content Block descendants. Content outside those blocks remains read-only, even when it looks like ordinary editable text.\n- Put reusable insertable options inside the Content Block\'s `ws:block-template` child. A template is source material, not editor content: editors cannot edit or delete it directly. When an editor inserts a template, its copy becomes a direct child of the Content Block and is editable.\n- Before handing off a page, verify with `inspect-instance` that the intended text, images, and links are inside a Content Block, and that templates include all required styling because Content-mode editors cannot use the Style panel.\n\n## Move elements\n\nCommands:\n\n- MCP tool: move-instance {"moves":"moves.json contents"}\n\nNotes:\n\n- Use `position: "end"` to append an instance. Repeating this for A and then B preserves the final order A, B.\n- A numeric `insertIndex` addresses the target parent\'s children before the moved instance is removed. Use it for exact placement; do not calculate the last index to append.\n- Moves in one `moves` array are applied sequentially in array order.\n\n## Clone element subtree\n\nCommands:\n\n- MCP tool: clone-instance {"sourceInstanceId":"","targetParentInstanceId":""}\n\n## Delete element subtree\n\nCommands:\n\n- MCP tool: delete-instance {"instanceIds":[""]}\n\n## List text/expression children\n\nCommands:\n\n- MCP tool: list-texts {"pagePath":"/"}\n\n## Update text child\n\nCommands:\n\n- MCP tool: update-text {"instanceId":"","childIndex":0,"text":"Launch faster"}\n\n## Replace bounded literal text\n\nCommands:\n\n- MCP tool: replace-text {"find":"Start free","replace":"Get started","match":"exact","pagePath":"/pricing","limit":20}\n\nNotes:\n\n- This changes only literal text children, never expression children. Scope it to pagePath or pageId and set a limit before a broad replacement.\n\n## Replace bounded static prop text\n\nCommands:\n\n- MCP tool: replace-prop-text {"find":"old.example.com","replace":"www.example.com","match":"substring","names":["href","code"],"limit":20}\n\nNotes:\n\n- This changes only static string props such as href, alt, aria-label, title, and HTML embed code. It never changes expressions, resources, actions, assets, or other dynamic bindings. Use names or instanceIds and a limit to narrow the change.\n\n## Replace bounded resource text\n\nCommands:\n\n- MCP tool: replace-resource-text {"find":"api.old.example.com","replace":"api.example.com","fields":["url"],"limit":20}\n\nNotes:\n\n- This changes resource names and fixed URL literals only. It skips dynamic URL expressions, headers, search parameters, request bodies, and GraphQL query code.\n\n## Update props\n\nCommands:\n\n- MCP tool: update-props {"updates":"props.json contents"}\n- MCP tool: replace-prop-text {"find":"Old label","replace":"New label","names":["aria-label","title"],"limit":20}\n\nNotes:\n\n- Use this for fixed prop values such as `aria-label`, `alt`, `id`, static `href`, and other direct string/number/boolean/json prop values.\n\n## Add JSON-LD structured data\n\nCommands:\n\n- MCP tool: components.get {"component":"JsonLd"}\n- MCP tool: insert-component {"parentInstanceId":"","component":"JsonLd"}\n- MCP tool: update-props {"updates":[{"instanceId":"","name":"code","type":"string","value":"{\\"@context\\":\\"https://schema.org\\",\\"@type\\":\\"Organization\\",\\"name\\":\\"Acme\\"}"}]}\n- MCP tool: audit {"scopes":["seo"],"pagePath":"/"}\n\nNotes:\n\n- Prefer placing `JsonLd` inside `HeadSlot`.\n- Store `code` as a JSON object or array encoded as a compact string. The Builder formats it for editing.\n- The semantic prop update rejects malformed JSON and structurally invalid fixed JSON-LD with a precise JSON path.\n- The SEO audit also warns about a missing top-level `@context`, unknown or superseded Schema.org terms, properties unsupported by the supplied type, and incompatible primitive value types.\n- Schema.org vocabulary findings are warnings because custom vocabularies and extensions remain valid. Dynamic JSON-LD is marked as skipped for rendered validation.\n- Do not use bindings just to set static text.\n\n## Delete props\n\nCommands:\n\n- MCP tool: delete-props {"deletions":"props.json contents"}\n\n## Bind props to expressions/resources/actions\n\nCommands:\n\n- MCP tool: bind-props {"bindings":"bindings.json contents"}\n\nNotes:\n\n- Use this only when the prop should remain dynamic: expression, resource, action, or an existing scoped runtime context value such as `system`.\n- For a fixed string value, use `update-props` with `type:"string"` and a direct `value` instead.\n\n## Read styles\n\nCommands:\n\n- MCP tool: get-styles {"instanceIds":[""],"includeTokens":true}\n\n## Update local styles\n\nCommands:\n\n- MCP tool: update-styles {"updates":"styles.json contents"}\n\n## Delete local styles\n\nCommands:\n\n- MCP tool: delete-styles {"deletions":"styles.json contents"}\n\n## Replace matching style values\n\nCommands:\n\n- MCP tool: replace-styles {"property":"color","fromValue":{"type":"keyword","value":"red"},"toValue":{"type":"keyword","value":"blue"}}\n\n## List design tokens\n\nCommands:\n\n- MCP tool: list-design-tokens {}\n- MCP tool: list-design-tokens {"withUsage":true}\n- MCP tool: list-design-tokens {"verbose":true}\n\nNotes:\n\n- The default response is compact and includes token id, name, declaration count, and optional usage count.\n- Use `verbose:true` only when you need the full inline style declarations.\n\n## Create design tokens\n\nCommands:\n\n- MCP tool: create-design-token {"tokens":"tokens.json contents"}\n\n## Update design token styles\n\nCommands:\n\n- MCP tool: update-design-token-styles {"designTokenId":"","updates":"styles.json contents"}\n\n## Delete design token styles\n\nCommands:\n\n- MCP tool: delete-design-token-styles {"designTokenId":"","deletions":"styles.json contents"}\n\n## Attach design token to instances\n\nCommands:\n\n- MCP tool: attach-design-token {"designTokenId":"","instanceIds":"instances.json contents"}\n\n## Detach design token from instances\n\nCommands:\n\n- MCP tool: detach-design-token {"designTokenId":"","instanceIds":"instances.json contents"}\n\n## Extract design token from local styles\n\nCommands:\n\n- MCP tool: extract-design-token {"instanceIds":[""],"name":"Brand Primary","removeLocalProps":["color"]}\n\n## List CSS variables\n\nCommands:\n\n- MCP tool: list-css-variables {"withUsage":true}\n\n## Define CSS variables\n\nCommands:\n\n- MCP tool: define-css-variable {"vars":"vars.json contents"}\n\n## Delete CSS variables\n\nCommands:\n\n- MCP tool: delete-css-variable {"names":"names.json contents","force":true}\n\n## Rewrite CSS variable references\n\nCommands:\n\n- MCP tool: rewrite-css-variable-refs {"map":"variables.json contents"}\n\n## List data variables\n\nCommands:\n\n- MCP tool: list-variables {}\n- MCP tool: list-variables {"scopeInstanceId":""}\n\nNotes:\n\n- Data variables live in the internal `dataSources` namespace.\n- For raw `snapshot`, request the public `variables` namespace rather than the internal `dataSources` name. Raw patch payloads still use `dataSources` when applying direct changes.\n- Scope variables to the instance where they should become available. Descendants can use them in expressions, and nested variables with the same name mask outer variables.\n\n## Create data variable\n\nCommands:\n\n- MCP tool: create-variable {"scopeInstanceId":"","name":"title","value":{"type":"string","value":"Hello"}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"count","value":{"type":"number","value":3}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"featured","value":{"type":"boolean","value":true}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"tags","value":{"type":"string[]","value":["news","product"]}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"filters","value":{"type":"json","value":{"tag":"news"}}}\n\nNotes:\n\n- Data variable values support `string`, `number`, `boolean`, `string[]`, and `json`.\n- Parameters are internal scoped runtime values provided by pages, collections, or components. They are not a public authoring surface: do not create, update, or delete parameter records. Use data variables/resources for user-authored data, and reference documented context values such as `system` only where they are already in scope.\n\n## Update data variable\n\nCommands:\n\n- MCP tool: update-variable {"dataSourceId":"","values":{"value":{"type":"json","value":{"count":1}}}}\n\n## Delete data variable\n\nCommands:\n\n- MCP tool: delete-variable {"dataSourceId":""}\n\n## List resources\n\nCommands:\n\n- MCP tool: list-resources {}\n- MCP tool: list-resources {"scopeInstanceId":""}\n\n## Create resource\n\nCommands:\n\n- MCP tool: create-resource {"resource":{"name":"Posts","method":"get","url":"https://api.example.com/posts","headers":[]}}\n- MCP tool: create-resource {"resource":{"name":"Posts","method":"get","url":"\\"https://api.example.com/posts?tag=\\" + filters.tag","headers":[]},"scopeInstanceId":"","dataSourceName":"posts"}\n- MCP tool: create-resource {"resource":{"name":"Filtered Posts","method":"get","url":"https://api.example.com/posts","searchParams":[{"name":"tag","value":"filters.tag"},{"name":"source","value":{"type":"literal","value":"website"}}],"headers":[{"name":"Authorization","value":"\\"Bearer \\" + auth.token"}]},"scopeInstanceId":"","dataSourceName":"posts"}\n- MCP tool: create-resource {"resource":{"name":"Post GraphQL","control":"graphql","method":"post","url":"https://api.example.com/graphql","headers":[{"name":"Content-Type","value":{"type":"literal","value":"application/json"}}],"body":"{ query: \\"query Post($slug: String!) { post(slug: $slug) { title } }\\", variables: { slug: system.params.slug } }"},"scopeInstanceId":"","dataSourceName":"post"}\n- MCP tool: create-resource {"resource":{"name":"Current Date","control":"system","method":"get","url":"/$resources/current-date","headers":[]},"scopeInstanceId":"","dataSourceName":"currentDate"}\n\nNotes:\n\n- Resource `url` accepts plain fixed URLs and paths such as `https://api.example.com/posts` and `/$resources/current-date`.\n- Resource `url` can also be a JavaScript expression when it is computed, such as `"https://api.example.com/posts?tag=" + filters.tag`.\n- Header values, search parameter values, and body accept expressions for dynamic values. For fixed text, use `{"type":"literal","value":"application/json"}`; Webstudio stores the required string expression for you.\n- Search parameter values, header values, and body expressions can read scoped variables and documented runtime context values such as `system` when they are available at the resource scope.\n- Add `scopeInstanceId` and `dataSourceName` when the resource result should be exposed as a scoped read data variable. Scoped resources are generated into the page resource `data` map and may be loaded during page rendering. Use this for read-oriented resources such as GET CMS/API data.\n- For submit/write/action resources, create the resource without `scopeInstanceId`, then bind a component prop such as a Form `action` with `bind-props` and `binding.type: "resource"`. Prop-bound resources are generated into the page resource `action` map instead of the read `data` map. Use this for POST, PUT, DELETE, webhooks, GraphQL submissions, and other explicit action flows.\n- Resource `method` can be `get`, `post`, `put`, or `delete`. Use GET for read data, POST for creates/GraphQL/webhooks/form submissions, PUT for full updates or replacements, and DELETE for deletion actions.\n- Optional `control` values are `graphql` and `system`. Use `graphql` for GraphQL-style requests, usually POST with a query body. Use `system` for built-in resources such as `"/$resources/sitemap.xml"`, `"/$resources/current-date"`, and `"/$resources/assets"` and when the resource should use the built-in `system` parameter. System fields are `system.origin`, `system.pathname`, `system.params`, and `system.search`.\n\n## Update resource\n\nCommands:\n\n- MCP tool: update-resource {"resourceId":"","values":{"url":"https://api.example.com/posts"}}\n- MCP tool: replace-resource-text {"find":"api.old.example.com","replace":"api.example.com","fields":["url"],"limit":20}\n\n## Delete resource\n\nCommands:\n\n- MCP tool: delete-resource {"resourceId":""}\n\n## List assets\n\nCommands:\n\n- MCP tool: list-assets {"withUsage":true}\n- MCP tool: list-assets {"verbose":true}\n\nNotes:\n\n- Compact results include each asset\'s folder id. Use `verbose:true` to include complete records for a page of assets, or `get-asset` to read one complete record including description, folder, creation time, and image/font metadata.\n- Image asset descriptions are the default alt text for asset-backed Image components.\n- To generate missing descriptions, inspect the image in its rendered page or asset source, write a concise description of its purpose, and save it on the asset rather than duplicating it on each Image instance.\n\n## Get asset\n\nCommands:\n\n- MCP tool: get-asset {"assetId":""}\n\n## List asset folders\n\nCommands:\n\n- MCP tool: list-asset-folders {}\n\n## Create asset folder\n\nCommands:\n\n- MCP tool: create-asset-folder {"name":"Marketing"}\n- MCP tool: create-asset-folder {"name":"Photos","parentId":""}\n\n## Update asset folder\n\nCommands:\n\n- MCP tool: update-asset-folder {"folderId":"","values":{"name":"Brand"}}\n- MCP tool: update-asset-folder {"folderId":"","values":{"parentId":""}}\n- MCP tool: update-asset-folder {"folderId":"","values":{"parentId":null}}\n\nNotes:\n\n- Updating `parentId` is the folder equivalent of cut and paste. Use `null` to move a folder to Root.\n\n## Duplicate asset folder\n\nCommands:\n\n- MCP tool: duplicate-asset-folder {"folderId":""}\n- MCP tool: duplicate-asset-folder {"folderId":"","parentId":""}\n\nNotes:\n\n- Duplication recursively copies descendant folders and assets. This is the folder equivalent of copy and paste.\n\n## Delete asset folder\n\nCommands:\n\n- MCP tool: delete-asset-folder {"folderId":""}\n\nNotes:\n\n- Deleting a folder recursively deletes its descendant folders and assets.\n\n## Update asset metadata\n\nCommands:\n\n- MCP tool: update-asset {"assetId":"","values":{"description":"Team collaborating around a whiteboard"}}\n- MCP tool: update-asset {"assetId":"","values":{"meta":{"family":"Rajdhani","style":"normal","weight":600}}}\n\nNotes:\n\n- Use an empty description only when the image is intentionally decorative.\n- Updating an image asset description updates the default alt text wherever that asset is used with an asset-backed alt prop.\n- Font metadata updates merge with the detected metadata and are validated before committing; use this to correct a family, style, or weight after upload.\n\n## Generate missing image descriptions with an agent\n\nCommands:\n\n- MCP tool: audit {"scopes":["accessibility"],"verbose":true}\n- MCP tool: set-image-descriptions {"updates":[{"assetId":"hero-id","description":"Team collaborating around a whiteboard"},{"assetId":"texture-id","decorative":true}]}\n- MCP tool: audit {"scopes":["accessibility"]}\n\nNotes:\n\n- Start from `missing-image-description` findings. Inspect each image in its rendered page context before writing text.\n- The vision-capable agent generates the wording; the CLI validates and stores it but does not contain its own vision model.\n- Use `decorative:true` only when the image adds no information. This intentionally stores an empty description so later audits do not report it as missing.\n- Re-run the accessibility audit after the update. Asset-backed Image components use the saved asset description as their default alt text.\n\n## Manage fonts\n\nCommands:\n\n- MCP tool: list-fonts {"includeSystem":true}\n- MCP tool: list-assets {"type":"font"}\n- MCP tool: upload-asset {"asset":{"name":"acme-sans.woff2","type":"font","format":"woff2","meta":{"family":"Acme Sans","style":"normal","weight":400}},"assetsDir":".webstudio/assets"}\n- MCP tool: update-styles {"updates":"styles.json contents"}\n\nNotes:\n\n- Use `list-fonts` to discover uploaded families and system stacks. Upload/delete fonts through the existing asset tools, then apply a family with a `font-family` style declaration.\n\n## Upload one asset\n\nCommands:\n\n- MCP tool: upload-asset {"asset":{"name":"image.png","type":"image","format":"png","meta":{"width":1200,"height":630}},"assetsDir":".webstudio/assets"}\n- MCP tool: upload-asset {"asset":{"name":"image.png","type":"image","format":"png","folderId":"","meta":{"width":1200,"height":630}},"assetsDir":".webstudio/assets"}\n\n## Upload asset batch\n\nCommands:\n\n- MCP tool: upload-assets {"assets":[{"name":"image.png","type":"image","format":"png","meta":{"width":1200,"height":630}}],"assetsDir":".webstudio/assets"}\n\n## Duplicate asset\n\nCommands:\n\n- MCP tool: duplicate-asset {"assetId":""}\n- MCP tool: duplicate-asset {"assetId":"","folderId":""}\n- MCP tool: duplicate-asset {"assetId":"","folderId":null}\n\nNotes:\n\n- Duplication is the asset equivalent of copy and paste. Updating `folderId` is the equivalent of cut and paste; use `null` for Root.\n\n## Download asset\n\nCommands:\n\n- MCP tool: download-asset {"assetId":""}\n\n## Find asset usage\n\nCommands:\n\n- MCP tool: find-asset-usage {"assetId":""}\n\n## Replace asset references\n\nCommands:\n\n- MCP tool: replace-asset {"fromAssetId":"","toAssetId":""}\n\n## Delete assets\n\nCommands:\n\n- MCP tool: delete-asset {"assetIdsOrPrefixes":[""],"force":true}\n\n## Publish project\n\nCommands:\n\n- webstudio publish deploy --target production --json\n\n## List publishes\n\nCommands:\n\n- webstudio publish list --json\n\n## Check publish job\n\nCommands:\n\n- webstudio publish status --job --json\n\n## Unpublish\n\nCommands:\n\n- webstudio publish unpublish --target production --confirm --json\n\n## List domains\n\nCommands:\n\n- webstudio domains list --json\n\n## Create domain\n\nCommands:\n\n- webstudio domains create --domain example.com --json\n\n## Update domain\n\nCommands:\n\n- webstudio domains update --domain-id --domain www.example.com --json\n\n## Delete domain\n\nCommands:\n\n- webstudio domains delete --domain-id --confirm --json\n\n## Verify domain\n\nCommands:\n\n- webstudio domains verify --domain-id --json\n\n## Make arbitrary store-level changes\n\nCommands:\n\n- MCP tool: inspect {}\n- MCP tool: snapshot {"include":[""]}\n- MCP tool: apply-patch {"baseVersion":"","transactions":"patch.json contents"}\n\nNotes:\n\n- Use only when no semantic command exists.\n\n## Manage marketplace metadata\n\nCommands:\n\n- MCP tool: get-marketplace-product {}\n- MCP tool: update-marketplace-product {"category":"pageTemplates","name":"Acme Template","thumbnailAssetId":"asset-id","author":"Acme Studio","email":"hello@example.com","website":"https://example.com","issues":"","description":"Reusable template project for Acme landing pages."}\n\nPatch namespaces:\n\n- marketplaceProduct\n\n## Search and inspect safely\n\nCommands:\n\n- MCP tool: search-project {"query":"pricing"}\n- MCP tool: search-project {"query":"api.example.com","scopes":["resources"]}\n- MCP tool: list-instances {"pagePath":"/","maxDepth":5}\n- MCP tool: inspect-instance {"instanceId":"","include":["props","styles","children"]}\n- MCP tool: list-texts {"pagePath":"/"}\n- MCP tool: list-assets {"withUsage":true}\n- MCP tool: find-asset-usage {"assetId":""}\n- MCP tool: snapshot {"include":["pages","instances","props","resources","assets"]}\n\nNotes:\n\n- Use `search-project` for query-driven lookup across labels, text, prop values, resource URLs, asset metadata, and styles. Use `audit` for project health findings.\n\n## Audit project quality\n\nCommands:\n\n- webstudio audit --json\n- webstudio audit --scopes accessibility --scopes seo --json\n- webstudio audit --page-path /pricing --json\n- webstudio audit --scopes accessibility --verbose --json\n- webstudio audit --rendered --page-path /pricing --json\n- webstudio audit --rendered --route-example post=/blog/hello --json\n- webstudio audit --rendered --image-domain images.example.com --json\n- MCP tool: audit {}\n- MCP tool: audit {"scopes":["accessibility","security"],"severities":["error","warning"]}\n- MCP tool: audit {"scopes":["accessibility"],"verbose":true}\n- MCP tool: audit {"scopes":["craft"],"verbose":true}\n- MCP tool: audit {"rendered":true,"verbose":true}\n\nNotes:\n\n- With no scopes, `audit` checks accessibility, security, SEO, performance settings, unused assets, ineffective Collection styles, non-GET resources exposed as render-time data, and unused or duplicate style data.\n- Craft is opt-in and read-only. Run `audit` with `scopes:["craft"]` to detect whether the project is not using Craft, partially compatible, or compatible with the versioned Craft 1.2 profile. `profileStatuses` includes the University-doc provenance and the smallest safe next action. The audit never installs Craft or changes a non-Craft project.\n- The `performance` scope reports disabled atomic CSS generation. A rendered audit also measures broken, eager below-fold, and oversized images, browser-marked render-blocking resources, and legacy font formats.\n- Rendered image and resource metrics run only when the selected scopes include `performance`; responsive layout dimensions remain available whenever `rendered:true` is requested.\n- Compact findings include stable ids, severity, message, and location. Use `--verbose` or `{"verbose":true}` for evidence, explanation, suggested remediation, skipped-check details, and manual-check workflows.\n- `summary` counts all findings before severity filtering and pagination.\n- `contractVersion` identifies the audit response contract. Handle a new value before assuming existing fields retain the same meaning.\n- Expression-, resource-, and parameter-backed values that cannot be checked reliably appear in `skippedChecks`; they are not treated as passing or failing.\n- Page filters apply to page-owned accessibility, security, and SEO checks. Asset and style usage remain project-wide to avoid false unused findings.\n- Continue paginated results with `cursor`. Restart the audit if the project version changes.\n- Verbose skipped-check and manual-check details are included on the first findings page only; their total counts remain available on every page.\n- `manualChecks` describes responsive, hierarchy, and contrast checks that require preview screenshots and vision.\n- Focused audits return only manual checks relevant to their selected scopes.\n- In a long-lived MCP session, `{"rendered":true}` reuses preview and screenshot\n tools to capture every static page at mobile, desktop, and Builder breakpoint\n edges. Compact output reports rendered check/issue/failure counts; verbose\n output includes screenshot paths and measured layout dimensions.\n- Dynamic route templates are skipped unless `--route-example =`\n (or MCP `routeExamples`) supplies a concrete path. The path must not contain\n unresolved `:` or `*` parameters.\n- Plans above 120 captures return a short-lived confirmation token. Review the\n unchanged plan, then rerun with `--confirm-large-run` and\n `--confirmation-token`.\n- Detailed rendered evidence is stored in a versioned manifest under\n `.webstudio/audits`; compact output includes its path and screenshot count.\n- Rendered checks also report broken images, eager images below the fold, and\n image sources more than 2x their rendered dimensions in both axes, including\n Webstudio instance ids and measured dimensions when available.\n- Rendered checks include sanitized Resource Timing evidence and report\n browser-marked render-blocking resources plus legacy `.ttf`, `.otf`, and\n `.woff` fonts without applying a universal transfer-size threshold.\n- Fix findings through semantic mutation commands, then rerun `audit` to confirm their deterministic finding ids disappeared.\n\n## Verify dynamic bindings\n\nCommands:\n\n- webstudio verify-bindings --json\n- MCP tool: verify-bindings {"pagePath":"/pricing"}\n- MCP tool: verify-bindings {"instanceId":"","limit":50}\n\nNotes:\n\n- Statically checks persisted text expressions, expression/action/resource/parameter props, resource expressions, and page metadata.\n- Findings distinguish invalid syntax, unknown or out-of-scope variables, stale internal data-source ids, and missing resource or parameter references.\n- Page and instance filters can be combined when the instance belongs to the selected page. Continue findings with `cursor`.\n- This operation does not resolve rendered values or execute external resources. Preview representative loading, empty, error, and populated states after static findings are fixed.\n\n## Refactor targeted content\n\nCommands:\n\n- MCP tool: list-instances {"pagePath":"/"}\n- MCP tool: list-texts {"pagePath":"/"}\n- MCP tool: update-text {"instanceId":"","childIndex":0,"text":"Launch faster"}\n- MCP tool: replace-text {"find":"Old headline","replace":"New headline","match":"exact","pagePath":"/pricing","limit":20}\n- MCP tool: update-props {"updates":"props.json contents"}\n- MCP tool: update-page {"pageId":"","values":{"title":"Pricing","meta":{"description":"Plans"}}}\n- MCP tool: update-resource {"resourceId":"","values":{"url":"https://api.example.com/posts"}}\n- MCP tool: replace-asset {"fromAssetId":"","toAssetId":""}\n- MCP tool: replace-styles {"property":"color","fromValue":{"type":"keyword","value":"red"},"toValue":{"type":"keyword","value":"blue"}}\n- MCP tool: rewrite-css-variable-refs {"map":"variables.json contents"}\n\nNotes:\n\n- Use focused reads first, then mutate only matching instances, props, metadata, resource URLs, assets, or style references. Use `replace-text` for bounded literal text changes, `replace-prop-text` for bounded static prop text, `replace-resource-text` for fixed resource names/URLs, and `update-text` for one known child or expressions.\n\n## Optimize existing project\n\nCommands:\n\n- MCP tool: list-pages {}\n- MCP tool: list-folders {}\n- MCP tool: update-page {"pageId":"","values":{"title":"Pricing","meta":{"description":"Plans"}}}\n- MCP tool: update-props {"updates":"props.json contents"}\n- MCP tool: list-breakpoints {}\n- MCP tool: update-breakpoint {"breakpointId":"tablet","values":{"maxWidth":1023}}\n- MCP tool: get-styles {"instanceIds":[""],"includeTokens":true}\n- MCP tool: update-styles {"updates":"styles.json contents"}\n- MCP tool: attach-design-token {"designTokenId":"","instanceIds":"instances.json contents"}\n- MCP tool: update-project-settings {"meta":{"siteName":"Acme"}}\n\nNotes:\n\n- Use this for SEO metadata, accessibility labels, responsive behavior, token consistency, and project settings.\n\n## Connect external data\n\nCommands:\n\n- MCP tool: create-variable {"scopeInstanceId":"","name":"title","value":{"type":"string","value":"Hello"}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"tags","value":{"type":"string[]","value":["news","product"]}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"filters","value":{"type":"json","value":{"tag":"news"}}}\n- MCP tool: create-resource {"resource":{"name":"Posts","method":"get","url":"https://api.example.com/posts","searchParams":[{"name":"tag","value":"filters.tag"},{"name":"source","value":{"type":"literal","value":"website"}}],"headers":[]},"scopeInstanceId":"","dataSourceName":"posts"}\n- MCP tool: create-resource {"resource":{"name":"Post GraphQL","control":"graphql","method":"post","url":"https://api.example.com/graphql","headers":[{"name":"Content-Type","value":{"type":"literal","value":"application/json"}}],"body":"{ query: \\"query Post($slug: String!) { post(slug: $slug) { title } }\\", variables: { slug: system.params.slug } }"},"scopeInstanceId":"","dataSourceName":"post"}\n- MCP tool: create-resource {"resource":{"name":"Current Date","control":"system","method":"get","url":"/$resources/current-date","headers":[]},"scopeInstanceId":"","dataSourceName":"currentDate"}\n- MCP tool: update-resource {"resourceId":"","values":{"url":"https://api.example.com/posts"}}\n- MCP tool: bind-props {"bindings":"bindings.json contents"}\n- MCP tool: insert-fragment {"parentInstanceId":"","fragment":"{/_ collection content _/}"}\n\nNotes:\n\n- Use this for CMS sections, blog listings, Ghost/headless CMS pages, n8n-style integrations, and API URLs built from variables.\n- For read data, expose GET resources as scoped data variables with `scopeInstanceId`/`dataSourceName` and read the loaded result from the resource result wrapper, usually `.data`.\n- For writes, webhooks, GraphQL submissions, and deletes, prefer unscoped resources bound to Form `action` props so they become action resources instead of auto-loaded read resources.\n- Use direct props for fixed values and prop bindings only when a prop must read a data variable, resource, action, or documented runtime context value such as `system`.\n\n## Render an array or object as repeated content\n\nCommands:\n\n- MCP tool: insert-collection {"parentInstanceId":"","data":{"type":"expression","value":"posts.data.items"},"itemFragment":"{expression`collectionItem.title`}"}\n- MCP tool: inspect-instance {"instanceId":"","include":["props","bindings","children"]}\n\nNotes:\n\n- Use Collection whenever an array or object from a resource or data variable should render a list, grid, cards, table rows, options, tabs, or other repeated UI.\n- Pass `insert-collection` the complete array or object. Do not pass the resource response wrapper or one indexed item. External resource arrays are commonly nested under the scoped resource result\'s `data` field or deeper.\n- Pass one repeated-item Webstudio JSX root. The command creates the Collection, its private current-item/current-key parameters, the iterable binding, and descendant item bindings atomically.\n- Collection renders the item root once for every entry. Use `expression` values such as `collectionItem.title` in descendant text and props. Object iteration also exposes `collectionItemKey`.\n- Wrap multiple repeated sibling instances in one Element inside Collection.\n- For repeated Radix items such as accordion items, tabs, or menu options, bind a stable unique id or slug to every required `value` prop.\n- See the [Collection documentation](https://docs.webstudio.is/university/core-components/collection) for the equivalent Builder workflow.\n\n## Support dynamic runtime behavior\n\nCommands:\n\n- MCP tool: integrate-runtime-ui {"parentInstanceId":"","resources":[{"resource":{"name":"Seats","method":"get","url":"https://api.example.com/seats","headers":[]},"dataSourceName":"Seats","exposeAsDataSource":true}],"structure":{"type":"collection","data":{"type":"expression","value":"Seats.data"},"itemFragment":{"children":[{"type":"id","value":"seat"}],"instances":[{"type":"instance","id":"seat","component":"Text","children":[{"type":"expression","value":"collectionItem.label"}]}],"props":[],"dataSources":[],"resources":[],"styleSources":[],"styleSourceSelections":[],"styles":[],"breakpoints":[],"assets":[]}},"retainedBehavior":[{"instanceId":"","responsibility":"Seat selection behavior"}]}\n- MCP tool: update-props {"updates":"props.json contents"}\n- MCP tool: bind-props {"bindings":"bindings.json contents"}\n- MCP tool: create-resource {"resource":{"name":"Seats","method":"get","url":"https://api.example.com/seats","headers":[]}}\n- MCP tool: snapshot {"include":["instances","props","resources"]}\n- MCP tool: apply-patch {"baseVersion":"","transactions":"patch.json contents"}\n\nNotes:\n\n- Use `integrate-runtime-ui` to create variables/resources, insert one editable fragment or Collection, and add safe data bindings in one transaction.\n- List existing script-owned responsibilities under `retainedBehavior`. The operation preserves those instances and never evaluates or accepts replacement script bodies.\n- `unsupportedConversions` records behavior that cannot be represented safely. Dry-run returns the complete transaction and the same retained/unsupported report without changing the project.\n- New actions and HtmlEmbed scripts are intentionally rejected. Create normal editable components and data bindings; keep opaque runtime behavior in existing script instances.\n\n## Build authenticated pages\n\nCommands:\n\n- MCP tool: meta.guide {"brief":"Build a Supabase-authenticated account page"}\n- MCP tool: create-page {"name":"Account","path":"/account"}\n- MCP tool: update-page {"pageId":"","values":{"meta":{"auth":{"method":"basic","login":"","password":""}}}}\n- MCP tool: create-resource {"resource":{"name":"Session","method":"get","url":"https://api.example.com/session","headers":[]}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"user","value":{"type":"json","value":{}}}\n- MCP tool: update-props {"updates":"props.json contents"}\n- MCP tool: bind-props {"bindings":"bindings.json contents"}\n\nNotes:\n\n- Inspect and reuse the project\'s existing auth convention before authoring. Do\n not add a second provider or session model implicitly.\n- Model signed-out, loading, signed-in, and failed-auth states explicitly.\n- Never store credentials, service-role keys, refresh tokens, private session\n values, or authenticated response bodies in project data, command output,\n screenshots, agent instructions, or error reports. Privileged provider calls\n and authorization enforcement belong server-side.\n- Basic auth is semantic today. Provider-specific Supabase/Firebase setup still\n uses the existing resource, variable, prop, binding, and embed tools; there is\n no provider-specific installer.\n\n## Generate from design input\n\nCommands:\n\n- MCP tool: meta.guide {"brief":"Recreate this Figma design as a responsive page"}\n- MCP tool: create-page {"name":"Landing","path":"/landing"}\n- MCP tool: create-design-token {"tokens":"tokens.json contents"}\n- MCP tool: define-css-variable {"vars":"vars.json contents"}\n- MCP tool: list-breakpoints {}\n- MCP tool: insert-fragment {"parentInstanceId":"","fragment":"Section copy"}\n- MCP tool: update-styles {"updates":[{"instanceId":"","breakpointId":"","property":"padding-left","value":{"type":"unit","unit":"px","value":24}}]}\n- MCP tool: preview.start {"host":"127.0.0.1","port":5173}\n- MCP tool: screenshot {"path":"/landing","output":"landing-desktop.png","viewport":{"width":1440,"height":900},"waitUntil":"load","waitForTimeout":250}\n- MCP tool: screenshot {"path":"/landing","output":"landing-mobile.png","viewport":{"width":390,"height":844},"waitUntil":"load","waitForTimeout":250}\n- MCP tool: screenshot {"baseUrl":"http://127.0.0.1:5177","path":"/landing","output":"landing-desktop.png","viewport":{"width":1440,"height":900},"waitUntil":"load","waitForTimeout":250}\n\nNotes:\n\n- Use this after the agent can inspect the supplied design. There is no direct\n Figma, screenshot, Inception, or `design.md` import command.\n- Inspect and reuse existing variables, tokens, styles, components, assets, and\n page patterns before authoring. Build semantic editable structure rather than\n flattening the design into an image or absolute-positioned approximation.\n- Verify one familiar viewport inside every distinct Builder breakpoint range,\n then run rendered audit and inspect the screenshots before completion.\n\n## Cross-project maintenance\n\nCommands:\n\n- webstudio mcp run .temp/projects.json\n- webstudio mcp run .temp/projects.json --dry-run\n- webstudio mcp run .temp/projects.json --approve-mutations --concurrency 2\n\nNotes:\n\n- Put shared `calls` and a `projects` array of independently linked project roots in the existing `mcp run` manifest. Project roots are relative to the manifest file.\n- Each project uses its own config, authentication, ProjectSession storage, checkpoint, and failure boundary. Confirmed successful calls are checkpointed. Reads can resume automatically; a mutation interrupted after dispatch is reported as ambiguous and is not replayed automatically, preventing silent duplicate writes.\n- Focus the manifest on bounded reads or audits first. Use per-call `dryRun`, global `--dry-run`, or explicitly approve committed mutations with `--approve-mutations` after reviewing the manifest.\n\n# Known CLI Gaps\n\n## Provider-specific authenticated pages\n\nMissing:\nCLI supports page basic auth and generic resources/props/embeds, but not guided Supabase/Firebase auth setup.\n\nCurrent fallback:\nCall `meta.guide` with the provider-authenticated page goal, then create the\npage, resources, variables, props, bindings, and embeds with existing semantic\ntools.\n\nSuggested commands:\n\n- setup-auth-page\n\n## Generate from design input\n\nMissing:\nNo command imports Figma, screenshots, Inception output, or design.md and turns it into pages/tokens/layout.\n\nCurrent fallback:\nCall `meta.guide` with the design-input goal, let the agent inspect the supplied\ndesign, then use semantic page, token, asset, fragment, style, preview,\nscreenshot, and audit tools. Use `apply-patch` only when no semantic operation\nfits.\n\nSuggested commands:\n\n- generate-from-design\n\n## Built-in cross-project maintenance\n\nMissing:\nPublic API and CLI intentionally operate on one configured project at a time; there is no built-in multi-project discovery or loop runner.\n\nCurrent fallback:\nRun the CLI from an external script that reconfigures one project/session at a time.\n\nSuggested commands:\n\n- none\n', + '# CLI API Use Cases\n\n## Link/configure one project\n\nCommands:\n\n- webstudio init --link --json\n\nNotes:\n\n- Writes local project id and global origin/token config.\n\n## Import synced project bundle into another project\n\nCommands:\n\n- webstudio sync\n- webstudio import --to \n- MCP tool: import {"to":""}\n\nNotes:\n\n- Imports local `.webstudio/data.json` into the destination project.\n- Destination share link must allow build/import access.\n- Use `--skip-assets` only when asset rows and files should not be imported.\n\n## Identify current token\n\nCommands:\n\n- MCP tool: whoami {}\n\n## Check token permissions\n\nCommands:\n\n- webstudio permissions --json\n\n## Inspect project/build/version\n\nCommands:\n\n- MCP tool: inspect {}\n- MCP tool: snapshot {"include":["pages","instances","styles"]}\n\n## Discover CLI/API capabilities\n\nCommands:\n\n- webstudio schema api\n- webstudio schema mcp\n- webstudio man --json\n- webstudio man llm --json\n- MCP tool: meta.index {}\n- MCP tool: meta.guide {"brief":"Create a pricing page"}\n- MCP tool: meta.get_more_tools {"brief":"update-styles"}\n- webstudio mcp list-resources\n- webstudio mcp read-resource webstudio://project/guide\n- webstudio mcp read-resource webstudio://project/expressions\n\nNotes:\n\n- Use `webstudio schema mcp` for a compact machine-readable MCP tool overview. Add `--verbose` or use focused `meta.get_more_tools` calls only when exact input schemas are needed.\n- Use focused MCP tools for discovery first: `meta.index`, `meta.guide`, `meta.get_more_tools`, `components.list`, `components.summary`, `components.search`, `components.get`, `templates.list`, and `templates.get`. Protocol clients can use `resources/list` and `resources/read`; shell agents can use `webstudio mcp list-resources` and `webstudio mcp read-resource `. Read longer resources such as `webstudio://project/tools` and `webstudio://project/components` only when focused tools are insufficient.\n- `components.summary` returns counts by default; request `{"detail":"components","limit":20}` for paginated entries. Registry list tools return compact metadata, while `components.get` and `templates.get` return focused full details.\n- Read `webstudio://project/expressions` before authoring unfamiliar computed text, prop bindings, resource expressions, actions, or Collection item bindings.\n- From a shell, call one MCP tool with the shortcut form `webstudio \'\'`, for example `webstudio components.summary`. The explicit equivalent is `webstudio mcp single-op-call \'\'`. Use `--input-file` for large payloads.\n\n## Inspect external shadcn registry items\n\nCommands:\n\n- webstudio registry inspect --source https://example.com/r/registry.json --item button --json\n- webstudio registry inspect --source ./registry.json --item dialog --json\n- webstudio registry inspect --source https://example.com/r/button.json --json\n\nNotes:\n\n- Reads a local or remote registry item without installing files or changing the configured Webstudio project.\n- Returns the item name, description, package and registry dependencies, file paths/targets, available docs, and a read-only compatibility report.\n- The report explicitly says whether installation or editable-component conversion is supported, lists declared requirements and manual steps, and says when arbitrary source code was not analyzed.\n- This is an inspection step only. It does not install files or change the configured project.\n\n## Understand what MCP can do\n\nCommands:\n\n- webstudio man mcp\n- MCP tool: meta.index {}\n- MCP tool: meta.guide {"brief":"What can Webstudio MCP do?"}\n\nMCP lets agents work on one configured Webstudio project. Agents can:\n\n- Inspect the linked project, token permissions, and latest editable build.\n- Read selected project data for audits, migrations, and repair.\n- Search labels, text, props, resource URLs, asset metadata, and styles.\n- Audit accessibility, security, SEO, performance settings, unused assets, ineffective Collection styles, and unused or duplicate style data.\n- Create and edit pages, folders, redirects, breakpoints, and page templates.\n- Create pages from reusable templates.\n- Update page metadata, SEO fields, auth settings, and marketplace metadata.\n- Insert components and styled JSX sections.\n- Create data-driven lists, grids, cards, and similar repeated UI from array or object data in one Collection operation.\n- Move, copy, wrap, unwrap, convert, rename, retag, and delete elements.\n- Update text, rich text, props, bindings, and actions.\n- Create and update local styles, design tokens, style sources, and CSS variables.\n- Create static data variables and JSON variables.\n- Create HTTP, GraphQL, and system resources.\n- Use system resources for sitemap, current date, and assets.\n- Bind resources to rendered data or form/action props.\n- Manage nested asset folders and upload, inspect, move, duplicate, download, replace, delete, and inspect usage for assets.\n- Publish, unpublish, inspect publish jobs, and manage custom domains.\n- Start preview, capture screenshots, compare screenshot diffs, and use OCR when installed.\n\n## Inspect and refresh MCP session cache\n\nCommands:\n\n- MCP tool: status {}\n- MCP tool: status {"verbose":true}\n- MCP tool: refresh {"namespaces":["pages","instances","styles"]}\n- MCP tool: reset-session {}\n\nNotes:\n\n- Use status before a task to understand the cached ProjectSession state.\n- Use status with `{"verbose":true}` only when debugging full namespaces, freshness, compatibility, or diagnostics.\n- Use refresh when project data may have changed outside the current MCP session.\n- Use reset-session when local cached state is corrupt or incompatible.\n\n## Visually verify rendered work with AI vision\n\nCommands:\n\n- MCP tool: preview.start {"host":"127.0.0.1","port":5173}\n- MCP tool: preview.status {}\n- MCP tool: preview.stop {}\n- MCP tool: screenshot {"path":"/","output":".webstudio/screenshots/home-current.png","viewport":{"width":1440,"height":900},"waitUntil":"load","waitForTimeout":250}\n- MCP tool: screenshot {"path":"/pricing","output":".webstudio/screenshots/pricing-current.png","viewport":{"width":1440,"height":900},"waitUntil":"load","waitForTimeout":250}\n- MCP tool: screenshot {"baseUrl":"http://127.0.0.1:5177","path":"/pricing","output":".webstudio/screenshots/pricing-current.png","viewport":{"width":1440,"height":900},"waitUntil":"load","waitForTimeout":250}\n- MCP tool: screenshot.diff {"baselinePath":".webstudio/screenshots/home-before.png","currentPath":".webstudio/screenshots/home-current.png","outputDir":".webstudio/screenshots/diff"}\n- MCP tool: screenshot.diff {"baselinePath":".webstudio/screenshots/home-before.png","currentPath":".webstudio/screenshots/home-current.png","outputDir":".webstudio/screenshots/diff","expectedText":["Pricing","Start free"]}\n- MCP tool: screenshot.diff {"baselinePath":".webstudio/screenshots/home-before.png","currentPath":".webstudio/screenshots/home-current.png","outputDir":".webstudio/screenshots/diff","expectedVisual":{"maxMismatchPercentage":2,"maxChangedRegions":3,"dominantColorChange":{"channel":"luminance","direction":"increase","minMagnitude":10}}}\n- MCP tool: screenshot.diff {"baselinePath":".webstudio/screenshots/pricing-before.png","currentPath":".webstudio/screenshots/pricing-current.png","outputDir":".webstudio/screenshots/diff"}\n- MCP tool: vision.install-ocr {"confirm":true}\n\nNotes:\n\n- Use this after page/content/style mutations and after generated project files are current so a vision-capable AI can see the production-like generated site.\n- For multi-page work, capture every changed page by `path` through the same preview server; no click navigation is required.\n- After MCP mutations, path screenshots regenerate/restart preview as needed before capture; when preview is fresh, repeated path screenshots reuse the running server.\n- Do not call `preview.start` through one-shot `webstudio mcp single-op-call`: it is long-lived. From a shell, use `webstudio mcp run` with preview.start, screenshot, and preview.stop in one shared process, or use a real long-running MCP client.\n- From one-shot shell calls or another process, pass `baseUrl` with `path` to capture an already-running preview/site without generating, building, starting, or restarting preview.\n- Use preview.stop only in the same long-running MCP server or `webstudio mcp run` process that started preview. A separate one-shot `single-op-call` process does not own another process\'s preview controller.\n- Use waitForSelector when the rendered app has a reliable ready marker, waitUntil:"networkidle" for network-heavy pages, and waitForTimeout only for final visual settling.\n- Preview installs generated app dependencies under `.webstudio/preview` and reuses them across regenerations.\n- Do not add generated-preview dependencies to the repository root `package.json` or `pnpm-lock.yaml`.\n- If dependency installation fails, check npm and network configuration, then reinstall or update the Webstudio CLI if the problem persists.\n- When a baseline exists, use screenshot.diff once per baseline/current page or viewport pair to get changed regions, OCR textAnalysis, and diff artifact paths before deciding whether the result matches. Pass expectedText for explicit pass/fail current-screen text assertions with found and missing text. Pass expectedVisual for pass/fail limits on pixel mismatch percentage, changed-region count, or the overall dominant color/brightness direction.\n- If screenshot.diff reports OCR unavailable and the user agrees to install it, call vision.install-ocr {"confirm":true}; otherwise continue with pixel diff and visual inspection.\n- Compare the PNG, OCR text evidence, and diff artifacts against the user\'s intent for layout, typography, colors, spacing, imagery, and responsive framing; then iterate with focused mutations.\n- Root CLI equivalent: `webstudio screenshot --path /pricing --output pricing.png` generates a temporary production preview, captures that route, and stops the server. For repeated captures, keep `webstudio preview` running and pass its absolute URL to `webstudio screenshot`.\n\n## List pages\n\nCommands:\n\n- MCP tool: list-pages {}\n- MCP tool: list-folders {}\n\n## Read page by id\n\nCommands:\n\n- MCP tool: get-page {"pageId":""}\n\n## Read page by path\n\nCommands:\n\n- MCP tool: get-page-by-path {"path":"/pricing"}\n\n## Create page\n\nCommands:\n\n- MCP tool: create-page {"name":"Pricing","path":"/pricing"}\n- MCP tool: create-page {"name":"Pricing","path":"/pricing","title":"Pricing","meta":{"description":"Plans for teams"}}\n\nNotes:\n\n- `name`, `path`, page `title`, and metadata text fields accept plain fixed values.\n- For computed page titles or metadata, send JavaScript expression code such as `pageTitle ?? "Pricing"`.\n\n## Update page settings/metadata\n\nCommands:\n\n- MCP tool: update-page {"pageId":"","values":{"title":"Pricing","meta":{"description":"Plans","status":200}}}\n- MCP tool: update-page {"pageId":"","values":{"meta":{"auth":{"method":"basic","login":"","password":""}}}}\n\nNotes:\n\n- Page `title` and metadata text fields accept plain fixed values.\n- For computed page titles or metadata, send JavaScript expression code such as `pageTitle ?? "Pricing"`.\n- Page `status` accepts a fixed HTTP status code as a number from 200 through 599 or a JavaScript expression string for a dynamic status.\n\n## Read project settings\n\nCommands:\n\n- MCP tool: get-project-settings {}\n\nNotes:\n\n- Read `meta.agentInstructions` before making project changes. It contains the project\'s own guidance for AI agents.\n- Agent instructions are shared project guidance. Do not store credentials or other secrets there.\n\n## Update project settings\n\nCommands:\n\n- MCP tool: update-project-settings {"meta":{"siteName":"Acme"}}\n- MCP tool: update-project-settings {"meta":{"agentInstructions":"Use existing design tokens and keep product copy concise."}}\n\n## Read marketplace product\n\nCommands:\n\n- MCP tool: get-marketplace-product {}\n\n## Update marketplace product\n\nCommands:\n\n- MCP tool: update-marketplace-product {"category":"pageTemplates","name":"Acme Template","thumbnailAssetId":"asset-id","author":"Acme Studio","email":"hello@example.com","website":"https://example.com","issues":"","description":"Reusable template project for Acme landing pages."}\n\n## List redirects\n\nCommands:\n\n- MCP tool: list-redirects {}\n\n## Create redirect\n\nCommands:\n\n- MCP tool: create-redirect {"old":"/old","new":"/new","status":301}\n\n## Update redirect\n\nCommands:\n\n- MCP tool: update-redirect {"old":"/old","values":{"new":"/newer","status":302}}\n- MCP tool: update-redirect {"old":"/old","values":{"status":null}}\n\n## Delete redirect\n\nCommands:\n\n- MCP tool: delete-redirect {"old":"/old"}\n\n## Set redirects\n\nCommands:\n\n- MCP tool: set-redirects {"redirects":[{"old":"/old","new":"/new","status":"301"}]}\n\n## List breakpoints\n\nCommands:\n\n- MCP tool: list-breakpoints {}\n\n## Create breakpoint\n\nCommands:\n\n- MCP tool: create-breakpoint {"label":"Tablet","maxWidth":991}\n\n## Update breakpoint\n\nCommands:\n\n- MCP tool: update-breakpoint {"breakpointId":"tablet","values":{"label":"Tablet","maxWidth":1023}}\n- MCP tool: update-breakpoint {"breakpointId":"tablet","values":{"condition":null,"minWidth":768}}\n- MCP tool: update-breakpoint {"breakpointId":"tablet","values":{"minWidth":null,"maxWidth":null,"condition":"(hover: hover)"}}\n\n## Delete breakpoint\n\nCommands:\n\n- MCP tool: delete-breakpoint {"breakpointId":"tablet"}\n\n## Duplicate page\n\nCommands:\n\n- MCP tool: duplicate-page {"pageId":"","name":"Pricing Copy","path":"/pricing-copy"}\n- MCP tool: duplicate-page {"pageId":"","name":"Paris","path":"/paris","substitutions":{"text":{"London":"Paris"},"variables":{"city":{"type":"string","value":"Paris"}}}}\n- webstudio duplicate-page --page --name Paris --path /paris --substitutions \'{"text":{"London":"Paris"},"variables":{"city":{"type":"string","value":"Paris"}}}\' --json\n\nNotes:\n\n- Text substitutions replace exact fixed text only in the duplicated page\'s text children, string props, title, and metadata.\n- Variable substitutions are keyed by copied source-variable name and use typed variable values. The operation rejects missing or ambiguous names without committing a partial duplicate.\n- Existing expressions and cloned variable/resource references keep their remapped ids.\n\n## List page templates\n\nCommands:\n\n- MCP tool: list-page-templates {}\n\n## Create page template\n\nCommands:\n\n- MCP tool: create-page-template {"name":"Landing Template","title":"Landing"}\n\n## Update page template\n\nCommands:\n\n- MCP tool: update-page-template {"templateId":"","values":{"name":"Article Template","meta":{"description":"Reusable article layout"}}}\n\n## Delete page template\n\nCommands:\n\n- MCP tool: delete-page-template {"templateId":""}\n\n## Duplicate page template\n\nCommands:\n\n- MCP tool: duplicate-page-template {"templateId":""}\n\n## Reorder page template\n\nCommands:\n\n- MCP tool: reorder-page-template {"sourceTemplateId":"","targetTemplateId":"","position":"before"}\n\n## Create page from template\n\nCommands:\n\n- MCP tool: create-page-from-template {"templateId":"","name":"Landing","path":"/landing"}\n\n## Delete page\n\nCommands:\n\n- MCP tool: delete-page {"pageId":""}\n\n## List folders\n\nCommands:\n\n- MCP tool: list-folders {}\n- MCP tool: list-pages {}\n\n## Create folder\n\nCommands:\n\n- MCP tool: create-folder {"name":"Blog","slug":"blog"}\n\n## Update folder\n\nCommands:\n\n- MCP tool: update-folder {"folderId":"","values":{"name":"Blog","slug":"blog"}}\n\n## Delete folder\n\nCommands:\n\n- MCP tool: delete-folder {"folderId":""}\n\n## List element instances\n\nCommands:\n\n- MCP tool: list-instances {"pagePath":"/","maxDepth":3}\n\n## Inspect one element instance\n\nCommands:\n\n- MCP tool: inspect-instance {"instanceId":"","include":["props","styles","children"]}\n\n## Insert authored JSX or one component template\n\nCommands:\n\n- MCP tool: insert-fragment {"parentInstanceId":"","fragment":"Product OS"}\n- MCP tool: insert-component {"parentInstanceId":"","component":"@webstudio-is/sdk-components-react-radix:Switch"}\n- MCP tool: insert-component {"parentInstanceId":"","component":"Form"}\n\nNotes:\n\n- Use MCP `insert-fragment` as the default way to author styled component trees. It converts JSX to a structured fragment before mutation.\n- Use only exact component ids returned by `components.search`, `components.get`, or `templates.get`. Never derive or guess component ids.\n- The `ws:` namespace contains specific Webstudio core components; it is not HTML-tag shorthand. Use `` for a native `div` and `` for a native form, never `` or ``.\n- For Webstudio\'s complete form structure, discover the Form component and insert its automatic template with `insert-component` using component `"Form"`.\n- MCP receives JSX as a JSON string because MCP arguments are JSON. The CLI converts it locally before the runtime mutation, so the project session receives structured Webstudio data, not JSX source.\n- In `insert-fragment` JSX, use `ws:style={css\\`...\\`}`for Webstudio-native CSS, or use React-style object syntax such as`style={{ padding: 24 }}` when that is simpler. Both forms create editable Webstudio style data.\n- Do not access host globals or dynamic code APIs in JSX fragments, including `process`, `globalThis`, `eval`, `Function`, or `constructor`.\n- Use Webstudio prop names such as `class` and `for`; do not use React aliases `className` or `htmlFor`.\n- Use Webstudio actions for event/action props, for example `onClick={new ActionValue(["event"], expression\\`console.log(event)\\`)}`. Do not pass JavaScript functions such as `onClick={() => ...}`.\n- Plain prop values must be JSON-compatible: `null`, strings, booleans, finite numbers, arrays, and plain objects. Do not pass `undefined`, `Symbol`, `BigInt`, `NaN`, `Infinity`, `Date`, `Map`, `Set`, class instances, or circular objects; omit the prop, use plain data, or use `expression`/`ActionValue` when the value is dynamic.\n- Template-backed components used in JSX must include required child/part components explicitly under the same parent structure as the template, for example ``.\n- Webstudio applies a registered template automatically when using `insert-component`, so composed components such as Switch include required child parts and styles.\n- Use `components.list`, `components.summary`, `components.search`, `components.get`, `templates.list`, and `templates.get` to discover known registry items, component ids, props, templates, insertability, and content model. Read `webstudio://project/components` only when those focused tools are insufficient.\n- Component/template registry items use a shadcn-compatible top-level shape plus Webstudio-specific superset metadata in `meta`. They are for Builder/MCP discovery, not a published shadcn install registry yet.\n- Known components with `contentModel.category: "none"` are not standalone-insertable; insert their root component template instead so required providers/parents are included.\n- Unknown custom component ids are a low-level extension mechanism, not a discovery fallback. Agents must not synthesize them.\n\n## Make a region editable in Content mode\n\nCommands:\n\n- MCP tool: insert-component {"parentInstanceId":"","component":"ws:block"}\n- MCP tool: inspect-instance {"instanceId":"","include":["children"]}\n\nNotes:\n\n- When a page will be handed to a Content-mode editor, wrap every region they should be able to edit in a Content Block (`ws:block`). Content-mode editors can edit text and supported props only in Content Block descendants. Content outside those blocks remains read-only, even when it looks like ordinary editable text.\n- Put reusable insertable options inside the Content Block\'s `ws:block-template` child. A template is source material, not editor content: editors cannot edit or delete it directly. When an editor inserts a template, its copy becomes a direct child of the Content Block and is editable.\n- Before handing off a page, verify with `inspect-instance` that the intended text, images, and links are inside a Content Block, and that templates include all required styling because Content-mode editors cannot use the Style panel.\n\n## Move elements\n\nCommands:\n\n- MCP tool: move-instance {"moves":"moves.json contents"}\n\nNotes:\n\n- Use `position: "end"` to append an instance. Repeating this for A and then B preserves the final order A, B.\n- A numeric `insertIndex` addresses the target parent\'s children before the moved instance is removed. Use it for exact placement; do not calculate the last index to append.\n- Moves in one `moves` array are applied sequentially in array order.\n\n## Clone element subtree\n\nCommands:\n\n- MCP tool: clone-instance {"sourceInstanceId":"","targetParentInstanceId":""}\n\n## Delete element subtree\n\nCommands:\n\n- MCP tool: delete-instance {"instanceIds":[""]}\n\n## List text/expression children\n\nCommands:\n\n- MCP tool: list-texts {"pagePath":"/"}\n\n## Update text child\n\nCommands:\n\n- MCP tool: update-text {"instanceId":"","childIndex":0,"text":"Launch faster"}\n\n## Replace bounded literal text\n\nCommands:\n\n- MCP tool: replace-text {"find":"Start free","replace":"Get started","match":"exact","pagePath":"/pricing","limit":20}\n\nNotes:\n\n- This changes only literal text children, never expression children. Scope it to pagePath or pageId and set a limit before a broad replacement.\n\n## Replace bounded static prop text\n\nCommands:\n\n- MCP tool: replace-prop-text {"find":"old.example.com","replace":"www.example.com","match":"substring","names":["href","code"],"limit":20}\n\nNotes:\n\n- This changes only static string props such as href, alt, aria-label, title, and HTML embed code. It never changes expressions, resources, actions, assets, or other dynamic bindings. Use names or instanceIds and a limit to narrow the change.\n\n## Replace bounded resource text\n\nCommands:\n\n- MCP tool: replace-resource-text {"find":"api.old.example.com","replace":"api.example.com","fields":["url"],"limit":20}\n\nNotes:\n\n- This changes resource names and fixed URL literals only. It skips dynamic URL expressions, headers, search parameters, request bodies, and GraphQL query code.\n\n## Update props\n\nCommands:\n\n- MCP tool: update-props {"updates":"props.json contents"}\n- MCP tool: replace-prop-text {"find":"Old label","replace":"New label","names":["aria-label","title"],"limit":20}\n\nNotes:\n\n- Use this for fixed prop values such as `aria-label`, `alt`, `id`, static `href`, and other direct string/number/boolean/json prop values.\n\n## Add JSON-LD structured data\n\nCommands:\n\n- MCP tool: components.get {"component":"JsonLd"}\n- MCP tool: insert-component {"parentInstanceId":"","component":"JsonLd"}\n- MCP tool: update-props {"updates":[{"instanceId":"","name":"code","type":"string","value":"{\\"@context\\":\\"https://schema.org\\",\\"@type\\":\\"Organization\\",\\"name\\":\\"Acme\\"}"}]}\n- MCP tool: audit {"scopes":["seo"],"pagePath":"/"}\n\nNotes:\n\n- Prefer placing `JsonLd` inside `HeadSlot`.\n- Store `code` as a JSON object or array encoded as a compact string. The Builder formats it for editing.\n- The semantic prop update rejects malformed JSON and structurally invalid fixed JSON-LD with a precise JSON path.\n- The SEO audit also warns about a missing top-level `@context`, unknown or superseded Schema.org terms, properties unsupported by the supplied type, and incompatible primitive value types.\n- Schema.org vocabulary findings are warnings because custom vocabularies and extensions remain valid. Dynamic JSON-LD is marked as skipped for rendered validation.\n- Do not use bindings just to set static text.\n\n## Delete props\n\nCommands:\n\n- MCP tool: delete-props {"deletions":"props.json contents"}\n\n## Bind props to expressions/resources/actions\n\nCommands:\n\n- MCP tool: bind-props {"bindings":"bindings.json contents"}\n\nNotes:\n\n- Use this only when the prop should remain dynamic: expression, resource, action, or an existing scoped runtime context value such as `system`.\n- For a fixed string value, use `update-props` with `type:"string"` and a direct `value` instead.\n\n## Read styles\n\nCommands:\n\n- MCP tool: get-styles {"instanceIds":[""],"includeTokens":true}\n\n## Update local styles\n\nCommands:\n\n- MCP tool: update-styles {"updates":"styles.json contents"}\n\n## Delete local styles\n\nCommands:\n\n- MCP tool: delete-styles {"deletions":"styles.json contents"}\n\n## Replace matching style values\n\nCommands:\n\n- MCP tool: replace-styles {"property":"color","fromValue":{"type":"keyword","value":"red"},"toValue":{"type":"keyword","value":"blue"}}\n\n## List design tokens\n\nCommands:\n\n- MCP tool: list-design-tokens {}\n- MCP tool: list-design-tokens {"withUsage":true}\n- MCP tool: list-design-tokens {"verbose":true}\n\nNotes:\n\n- The default response is compact and includes token id, name, declaration count, and optional usage count.\n- Use `verbose:true` only when you need the full inline style declarations.\n\n## Create design tokens\n\nCommands:\n\n- MCP tool: create-design-token {"tokens":"tokens.json contents"}\n\n## Update design token styles\n\nCommands:\n\n- MCP tool: update-design-token-styles {"designTokenId":"","updates":"styles.json contents"}\n\n## Delete design token styles\n\nCommands:\n\n- MCP tool: delete-design-token-styles {"designTokenId":"","deletions":"styles.json contents"}\n\n## Attach design token to instances\n\nCommands:\n\n- MCP tool: attach-design-token {"designTokenId":"","instanceIds":"instances.json contents"}\n\n## Detach design token from instances\n\nCommands:\n\n- MCP tool: detach-design-token {"designTokenId":"","instanceIds":"instances.json contents"}\n\n## Extract design token from local styles\n\nCommands:\n\n- MCP tool: extract-design-token {"instanceIds":[""],"name":"Brand Primary","removeLocalProps":["color"]}\n\n## List CSS variables\n\nCommands:\n\n- MCP tool: list-css-variables {"withUsage":true}\n\n## Define CSS variables\n\nCommands:\n\n- MCP tool: define-css-variable {"vars":"vars.json contents"}\n\n## Delete CSS variables\n\nCommands:\n\n- MCP tool: delete-css-variable {"names":"names.json contents","force":true}\n\n## Rewrite CSS variable references\n\nCommands:\n\n- MCP tool: rewrite-css-variable-refs {"map":"variables.json contents"}\n\n## List data variables\n\nCommands:\n\n- MCP tool: list-variables {}\n- MCP tool: list-variables {"scopeInstanceId":""}\n\nNotes:\n\n- Data variables live in the internal `dataSources` namespace.\n- For raw `snapshot`, request the public `variables` namespace rather than the internal `dataSources` name. Raw patch payloads still use `dataSources` when applying direct changes.\n- Scope variables to the instance where they should become available. Descendants can use them in expressions, and nested variables with the same name mask outer variables.\n\n## Create data variable\n\nCommands:\n\n- MCP tool: create-variable {"scopeInstanceId":"","name":"title","value":{"type":"string","value":"Hello"}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"count","value":{"type":"number","value":3}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"featured","value":{"type":"boolean","value":true}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"tags","value":{"type":"string[]","value":["news","product"]}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"filters","value":{"type":"json","value":{"tag":"news"}}}\n\nNotes:\n\n- Data variable values support `string`, `number`, `boolean`, `string[]`, and `json`.\n- Parameters are internal scoped runtime values provided by pages, collections, or components. They are not a public authoring surface: do not create, update, or delete parameter records. Use data variables/resources for user-authored data, and reference documented context values such as `system` only where they are already in scope.\n\n## Update data variable\n\nCommands:\n\n- MCP tool: update-variable {"dataSourceId":"","values":{"value":{"type":"json","value":{"count":1}}}}\n\n## Delete data variable\n\nCommands:\n\n- MCP tool: delete-variable {"dataSourceId":""}\n\n## List resources\n\nCommands:\n\n- MCP tool: list-resources {}\n- MCP tool: list-resources {"scopeInstanceId":""}\n\n## Create resource\n\nCommands:\n\n- MCP tool: create-resource {"resource":{"name":"Posts","method":"get","url":"https://api.example.com/posts","headers":[]}}\n- MCP tool: create-resource {"resource":{"name":"Posts","method":"get","url":"\\"https://api.example.com/posts?tag=\\" + filters.tag","headers":[]},"scopeInstanceId":"","dataSourceName":"posts"}\n- MCP tool: create-resource {"resource":{"name":"Filtered Posts","method":"get","url":"https://api.example.com/posts","searchParams":[{"name":"tag","value":"filters.tag"},{"name":"source","value":{"type":"literal","value":"website"}}],"headers":[{"name":"Authorization","value":"\\"Bearer \\" + auth.token"}]},"scopeInstanceId":"","dataSourceName":"posts"}\n- MCP tool: create-resource {"resource":{"name":"Post GraphQL","control":"graphql","method":"post","url":"https://api.example.com/graphql","headers":[{"name":"Content-Type","value":{"type":"literal","value":"application/json"}}],"body":"{ query: \\"query Post($slug: String!) { post(slug: $slug) { title } }\\", variables: { slug: system.params.slug } }"},"scopeInstanceId":"","dataSourceName":"post"}\n- MCP tool: create-resource {"resource":{"name":"Current Date","control":"system","method":"get","url":"/$resources/current-date","headers":[]},"scopeInstanceId":"","dataSourceName":"currentDate"}\n\nNotes:\n\n- Resource `url` accepts plain fixed URLs and paths such as `https://api.example.com/posts` and `/$resources/current-date`.\n- Resource `url` can also be a JavaScript expression when it is computed, such as `"https://api.example.com/posts?tag=" + filters.tag`.\n- Header values, search parameter values, and body accept expressions for dynamic values. For fixed text, use `{"type":"literal","value":"application/json"}`; Webstudio stores the required string expression for you.\n- Search parameter values, header values, and body expressions can read scoped variables and documented runtime context values such as `system` when they are available at the resource scope.\n- Add `scopeInstanceId` and `dataSourceName` when the resource result should be exposed as a scoped read data variable. Scoped resources are generated into the page resource `data` map and may be loaded during page rendering. Use this for read-oriented resources such as GET CMS/API data.\n- For submit/write/action resources, create the resource without `scopeInstanceId`, then bind a component prop such as a Form `action` with `bind-props` and `binding.type: "resource"`. Prop-bound resources are generated into the page resource `action` map instead of the read `data` map. Use this for POST, PUT, DELETE, webhooks, GraphQL submissions, and other explicit action flows.\n- Resource `method` can be `get`, `post`, `put`, or `delete`. Use GET for read data, POST for creates/GraphQL/webhooks/form submissions, PUT for full updates or replacements, and DELETE for deletion actions.\n- Optional `control` values are `graphql` and `system`. Use `graphql` for GraphQL-style requests, usually POST with a query body. Use `system` for built-in resources such as `"/$resources/sitemap.xml"`, `"/$resources/current-date"`, and `"/$resources/assets"` and when the resource should use the built-in `system` parameter. System fields are `system.origin`, `system.pathname`, `system.params`, and `system.search`.\n\n## Update resource\n\nCommands:\n\n- MCP tool: update-resource {"resourceId":"","values":{"url":"https://api.example.com/posts"}}\n- MCP tool: replace-resource-text {"find":"api.old.example.com","replace":"api.example.com","fields":["url"],"limit":20}\n\n## Query Markdown assets\n\nCommands:\n\n- MCP tool: get-asset-field-catalog {}\n- MCP tool: validate-asset-query {"query":"*[_type == \\"asset.file\\" && extension == \\"md\\" && properties.draft != true] | order(properties.publishedAt desc, _id asc)[0...20]"}\n- MCP tool: create-assets-resource {"name":"All assets","scopeInstanceId":"","dataSourceName":"assets"}\n- MCP tool: create-assets-resource {"name":"Published posts","scopeInstanceId":"","dataSourceName":"posts","query":{"groq":"*[_type == \\"asset.file\\" && extension == \\"md\\" && properties.draft != true] | order(properties.publishedAt desc, _id asc)[0...20]{_id, path, properties, excerpt}","resultLimit":20,"content":{"mode":"none"}}}\n- MCP tool: create-assets-resource {"name":"Post by slug","scopeInstanceId":"","dataSourceName":"post","query":{"groq":"*[_type == \\"asset.file\\" && extension == \\"md\\" && properties.slug == $slug][0]","parameters":[{"name":"slug","value":"system.params.slug"}],"resultLimit":1,"content":{"mode":"markdown-body","maxBytes":1048576}}}\n- MCP tool: list-assets-resources {}\n- MCP tool: get-assets-resource {"resourceId":""}\n- MCP tool: update-assets-resource {"resourceId":"","values":{"query":null}}\n- MCP tool: preview-asset-query {"query":"*[_type == \\"asset.file\\" && properties.slug == $slug][0]","parameters":{"slug":"hello-world"},"resultLimit":1,"content":{"mode":"markdown-body","maxBytes":1048576}}\n- MCP tool: get-asset-resource-index-status {"resourceId":""}\n- MCP tool: rebuild-asset-resource-index {"resourceId":""}\n\nNotes:\n\n- Read the field catalog before authoring unfamiliar queries. It includes dynamic schema-less frontmatter paths such as `properties.author.name`, observed types, optionality, and mixed-type state without downloading Markdown files.\n- Parameter bindings on a saved resource are Webstudio expressions evaluated at render time. Preview parameters are concrete JSON values for one test execution.\n- Use `content.mode:"none"` for listings. Use `markdown-body` or `full` only after a query selects the intended file, normally with `resultLimit:1` for a slug detail route.\n- Assets remains one resource type. Omit `query` for the legacy fetch-all request and response; provide `query` explicitly to enable GROQ. Set `values.query:null` to return to fetch-all mode.\n- `create-assets-resource` and `update-assets-resource` are the semantic authoring path. Do not construct the internal query URL, headers, or body expression manually.\n- Check index status after saving. Request an explicit rebuild only for recovery; normal resource and asset mutations schedule index maintenance automatically.\n\n## Delete resource\n\nCommands:\n\n- MCP tool: delete-resource {"resourceId":""}\n\n## List assets\n\nCommands:\n\n- MCP tool: list-assets {"withUsage":true}\n- MCP tool: list-assets {"verbose":true}\n\nNotes:\n\n- Compact results include each asset\'s folder id. Use `verbose:true` to include complete records for a page of assets, or `get-asset` to read one complete record including description, folder, creation time, and image/font metadata.\n- Image asset descriptions are the default alt text for asset-backed Image components.\n- To generate missing descriptions, inspect the image in its rendered page or asset source, write a concise description of its purpose, and save it on the asset rather than duplicating it on each Image instance.\n\n## Get asset\n\nCommands:\n\n- MCP tool: get-asset {"assetId":""}\n\n## List asset folders\n\nCommands:\n\n- MCP tool: list-asset-folders {}\n\n## Create asset folder\n\nCommands:\n\n- MCP tool: create-asset-folder {"name":"Marketing"}\n- MCP tool: create-asset-folder {"name":"Photos","parentId":""}\n\n## Update asset folder\n\nCommands:\n\n- MCP tool: update-asset-folder {"folderId":"","values":{"name":"Brand"}}\n- MCP tool: update-asset-folder {"folderId":"","values":{"parentId":""}}\n- MCP tool: update-asset-folder {"folderId":"","values":{"parentId":null}}\n\nNotes:\n\n- Updating `parentId` is the folder equivalent of cut and paste. Use `null` to move a folder to Root.\n\n## Duplicate asset folder\n\nCommands:\n\n- MCP tool: duplicate-asset-folder {"folderId":""}\n- MCP tool: duplicate-asset-folder {"folderId":"","parentId":""}\n\nNotes:\n\n- Duplication recursively copies descendant folders and assets. This is the folder equivalent of copy and paste.\n\n## Delete asset folder\n\nCommands:\n\n- MCP tool: delete-asset-folder {"folderId":""}\n\nNotes:\n\n- Deleting a folder recursively deletes its descendant folders and assets.\n\n## Update asset metadata\n\nCommands:\n\n- MCP tool: update-asset {"assetId":"","values":{"description":"Team collaborating around a whiteboard"}}\n- MCP tool: update-asset {"assetId":"","values":{"meta":{"family":"Rajdhani","style":"normal","weight":600}}}\n\nNotes:\n\n- Use an empty description only when the image is intentionally decorative.\n- Updating an image asset description updates the default alt text wherever that asset is used with an asset-backed alt prop.\n- Font metadata updates merge with the detected metadata and are validated before committing; use this to correct a family, style, or weight after upload.\n\n## Generate missing image descriptions with an agent\n\nCommands:\n\n- MCP tool: audit {"scopes":["accessibility"],"verbose":true}\n- MCP tool: set-image-descriptions {"updates":[{"assetId":"hero-id","description":"Team collaborating around a whiteboard"},{"assetId":"texture-id","decorative":true}]}\n- MCP tool: audit {"scopes":["accessibility"]}\n\nNotes:\n\n- Start from `missing-image-description` findings. Inspect each image in its rendered page context before writing text.\n- The vision-capable agent generates the wording; the CLI validates and stores it but does not contain its own vision model.\n- Use `decorative:true` only when the image adds no information. This intentionally stores an empty description so later audits do not report it as missing.\n- Re-run the accessibility audit after the update. Asset-backed Image components use the saved asset description as their default alt text.\n\n## Manage fonts\n\nCommands:\n\n- MCP tool: list-fonts {"includeSystem":true}\n- MCP tool: list-assets {"type":"font"}\n- MCP tool: upload-asset {"asset":{"name":"acme-sans.woff2","type":"font","format":"woff2","meta":{"family":"Acme Sans","style":"normal","weight":400}},"assetsDir":".webstudio/assets"}\n- MCP tool: update-styles {"updates":"styles.json contents"}\n\nNotes:\n\n- Use `list-fonts` to discover uploaded families and system stacks. Upload/delete fonts through the existing asset tools, then apply a family with a `font-family` style declaration.\n\n## Upload one asset\n\nCommands:\n\n- MCP tool: upload-asset {"asset":{"name":"image.png","type":"image","format":"png","meta":{"width":1200,"height":630}},"assetsDir":".webstudio/assets"}\n- MCP tool: upload-asset {"asset":{"name":"image.png","type":"image","format":"png","folderId":"","meta":{"width":1200,"height":630}},"assetsDir":".webstudio/assets"}\n\n## Upload asset batch\n\nCommands:\n\n- MCP tool: upload-assets {"assets":[{"name":"image.png","type":"image","format":"png","meta":{"width":1200,"height":630}}],"assetsDir":".webstudio/assets"}\n\n## Duplicate asset\n\nCommands:\n\n- MCP tool: duplicate-asset {"assetId":""}\n- MCP tool: duplicate-asset {"assetId":"","folderId":""}\n- MCP tool: duplicate-asset {"assetId":"","folderId":null}\n\nNotes:\n\n- Duplication is the asset equivalent of copy and paste. Updating `folderId` is the equivalent of cut and paste; use `null` for Root.\n\n## Download asset\n\nCommands:\n\n- MCP tool: download-asset {"assetId":""}\n\n## Find asset usage\n\nCommands:\n\n- MCP tool: find-asset-usage {"assetId":""}\n\n## Replace asset references\n\nCommands:\n\n- MCP tool: replace-asset {"fromAssetId":"","toAssetId":""}\n\n## Delete assets\n\nCommands:\n\n- MCP tool: delete-asset {"assetIdsOrPrefixes":[""],"force":true}\n\n## Publish project\n\nCommands:\n\n- webstudio publish deploy --target production --json\n\n## List publishes\n\nCommands:\n\n- webstudio publish list --json\n\n## Check publish job\n\nCommands:\n\n- webstudio publish status --job --json\n\n## Unpublish\n\nCommands:\n\n- webstudio publish unpublish --target production --confirm --json\n\n## List domains\n\nCommands:\n\n- webstudio domains list --json\n\n## Create domain\n\nCommands:\n\n- webstudio domains create --domain example.com --json\n\n## Update domain\n\nCommands:\n\n- webstudio domains update --domain-id --domain www.example.com --json\n\n## Delete domain\n\nCommands:\n\n- webstudio domains delete --domain-id --confirm --json\n\n## Verify domain\n\nCommands:\n\n- webstudio domains verify --domain-id --json\n\n## Make arbitrary store-level changes\n\nCommands:\n\n- MCP tool: inspect {}\n- MCP tool: snapshot {"include":[""]}\n- MCP tool: apply-patch {"baseVersion":"","transactions":"patch.json contents"}\n\nNotes:\n\n- Use only when no semantic command exists.\n\n## Manage marketplace metadata\n\nCommands:\n\n- MCP tool: get-marketplace-product {}\n- MCP tool: update-marketplace-product {"category":"pageTemplates","name":"Acme Template","thumbnailAssetId":"asset-id","author":"Acme Studio","email":"hello@example.com","website":"https://example.com","issues":"","description":"Reusable template project for Acme landing pages."}\n\nPatch namespaces:\n\n- marketplaceProduct\n\n## Search and inspect safely\n\nCommands:\n\n- MCP tool: search-project {"query":"pricing"}\n- MCP tool: search-project {"query":"api.example.com","scopes":["resources"]}\n- MCP tool: list-instances {"pagePath":"/","maxDepth":5}\n- MCP tool: inspect-instance {"instanceId":"","include":["props","styles","children"]}\n- MCP tool: list-texts {"pagePath":"/"}\n- MCP tool: list-assets {"withUsage":true}\n- MCP tool: find-asset-usage {"assetId":""}\n- MCP tool: snapshot {"include":["pages","instances","props","resources","assets"]}\n\nNotes:\n\n- Use `search-project` for query-driven lookup across labels, text, prop values, resource URLs, asset metadata, and styles. Use `audit` for project health findings.\n\n## Audit project quality\n\nCommands:\n\n- webstudio audit --json\n- webstudio audit --scopes accessibility --scopes seo --json\n- webstudio audit --page-path /pricing --json\n- webstudio audit --scopes accessibility --verbose --json\n- webstudio audit --rendered --page-path /pricing --json\n- webstudio audit --rendered --route-example post=/blog/hello --json\n- webstudio audit --rendered --image-domain images.example.com --json\n- MCP tool: audit {}\n- MCP tool: audit {"scopes":["accessibility","security"],"severities":["error","warning"]}\n- MCP tool: audit {"scopes":["accessibility"],"verbose":true}\n- MCP tool: audit {"scopes":["craft"],"verbose":true}\n- MCP tool: audit {"rendered":true,"verbose":true}\n\nNotes:\n\n- With no scopes, `audit` checks accessibility, security, SEO, performance settings, unused assets, ineffective Collection styles, non-GET resources exposed as render-time data, and unused or duplicate style data.\n- Craft is opt-in and read-only. Run `audit` with `scopes:["craft"]` to detect whether the project is not using Craft, partially compatible, or compatible with the versioned Craft 1.2 profile. `profileStatuses` includes the University-doc provenance and the smallest safe next action. The audit never installs Craft or changes a non-Craft project.\n- The `performance` scope reports disabled atomic CSS generation. A rendered audit also measures broken, eager below-fold, and oversized images, browser-marked render-blocking resources, and legacy font formats.\n- Rendered image and resource metrics run only when the selected scopes include `performance`; responsive layout dimensions remain available whenever `rendered:true` is requested.\n- Compact findings include stable ids, severity, message, and location. Use `--verbose` or `{"verbose":true}` for evidence, explanation, suggested remediation, skipped-check details, and manual-check workflows.\n- `summary` counts all findings before severity filtering and pagination.\n- `contractVersion` identifies the audit response contract. Handle a new value before assuming existing fields retain the same meaning.\n- Expression-, resource-, and parameter-backed values that cannot be checked reliably appear in `skippedChecks`; they are not treated as passing or failing.\n- Page filters apply to page-owned accessibility, security, and SEO checks. Asset and style usage remain project-wide to avoid false unused findings.\n- Continue paginated results with `cursor`. Restart the audit if the project version changes.\n- Verbose skipped-check and manual-check details are included on the first findings page only; their total counts remain available on every page.\n- `manualChecks` describes responsive, hierarchy, and contrast checks that require preview screenshots and vision.\n- Focused audits return only manual checks relevant to their selected scopes.\n- In a long-lived MCP session, `{"rendered":true}` reuses preview and screenshot\n tools to capture every static page at mobile, desktop, and Builder breakpoint\n edges. Compact output reports rendered check/issue/failure counts; verbose\n output includes screenshot paths and measured layout dimensions.\n- Dynamic route templates are skipped unless `--route-example =`\n (or MCP `routeExamples`) supplies a concrete path. The path must not contain\n unresolved `:` or `*` parameters.\n- Plans above 120 captures return a short-lived confirmation token. Review the\n unchanged plan, then rerun with `--confirm-large-run` and\n `--confirmation-token`.\n- Detailed rendered evidence is stored in a versioned manifest under\n `.webstudio/audits`; compact output includes its path and screenshot count.\n- Rendered checks also report broken images, eager images below the fold, and\n image sources more than 2x their rendered dimensions in both axes, including\n Webstudio instance ids and measured dimensions when available.\n- Rendered checks include sanitized Resource Timing evidence and report\n browser-marked render-blocking resources plus legacy `.ttf`, `.otf`, and\n `.woff` fonts without applying a universal transfer-size threshold.\n- Fix findings through semantic mutation commands, then rerun `audit` to confirm their deterministic finding ids disappeared.\n\n## Verify dynamic bindings\n\nCommands:\n\n- webstudio verify-bindings --json\n- MCP tool: verify-bindings {"pagePath":"/pricing"}\n- MCP tool: verify-bindings {"instanceId":"","limit":50}\n\nNotes:\n\n- Statically checks persisted text expressions, expression/action/resource/parameter props, resource expressions, and page metadata.\n- Findings distinguish invalid syntax, unknown or out-of-scope variables, stale internal data-source ids, and missing resource or parameter references.\n- Page and instance filters can be combined when the instance belongs to the selected page. Continue findings with `cursor`.\n- This operation does not resolve rendered values or execute external resources. Preview representative loading, empty, error, and populated states after static findings are fixed.\n\n## Refactor targeted content\n\nCommands:\n\n- MCP tool: list-instances {"pagePath":"/"}\n- MCP tool: list-texts {"pagePath":"/"}\n- MCP tool: update-text {"instanceId":"","childIndex":0,"text":"Launch faster"}\n- MCP tool: replace-text {"find":"Old headline","replace":"New headline","match":"exact","pagePath":"/pricing","limit":20}\n- MCP tool: update-props {"updates":"props.json contents"}\n- MCP tool: update-page {"pageId":"","values":{"title":"Pricing","meta":{"description":"Plans"}}}\n- MCP tool: update-resource {"resourceId":"","values":{"url":"https://api.example.com/posts"}}\n- MCP tool: replace-asset {"fromAssetId":"","toAssetId":""}\n- MCP tool: replace-styles {"property":"color","fromValue":{"type":"keyword","value":"red"},"toValue":{"type":"keyword","value":"blue"}}\n- MCP tool: rewrite-css-variable-refs {"map":"variables.json contents"}\n\nNotes:\n\n- Use focused reads first, then mutate only matching instances, props, metadata, resource URLs, assets, or style references. Use `replace-text` for bounded literal text changes, `replace-prop-text` for bounded static prop text, `replace-resource-text` for fixed resource names/URLs, and `update-text` for one known child or expressions.\n\n## Optimize existing project\n\nCommands:\n\n- MCP tool: list-pages {}\n- MCP tool: list-folders {}\n- MCP tool: update-page {"pageId":"","values":{"title":"Pricing","meta":{"description":"Plans"}}}\n- MCP tool: update-props {"updates":"props.json contents"}\n- MCP tool: list-breakpoints {}\n- MCP tool: update-breakpoint {"breakpointId":"tablet","values":{"maxWidth":1023}}\n- MCP tool: get-styles {"instanceIds":[""],"includeTokens":true}\n- MCP tool: update-styles {"updates":"styles.json contents"}\n- MCP tool: attach-design-token {"designTokenId":"","instanceIds":"instances.json contents"}\n- MCP tool: update-project-settings {"meta":{"siteName":"Acme"}}\n\nNotes:\n\n- Use this for SEO metadata, accessibility labels, responsive behavior, token consistency, and project settings.\n\n## Connect external data\n\nCommands:\n\n- MCP tool: create-variable {"scopeInstanceId":"","name":"title","value":{"type":"string","value":"Hello"}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"tags","value":{"type":"string[]","value":["news","product"]}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"filters","value":{"type":"json","value":{"tag":"news"}}}\n- MCP tool: create-resource {"resource":{"name":"Posts","method":"get","url":"https://api.example.com/posts","searchParams":[{"name":"tag","value":"filters.tag"},{"name":"source","value":{"type":"literal","value":"website"}}],"headers":[]},"scopeInstanceId":"","dataSourceName":"posts"}\n- MCP tool: create-resource {"resource":{"name":"Post GraphQL","control":"graphql","method":"post","url":"https://api.example.com/graphql","headers":[{"name":"Content-Type","value":{"type":"literal","value":"application/json"}}],"body":"{ query: \\"query Post($slug: String!) { post(slug: $slug) { title } }\\", variables: { slug: system.params.slug } }"},"scopeInstanceId":"","dataSourceName":"post"}\n- MCP tool: create-resource {"resource":{"name":"Current Date","control":"system","method":"get","url":"/$resources/current-date","headers":[]},"scopeInstanceId":"","dataSourceName":"currentDate"}\n- MCP tool: update-resource {"resourceId":"","values":{"url":"https://api.example.com/posts"}}\n- MCP tool: bind-props {"bindings":"bindings.json contents"}\n- MCP tool: insert-fragment {"parentInstanceId":"","fragment":"{/_ collection content _/}"}\n\nNotes:\n\n- Use this for CMS sections, blog listings, Ghost/headless CMS pages, n8n-style integrations, and API URLs built from variables.\n- For read data, expose GET resources as scoped data variables with `scopeInstanceId`/`dataSourceName` and read the loaded result from the resource result wrapper, usually `.data`.\n- For writes, webhooks, GraphQL submissions, and deletes, prefer unscoped resources bound to Form `action` props so they become action resources instead of auto-loaded read resources.\n- Use direct props for fixed values and prop bindings only when a prop must read a data variable, resource, action, or documented runtime context value such as `system`.\n\n## Render an array or object as repeated content\n\nCommands:\n\n- MCP tool: insert-collection {"parentInstanceId":"","data":{"type":"expression","value":"posts.data.items"},"itemFragment":"{expression`collectionItem.title`}"}\n- MCP tool: inspect-instance {"instanceId":"","include":["props","bindings","children"]}\n\nNotes:\n\n- Use Collection whenever an array or object from a resource or data variable should render a list, grid, cards, table rows, options, tabs, or other repeated UI.\n- Pass `insert-collection` the complete array or object. Do not pass the resource response wrapper or one indexed item. External resource arrays are commonly nested under the scoped resource result\'s `data` field or deeper.\n- Pass one repeated-item Webstudio JSX root. The command creates the Collection, its private current-item/current-key parameters, the iterable binding, and descendant item bindings atomically.\n- Collection renders the item root once for every entry. Use `expression` values such as `collectionItem.title` in descendant text and props. Object iteration also exposes `collectionItemKey`.\n- Wrap multiple repeated sibling instances in one Element inside Collection.\n- For repeated Radix items such as accordion items, tabs, or menu options, bind a stable unique id or slug to every required `value` prop.\n- See the [Collection documentation](https://docs.webstudio.is/university/core-components/collection) for the equivalent Builder workflow.\n\n## Support dynamic runtime behavior\n\nCommands:\n\n- MCP tool: integrate-runtime-ui {"parentInstanceId":"","resources":[{"resource":{"name":"Seats","method":"get","url":"https://api.example.com/seats","headers":[]},"dataSourceName":"Seats","exposeAsDataSource":true}],"structure":{"type":"collection","data":{"type":"expression","value":"Seats.data"},"itemFragment":{"children":[{"type":"id","value":"seat"}],"instances":[{"type":"instance","id":"seat","component":"Text","children":[{"type":"expression","value":"collectionItem.label"}]}],"props":[],"dataSources":[],"resources":[],"styleSources":[],"styleSourceSelections":[],"styles":[],"breakpoints":[],"assets":[]}},"retainedBehavior":[{"instanceId":"","responsibility":"Seat selection behavior"}]}\n- MCP tool: update-props {"updates":"props.json contents"}\n- MCP tool: bind-props {"bindings":"bindings.json contents"}\n- MCP tool: create-resource {"resource":{"name":"Seats","method":"get","url":"https://api.example.com/seats","headers":[]}}\n- MCP tool: snapshot {"include":["instances","props","resources"]}\n- MCP tool: apply-patch {"baseVersion":"","transactions":"patch.json contents"}\n\nNotes:\n\n- Use `integrate-runtime-ui` to create variables/resources, insert one editable fragment or Collection, and add safe data bindings in one transaction.\n- List existing script-owned responsibilities under `retainedBehavior`. The operation preserves those instances and never evaluates or accepts replacement script bodies.\n- `unsupportedConversions` records behavior that cannot be represented safely. Dry-run returns the complete transaction and the same retained/unsupported report without changing the project.\n- New actions and HtmlEmbed scripts are intentionally rejected. Create normal editable components and data bindings; keep opaque runtime behavior in existing script instances.\n\n## Build authenticated pages\n\nCommands:\n\n- MCP tool: meta.guide {"brief":"Build a Supabase-authenticated account page"}\n- MCP tool: create-page {"name":"Account","path":"/account"}\n- MCP tool: update-page {"pageId":"","values":{"meta":{"auth":{"method":"basic","login":"","password":""}}}}\n- MCP tool: create-resource {"resource":{"name":"Session","method":"get","url":"https://api.example.com/session","headers":[]}}\n- MCP tool: create-variable {"scopeInstanceId":"","name":"user","value":{"type":"json","value":{}}}\n- MCP tool: update-props {"updates":"props.json contents"}\n- MCP tool: bind-props {"bindings":"bindings.json contents"}\n\nNotes:\n\n- Inspect and reuse the project\'s existing auth convention before authoring. Do\n not add a second provider or session model implicitly.\n- Model signed-out, loading, signed-in, and failed-auth states explicitly.\n- Never store credentials, service-role keys, refresh tokens, private session\n values, or authenticated response bodies in project data, command output,\n screenshots, agent instructions, or error reports. Privileged provider calls\n and authorization enforcement belong server-side.\n- Basic auth is semantic today. Provider-specific Supabase/Firebase setup still\n uses the existing resource, variable, prop, binding, and embed tools; there is\n no provider-specific installer.\n\n## Generate from design input\n\nCommands:\n\n- MCP tool: meta.guide {"brief":"Recreate this Figma design as a responsive page"}\n- MCP tool: create-page {"name":"Landing","path":"/landing"}\n- MCP tool: create-design-token {"tokens":"tokens.json contents"}\n- MCP tool: define-css-variable {"vars":"vars.json contents"}\n- MCP tool: list-breakpoints {}\n- MCP tool: insert-fragment {"parentInstanceId":"","fragment":"Section copy"}\n- MCP tool: update-styles {"updates":[{"instanceId":"","breakpointId":"","property":"padding-left","value":{"type":"unit","unit":"px","value":24}}]}\n- MCP tool: preview.start {"host":"127.0.0.1","port":5173}\n- MCP tool: screenshot {"path":"/landing","output":"landing-desktop.png","viewport":{"width":1440,"height":900},"waitUntil":"load","waitForTimeout":250}\n- MCP tool: screenshot {"path":"/landing","output":"landing-mobile.png","viewport":{"width":390,"height":844},"waitUntil":"load","waitForTimeout":250}\n- MCP tool: screenshot {"baseUrl":"http://127.0.0.1:5177","path":"/landing","output":"landing-desktop.png","viewport":{"width":1440,"height":900},"waitUntil":"load","waitForTimeout":250}\n\nNotes:\n\n- Use this after the agent can inspect the supplied design. There is no direct\n Figma, screenshot, Inception, or `design.md` import command.\n- Inspect and reuse existing variables, tokens, styles, components, assets, and\n page patterns before authoring. Build semantic editable structure rather than\n flattening the design into an image or absolute-positioned approximation.\n- Verify one familiar viewport inside every distinct Builder breakpoint range,\n then run rendered audit and inspect the screenshots before completion.\n\n## Cross-project maintenance\n\nCommands:\n\n- webstudio mcp run .temp/projects.json\n- webstudio mcp run .temp/projects.json --dry-run\n- webstudio mcp run .temp/projects.json --approve-mutations --concurrency 2\n\nNotes:\n\n- Put shared `calls` and a `projects` array of independently linked project roots in the existing `mcp run` manifest. Project roots are relative to the manifest file.\n- Each project uses its own config, authentication, ProjectSession storage, checkpoint, and failure boundary. Confirmed successful calls are checkpointed. Reads can resume automatically; a mutation interrupted after dispatch is reported as ambiguous and is not replayed automatically, preventing silent duplicate writes.\n- Focus the manifest on bounded reads or audits first. Use per-call `dryRun`, global `--dry-run`, or explicitly approve committed mutations with `--approve-mutations` after reviewing the manifest.\n\n# Known CLI Gaps\n\n## Provider-specific authenticated pages\n\nMissing:\nCLI supports page basic auth and generic resources/props/embeds, but not guided Supabase/Firebase auth setup.\n\nCurrent fallback:\nCall `meta.guide` with the provider-authenticated page goal, then create the\npage, resources, variables, props, bindings, and embeds with existing semantic\ntools.\n\nSuggested commands:\n\n- setup-auth-page\n\n## Generate from design input\n\nMissing:\nNo command imports Figma, screenshots, Inception output, or design.md and turns it into pages/tokens/layout.\n\nCurrent fallback:\nCall `meta.guide` with the design-input goal, let the agent inspect the supplied\ndesign, then use semantic page, token, asset, fragment, style, preview,\nscreenshot, and audit tools. Use `apply-patch` only when no semantic operation\nfits.\n\nSuggested commands:\n\n- generate-from-design\n\n## Built-in cross-project maintenance\n\nMissing:\nPublic API and CLI intentionally operate on one configured project at a time; there is no built-in multi-project discovery or loop runner.\n\nCurrent fallback:\nRun the CLI from an external script that reconfigures one project/session at a time.\n\nSuggested commands:\n\n- none\n', "manual-api": '# Webstudio API CLI Manual\n\nThe API commands operate on the single project configured by:\n\n- .webstudio/config.json: projectId\n- global Webstudio config: origin and token\n\n- Pass --json to API/discovery commands that support it. Do not add --json to top-level commands unless their help/schema documents it.\n- Never pass a project id. Commands use configured project only.\n- Read ids before writing. Do not invent ids for existing records.\n- stdout is one JSON object. stderr is diagnostics.\n- Prefer MCP semantic tools for detailed project edits. Use MCP apply-patch only when no semantic tool exists.\n\n## Start\n\n{{start}}\n\n## Read First\n\n{{readFirst}}\n\n## Project Session Cache\n\n- CLI commands use one local ProjectSession snapshot for the configured project.\n- Local-capable reads use cached namespaces when compatible and fetch only missing or stale namespaces.\n- Local-capable mutations build patches from the local snapshot, then commit with the cached build version.\n- Successful mutation commits update the local snapshot only after the remote commit succeeds.\n- Server-only commands run remotely and invalidate/refetch namespaces declared by the operation catalog.\n- Use --refresh on local-capable commands to refresh required namespaces before running.\n- Successful JSON responses include compact meta.session with operationId, buildId, version, source, committed, namespaceCounts, diagnosticCount, non-empty diagnostic summaries, and optional compatibilityVersion.\n\n## CLI Capability Inventory\n\nFor a short end-consumer summary of what MCP can do, see\n`manual mcp` / `webstudio man mcp`. The MCP inventory describes the same\nproject, page, element, style, data, asset, publish, domain, and visual\nverification capabilities without internal command names.\n\n### Top-Level Commands\n\n{{topLevelCapabilityIndex}}\n\n### High-Level API Commands By Area\n\n{{apiCapabilityIndex}}\n\n### MCP Tool Operations\n\nThese are MCP tools. From a shell, call them with the shortcut form `webstudio \'\'` or with the explicit form `webstudio mcp single-op-call \'\'`:\n\n{{mcpOnlyCommandIndex}}\n\n## Task Recipes\n\n{{taskRecipeIndex}}\n\n## Use Case Index\n\n{{useCaseIndex}}\n\n## Known CLI Gaps\n\n{{knownCliGapIndex}}\n\n## Input File Shapes\n\n{{inputFileShapeIndex}}\n\n## Raw Patch Fallback\n\napply-patch accepts either BuildPatchTransaction[] or { "transactions": BuildPatchTransaction[] }.\n\nEach transaction has:\n\n{\n"id": "patch-transaction-label",\n"payload": [\n{\n"namespace": "projectSettings",\n"patches": [\n{ "op": "replace", "path": ["meta", "siteName"], "value": "New Site" }\n]\n}\n]\n}\n\nThe transaction id is a patch label used for optimistic synchronization. It is\nnot a Builder record id. Do not invent ids for pages, instances, props,\nbreakpoints, resources, variables, folders, assets, or other project records.\n\nPatch paths are JSON-patch-like paths into Builder store data. Map-like namespaces use ids as the first path item.\n\nSupported namespaces:\n\n- pages: redirects, page records, and folders\n- projectSettings: project-wide metadata and compiler settings\n- instances: element instances and children, including text/expression children\n- props: element props, bindings, page references, resource bindings\n- styles: CSS declarations keyed by style declaration key\n- styleSources: local style sources and reusable design tokens\n- styleSourceSelections: instance-to-style-source connections\n- dataSources: data variables, parameters, and resource data sources\n- resources: data resource definitions\n- assets: project asset records handled by the existing asset patch path\n- breakpoints: responsive breakpoints\n- marketplaceProduct: marketplace metadata\n\n## Data Sources\n\n`dataSources` is the internal Builder namespace for variables. Public API, CLI,\nand MCP tools expose it through two user-facing groups:\n\n- data variables: `list-variables`, `create-variable`, `update-variable`, and\n `delete-variable`\n- data resources: `list-resources`, `create-resource`, `update-resource`, and\n `delete-resource`\n\nFor raw `snapshot`, request the public `variables` namespace rather than the\ninternal `dataSources` name. Raw patch payloads still use `dataSources` when\napplying direct changes.\n\nVariables can be scoped to an instance. Expressions under that instance can use\nthe variable by name; nested variables with the same name mask outer variables.\nVariable values support `string`, `number`, `boolean`, `string[]`, and `json`.\nUse `string[]` for lists of strings such as tags or selected categories; use\n`json` for objects, arrays with mixed shapes, or nested API filter state.\nParameters are internal scoped runtime values provided by pages, collections,\nor components. They are not a public authoring surface: do not create, update,\nor delete parameter records. Public tools should preserve existing parameter\nrecords and may reference documented context values such as `system` in\nexpressions where they are already in scope.\n\nResource `url` accepts plain fixed URLs and paths, for example\n`https://api.example.com/posts` or `/$resources/current-date`. Dynamic URLs can\ncombine strings and variables, for example\n`"https://api.example.com/posts?tag=" + filters.tag`. Prefer `searchParams` for\nquery parameters that should be encoded separately:\n`[{ "name": "tag", "value": "filters.tag" }]`. Header values, search parameter\nvalues, and bodies are expressions for dynamic content. For fixed text, use\n`{ "type": "literal", "value": "application/json" }`; Webstudio stores the\nrequired string expression. Headers can still read variables such as\n`"Bearer " + auth.token`, and GraphQL bodies can return objects such as\n`{ query: "...", variables: { slug: system.params.slug } }`.\n\nCreate a GET resource with `scopeInstanceId` when the fetched resource result\nshould be available as a read data variable. Scoped GET resources default to\n`exposeAsDataSource: true`, are generated into the page resource `data` map,\nand may be loaded during page rendering. Use `dataSourceName` to choose the\nvariable name.\n\nFor submit/write/action resources, create the resource without\n`scopeInstanceId`, then bind a component prop such as a Form `action` to the\nresource with `bind-props` and `binding.type: "resource"`. Prop-bound resources\nare generated into the page resource `action` map instead of the read `data`\nmap. Use this shape for POST, PUT, DELETE, webhook, and other resources that\nshould run only from an explicit form/action flow, not merely because the page\nrendered.\n\nPOST, PUT, and DELETE resources default to `exposeAsDataSource: false` even\nwhen a scope is supplied. Set `exposeAsDataSource: true` only when a write-method\nresource intentionally provides render-time data, such as a read-only GraphQL\nPOST query. A scope is required, and the result includes a warning because the\nrequest may execute during page rendering. Set `exposeAsDataSource: false` on\n`update-resource` to detach an existing render-time data source.\n\nResource `method` can be `get`, `post`, `put`, or `delete`. Use GET for read\ndata. Use POST for creates, GraphQL requests, webhooks, and form submissions.\nUse PUT for full updates/replacements. Use DELETE for deletion actions.\nOptional `control` values are `graphql` and `system`: `graphql` marks a\nGraphQL-style resource, usually POST with a query body; `system` marks a\nresource intended to use the built-in `system` parameter or one of the built-in\nlocal resource URLs: `"/$resources/sitemap.xml"`,\n`"/$resources/current-date"`, and `"/$resources/assets"`. The system parameter\nfields are `system.origin`, `system.pathname`, `system.params`, and\n`system.search`.\n\nUse prop bindings for dynamic values that read variables or resources; use\ndirect props for static values.\n\nCommit raw patch:\n\nMCP tool: apply-patch\n\n## Raw Patch Examples\n\nRename the site:\n\n[\n{\n"id": "patch-site-name",\n"payload": [\n{\n"namespace": "projectSettings",\n"patches": [\n{ "op": "add", "path": ["meta", "siteName"], "value": "Acme Studio" }\n]\n}\n]\n}\n]\n\nUpdate page title metadata:\n\n[\n{\n"id": "patch-page-title",\n"payload": [\n{\n"namespace": "pages",\n"patches": [\n{ "op": "replace", "path": ["pages", "page-id", "meta", "title"], "value": "Pricing" }\n]\n}\n]\n}\n]\n\nUpdate a text child on an element:\n\n[\n{\n"id": "patch-text",\n"payload": [\n{\n"namespace": "instances",\n"patches": [\n{ "op": "replace", "path": ["instance-id", "children", 0, "value"], "value": "Launch faster" }\n]\n}\n]\n}\n]\n\nCreate records with semantic operations such as create-variable,\ncreate-resource, create-design-token, create-page, create-folder,\nand create-breakpoint. Raw patch rejects generated record\ncreation, collection replacement, record replacement with a different `id`, and\nrecord id field mutations in id-keyed namespaces because Webstudio must generate\nand preserve record ids.\n\n## Safety Rules\n\n- For MCP apply-patch, read the latest version with MCP snapshot before writing.\n- Reuse ids from MCP snapshot output when updating existing records.\n- Do not create generated records, replace generated record collections, replace records with different ids, or mutate record id fields with raw patch. Use semantic create operations so Webstudio generates ids.\n- If apply-patch reports a version conflict, read the latest build and regenerate the patch.\n- Prefer semantic MCP read tools for discovery, then use MCP snapshot for exact patch paths.\n\n## Command Index\n\n{{commandIndex}}\n', "manual-llm": - '# Webstudio CLI Manual for LLMs\n\nUse this order. Stop only when a command returns ok:false.\n\nIf you are inside the Webstudio monorepo, the first command discovery should use\nthe local CLI exactly as `node packages/cli/local.js ...` from the repo root. Do\nnot use `packages/cli/bin.js` for local source-tree work; it is the packaged\nbuild entry and may use stale built output. Do not use `pnpm exec webstudio`,\n`pnpm --filter webstudio exec webstudio`, or a global `webstudio`: they can\nresolve an older binary.\n\nFor delegated design-system or “use every component” tasks, skip the generic warm-up sequence and start with exactly one MCP command: `webstudio workflow.next \'{"goal":"design-system-page"}\'`. Report that returned checkpoint to the parent/user and stop until continued.\n\n## Connect an MCP client\n\nWhen the user asks to connect the current folder to Webstudio, run the command\nfor the agent client you are currently using:\n\n- Claude Code: `webstudio connect claude`\n- Codex: `webstudio connect codex`\n- Cursor: `webstudio connect cursor`\n- VS Code or GitHub Copilot: `webstudio connect vscode`\n\nRun the command from the linked project root. If the folder is not linked, ask\nfor an editable Builder share link and run\n`webstudio init --link --json`, followed by `webstudio sync`, then\nretry `webstudio connect `. Treat the share link as a credential and do\nnot include it in committed files, logs, screenshots, or issue reports.\n\n`connect` verifies project access before changing client configuration. For\nClaude Code, Cursor, and VS Code it safely merges the `webstudio` server into\nthe client\'s project configuration. For Codex it runs both `codex mcp add` and\n`codex mcp get webstudio`; do not repeat those commands separately. Follow the\nreload, restart, or approval instruction printed by `connect`, then verify the\nloaded MCP connection by asking the client to use Webstudio MCP and list the\nproject pages. Use `--print` only to inspect the generated setup without\nchanging configuration or requiring project access.\n\n## Always\n\n1. webstudio permissions --json\n2. For bounded shell workflows, call MCP tools directly through the CLI shortcut, for example `webstudio meta.index` or `webstudio insert-fragment \'\' --dry-run`. The explicit form `webstudio mcp single-op-call \'\'` is equivalent and useful when you need to make the MCP boundary obvious. Use `webstudio mcp run \'[{"tool":"components.find","input":{"brief":"button"}}]\'` for multiple calls in one shared CLI session. Use a normal JSON file path for large batches. Use long-running `webstudio mcp` only when your environment is a real MCP client. Do not manually send raw JSON-RPC to `webstudio mcp` from a shell or PTY.\n3. Read MCP `meta.index`, for example `webstudio meta.index`.\n4. Use focused MCP calls with concrete JSON: `webstudio meta.guide \'{"brief":"Create a design system page using every component"}\'`, `webstudio meta.get_more_tools \'{"tools":["insert-fragment"]}\'`, `webstudio components.list \'{"source":"all"}\'`, `webstudio components.coverage-plan`, `webstudio components.search \'{"brief":"radix select"}\'`, `webstudio components.get \'{"component":"@webstudio-is/sdk-components-react-radix:Select"}\'`, `webstudio templates.list`, and `webstudio templates.get \'{"component":"@webstudio-is/sdk-components-react-radix:Select"}\'`.\n5. Read overview resources `webstudio://project/tools-overview` or `webstudio://project/components-overview` when useful. Read full resources `webstudio://project/tools` or `webstudio://project/components` only when focused tools are insufficient.\n6. Pick focused MCP read tool.\n7. Pick semantic MCP write tool.\n\nUse `webstudio schema mcp` for a compact MCP tool overview. Add `--verbose` only when exact input schemas for all tools are truly needed; otherwise prefer focused `meta.get_more_tools` and `components.*` calls.\n\nRun these commands from the linked project root. Use the MCP startup status line\'s absolute root for local files; write temporary scripts and artifacts under `/.temp`, not under a parent workspace.\n\nMonorepo quick path for a simple styled section:\n\n```sh\nnode packages/cli/local.js mcp single-op-call meta.index\nnode packages/cli/local.js mcp single-op-call meta.get_more_tools \'{"tools":["insert-fragment"]}\'\nnode packages/cli/local.js mcp single-op-call insert-fragment \'{"parentInstanceId":"parent-id","fragment":"Launch KitA focused section created with Webstudio JSX.Get started"}\' --dry-run\n```\n\nThe same local shortcut form is shorter and preferred for simple shell steps:\n\n```sh\nnode packages/cli/local.js meta.index\nnode packages/cli/local.js meta.get_more_tools \'{"tools":["insert-fragment"]}\'\nnode packages/cli/local.js insert-fragment \'{"parentInstanceId":"parent-id","fragment":"Launch KitA focused section created with Webstudio JSX.Get started"}\' --dry-run\n```\n\nFor this simple path, do not grep source files, dump full MCP resources, or write parser scripts first. Use `list-pages`, `get-page-by-path`, or `list-instances` only to get the target `parentInstanceId`.\n\nWhen authoring JSX for `insert-fragment`, use Webstudio component helpers and Webstudio style syntax. Use `ws:style={css\\`...\\`}`for Webstudio-native CSS, or use React-style object syntax such as`style={{ padding: 24 }}` when that is simpler. Both forms create editable Webstudio style data.\n\nWhen the task says another user will edit a page in Content mode, use a Content Block (`ws:block`) around every editable region. Content-mode users can edit text and supported props only in descendants of that block; content outside it is read-only. Put reusable insertable options in the block\'s `ws:block-template` child. Do not put intended editor content inside that template container: templates are protected source material, while an inserted template copy becomes an editable direct child of the Content Block. Verify this structure before handoff.\n\nDo not access host globals or dynamic code APIs in JSX fragments, including `process`, `globalThis`, `eval`, `Function`, or `constructor`. JSX fragments are declarative project data; use the built-in Webstudio helpers instead.\n\nUse Webstudio prop names in JSX: `class`, `for`, `aria-label`, and other HTML/Webstudio names. Do not use React-only aliases such as `className` or `htmlFor`; the runtime rejects them with the Webstudio prop name to use.\n\nUse Webstudio actions for event/action props. Do not pass JavaScript functions such as `onClick={() => ...}`; the runtime rejects them because functions cannot be persisted as Webstudio project data.\n\n```tsx\n\n Open\n\n```\n\nPlain JSX prop values must be JSON-compatible: `null`, strings, booleans, finite numbers, arrays, and plain objects. Do not pass `undefined`, `Symbol`, `BigInt`, `NaN`, `Infinity`, `Date`, `Map`, `Set`, class instances, or circular objects; omit the prop, use plain data, or use `expression`/`ActionValue` when the value is dynamic.\n\nIf a component has a registered template with required parts, JSX must include those parts explicitly under the same parent structure as the template, for example ``. Use `insert-component` when you want Webstudio to apply one component template automatically.\n\n## Animation Components\n\nBefore creating animation examples, inspect the exact components with focused discovery:\n\n```sh\nwebstudio components.get \'{"component":"@webstudio-is/sdk-components-animation:AnimateChildren"}\'\nwebstudio components.get \'{"component":"@webstudio-is/sdk-components-animation:AnimateText"}\'\nwebstudio components.get \'{"component":"@webstudio-is/sdk-components-animation:StaggerAnimation"}\'\nwebstudio components.get \'{"component":"@webstudio-is/sdk-components-animation:VideoAnimation"}\'\n```\n\nUse Animation Group (`AnimateChildren`) as the controller. Put normal instances directly inside it, or put Text Animation, Stagger Animation, or Video Animation directly inside it. Text, Stagger, and Video are helper components with `contentModel.category: "none"` and should not be used as standalone section roots.\n\nDefine timing and CSS changes on the Animation Group `action` prop. Use `type:"view"` for viewport entry/exit progress and `type:"scroll"` for scroll-progress timelines. For in animations, keep the canvas styles as the final state and use `fill:"backwards"` with keyframes that describe the starting state. For out animations, use `fill:"forwards"` with keyframes that describe the ending state.\n\nText Animation settings: `slidingWindow` defaults to `5`, `easing` defaults to `linear`, and `splitBy` defaults to `char`. Use `splitBy:"space"` for word-by-word animation. The parent Animation Group keyframes provide the actual opacity, translate, scale, or other styles.\n\nStagger Animation settings: `slidingWindow` defaults to `1` and `easing` defaults to `linear`. It applies parent Animation Group progress across its direct children. Use `slidingWindow:0` for instant sequential steps, `1` for one child at a time, and values above `1` for overlapping waves.\n\nVideo Animation settings: `timeline` is a boolean. Prefer `insert-component` for Video Animation so the Video child template is inserted, then configure the Video child asset/source. Use short, seek-friendly videos for smooth scroll-linked playback.\n\nUse JSX fragments for authored animation structures when you need styled, editable examples. Put the final visual state in `ws:style` and put the starting or ending animated state in the Animation Group `action` keyframes. Include an explicit `offset` on every keyframe: use `offset: 0` for starting-state keyframes with `fill:"backwards"` and `offset: 1` for ending-state keyframes with `fill:"forwards"`.\n\n```tsx\n\n \n Launch metrics\n \n A polished card that fades up as it enters the viewport.\n \n \n\n```\n\nFor Text Animation, keep `animation.AnimateText` as the direct child of Animation Group and place the text-containing element inside it:\n\n```tsx\n\n \n Animate words with controlled rhythm\n \n\n```\n\nFor Stagger Animation, put the repeated cards or rows directly inside `animation.StaggerAnimation`:\n\n```tsx\n\n \n \n Plan\n \n \n Build\n \n \n Launch\n \n \n\n```\n\nFor Video Animation, use the registered template via `insert-component` when possible. If you author JSX, include the Video child explicitly:\n\n```tsx\n\n \n <$.Video\n preload="auto"\n autoPlay={true}\n muted={true}\n playsInline={true}\n crossOrigin="anonymous"\n />\n \n\n```\n\n## Command Surface Boundary\n\n- Use top-level `webstudio ...` shell commands for setup, sync/import/build/preview/screenshot, permissions, publish/domains, schema, registry inspection, man, and starting MCP.\n- Use MCP tools for Builder project data manipulation: pages, instances/components, props, text, styles, tokens, variables, resources, assets, breakpoints, redirects, and raw patches.\n- From a shell, call MCP tools with the shortcut form `webstudio \'\'`, for example `webstudio insert-fragment \'\' --dry-run`. The explicit equivalent is `webstudio mcp single-op-call \'\'`. Use `--input-file` for large payloads.\n- Inside the Webstudio monorepo, call the local CLI as its own command: `node packages/cli/local.js ...`. Do not wrap the CLI call in `pwd && ...`, command substitution, `pnpm exec webstudio`, `pnpm --filter webstudio exec webstudio`, or a global `webstudio`.\n- For experiments, pass `--dry-run` to local-capable mutation calls. Read the computed transaction from `meta.session.transaction` and its base build version from `meta.session.version`. Copying a `.webstudio` folder is not an isolated project clone; `.webstudio/config.json` still points to the same remote project, so non-dry-run mutations can commit to that project.\n- For bounded multi-step shell work, run inline JSON with `webstudio mcp run \'[{"tool":"components.find","input":{"brief":"button"}}]\'`; this reuses one CLI session without raw JSON-RPC. For large batches, write `{ "calls": [{ "tool": "..." }] }` to a normal JSON file and run `webstudio mcp run .temp/mcp-calls.json`.\n- Use JSON strings for `brief` fields. Never pass boolean flags such as `{"brief":true}`.\n- Treat `webstudio mcp single-op-call` and `webstudio mcp run` stderr lines as progress checkpoints; stdout remains JSON on both success and failure. On failure, parse stdout for `{ "ok": false, "error": { "code": "...", "message": "..." } }` before deciding what to fix.\n- If a CLI/MCP tool crashes, hangs, gives a confusing error, needs an undocumented workaround, or forces source-code inspection for normal usage, ask the user to report it in Discord `#help` at https://wstd.us/community. Give them a complete copy-paste report with the goal, expected behavior, actual error, exact command/tool call, stdout JSON, stderr/lifecycle logs, environment, workaround, and secrets redacted.\n- Run one-shot `webstudio mcp single-op-call` commands sequentially against a linked `.webstudio` folder. If a command returns `PROJECT_SESSION_BUSY`, another CLI/MCP process is updating the local session; wait a moment and retry sequentially.\n- In delegated or non-streaming agent environments, do not batch many MCP calls silently and do not wrap many shortcut or `webstudio mcp single-op-call` commands in a shell loop. Treat each parent-visible checkpoint as the unit of work. If the parent asks for status within 30 seconds, run exactly one shortcut command such as `webstudio meta.index` or one explicit `webstudio mcp single-op-call` command, report that command/result, then wait for the parent to continue. Do not take a broad task such as creating a full design-system page as one execution unit. Call `workflow.next {"goal":"design-system-page"}`, report the returned phase/checkpoint, wait until the parent continues, call `checkpoint.ack {"reported":true,"continueAfterReport":true,"summary":""}`, complete exactly that bounded phase, and return: discovery, page creation, one dry-run JSX section, one committed JSX section, one `components.coverage-insert-next` call, or one presentation pass. Phase commands do not include nextPhase in their own output. After the parent continues, acknowledge the previous checkpoint first, then call `workflow.next` with the next phase. For all-component design-system pages, checkpoint after workflow planning, discovery, page creation, call `components.coverage-insert-next` once before checkpointing again, then finish with `workflow.next {"goal":"design-system-page","phase":"presentation-pass"}`. Coverage 72/72 is necessary but not sufficient: the page must be organized into styled, real-world examples, not raw unstyled component dumps.\n- For design-system or “use every component” tasks, start with compact `webstudio components.coverage-plan`, checkpoint, then request component coverage details with `webstudio components.coverage-plan \'{"detail":"roots","offset":20}\'` or `{"detail":"parts"}` only when needed. Do not pass `detail` to `list-pages`; use `list-pages {}` or `get-page-by-path` for page lookup.\n- MCP tool shortcuts are only for MCP tools. If a shortcut is ambiguous with a real top-level command, the real top-level command wins; use `webstudio mcp single-op-call \'\'` to force the MCP path.\n\n## LLM Implementation Process\n\nUse this process for user requests that change Webstudio content, layout, styles, assets, pages, redirects, resources, or publishing state:\n\n1. Discover capabilities with `webstudio man --json`, `webstudio schema api`, `webstudio schema mcp`, MCP `meta.index`, `meta.guide`, `meta.get_more_tools`, `components.list`, `components.summary`, `components.coverage-plan`, `components.search`, `components.get`, `templates.list`, and `templates.get`. From a shell, prefer shortcut calls such as `webstudio meta.index` and `webstudio components.search \'{"brief":"button"}\'` for these focused tool calls; use `webstudio mcp single-op-call` when you need the explicit MCP form. Read full resources such as `webstudio://project/tools` and `webstudio://project/components` only when needed. Do not write scripts to parse full MCP discovery JSON for normal lookup.\n2. Inspect current project state with semantic reads such as `get-project-settings`, `list-pages`, `get-page-by-path`, `list-instances`, `inspect-instance`, `get-styles`, `list-assets`, `list-breakpoints`, and `snapshot` only when needed. Before changing a project, read `get-project-settings` and follow any non-empty `meta.agentInstructions`. These are shared project instructions, not a place for secrets.\n3. Mutate the Webstudio project with semantic MCP write tools first. Prefer MCP `insert-fragment` for authored/styled sections, use `insert-component` only for one automatic component template, then `update-text`, `update-props`, `update-styles`, `upload-asset`, `create-page`, and page/project settings tools over raw patches.\n4. Use `apply-patch` only when no semantic tool covers the required change, and only after reading the latest snapshot/version.\n5. For visual/design work, regenerate or preview the generated app, capture a screenshot, inspect it with vision, and iterate before final response.\n6. Report what changed and what verification ran.\n\n## Visual Design Workflow\n\nFor requests involving visible HTML/CSS, layout, typography, colors, imagery, responsive behavior, or screenshots:\n\n1. Read editable Webstudio structure first: pages, instances, props, styles, breakpoints, assets, and relevant text.\n2. Do not use generated route/component files as the source of truth for editable content.\n3. Make edits through Webstudio semantic commands/MCP tools so the result stays editable in Builder and survives the next `webstudio build`.\n4. Keep generated project files current, start preview, and capture the changed page with `screenshot`.\n5. Use `screenshot.diff` when a baseline exists and inspect screenshot/diff artifacts with vision before finishing.\n6. If vision or screenshot tooling is unavailable, state that explicitly and explain what fallback verification was used.\n\n## Responsive Verification Workflow\n\nFor responsive page work, use Builder breakpoints as the source of truth:\n\n1. Read breakpoints with `list-breakpoints` before deciding responsive behavior.\n2. Apply responsive styles with existing Builder breakpoint ids; do not invent CSS media queries or breakpoint names when Webstudio breakpoint data exists.\n3. Pick screenshot viewport widths from the project breakpoints: include a desktop width, each defined max-width or min-width edge, and a narrow mobile width.\n4. Capture each viewport with `screenshot`, for example `{"path":"/","output":"home-375.png","viewport":{"width":375,"height":812}}` and `{"path":"/","output":"home-1440.png","viewport":{"width":1440,"height":900}}`.\n5. Inspect every viewport screenshot with vision before finishing, checking layout, overflow, hidden content, text wrapping, and breakpoint-specific style changes.\n6. If any viewport fails, update styles through semantic Webstudio tools and repeat screenshots for the affected breakpoints.\n\n## Generated Files Guardrails\n\n- Do not edit `app/__generated__`, generated route files, generated page files, generated CSS, or build output for normal Webstudio content/design requests.\n- Do not replace generated page components with handcrafted app code unless the user explicitly asks for code-only export customization.\n- Generated files are build artifacts and may be overwritten by `webstudio build`.\n- If a task truly requires generated app customization, keep it outside `app/__generated__` where possible and explain that it is not editable Webstudio content.\n\n## Values vs Bindings\n\nBefore authoring unfamiliar expressions, read `webstudio://project/expressions` with MCP `resources/read` or `webstudio mcp read-resource webstudio://project/expressions`. It documents the supported expression subset, method allowlist, scope, Collection context, and validation limits.\n\n- Use direct value tools for fixed content. For one visible text child, use `update-text` with plain `text`. For a bounded multi-instance literal replacement, use `replace-text` with `find`, `replace`, `pagePath` or `pageId`, and `limit`; it does not change expression children. Use `replace-prop-text` for bounded changes inside static string props, optionally limited to prop names or instance ids; it never changes dynamic bindings. For static props such as `aria-label`, `alt`, `id`, `class`, `href`, or button labels stored as props, use `update-props` with the prop\'s direct type/value.\n- Use `bind-props` only when the prop must stay dynamic: an expression, resource result, action, or existing scoped runtime context such as `system`. Do not use `bind-props` just to set a fixed string.\n- Direct prop string example: `{"updates":[{"instanceId":"button-id","name":"aria-label","type":"string","value":"Open menu"}]}`.\n- Expression binding example: `{"bindings":[{"instanceId":"link-id","name":"href","binding":{"type":"expression","value":"currentPost.url"}}]}`.\n- Page metadata fields such as `title`, `description`, `language`, `redirect`, and custom meta content accept plain fixed text. For computed values, pass JavaScript expression code such as `pageTitle ?? "Pricing | Acme"`.\n- Page `status` accepts a fixed HTTP status code as a number from 200 through 599, for example `302`. For a dynamic status, pass JavaScript expression code such as `system.status`.\n- Page metadata update example: use `update-page` with `{"pageId":"page-id","values":{"title":"Pricing | Acme","meta":{"description":"Plans for teams"}}}`.\n- Draft a page with `update-page` and `{"pageId":"page-id","values":{"isDraft":true}}`. It remains editable and previewable but is omitted from every publish target, including staging, and from sitemap output.\n- Stage a draft page for a future publish with `{"pageId":"page-id","values":{"isDraft":false}}`. This clears draft state but does not deploy the site. The home page and `/*` catch-all page cannot be drafts.\n- Resource `url` accepts plain fixed URLs and paths. For computed URLs, pass JavaScript expression code such as `"https://api.example.com/items?tag=" + filters.tag`. Resource header values, search parameter values, and text bodies accept expressions for dynamic values; for fixed text, use `{ "type": "literal", "value": "application/json" }`.\n- Resource update example: use `update-resource` with `{"resourceId":"resource-id","values":{"url":"https://api.example.com/items"}}`.\n- Data variable values support `string`, `number`, `boolean`, `string[]`, and `json`. Use `string[]` only for arrays where every item is a string; use `json` for objects, mixed arrays, filters, and nested data.\n- Parameters are internal scoped runtime values from pages, collections, or components. They are not a public authoring surface: do not create, update, or delete parameter records. Public tools should preserve existing parameter records and may reference documented context values such as `system` in expressions where they are already in scope.\n- Use scoped resources for read data. A GET resource created with `scopeInstanceId`/`dataSourceName` defaults to `exposeAsDataSource:true`, becomes a scoped resource data variable, is generated into the page resource `data` map, and may be loaded while rendering the page. Read the loaded resource result from its wrapper, usually `.data`.\n- Use prop-bound resources for actions. A resource created without `scopeInstanceId` and bound to a component prop such as Form `action` with `bind-props` and `binding.type: "resource"` becomes an action resource in the page resource `action` map. Use this for POST, PUT, DELETE, webhooks, GraphQL submissions, and anything that should run only from an explicit form/action flow.\n- POST, PUT, and DELETE resources default to `exposeAsDataSource:false`, even with a scope. Set `exposeAsDataSource:true` only for an intentional render-time read such as a GraphQL POST query; provide `scopeInstanceId` and inspect the returned warning. Set it to `false` during `update-resource` to detach existing render-time exposure.\n- For dynamic resource query parameters prefer `searchParams`, for example `{"name":"tag","value":"filters.tag"}`. Use `{"type":"literal","value":"website"}` for fixed request text. Header values can be expressions such as `"\\"Bearer \\" + auth.token"`. Body can be an object expression, including GraphQL payloads such as `{ query: "...", variables: { slug: system.params.slug } }`.\n- Resource methods are `get`, `post`, `put`, and `delete`. Optional resource controls are `graphql` and `system`. Use `control:"graphql"` for GraphQL POST resources with query bodies. Use `control:"system"` for built-in local resource URLs such as `"/$resources/current-date"` and for resources reading the built-in `system` parameter. The built-in system fields are `system.origin`, `system.pathname`, `system.params`, and `system.search`; do not use `system.path`.\n- Whenever an array or object from a resource or data variable should render repeated UI, call `insert-collection` with the complete iterable and one repeated-item JSX root. The command creates the Collection, private item parameters, iterable binding, and descendant item bindings atomically. Use `collectionItem` and `collectionItemKey` expressions in the item JSX. Wrap multiple repeated siblings in one Element, and give repeated Radix items stable unique `value` bindings.\n- Expressions are single JavaScript expressions, not statements or functions. Functions, arrow functions, classes, `new`, `this`, `await`, imports, arbitrary calls, increment/decrement, and assignment outside actions are unsupported. Prefer optional chaining, nullish coalescing, ternaries, property/index access, operators, and the documented string/array methods.\n\n## Pick Read Command\n\n{{readFirst}}\n\n## Pick Write Command\n\n{{taskRecipeIndex}}\n\n## Raw Patch Only If Needed\n\n1. Use MCP tool: snapshot.\n2. Write BuildPatchTransaction[].\n3. Use MCP tool: apply-patch.\n\n## MCP Argument Examples\n\nMCP tools receive JSON argument objects, not CLI flags. Use these shapes:\n\n{{mcpArgumentExampleIndex}}\n\n## Rules\n\n- Never guess ids for existing records. Read them first.\n- Never use project ids from user input. Commands use the configured project.\n- Use --refresh before a local-capable command when cached data may be stale.\n- Pass --json only to commands whose help/schema documents it. Do not add --json to top-level commands such as sync unless supported.\n- On VERSION_CONFLICT, read MCP snapshot again, regenerate the patch, then retry.\n- Treat stdout JSON as the API contract and stderr as diagnostics.\n- For visual/design work, verify the rendered result with vision before finishing.\n- Do not edit generated files for normal Webstudio content/design requests.\n- Use direct values for static strings and bindings only for dynamic expressions/resources/actions.\n- Use plain fixed text where documented. Only encode a quoted JavaScript string literal when a field is explicitly documented as an expression-only value.\n- Confirm destructive commands with --confirm only when user requested deletion/unpublish/replacement.\n- Use webstudio schema api for machine-readable top-level command metadata and webstudio schema mcp for MCP tool schemas.\n\n## Known Gaps\n\n{{knownCliGapIndex}}\n', + '# Webstudio CLI Manual for LLMs\n\nUse this order. Stop only when a command returns ok:false.\n\nIf you are inside the Webstudio monorepo, the first command discovery should use\nthe local CLI exactly as `node packages/cli/local.js ...` from the repo root. Do\nnot use `packages/cli/bin.js` for local source-tree work; it is the packaged\nbuild entry and may use stale built output. Do not use `pnpm exec webstudio`,\n`pnpm --filter webstudio exec webstudio`, or a global `webstudio`: they can\nresolve an older binary.\n\nFor delegated design-system or “use every component” tasks, skip the generic warm-up sequence and start with exactly one MCP command: `webstudio workflow.next \'{"goal":"design-system-page"}\'`. Report that returned checkpoint to the parent/user and stop until continued.\n\n## Connect an MCP client\n\nWhen the user asks to connect the current folder to Webstudio, run the command\nfor the agent client you are currently using:\n\n- Claude Code: `webstudio connect claude`\n- Codex: `webstudio connect codex`\n- Cursor: `webstudio connect cursor`\n- VS Code or GitHub Copilot: `webstudio connect vscode`\n\nRun the command from the linked project root. If the folder is not linked, ask\nfor an editable Builder share link and run\n`webstudio init --link --json`, followed by `webstudio sync`, then\nretry `webstudio connect `. Treat the share link as a credential and do\nnot include it in committed files, logs, screenshots, or issue reports.\n\n`connect` verifies project access before changing client configuration. For\nClaude Code, Cursor, and VS Code it safely merges the `webstudio` server into\nthe client\'s project configuration. For Codex it runs both `codex mcp add` and\n`codex mcp get webstudio`; do not repeat those commands separately. Follow the\nreload, restart, or approval instruction printed by `connect`, then verify the\nloaded MCP connection by asking the client to use Webstudio MCP and list the\nproject pages. Use `--print` only to inspect the generated setup without\nchanging configuration or requiring project access.\n\n## Always\n\n1. webstudio permissions --json\n2. For bounded shell workflows, call MCP tools directly through the CLI shortcut, for example `webstudio meta.index` or `webstudio insert-fragment \'\' --dry-run`. The explicit form `webstudio mcp single-op-call \'\'` is equivalent and useful when you need to make the MCP boundary obvious. Use `webstudio mcp run \'[{"tool":"components.find","input":{"brief":"button"}}]\'` for multiple calls in one shared CLI session. Use a normal JSON file path for large batches. Use long-running `webstudio mcp` only when your environment is a real MCP client. Do not manually send raw JSON-RPC to `webstudio mcp` from a shell or PTY.\n3. Read MCP `meta.index`, for example `webstudio meta.index`.\n4. Use focused MCP calls with concrete JSON: `webstudio meta.guide \'{"brief":"Create a design system page using every component"}\'`, `webstudio meta.get_more_tools \'{"tools":["insert-fragment"]}\'`, `webstudio components.list \'{"source":"all"}\'`, `webstudio components.coverage-plan`, `webstudio components.search \'{"brief":"radix select"}\'`, `webstudio components.get \'{"component":"@webstudio-is/sdk-components-react-radix:Select"}\'`, `webstudio templates.list`, and `webstudio templates.get \'{"component":"@webstudio-is/sdk-components-react-radix:Select"}\'`.\n5. Read overview resources `webstudio://project/tools-overview` or `webstudio://project/components-overview` when useful. Read full resources `webstudio://project/tools` or `webstudio://project/components` only when focused tools are insufficient.\n6. Pick focused MCP read tool.\n7. Pick semantic MCP write tool.\n\nUse `webstudio schema mcp` for a compact MCP tool overview. Add `--verbose` only when exact input schemas for all tools are truly needed; otherwise prefer focused `meta.get_more_tools` and `components.*` calls.\n\nRun these commands from the linked project root. Use the MCP startup status line\'s absolute root for local files; write temporary scripts and artifacts under `/.temp`, not under a parent workspace.\n\nMonorepo quick path for a simple styled section:\n\n```sh\nnode packages/cli/local.js mcp single-op-call meta.index\nnode packages/cli/local.js mcp single-op-call meta.get_more_tools \'{"tools":["insert-fragment"]}\'\nnode packages/cli/local.js mcp single-op-call insert-fragment \'{"parentInstanceId":"parent-id","fragment":"Launch KitA focused section created with Webstudio JSX.Get started"}\' --dry-run\n```\n\nThe same local shortcut form is shorter and preferred for simple shell steps:\n\n```sh\nnode packages/cli/local.js meta.index\nnode packages/cli/local.js meta.get_more_tools \'{"tools":["insert-fragment"]}\'\nnode packages/cli/local.js insert-fragment \'{"parentInstanceId":"parent-id","fragment":"Launch KitA focused section created with Webstudio JSX.Get started"}\' --dry-run\n```\n\nFor this simple path, do not grep source files, dump full MCP resources, or write parser scripts first. Use `list-pages`, `get-page-by-path`, or `list-instances` only to get the target `parentInstanceId`.\n\nWhen authoring JSX for `insert-fragment`, use Webstudio component helpers and Webstudio style syntax. Use `ws:style={css\\`...\\`}`for Webstudio-native CSS, or use React-style object syntax such as`style={{ padding: 24 }}` when that is simpler. Both forms create editable Webstudio style data.\n\nWhen the task says another user will edit a page in Content mode, use a Content Block (`ws:block`) around every editable region. Content-mode users can edit text and supported props only in descendants of that block; content outside it is read-only. Put reusable insertable options in the block\'s `ws:block-template` child. Do not put intended editor content inside that template container: templates are protected source material, while an inserted template copy becomes an editable direct child of the Content Block. Verify this structure before handoff.\n\nDo not access host globals or dynamic code APIs in JSX fragments, including `process`, `globalThis`, `eval`, `Function`, or `constructor`. JSX fragments are declarative project data; use the built-in Webstudio helpers instead.\n\nUse Webstudio prop names in JSX: `class`, `for`, `aria-label`, and other HTML/Webstudio names. Do not use React-only aliases such as `className` or `htmlFor`; the runtime rejects them with the Webstudio prop name to use.\n\nUse Webstudio actions for event/action props. Do not pass JavaScript functions such as `onClick={() => ...}`; the runtime rejects them because functions cannot be persisted as Webstudio project data.\n\n```tsx\n\n Open\n\n```\n\nPlain JSX prop values must be JSON-compatible: `null`, strings, booleans, finite numbers, arrays, and plain objects. Do not pass `undefined`, `Symbol`, `BigInt`, `NaN`, `Infinity`, `Date`, `Map`, `Set`, class instances, or circular objects; omit the prop, use plain data, or use `expression`/`ActionValue` when the value is dynamic.\n\nIf a component has a registered template with required parts, JSX must include those parts explicitly under the same parent structure as the template, for example ``. Use `insert-component` when you want Webstudio to apply one component template automatically.\n\n## Animation Components\n\nBefore creating animation examples, inspect the exact components with focused discovery:\n\n```sh\nwebstudio components.get \'{"component":"@webstudio-is/sdk-components-animation:AnimateChildren"}\'\nwebstudio components.get \'{"component":"@webstudio-is/sdk-components-animation:AnimateText"}\'\nwebstudio components.get \'{"component":"@webstudio-is/sdk-components-animation:StaggerAnimation"}\'\nwebstudio components.get \'{"component":"@webstudio-is/sdk-components-animation:VideoAnimation"}\'\n```\n\nUse Animation Group (`AnimateChildren`) as the controller. Put normal instances directly inside it, or put Text Animation, Stagger Animation, or Video Animation directly inside it. Text, Stagger, and Video are helper components with `contentModel.category: "none"` and should not be used as standalone section roots.\n\nDefine timing and CSS changes on the Animation Group `action` prop. Use `type:"view"` for viewport entry/exit progress and `type:"scroll"` for scroll-progress timelines. For in animations, keep the canvas styles as the final state and use `fill:"backwards"` with keyframes that describe the starting state. For out animations, use `fill:"forwards"` with keyframes that describe the ending state.\n\nText Animation settings: `slidingWindow` defaults to `5`, `easing` defaults to `linear`, and `splitBy` defaults to `char`. Use `splitBy:"space"` for word-by-word animation. The parent Animation Group keyframes provide the actual opacity, translate, scale, or other styles.\n\nStagger Animation settings: `slidingWindow` defaults to `1` and `easing` defaults to `linear`. It applies parent Animation Group progress across its direct children. Use `slidingWindow:0` for instant sequential steps, `1` for one child at a time, and values above `1` for overlapping waves.\n\nVideo Animation settings: `timeline` is a boolean. Prefer `insert-component` for Video Animation so the Video child template is inserted, then configure the Video child asset/source. Use short, seek-friendly videos for smooth scroll-linked playback.\n\nUse JSX fragments for authored animation structures when you need styled, editable examples. Put the final visual state in `ws:style` and put the starting or ending animated state in the Animation Group `action` keyframes. Include an explicit `offset` on every keyframe: use `offset: 0` for starting-state keyframes with `fill:"backwards"` and `offset: 1` for ending-state keyframes with `fill:"forwards"`.\n\n```tsx\n\n \n Launch metrics\n \n A polished card that fades up as it enters the viewport.\n \n \n\n```\n\nFor Text Animation, keep `animation.AnimateText` as the direct child of Animation Group and place the text-containing element inside it:\n\n```tsx\n\n \n Animate words with controlled rhythm\n \n\n```\n\nFor Stagger Animation, put the repeated cards or rows directly inside `animation.StaggerAnimation`:\n\n```tsx\n\n \n \n Plan\n \n \n Build\n \n \n Launch\n \n \n\n```\n\nFor Video Animation, use the registered template via `insert-component` when possible. If you author JSX, include the Video child explicitly:\n\n```tsx\n\n \n <$.Video\n preload="auto"\n autoPlay={true}\n muted={true}\n playsInline={true}\n crossOrigin="anonymous"\n />\n \n\n```\n\n## Command Surface Boundary\n\n- Use top-level `webstudio ...` shell commands for setup, sync/import/build/preview/screenshot, permissions, publish/domains, schema, registry inspection, man, and starting MCP.\n- Use MCP tools for Builder project data manipulation: pages, instances/components, props, text, styles, tokens, variables, resources, assets, breakpoints, redirects, and raw patches.\n- From a shell, call MCP tools with the shortcut form `webstudio \'\'`, for example `webstudio insert-fragment \'\' --dry-run`. The explicit equivalent is `webstudio mcp single-op-call \'\'`. Use `--input-file` for large payloads.\n- Inside the Webstudio monorepo, call the local CLI as its own command: `node packages/cli/local.js ...`. Do not wrap the CLI call in `pwd && ...`, command substitution, `pnpm exec webstudio`, `pnpm --filter webstudio exec webstudio`, or a global `webstudio`.\n- For experiments, pass `--dry-run` to local-capable mutation calls. Read the computed transaction from `meta.session.transaction` and its base build version from `meta.session.version`. Copying a `.webstudio` folder is not an isolated project clone; `.webstudio/config.json` still points to the same remote project, so non-dry-run mutations can commit to that project.\n- For bounded multi-step shell work, run inline JSON with `webstudio mcp run \'[{"tool":"components.find","input":{"brief":"button"}}]\'`; this reuses one CLI session without raw JSON-RPC. For large batches, write `{ "calls": [{ "tool": "..." }] }` to a normal JSON file and run `webstudio mcp run .temp/mcp-calls.json`.\n- Use JSON strings for `brief` fields. Never pass boolean flags such as `{"brief":true}`.\n- Treat `webstudio mcp single-op-call` and `webstudio mcp run` stderr lines as progress checkpoints; stdout remains JSON on both success and failure. On failure, parse stdout for `{ "ok": false, "error": { "code": "...", "message": "..." } }` before deciding what to fix.\n- If a CLI/MCP tool crashes, hangs, gives a confusing error, needs an undocumented workaround, or forces source-code inspection for normal usage, ask the user to report it in Discord `#help` at https://wstd.us/community. Give them a complete copy-paste report with the goal, expected behavior, actual error, exact command/tool call, stdout JSON, stderr/lifecycle logs, environment, workaround, and secrets redacted.\n- Run one-shot `webstudio mcp single-op-call` commands sequentially against a linked `.webstudio` folder. If a command returns `PROJECT_SESSION_BUSY`, another CLI/MCP process is updating the local session; wait a moment and retry sequentially.\n- In delegated or non-streaming agent environments, do not batch many MCP calls silently and do not wrap many shortcut or `webstudio mcp single-op-call` commands in a shell loop. Treat each parent-visible checkpoint as the unit of work. If the parent asks for status within 30 seconds, run exactly one shortcut command such as `webstudio meta.index` or one explicit `webstudio mcp single-op-call` command, report that command/result, then wait for the parent to continue. Do not take a broad task such as creating a full design-system page as one execution unit. Call `workflow.next {"goal":"design-system-page"}`, report the returned phase/checkpoint, wait until the parent continues, call `checkpoint.ack {"reported":true,"continueAfterReport":true,"summary":""}`, complete exactly that bounded phase, and return: discovery, page creation, one dry-run JSX section, one committed JSX section, one `components.coverage-insert-next` call, or one presentation pass. Phase commands do not include nextPhase in their own output. After the parent continues, acknowledge the previous checkpoint first, then call `workflow.next` with the next phase. For all-component design-system pages, checkpoint after workflow planning, discovery, page creation, call `components.coverage-insert-next` once before checkpointing again, then finish with `workflow.next {"goal":"design-system-page","phase":"presentation-pass"}`. Coverage 72/72 is necessary but not sufficient: the page must be organized into styled, real-world examples, not raw unstyled component dumps.\n- For design-system or “use every component” tasks, start with compact `webstudio components.coverage-plan`, checkpoint, then request component coverage details with `webstudio components.coverage-plan \'{"detail":"roots","offset":20}\'` or `{"detail":"parts"}` only when needed. Do not pass `detail` to `list-pages`; use `list-pages {}` or `get-page-by-path` for page lookup.\n- MCP tool shortcuts are only for MCP tools. If a shortcut is ambiguous with a real top-level command, the real top-level command wins; use `webstudio mcp single-op-call \'\'` to force the MCP path.\n\n## LLM Implementation Process\n\nUse this process for user requests that change Webstudio content, layout, styles, assets, pages, redirects, resources, or publishing state:\n\n1. Discover capabilities with `webstudio man --json`, `webstudio schema api`, `webstudio schema mcp`, MCP `meta.index`, `meta.guide`, `meta.get_more_tools`, `components.list`, `components.summary`, `components.coverage-plan`, `components.search`, `components.get`, `templates.list`, and `templates.get`. From a shell, prefer shortcut calls such as `webstudio meta.index` and `webstudio components.search \'{"brief":"button"}\'` for these focused tool calls; use `webstudio mcp single-op-call` when you need the explicit MCP form. Read full resources such as `webstudio://project/tools` and `webstudio://project/components` only when needed. Do not write scripts to parse full MCP discovery JSON for normal lookup.\n2. Inspect current project state with semantic reads such as `get-project-settings`, `list-pages`, `get-page-by-path`, `list-instances`, `inspect-instance`, `get-styles`, `list-assets`, `list-breakpoints`, and `snapshot` only when needed. Before changing a project, read `get-project-settings` and follow any non-empty `meta.agentInstructions`. These are shared project instructions, not a place for secrets.\n3. Mutate the Webstudio project with semantic MCP write tools first. Prefer MCP `insert-fragment` for authored/styled sections, use `insert-component` only for one automatic component template, then `update-text`, `update-props`, `update-styles`, `upload-asset`, `create-page`, and page/project settings tools over raw patches.\n4. Use `apply-patch` only when no semantic tool covers the required change, and only after reading the latest snapshot/version.\n5. For visual/design work, regenerate or preview the generated app, capture a screenshot, inspect it with vision, and iterate before final response.\n6. Report what changed and what verification ran.\n\n## Visual Design Workflow\n\nFor requests involving visible HTML/CSS, layout, typography, colors, imagery, responsive behavior, or screenshots:\n\n1. Read editable Webstudio structure first: pages, instances, props, styles, breakpoints, assets, and relevant text.\n2. Do not use generated route/component files as the source of truth for editable content.\n3. Make edits through Webstudio semantic commands/MCP tools so the result stays editable in Builder and survives the next `webstudio build`.\n4. Keep generated project files current, start preview, and capture the changed page with `screenshot`.\n5. Use `screenshot.diff` when a baseline exists and inspect screenshot/diff artifacts with vision before finishing.\n6. If vision or screenshot tooling is unavailable, state that explicitly and explain what fallback verification was used.\n\n## Responsive Verification Workflow\n\nFor responsive page work, use Builder breakpoints as the source of truth:\n\n1. Read breakpoints with `list-breakpoints` before deciding responsive behavior.\n2. Apply responsive styles with existing Builder breakpoint ids; do not invent CSS media queries or breakpoint names when Webstudio breakpoint data exists.\n3. Pick screenshot viewport widths from the project breakpoints: include a desktop width, each defined max-width or min-width edge, and a narrow mobile width.\n4. Capture each viewport with `screenshot`, for example `{"path":"/","output":"home-375.png","viewport":{"width":375,"height":812}}` and `{"path":"/","output":"home-1440.png","viewport":{"width":1440,"height":900}}`.\n5. Inspect every viewport screenshot with vision before finishing, checking layout, overflow, hidden content, text wrapping, and breakpoint-specific style changes.\n6. If any viewport fails, update styles through semantic Webstudio tools and repeat screenshots for the affected breakpoints.\n\n## Generated Files Guardrails\n\n- Do not edit `app/__generated__`, generated route files, generated page files, generated CSS, or build output for normal Webstudio content/design requests.\n- Do not replace generated page components with handcrafted app code unless the user explicitly asks for code-only export customization.\n- Generated files are build artifacts and may be overwritten by `webstudio build`.\n- If a task truly requires generated app customization, keep it outside `app/__generated__` where possible and explain that it is not editable Webstudio content.\n\n## Values vs Bindings\n\nBefore authoring unfamiliar expressions, read `webstudio://project/expressions` with MCP `resources/read` or `webstudio mcp read-resource webstudio://project/expressions`. It documents the supported expression subset, method allowlist, scope, Collection context, and validation limits.\n\n- Use direct value tools for fixed content. For one visible text child, use `update-text` with plain `text`. For a bounded multi-instance literal replacement, use `replace-text` with `find`, `replace`, `pagePath` or `pageId`, and `limit`; it does not change expression children. Use `replace-prop-text` for bounded changes inside static string props, optionally limited to prop names or instance ids; it never changes dynamic bindings. For static props such as `aria-label`, `alt`, `id`, `class`, `href`, or button labels stored as props, use `update-props` with the prop\'s direct type/value.\n- Use `bind-props` only when the prop must stay dynamic: an expression, resource result, action, or existing scoped runtime context such as `system`. Do not use `bind-props` just to set a fixed string.\n- Direct prop string example: `{"updates":[{"instanceId":"button-id","name":"aria-label","type":"string","value":"Open menu"}]}`.\n- Expression binding example: `{"bindings":[{"instanceId":"link-id","name":"href","binding":{"type":"expression","value":"currentPost.url"}}]}`.\n- Page metadata fields such as `title`, `description`, `language`, `redirect`, and custom meta content accept plain fixed text. For computed values, pass JavaScript expression code such as `pageTitle ?? "Pricing | Acme"`.\n- Page `status` accepts a fixed HTTP status code as a number from 200 through 599, for example `302`. For a dynamic status, pass JavaScript expression code such as `system.status`.\n- Page metadata update example: use `update-page` with `{"pageId":"page-id","values":{"title":"Pricing | Acme","meta":{"description":"Plans for teams"}}}`.\n- Draft a page with `update-page` and `{"pageId":"page-id","values":{"isDraft":true}}`. It remains editable and previewable but is omitted from every publish target, including staging, and from sitemap output.\n- Stage a draft page for a future publish with `{"pageId":"page-id","values":{"isDraft":false}}`. This clears draft state but does not deploy the site. The home page and `/*` catch-all page cannot be drafts.\n- Resource `url` accepts plain fixed URLs and paths. For computed URLs, pass JavaScript expression code such as `"https://api.example.com/items?tag=" + filters.tag`. Resource header values, search parameter values, and text bodies accept expressions for dynamic values; for fixed text, use `{ "type": "literal", "value": "application/json" }`.\n- Resource update example: use `update-resource` with `{"resourceId":"resource-id","values":{"url":"https://api.example.com/items"}}`.\n- Assets is one system resource. `create-assets-resource` without `query` preserves the existing fetch-all behavior and response shape. Add an explicit query object only when filtering, projection, limits, parameters, or selected-file hydration are required.\n- For a Markdown-backed blog, read `get-asset-field-catalog`, validate GROQ with `validate-asset-query`, then call `create-assets-resource` or `update-assets-resource`. Set `values.query:null` to return an existing resource to legacy fetch-all mode.\n- Use one metadata-only Assets resource with query configuration for listings with `content:{"mode":"none"}`. Use a separate Assets resource with a `$slug` query for the detail route, a runtime binding such as `{"name":"slug","value":"system.params.slug"}`, `resultLimit:1`, and `content:{"mode":"markdown-body"}`.\n- Use `preview-asset-query` with concrete JSON parameter values before binding the saved resource. Inspect saved mode and configuration with `list-assets-resources` or `get-assets-resource`, and read `get-asset-resource-index-status` after query mutations. Use `rebuild-asset-resource-index` only for explicit recovery.\n- Data variable values support `string`, `number`, `boolean`, `string[]`, and `json`. Use `string[]` only for arrays where every item is a string; use `json` for objects, mixed arrays, filters, and nested data.\n- Parameters are internal scoped runtime values from pages, collections, or components. They are not a public authoring surface: do not create, update, or delete parameter records. Public tools should preserve existing parameter records and may reference documented context values such as `system` in expressions where they are already in scope.\n- Use scoped resources for read data. A GET resource created with `scopeInstanceId`/`dataSourceName` defaults to `exposeAsDataSource:true`, becomes a scoped resource data variable, is generated into the page resource `data` map, and may be loaded while rendering the page. Read the loaded resource result from its wrapper, usually `.data`.\n- Use prop-bound resources for actions. A resource created without `scopeInstanceId` and bound to a component prop such as Form `action` with `bind-props` and `binding.type: "resource"` becomes an action resource in the page resource `action` map. Use this for POST, PUT, DELETE, webhooks, GraphQL submissions, and anything that should run only from an explicit form/action flow.\n- POST, PUT, and DELETE resources default to `exposeAsDataSource:false`, even with a scope. Set `exposeAsDataSource:true` only for an intentional render-time read such as a GraphQL POST query; provide `scopeInstanceId` and inspect the returned warning. Set it to `false` during `update-resource` to detach existing render-time exposure.\n- For dynamic resource query parameters prefer `searchParams`, for example `{"name":"tag","value":"filters.tag"}`. Use `{"type":"literal","value":"website"}` for fixed request text. Header values can be expressions such as `"\\"Bearer \\" + auth.token"`. Body can be an object expression, including GraphQL payloads such as `{ query: "...", variables: { slug: system.params.slug } }`.\n- Resource methods are `get`, `post`, `put`, and `delete`. Optional resource controls are `graphql` and `system`. Use `control:"graphql"` for GraphQL POST resources with query bodies. Use `control:"system"` for built-in local resource URLs such as `"/$resources/current-date"` and for resources reading the built-in `system` parameter. The built-in system fields are `system.origin`, `system.pathname`, `system.params`, and `system.search`; do not use `system.path`.\n- Whenever an array or object from a resource or data variable should render repeated UI, call `insert-collection` with the complete iterable and one repeated-item JSX root. The command creates the Collection, private item parameters, iterable binding, and descendant item bindings atomically. Use `collectionItem` and `collectionItemKey` expressions in the item JSX. Wrap multiple repeated siblings in one Element, and give repeated Radix items stable unique `value` bindings.\n- Expressions are single JavaScript expressions, not statements or functions. Functions, arrow functions, classes, `new`, `this`, `await`, imports, arbitrary calls, increment/decrement, and assignment outside actions are unsupported. Prefer optional chaining, nullish coalescing, ternaries, property/index access, operators, and the documented string/array methods.\n\n## Pick Read Command\n\n{{readFirst}}\n\n## Pick Write Command\n\n{{taskRecipeIndex}}\n\n## Raw Patch Only If Needed\n\n1. Use MCP tool: snapshot.\n2. Write BuildPatchTransaction[].\n3. Use MCP tool: apply-patch.\n\n## MCP Argument Examples\n\nMCP tools receive JSON argument objects, not CLI flags. Use these shapes:\n\n{{mcpArgumentExampleIndex}}\n\n## Rules\n\n- Never guess ids for existing records. Read them first.\n- Never use project ids from user input. Commands use the configured project.\n- Use --refresh before a local-capable command when cached data may be stale.\n- Pass --json only to commands whose help/schema documents it. Do not add --json to top-level commands such as sync unless supported.\n- On VERSION_CONFLICT, read MCP snapshot again, regenerate the patch, then retry.\n- Treat stdout JSON as the API contract and stderr as diagnostics.\n- For visual/design work, verify the rendered result with vision before finishing.\n- Do not edit generated files for normal Webstudio content/design requests.\n- Use direct values for static strings and bindings only for dynamic expressions/resources/actions.\n- Use plain fixed text where documented. Only encode a quoted JavaScript string literal when a field is explicitly documented as an expression-only value.\n- Confirm destructive commands with --confirm only when user requested deletion/unpublish/replacement.\n- Use webstudio schema api for machine-readable top-level command metadata and webstudio schema mcp for MCP tool schemas.\n\n## Known Gaps\n\n{{knownCliGapIndex}}\n', "manual-mcp": '# Webstudio MCP Manual\n\n`webstudio mcp` starts a stdio MCP server for real MCP clients. Shell users can call MCP tools with the shortcut form `webstudio \'\'`, for example `webstudio meta.index` or `webstudio insert-fragment \'\' --dry-run`. `webstudio mcp single-op-call` is the explicit equivalent and prints the structured JSON result. `webstudio mcp run` runs multiple MCP tool calls from inline JSON or a normal JSON file in one shared CLI session. Do not manually type or pipe raw JSON-RPC frames into `webstudio mcp` from an interactive shell or PTY.\n\n## Startup\n\nIf you are already working with an agent, ask it to connect the current folder\nto Webstudio. Give the editable Builder share link only when the trusted agent\nasks for it. The agent should perform the steps below, follow any\nclient-specific output from `webstudio connect`, and verify the connection by\nlisting the project pages. Treat the share link as a credential: do not include\nit in committed files, screenshots, logs, or issue reports.\n\n1. Configure a project with `webstudio init --link --json`.\n2. Synchronize it with `webstudio sync`.\n3. Generate the client configuration with `webstudio connect claude`,\n `webstudio connect codex`, `webstudio connect cursor`, or\n `webstudio connect vscode`. Use `--print` to preview the generated setup.\n4. Check capabilities with `webstudio permissions --json`.\n5. For shell-driven agents, use shortcut calls such as `webstudio meta.index` and `webstudio insert-fragment \'\' --dry-run` for individual MCP tool calls. Use the explicit equivalent `webstudio mcp single-op-call \'\'` when you need to force the MCP path, or `webstudio mcp run \'[{"tool":"components.find","input":{"brief":"button"}}]\'` for bounded multi-call workflows. Use `webstudio mcp run .temp/mcp-calls.json` for large batches.\n6. For MCP clients, start the server with `webstudio mcp`.\n7. Start discovery with `meta.index`, then call focused tools with concrete JSON, for example `webstudio mcp single-op-call meta.guide \'{"brief":"Create a design system page using every component"}\'`.\n\nAfter linking and synchronization, run `webstudio connect `, reload or\nrestart the client, then ask the agent to use Webstudio MCP to list the project\npages. The supported client names are `claude`, `codex`, `cursor`, and `vscode`.\nFor Codex, `connect` registers and verifies the server through the Codex CLI;\nthe resulting server appears in Codex settings. Use `--print` to inspect the\nexact registration command without changing configuration or requiring project\naccess. Before changing client configuration, `connect` verifies that the saved\nproject endpoint is reachable and its credential is accepted. If verification\nfails, follow the specific connection, compatibility, or relinking guidance in\nthe error.\n\nStart MCP from the linked Webstudio project root. The lifecycle status line prints that absolute root; create local scripts, screenshots, and temporary artifacts under that root, for example `/.temp/script.mjs`. If the shell starts in a parent workspace, `cd` into the project root first or use absolute paths.\n\nWhen developing inside the Webstudio monorepo, start the local CLI exactly as `node packages/cli/local.js mcp` from the repo root. Do not use `pnpm exec webstudio`, `pnpm --filter webstudio exec webstudio`, or a global `webstudio`: they can resolve an older binary.\n\nWhile the server is running, stdout is reserved for MCP JSON-RPC messages. Do not print human text from the server process. The server advertises MCP `logging` capability and emits sparse `notifications/message` logs for ready state and tool lifecycle checkpoints such as `tool preview.start started`, `tool preview.start still running after 10000ms`, and `tool preview.start succeeded in 1234ms`; stderr also mirrors these sparse lifecycle fallback lines prefixed with `[webstudio mcp]`.\n\n## One-Shot Tool Calls\n\nUse the shortcut `webstudio \'\'` when you are operating from a shell and need one MCP tool result. The explicit form `webstudio mcp single-op-call \'\'` is equivalent and avoids writing temporary Node.js stdio client scripts.\n\nExamples:\n\n```sh\nwebstudio mcp single-op-call meta.index\nwebstudio mcp single-op-call meta.guide \'{"brief":"Create a design system page using every component"}\'\nwebstudio mcp single-op-call meta.get_more_tools \'{"tools":["insert-fragment"]}\'\nwebstudio mcp single-op-call components.list \'{"source":"all"}\'\nwebstudio mcp single-op-call components.coverage-plan\nwebstudio mcp single-op-call components.search \'{"brief":"radix select"}\'\nwebstudio mcp single-op-call components.get \'{"component":"@webstudio-is/sdk-components-react-radix:Select"}\'\nwebstudio mcp single-op-call templates.list\nwebstudio mcp single-op-call templates.get \'{"component":"@webstudio-is/sdk-components-react-radix:Select"}\'\nwebstudio mcp single-op-call insert-fragment --input-file .temp/insert-fragment.json\n```\n\nShortcut equivalents:\n\n```sh\nwebstudio meta.index\nwebstudio meta.guide \'{"brief":"Create a design system page using every component"}\'\nwebstudio meta.get_more_tools \'{"tools":["insert-fragment"]}\'\nwebstudio components.list \'{"source":"all"}\'\nwebstudio components.coverage-plan\nwebstudio components.search \'{"brief":"radix select"}\'\nwebstudio components.get \'{"component":"@webstudio-is/sdk-components-react-radix:Select"}\'\nwebstudio templates.list\nwebstudio templates.get \'{"component":"@webstudio-is/sdk-components-react-radix:Select"}\'\nwebstudio insert-fragment --input-file .temp/insert-fragment.json\n```\n\nRules:\n\n- Inside the Webstudio monorepo, replace `webstudio` in the examples above with `node packages/cli/local.js`, for example `node packages/cli/local.js meta.index`.\n- For a simple authored/styled section, run `meta.index`, then `meta.get_more_tools \'{"tools":["insert-fragment"]}\'`, then `insert-fragment`. Do not grep source files, dump full MCP resources, or write parser scripts first.\n- In `insert-fragment` JSX, use `ws:style={css\\`...\\`}`for Webstudio-native CSS, or use React-style object syntax such as`style={{ padding: 24 }}` when that is simpler. Both forms create editable Webstudio style data.\n- Prefer JSX for authored/styled content. Common `insert-fragment` inputs:\n\n```jsonl\n{"parentInstanceId":"root-id","fragment":"Northstar Product OSReusable patterns for teams."}\n{"parentInstanceId":"root-id","fragment":"Operations ConsoleReact-style object styles become editable Webstudio styles."}\n{"parentInstanceId":"root-id","fragment":"Track launch"}\n{"parentInstanceId":"root-id","fragment":""}\n```\n\n- Do not access host globals or dynamic code APIs in JSX fragments, including `process`, `globalThis`, `eval`, `Function`, or `constructor`.\n- Use Webstudio prop names such as `class` and `for`; do not use React aliases `className` or `htmlFor`.\n- Use Webstudio actions for event/action props, for example `onClick={new ActionValue(["event"], expression\\`console.log(event)\\`)}`. Do not pass JavaScript functions such as `onClick={() => ...}`.\n- Plain prop values must be JSON-compatible: `null`, strings, booleans, finite numbers, arrays, and plain objects. Do not pass `undefined`, `Symbol`, `BigInt`, `NaN`, `Infinity`, `Date`, `Map`, `Set`, class instances, or circular objects; omit the prop, use plain data, or use `expression`/`ActionValue` when the value is dynamic.\n- Template-backed components used in JSX must include required child/part components explicitly under the same parent structure as the template, for example ``. Use `insert-component` when you want one automatic registered component template.\n- The positional input is JSON and defaults to `{}`.\n- Use `--input-file` for large mutation payloads.\n- Use `--dry-run` with local-capable mutation tools when you need a patch plan without committing. The computed transaction is returned in `meta.session.transaction`, and `meta.session.version` is its base build version. Copying a `.webstudio` folder is not an isolated project clone; `.webstudio/config.json` still points to the same remote project, so non-dry-run mutations can commit to that project.\n- The command prints JSON to stdout for both success and failure. Success uses the same `structuredContent` shape MCP tools return: `{ "ok": true, "data": ..., "meta": ... }`. Failure prints `{ "ok": false, "error": { "code": "...", "message": "..." }, "meta": ... }` and exits nonzero.\n- The command writes sparse progress to stderr, including start, success/failure, elapsed time, and committed status when the tool returns session metadata.\n- Invalid argument types fail loudly with path-specific messages, for example `meta.guide input.brief must be a string when provided`.\n- Run one-shot shortcut or `mcp single-op-call` commands sequentially against the same linked `.webstudio` folder. If you receive `PROJECT_SESSION_BUSY`, another CLI/MCP process is updating the local session; wait a moment and retry sequentially.\n- To work with another previously linked project without changing the directory\'s default link, start MCP or a shell call with `--project `, for example `webstudio mcp --project ` or `webstudio mcp single-op-call list-pages --project `. Selected projects use isolated local session and checkpoint files.\n- If you are a delegated agent and your parent cannot see live stderr/stdout, do not run a long sequence of shortcut or `mcp single-op-call` commands silently and do not wrap many calls in a shell loop. Treat each parent-visible checkpoint as the unit of work. If the parent asks for status within 30 seconds, run exactly one `webstudio ` or `webstudio mcp single-op-call` command, report that command/result, then wait before the next MCP command. For all-component design-system pages, checkpoint after discovery, checkpoint after page creation, call `components.coverage-insert-next` once before checkpointing again, then finish with the `presentation-pass` workflow phase. Coverage alone is not completion; organize examples into styled sections/cards.\n\n## Reporting CLI/MCP Issues\n\nIf a CLI/MCP tool gives a confusing error, crashes, hangs, produces invalid output, requires an undocumented workaround, or makes you inspect source code to understand normal usage, ask the user to report it in the Webstudio Discord `#help` channel: https://wstd.us/community.\n\nGive the user a complete copy-paste report. Include only non-secret values: never include auth tokens, private URLs, cookies, API keys, passwords, or proprietary project data. Redact them as ``.\n\nCopy-paste template:\n\n````md\nWebstudio CLI/MCP issue report\n\nWhat I was trying to do:\n\n\nWhat I expected:\n\n\nWhat happened instead:\n\n\nCommand/tool used:\n\n```sh\n\n```\n\nStructured output / error:\n\n```json\n\n```\n\nStderr / lifecycle logs:\n\n```txt\n\n```\n\nEnvironment:\n\n- CLI command path: \n- Webstudio CLI version: \n- OS: \n- Node version: \n- Project/session state: \n\nWorkaround tried:\n\n\nWhy this should be improved:\n\n````\n\n## Shared-Session Shell Runs\n\nUse `webstudio mcp run \'[{"tool":"components.find","input":{"brief":"button"}}]\'` when you are operating from a shell and need several MCP tool calls to share one CLI session without hand-writing JSON-RPC. For large batches, pass a normal JSON file path such as `.temp/mcp-calls.json`. Do not use shell process substitution like `<(...)`; use inline JSON or a real file.\n\nUse `mcp run` for long-lived tools such as `preview.start`. A one-shot `mcp single-op-call preview.start` cannot keep ownership of a preview server for a later screenshot or stop call. Put `preview.start`, `screenshot`, and `preview.stop` in one shared `mcp run` process, or use a real long-running MCP client.\n\nInput shape:\n\n```json\n{\n "calls": [\n { "tool": "meta.index" },\n { "tool": "components.find", "input": { "brief": "radix select" } }\n ]\n}\n```\n\nRules:\n\n- The command prints JSON to stdout for both success and failure. It stops at the first failed call and prints partial results in `{ "ok": false, "error": ..., "data": { "completedCalls": ..., "results": [...] }, "meta": ... }`, then exits nonzero.\n- If a call returns `checkpoint.required`, read-only discovery and inspection remain available, but mutations and state-changing session tools return `CHECKPOINT_REQUIRED`. Stop and report the checkpoint to the parent/user. Only after the parent/user continues, call `checkpoint.ack {"reported":true,"continueAfterReport":true,"summary":""}` before continuing mutations.\n- For `mcp single-op-call`, checkpoint requirements persist across later one-shot CLI processes until you call `checkpoint.ack {"reported":true,"continueAfterReport":true,"summary":""}`.\n- Use this instead of manually sending JSON-RPC frames to `webstudio mcp` from a shell.\n\n### Cross-project batches\n\nAdd `projects` to the same `mcp run` manifest to run focused reads, audits, or dry runs across independently linked project roots:\n\n```json\n{\n "concurrency": 2,\n "calls": [\n { "tool": "status" },\n { "tool": "audit", "input": {} },\n {\n "tool": "update-project-settings",\n "input": { "meta": { "siteName": "Reviewed" } },\n "dryRun": true\n }\n ],\n "projects": [\n { "id": "site-a", "root": "../site-a" },\n { "id": "site-b", "root": "../site-b" }\n ]\n}\n```\n\nProject roots and an optional `progressFile` are resolved relative to the manifest file. Each project may provide its own `calls` instead of using the top-level calls. Each root must already be linked with its own `.webstudio/config.json`; the runner creates an independently authenticated ProjectSession and uses root-scoped session, audit, preview-data, and checkpoint paths without changing the process working directory.\n\nConcurrency defaults to 2, is capped at 16, and can be set in the manifest or overridden with `--concurrency`. A failure is reported for that project while other projects continue. Progress is saved after every successful call; rerunning with the default `--resume` skips completed projects and starts failed projects after their last confirmed successful call. Reads and dry runs may be retried. A committed mutation interrupted after dispatch is marked `AMBIGUOUS_MUTATION_RESULT` and is never replayed automatically; inspect that project before deciding how to continue. Use `--no-resume` only to intentionally start the complete manifest over.\n\nCommitted mutation tools are rejected in a projects batch unless the command includes `--approve-mutations`. Review the complete manifest before granting approval. `--dry-run` applies to every call and does not require mutation approval. The final stdout object is compact: project counts, one status/error record per project, elapsed time, and the progress-file path rather than every tool result.\n\n## Discovery\n\nUse MCP itself after startup, or call the same tools with `webstudio mcp single-op-call`:\n\n- `tools/list`: machine-readable available tools\n- `resources/list`: available overview and full JSON resources\n- `meta.index`: concise capability catalog\n- `meta.guide`: workflow for a user goal; call with a string brief such as `{"brief":"Create a pricing page"}`\n- `meta.get_more_tools`: detailed params, examples, namespaces, and local/server behavior; prefer exact names such as `{"tools":["insert-fragment"]}` when you know them\n- `components.list`: compact registry metadata for visible components and templates; use a focused get tool for complete details\n- `components.summary`: component counts by default; use `{"detail":"components","limit":20}` for paginated entries\n- `components.coverage-plan`: compact paged plan for design-system coverage tasks that need every component; default returns counts plus the first root page, use `{"detail":"roots"}`, `{"detail":"parts"}`, or `{"detail":"full"}` for more\n- `components.coverage-status`: page-specific covered/missing component report with `missingRoots` and `missingParts`\n- `components.search`: focused component/template search by id, namespace, label, category, or content model\n- `components.find`: compatibility alias for focused component search\n- `components.get`: full metadata for one component id\n- `templates.list`: compact metadata for template-backed insertions only\n- `templates.get`: full registry item and payload metadata for one template\n\nComponent and template registry items use a shadcn-compatible top-level shape plus Webstudio-specific superset metadata in `meta`. Use `meta.runtime` for component ids, props, states, content model, and source identity; `meta.authoring` for composition and accessibility guidance; and `meta.builder` for template insertion details and expected project-data namespaces. These items are for Builder/MCP discovery and are not a published shadcn install registry yet.\n\nPrefer the focused `components.*` tools over dumping `webstudio://project/components`. Do not write local scripts to parse full MCP discovery JSON for common component lookup.\nFor “use every component” or design-system pages, start with compact `components.coverage-plan`, checkpoint, then page through roots/parts instead of dumping the full catalog.\n\n## Consumer Capabilities\n\nMCP lets agents work on one configured Webstudio project at a time. In consumer\nterms, agents can:\n\n- Check which project they are connected to.\n- Check what the share link is allowed to do.\n- Inspect project metadata and the latest editable build.\n- Read selected project data for audits and repair.\n- Apply precise project changes against a known version.\n- List, inspect, create, update, delete, duplicate, copy, and reorder pages.\n- Set the home page.\n- Preserve old page paths for redirects or history.\n- Read and update page titles, descriptions, metadata, auth settings, and SEO fields.\n- List, create, update, duplicate, move, and delete page folders.\n- List, create, update, delete, duplicate, reorder, and reuse page templates.\n- Create pages from reusable templates.\n- Read and update project site settings.\n- Read and update marketplace product metadata.\n- List, create, update, delete, and replace redirects.\n- List, create, update, and delete responsive breakpoints.\n- List and inspect page elements.\n- Insert registered components.\n- Insert styled JSX fragments.\n- Move, reparent, clone, duplicate, wrap, unwrap, convert, rename, retag, and delete elements.\n- Fill grid cells.\n- List and update text children.\n- Update plain text and expression text.\n- Update structured rich text.\n- Add, update, delete, and bind element props.\n- Bind props to expressions, resources, actions, and runtime system values.\n- Read, add, update, delete, and replace local styles.\n- Update selected style-source styles.\n- List, create, update, attach, detach, extract, duplicate, rename, lock, unlock, reorder, clear, and delete design tokens and style sources.\n- List, define, rename, delete, and rewrite CSS variables.\n- List, create, update, and delete static data variables.\n- Create string, number, boolean, string list, and JSON variables.\n- Delete unused data variables.\n- List, create, update, upsert, bind, and delete resources.\n- Create HTTP resources.\n- Create GraphQL resources.\n- Create system resources.\n- Use built-in system resources for sitemap, current date, and assets.\n- List and inspect complete asset metadata; upload, download, update, move, duplicate, find usage for, replace, and delete assets.\n- List, create, rename, move, recursively duplicate, and recursively delete nested asset folders.\n- Publish to staging or production.\n- Publish to selected domains.\n- List publish builds.\n- Check publish job status.\n- Unpublish staging or production deployments.\n- List, create, update, delete, and verify custom domains.\n- Start and stop preview.\n- Capture screenshots of generated pages.\n- Compare screenshots against baselines.\n- Install OCR support for richer visual checks.\n\nUseful resources:\n\n- `webstudio://project/status`: compact current ProjectSession status\n- `webstudio://project/tools-overview`: small operation overview by capability area\n- `webstudio://project/components-overview`: small component overview with ids, labels, namespaces, and categories\n- `webstudio://project/tools`: full operation catalog; read only when focused metadata is insufficient\n- `webstudio://project/components`: full component catalog with props, states, and content model composition constraints; read only when `components.summary`, `components.find`, and `components.get` are insufficient\n- `webstudio://project/guide`: concise discovery guide\n- `webstudio://project/expressions`: expression syntax, scope, supported methods, bindings, Collection iteration context, and verification\n- `webstudio://project/accessibility-review`: evidence-based LLM accessibility-review workflow using project checks, preview, and screenshots\n\n## MCP SDK Client Imports\n\nWhen writing a local Node.js MCP client script, use the official MCP SDK package and these exact ESM imports:\n\nInside the Webstudio monorepo this package is available at the repo root. In another project, install it first with `pnpm add -D @modelcontextprotocol/sdk`.\n\n```js\nimport { Client } from "@modelcontextprotocol/sdk/client/index.js";\nimport { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";\nimport { LoggingMessageNotificationSchema } from "@modelcontextprotocol/sdk/types.js";\n```\n\nMinimal stdio client for the local Webstudio CLI:\n\n```js\nconst client = new Client({ name: "webstudio-agent", version: "1.0.0" });\n\nclient.setNotificationHandler(\n LoggingMessageNotificationSchema,\n (notification) => {\n console.error(`[mcp] ${notification.params.data}`);\n }\n);\n\nconst transport = new StdioClientTransport({\n command: "node",\n args: ["packages/cli/local.js", "mcp"],\n cwd: process.cwd(),\n stderr: "inherit",\n});\n\nawait client.connect(transport);\n\nconst index = await client.callTool({\n name: "meta.index",\n arguments: {},\n});\nconsole.log(JSON.stringify(index.structuredContent, null, 2));\n\nawait client.close();\n```\n\nUse `node packages/cli/local.js mcp` from the Webstudio monorepo root for local development, or `webstudio mcp` from a linked project where the CLI is installed. Keep stdout for JSON-RPC/structured results and surface MCP logging notifications or stderr lifecycle lines as progress.\n\n## Core Rules\n\n- stdout is reserved for MCP JSON-RPC while the server is running.\n- Operate on the configured project only.\n- Read ids before writing.\n- Prefer semantic tools over `apply-patch`.\n- Use `status` and `refresh` when cached namespaces may be stale. Pass `status {"verbose":true}` only when debugging full namespace arrays, freshness, compatibility, or diagnostic details.\n- A mutation is durable only when `meta.session.committed` is true.\n- For visual/design work, verify the rendered result with vision before finishing.\n\n## Vision Verification Loop\n\nVision-capable AI can use MCP to see what it is building:\n\n{{mcpVisionVerificationLoopMarkdown}}\n\nGenerated app setup:\n\n{{mcpGeneratedAppDependencyNotes}}\n\n## MCP Argument Examples\n\nMCP tools receive JSON argument objects:\n\n{{mcpArgumentExampleIndex}}\n\n## Screenshot Verification\n\n{{screenshotVerificationSummary}}\n', "mcp-startup-epilogue": @@ -83,6 +83,10 @@ export const cliDocSections = { 'Stage a draft page for a future publish with {"pageId":"page-id","values":{"isDraft":false}}. This clears draft state but does not deploy the site. The home page and /* catch-all page cannot be drafts.', 'Resource url accepts plain fixed URLs and paths. For computed URLs, pass JavaScript expression code such as "https://api.example.com/items?tag=" + filters.tag. Resource header values, search parameter values, and text bodies accept expressions for dynamic values; for fixed text, use { "type": "literal", "value": "application/json" }.', 'Resource update example: use update-resource with {"resourceId":"resource-id","values":{"url":"https://api.example.com/items"}}.', + "Assets is one system resource. create-assets-resource without query preserves the existing fetch-all behavior and response shape. Add an explicit query object only when filtering, projection, limits, parameters, or selected-file hydration are required.", + "For a Markdown-backed blog, read get-asset-field-catalog, validate GROQ with validate-asset-query, then call create-assets-resource or update-assets-resource. Set values.query:null to return an existing resource to legacy fetch-all mode.", + 'Use one metadata-only Assets resource with query configuration for listings with content:{"mode":"none"}. Use a separate Assets resource with a $slug query for the detail route, a runtime binding such as {"name":"slug","value":"system.params.slug"}, resultLimit:1, and content:{"mode":"markdown-body"}.', + "Use preview-asset-query with concrete JSON parameter values before binding the saved resource. Inspect saved mode and configuration with list-assets-resources or get-assets-resource, and read get-asset-resource-index-status after query mutations. Use rebuild-asset-resource-index only for explicit recovery.", "Data variable values support string, number, boolean, string[], and json. Use string[] only for arrays where every item is a string; use json for objects, mixed arrays, filters, and nested data.", "Parameters are internal scoped runtime values from pages, collections, or components. They are not a public authoring surface: do not create, update, or delete parameter records. Public tools should preserve existing parameter records and may reference documented context values such as system in expressions where they are already in scope.", "Use scoped resources for read data. A GET resource created with scopeInstanceId/dataSourceName defaults to exposeAsDataSource:true, becomes a scoped resource data variable, is generated into the page resource data map, and may be loaded while rendering the page. Read the loaded resource result from its wrapper, usually .data.", From fd3d372f8d9b71388094e7a5540643066b52bacd Mon Sep 17 00:00:00 2001 From: Oleg Isonen Date: Wed, 22 Jul 2026 19:51:54 +0100 Subject: [PATCH 14/14] fix: reuse asset bucket for resource indexes --- apps/builder/.env | 1 + apps/builder/app/shared/asset-client.ts | 4 +- .../docs/asset-resource-architecture.md | 14 +++-- .../asset-resource/src/index-storage.test.ts | 1 + packages/asset-resource/src/index-storage.ts | 1 + packages/asset-uploader/src/clients/s3/s3.ts | 55 ++++++++----------- .../src/clients/storage-separation.test.ts | 54 ++++++++++++------ 7 files changed, 73 insertions(+), 57 deletions(-) diff --git a/apps/builder/.env b/apps/builder/.env index 40713faacf83..1a61a8efd6f8 100644 --- a/apps/builder/.env +++ b/apps/builder/.env @@ -24,6 +24,7 @@ POSTGREST_API_KEY= # S3 envs # S3_BUCKET= +# Optional dedicated bucket for resource indexes; defaults to S3_BUCKET # S3_RESOURCE_INDEX_BUCKET= # S3_REGION= # S3_ACL="public-read" diff --git a/apps/builder/app/shared/asset-client.ts b/apps/builder/app/shared/asset-client.ts index 2dae28161e35..0d2b9f616406 100644 --- a/apps/builder/app/shared/asset-client.ts +++ b/apps/builder/app/shared/asset-client.ts @@ -48,9 +48,7 @@ export const createAssetClientWithResourceIndexStore = () => { client.resourceIndexStore === undefined || client.resourceIndexStore.read === undefined ) { - throw new Error( - "Private asset resource index storage is not configured. Set S3_RESOURCE_INDEX_BUCKET." - ); + throw new Error("Private asset resource index storage is not configured"); } return client as AssetClientWithReadableResourceIndexStore; }; diff --git a/apps/builder/docs/asset-resource-architecture.md b/apps/builder/docs/asset-resource-architecture.md index 005a71ffc5c5..a282062b74fa 100644 --- a/apps/builder/docs/asset-resource-architecture.md +++ b/apps/builder/docs/asset-resource-architecture.md @@ -263,14 +263,16 @@ the resource stale until a matching revision can be activated. Publication never consumes a stale revision: it first reconciles the current query against the publication snapshot, or builds a historical query snapshot in memory. -Private index artifacts use a dedicated S3-compatible R2 store. The adapter +Private index artifacts use the configured S3-compatible R2 store under the +reserved `resource-indexes/` namespace. The adapter writes with `If-None-Match: *`, records the index checksum as object metadata, and verifies that checksum with `HEAD` when an idempotent write finds an -existing object. It sets no public ACL and must use a bucket with no public -object access, separate from public asset-delivery storage. Builder configures -that bucket with `S3_RESOURCE_INDEX_BUCKET`; query-index operations fail closed -when remote asset storage is enabled without it. Filesystem development stores -indexes below `private/asset-resource-indexes`, outside the public asset tree. +existing object. It sets no public ACL. The bucket is not exposed directly; +public asset delivery is mediated by Webstudio and must not expose the reserved +namespace. `S3_RESOURCE_INDEX_BUCKET` may select a dedicated bucket but defaults +to `S3_BUCKET`, preserving existing deployments that use a separate storage +boundary without requiring one. Filesystem development stores indexes below +`private/asset-resource-indexes`, outside the public asset tree. V1 does not deduplicate index objects across resources: `resourceId` is part of both object identity and the revision primary key. Cleanup therefore never diff --git a/packages/asset-resource/src/index-storage.test.ts b/packages/asset-resource/src/index-storage.test.ts index 3beaad69d85d..f5ba44771bdd 100644 --- a/packages/asset-resource/src/index-storage.test.ts +++ b/packages/asset-resource/src/index-storage.test.ts @@ -31,6 +31,7 @@ describe("resource index persistence", () => { index, }); + expect(result.key).toMatch(/^resource-indexes\//); expect(result).toEqual({ key: getAssetResourceIndexObjectKey({ projectId: "project/../private", diff --git a/packages/asset-resource/src/index-storage.ts b/packages/asset-resource/src/index-storage.ts index f634ccc98fe4..4289b23c3954 100644 --- a/packages/asset-resource/src/index-storage.ts +++ b/packages/asset-resource/src/index-storage.ts @@ -36,6 +36,7 @@ export const getAssetResourceIndexObjectKey = ({ index: AssetResourceIndexV1; }) => [ + "resource-indexes", "projects", encodeKeySegment(projectId), "resources", diff --git a/packages/asset-uploader/src/clients/s3/s3.ts b/packages/asset-uploader/src/clients/s3/s3.ts index 2c2a879bad03..b28448035d4a 100644 --- a/packages/asset-uploader/src/clients/s3/s3.ts +++ b/packages/asset-uploader/src/clients/s3/s3.ts @@ -20,12 +20,7 @@ type S3ClientOptions = { }; export const createS3Client = (options: S3ClientOptions): AssetClient => { - const resourceIndexBucket = options.resourceIndexBucket; - if (resourceIndexBucket === options.bucket) { - throw new Error( - "Resource indexes require a bucket distinct from the public asset bucket" - ); - } + const resourceIndexBucket = options.resourceIndexBucket ?? options.bucket; const signer = new SignatureV4({ credentials: { accessKeyId: options.accessKeyId, @@ -60,33 +55,29 @@ export const createS3Client = (options: S3ClientOptions): AssetClient => { }; return { - ...(resourceIndexBucket === undefined - ? {} - : { - resourceIndexStore: { - putIfAbsent: (object) => - putImmutableObjectToS3({ - signer, - endpoint: options.endpoint, - bucket: resourceIndexBucket, - object, - }), - read: (key) => - readFromS3({ - signer, - name: key, - endpoint: options.endpoint, - bucket: resourceIndexBucket, - }), - delete: (key) => - deleteImmutableObjectFromS3({ - signer, - endpoint: options.endpoint, - bucket: resourceIndexBucket, - key, - }), - }, + resourceIndexStore: { + putIfAbsent: (object) => + putImmutableObjectToS3({ + signer, + endpoint: options.endpoint, + bucket: resourceIndexBucket, + object, }), + read: (key) => + readFromS3({ + signer, + name: key, + endpoint: options.endpoint, + bucket: resourceIndexBucket, + }), + delete: (key) => + deleteImmutableObjectFromS3({ + signer, + endpoint: options.endpoint, + bucket: resourceIndexBucket, + key, + }), + }, uploadFile, readFile: (name, range) => readFromS3({ diff --git a/packages/asset-uploader/src/clients/storage-separation.test.ts b/packages/asset-uploader/src/clients/storage-separation.test.ts index 3fff7579c4b3..583c2b5bf132 100644 --- a/packages/asset-uploader/src/clients/storage-separation.test.ts +++ b/packages/asset-uploader/src/clients/storage-separation.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { createFsClient } from "./fs/fs"; import { createS3Client } from "./s3/s3"; @@ -10,8 +10,35 @@ const s3Options = { bucket: "public-assets", maxUploadSize: 1, }; +const indexObject = { + key: "resource-indexes/projects/project-1/index.json", + data: new TextEncoder().encode("{}"), + checksum: `sha256:${"a".repeat(64)}`, + contentType: "application/json" as const, +}; + +const expectIndexBucket = async ( + options: Parameters[0], + bucket: string +) => { + const fetch = vi.fn( + async (_input: string | URL | Request) => + new Response(null, { status: 200 }) + ); + vi.stubGlobal("fetch", fetch); + const store = createS3Client(options).resourceIndexStore; + expect(store).toBeDefined(); + + await store?.putIfAbsent(indexObject); + + expect(fetch.mock.calls[0]?.[0].toString()).toBe( + `https://storage.example/${bucket}/resource-indexes%2Fprojects%2Fproject-1%2Findex.json` + ); +}; + +afterEach(() => vi.unstubAllGlobals()); -describe("resource index storage separation", () => { +describe("resource index storage", () => { test("does not put filesystem indexes in the asset directory by default", () => { expect( createFsClient({ fileDirectory: "public/assets", maxUploadSize: 1 }) @@ -26,19 +53,14 @@ describe("resource index storage separation", () => { ).toBeDefined(); }); - test("requires a distinct S3 bucket before exposing an index store", () => { - expect(createS3Client(s3Options).resourceIndexStore).toBeUndefined(); - expect(() => - createS3Client({ - ...s3Options, - resourceIndexBucket: s3Options.bucket, - }) - ).toThrow("distinct from the public asset bucket"); - expect( - createS3Client({ - ...s3Options, - resourceIndexBucket: "private-indexes", - }).resourceIndexStore - ).toBeDefined(); + test("uses the configured S3 bucket for the private index store", async () => { + await expectIndexBucket(s3Options, s3Options.bucket); + }); + + test("preserves an optional dedicated S3 index bucket", async () => { + await expectIndexBucket( + { ...s3Options, resourceIndexBucket: "private-indexes" }, + "private-indexes" + ); }); });