diff --git a/apps/sim/app/api/files/public/[token]/content/route.ts b/apps/sim/app/api/files/public/[token]/content/route.ts index bd877ef5508..49f7d8104e2 100644 --- a/apps/sim/app/api/files/public/[token]/content/route.ts +++ b/apps/sim/app/api/files/public/[token]/content/route.ts @@ -11,6 +11,7 @@ import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { enforcePublicFileRateLimit } from '@/lib/public-shares/rate-limit' import { resolveActiveShareByToken } from '@/lib/public-shares/share-manager' import { downloadFile } from '@/lib/uploads/core/storage-service' +import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative' import { createErrorResponse, createFileResponse, @@ -31,8 +32,10 @@ const logger = createLogger('PublicFileContentAPI') * * Generated office docs are stored as source; {@link resolveServableDoc} swaps in * their prebuilt compiled binary (read-only, never compiles). Uploaded binaries - * pass through untouched. A generated doc whose compiled artifact isn't built yet - * returns 409 rather than serving raw source under a binary content type. + * pass through untouched, except under `preview=1`, where a format no browser + * decodes is substituted with a renderable derivative. A generated doc whose + * compiled artifact isn't built yet returns 409 rather than serving raw source + * under a binary content type. */ export const GET = withRouteHandler( async (request: NextRequest, context: { params: Promise<{ token: string }> }) => { @@ -45,6 +48,7 @@ export const GET = withRouteHandler( const parsed = await parseRequest(getPublicFileContentContract, request, context) if (!parsed.success) return parsed.response const { token } = parsed.data.params + const preview = parsed.data.query.preview === '1' const resolved = await resolveActiveShareByToken(token) if (!resolved) { @@ -77,12 +81,20 @@ export const GET = withRouteHandler( ) } - const buffer = servable.kind === 'artifact' ? servable.buffer : raw // This response is `nosniff`, so a stored `application/octet-stream` refuses to render // even though the bytes are fine. Resolving from the filename also keeps this route on // the same inline allowlist as the workspace serve route. - const contentType = - servable.kind === 'artifact' ? servable.contentType : getContentType(file.originalName) + let buffer = raw + let contentType = getContentType(file.originalName) + if (servable.kind === 'artifact') { + buffer = servable.buffer + contentType = servable.contentType + } else if (preview) { + // Only for a render request: the Download button omits `preview`, so a saved + // file is always the bytes that were shared. + const image = await resolveServableImageBytes(raw, file.key) + if (image) ({ buffer, contentType } = image) + } logger.info('Public shared file served', { token, key: file.key, size: buffer.length }) diff --git a/apps/sim/app/api/files/serve/[...path]/route.ts b/apps/sim/app/api/files/serve/[...path]/route.ts index 4b29a568e78..b8ed3154eab 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.ts @@ -13,6 +13,7 @@ import { CopilotFiles, isUsingCloudStorage } from '@/lib/uploads' import type { StorageContext } from '@/lib/uploads/config' import { parseWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { downloadFile } from '@/lib/uploads/core/storage-service' +import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative' import { inferContextFromKey } from '@/lib/uploads/utils/file-utils' import { verifyFileAccess } from '@/app/api/files/authorization' import { @@ -25,20 +26,43 @@ import { const logger = createLogger('FilesServeAPI') +interface ServeOptions { + /** `raw=1` — bypass all resolution and serve the stored source as-is. */ + raw: boolean + /** `preview=1` — the caller renders these bytes rather than saving them. */ + preview: boolean + /** `v=` — the URL addresses content-immutable bytes. */ + versioned: boolean +} + /** - * Resolves the bytes + content type to serve for a stored file via the shared - * {@link resolveServableDocBytes} (generated docs → compiled artifact). `raw=1` - * bypasses resolution and serves the stored source as-is. + * Resolves the bytes + content type to serve for a stored file. + * + * Document compilation is unconditional: a generated `.docx`/`.xlsx`/`.pptx` is + * stored as source, so the compiled artifact *is* the file, and every download + * routes through here. An image derivative is the opposite — the stored bytes are + * the file — so it is served only when the caller asked to preview, never when it + * asked to download. */ -async function compileDocumentIfNeeded( - buffer: Buffer, - filename: string, - workspaceId: string | undefined, - raw: boolean, - ownerKey: string | undefined, +async function resolveServableBytes(params: { + buffer: Buffer + filename: string + storageKey: string + workspaceId: string | undefined + options: ServeOptions + ownerKey: string | undefined signal: AbortSignal | undefined -): Promise<{ buffer: Buffer; contentType: string }> { - if (raw) return { buffer, contentType: getContentType(filename) } +}): Promise<{ buffer: Buffer; contentType: string }> { + const { buffer, filename, storageKey, workspaceId, options, ownerKey, signal } = params + if (options.raw) return { buffer, contentType: getContentType(filename) } + + if (options.preview) { + // Images resolve independently of the document path: a HEIF has no compiled-source + // concept, so it never reaches the doc branch. + const image = await resolveServableImageBytes(buffer, storageKey) + if (image) return image + } + return resolveServableDocBytes({ rawBuffer: buffer, fileName: filename, @@ -117,10 +141,14 @@ export const GET = withRouteHandler( const query = fileServeQuerySchema.parse({ raw: request.nextUrl.searchParams.get('raw'), + preview: request.nextUrl.searchParams.get('preview'), v: request.nextUrl.searchParams.get('v'), }) - const raw = query.raw === '1' - const versioned = query.v != null + const options: ServeOptions = { + raw: query.raw === '1', + preview: query.preview === '1', + versioned: query.v != null, + } const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) @@ -135,10 +163,10 @@ export const GET = withRouteHandler( const userId = authResult.userId if (isUsingCloudStorage()) { - return await handleCloudProxy(cloudKey, userId, raw, versioned, request.signal) + return await handleCloudProxy(cloudKey, userId, options, request.signal) } - return await handleLocalFile(cloudKey, userId, raw, versioned, request.signal) + return await handleLocalFile(cloudKey, userId, options, request.signal) } catch (error) { // An in-progress/incomplete doc source fails to compile — this is expected // mid-generation, not a server fault. Return 409 (not 500) so it isn't an @@ -165,8 +193,7 @@ export const GET = withRouteHandler( async function handleLocalFile( filename: string, userId: string, - raw: boolean, - versioned: boolean, + options: ServeOptions, signal: AbortSignal | undefined ): Promise { const ownerKey = `user:${userId}` @@ -198,14 +225,15 @@ async function handleLocalFile( const segment = filename.split('/').pop() || filename const displayName = stripStorageKeyPrefix(segment) const workspaceId = getWorkspaceIdForCompile(filename) - const { buffer: fileBuffer, contentType } = await compileDocumentIfNeeded( - rawBuffer, - displayName, + const { buffer: fileBuffer, contentType } = await resolveServableBytes({ + buffer: rawBuffer, + filename: displayName, + storageKey: filename, workspaceId, - raw, + options, ownerKey, - signal - ) + signal, + }) logger.info('Local file served', { userId, filename, size: fileBuffer.length }) @@ -213,7 +241,7 @@ async function handleLocalFile( buffer: fileBuffer, contentType, filename: displayName, - cacheControl: resolveServeCacheControl(versioned, contextParam), + cacheControl: resolveServeCacheControl(options.versioned, contextParam), }) } catch (error) { logger.error('Error reading local file:', error) @@ -224,9 +252,8 @@ async function handleLocalFile( async function handleCloudProxy( cloudKey: string, userId: string, - raw = false, - versioned = false, - signal: AbortSignal | undefined = undefined + options: ServeOptions, + signal: AbortSignal | undefined ): Promise { const ownerKey = `user:${userId}` try { @@ -260,14 +287,15 @@ async function handleCloudProxy( const segment = cloudKey.split('/').pop() || 'download' const displayName = stripStorageKeyPrefix(segment) const workspaceId = getWorkspaceIdForCompile(cloudKey) - const { buffer: fileBuffer, contentType } = await compileDocumentIfNeeded( - rawBuffer, - displayName, + const { buffer: fileBuffer, contentType } = await resolveServableBytes({ + buffer: rawBuffer, + filename: displayName, + storageKey: cloudKey, workspaceId, - raw, + options, ownerKey, - signal - ) + signal, + }) logger.info('Cloud file served', { userId, @@ -280,7 +308,7 @@ async function handleCloudProxy( buffer: fileBuffer, contentType, filename: displayName, - cacheControl: resolveServeCacheControl(versioned, context), + cacheControl: resolveServeCacheControl(options.versioned, context), }) } catch (error) { logger.error('Error downloading from cloud storage:', error) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-category.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-category.test.ts index cd3ff2b3b6a..00480f609dc 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-category.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-category.test.ts @@ -227,13 +227,15 @@ describe('resolveFileCategory — formats accepted on upload must be previewable ['image.bmp', 'image/bmp'], ['image.avif', 'image/avif'], ['favicon.ico', 'image/x-icon'], + ['photo.heic', 'image/heic'], + ['photo.heif', 'image/heif'], ])('%s previews as an image', (filename, mimeType) => { expect(resolveFileCategory(mimeType, filename)).toBe('image-previewable') expect(resolveFileCategory('application/octet-stream', filename)).toBe('image-previewable') }) - it.each(['image.tiff', 'photo.heic'])( - '%s stays unsupported — no browser renders it in an ', + it.each(['image.tiff', 'scan.tif'])( + '%s stays unsupported — nothing decodes it on either side', (filename) => { expect(resolveFileCategory(null, filename)).toBe('unsupported') } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-category.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-category.ts index 22fa9703ec9..485ebdeda56 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-category.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-category.ts @@ -44,10 +44,12 @@ const IFRAME_PREVIEWABLE_MIME_TYPES = new Set([ const IFRAME_PREVIEWABLE_EXTENSIONS = new Set(['pdf']) /** - * Image formats every supported browser decodes natively. `.tif`/`.tiff` and - * `.heic`/`.heif` are accepted uploads but deliberately absent — no browser renders - * them in an ``, so they stay on the download-only path rather than showing a - * broken image. + * Formats the image viewer can show. Most are decoded by the browser directly; + * `.heic`/`.heif` are not — no browser outside Safari renders them — but the serve + * route transcodes them to JPEG, so an `` pointed at it works. + * + * `.tif`/`.tiff` are accepted uploads and deliberately absent: nothing decodes them + * on either side, so they stay download-only rather than showing a broken image. */ const IMAGE_PREVIEWABLE_MIME_TYPES = new Set([ 'image/png', @@ -58,6 +60,8 @@ const IMAGE_PREVIEWABLE_MIME_TYPES = new Set([ 'image/bmp', 'image/x-icon', 'image/vnd.microsoft.icon', + 'image/heic', + 'image/heif', ]) const IMAGE_PREVIEWABLE_EXTENSIONS = new Set([ 'png', @@ -68,6 +72,8 @@ const IMAGE_PREVIEWABLE_EXTENSIONS = new Set([ 'avif', 'bmp', 'ico', + 'heic', + 'heif', ]) const AUDIO_PREVIEWABLE_MIME_TYPES = new Set([ diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index 9b0b6af14dd..ee7919c357c 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -4,7 +4,7 @@ import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react' import { Music } from '@sim/emcn/icons' import dynamic from 'next/dynamic' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import { getFileExtension, resolveMediaMimeType } from '@/lib/uploads/utils/file-utils' +import { resolveMediaMimeType } from '@/lib/uploads/utils/file-utils' import { useWorkspaceFileBinary, useWorkspaceFileContent, @@ -28,6 +28,7 @@ import { PreviewErrorBoundary, PreviewLoadingFrame, resolvePreviewError, + UnsupportedPreview, } from './preview-shared' import { TextEditor } from './text-editor' import { useDocPreviewBinary } from './use-doc-preview-binary' @@ -170,7 +171,7 @@ function FileViewerContent({ // browser. CsvTablePreview's streamed fallback is workspace-only, so on the // read-only public path a large CSV is download-only. if (isCsvStreamOnly(file)) { - return + return } // Markdown renders through the inline rich editor (non-editable) so the public share // surface matches the in-app reading experience; canEdit={false} disables autosave, @@ -259,7 +260,7 @@ function FileViewerContent({ return } - return + return } /** @@ -421,22 +422,3 @@ const MediaPreview = memo(function MediaPreview({ ) }) - -const UnsupportedPreview = memo(function UnsupportedPreview({ - file, -}: { - file: WorkspaceFileRecord -}) { - const ext = getFileExtension(file.name) - - return ( -
-

- Preview not available{ext ? ` for .${ext} files` : ' for this file'} -

-

- Use the download button to view this file -

-
- ) -}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/image-preview.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/image-preview.test.tsx new file mode 100644 index 00000000000..17f3885d081 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/image-preview.test.tsx @@ -0,0 +1,88 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { ImagePreview } from './image-preview' + +const file = { + id: 'file-1', + workspaceId: 'ws-1', + name: 'photo.heic', + key: 'workspace/ws-1/photo.heic', + path: '', + size: 1024, + type: 'image/heic', + uploadedBy: 'user-1', + folderId: null, + uploadedAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), +} satisfies WorkspaceFileRecord + +let container: HTMLDivElement +let root: Root + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true + // ZoomablePreview measures its content through one; jsdom has no implementation. + globalThis.ResizeObserver = class { + observe() {} + unobserve() {} + disconnect() {} + } + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) +}) + +afterEach(() => { + act(() => root.unmount()) + container.remove() +}) + +function render(record: WorkspaceFileRecord = file) { + act(() => root.render()) +} + +describe('ImagePreview', () => { + it('requests the preview derivative rather than the stored bytes', () => { + render() + + const src = container.querySelector('img')?.getAttribute('src') ?? '' + expect(src).toContain('preview=1') + expect(src).not.toContain('raw=1') + }) + + it('falls back to the unsupported state when the image fails to decode', () => { + render() + + const img = container.querySelector('img') + expect(img).not.toBeNull() + + act(() => { + img?.dispatchEvent(new Event('error')) + }) + + expect(container.querySelector('img')).toBeNull() + expect(container.textContent).toContain('Preview not available') + }) + + it('retries after an overwrite, which mints a new storage key and remounts', () => { + act(() => root.render()) + + act(() => { + container.querySelector('img')?.dispatchEvent(new Event('error')) + }) + expect(container.textContent).toContain('Preview not available') + + // Every content write mints a fresh key, so the parent's `key={file.key}` + // remounts this and the previous bytes' outcome cannot stick. + const overwritten = { ...file, key: 'workspace/ws-1/photo-v2.heic' } + act(() => root.render()) + + expect(container.querySelector('img')).not.toBeNull() + expect(container.textContent).not.toContain('Preview not available') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/image-preview.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/image-preview.tsx index d65fd1d928b..4c6659c2975 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/image-preview.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/image-preview.tsx @@ -3,18 +3,25 @@ import { memo, useState } from 'react' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { useFileContentSource } from '@/hooks/use-file-content-source' -import { PREVIEW_LOADING_OVERLAY } from './preview-shared' +import { PREVIEW_LOADING_OVERLAY, UnsupportedPreview } from './preview-shared' import { ZoomablePreview } from './zoomable-preview' export const ImagePreview = memo(function ImagePreview({ file }: { file: WorkspaceFileRecord }) { const source = useFileContentSource() - const [hasSettled, setHasSettled] = useState(false) - // Version the URL on updatedAt: overwrites keep the same storage key, so an unversioned - // URL would resolve to a previously cached copy instead of the rewritten bytes. + /** `v` busts the browser cache across content writes; `preview` lets the server + * substitute a renderable JPEG for a HEIC. */ const serveUrl = source.buildUrl(file.key, { version: Number(new Date(file.updatedAt)) || file.size, + preview: true, }) + const [status, setStatus] = useState<'loading' | 'loaded' | 'error'>('loading') + + /** A derivative the server declined to build, or corrupt bytes — show the download + * fallback rather than a broken image. Content writes mint a new storage key, so + * the parent's `key={file.key}` resets this. */ + if (status === 'error') return + return (
@@ -24,11 +31,11 @@ export const ImagePreview = memo(function ImagePreview({ file }: { file: Workspa className='max-h-full max-w-full select-none rounded-md object-contain' draggable={false} loading='eager' - onLoad={() => setHasSettled(true)} - onError={() => setHasSettled(true)} + onLoad={() => setStatus('loaded')} + onError={() => setStatus('error')} /> - {!hasSettled && PREVIEW_LOADING_OVERLAY} + {status === 'loading' && PREVIEW_LOADING_OVERLAY}
) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/preview-shared.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/preview-shared.tsx index 041a0225074..0dc45d78749 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/preview-shared.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/preview-shared.tsx @@ -1,12 +1,33 @@ 'use client' -import { Component, type ErrorInfo, type ReactNode } from 'react' +import { Component, type ErrorInfo, memo, type ReactNode } from 'react' import { cn } from '@sim/emcn' import { Loader } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' +import { getFileExtension } from '@/lib/uploads/utils/file-utils' const logger = createLogger('FilePreview') +/** + * Terminal fallback for a file this app cannot render — either the format has no + * viewer at all, or a viewer that was expected to work failed (e.g. a HEIC whose + * server-side derivative could not be produced). + */ +export const UnsupportedPreview = memo(function UnsupportedPreview({ name }: { name: string }) { + const ext = getFileExtension(name) + + return ( +
+

+ Preview not available{ext ? ` for .${ext} files` : ' for this file'} +

+

+ Use the download button to view this file +

+
+ ) +}) + export function PreviewError({ label, error }: { label: string; error: string }) { return (
diff --git a/apps/sim/hooks/use-file-content-source.tsx b/apps/sim/hooks/use-file-content-source.tsx index c388987a216..bcdd1d3f493 100644 --- a/apps/sim/hooks/use-file-content-source.tsx +++ b/apps/sim/hooks/use-file-content-source.tsx @@ -9,6 +9,12 @@ import { export interface FileContentUrlOptions { /** Request the uncompiled source instead of the rendered/compiled bytes. */ raw?: boolean + /** + * Declare the bytes are being rendered, not downloaded, so the server may substitute a + * browser-renderable derivative for a format no browser decodes (HEIC). Downloads must + * leave this off — they need the stored bytes. + */ + preview?: boolean /** Content version (e.g. the record's `updatedAt`) — makes the URL cacheable/immutable. */ version?: string | number /** Append a timestamp cache-buster when there is no `version`. */ @@ -67,6 +73,7 @@ function buildServeUrl(key: string, opts?: FileContentUrlOptions): string { if (opts?.version != null) params.push(`v=${encodeURIComponent(String(opts.version))}`) else if (opts?.bust) params.push(`t=${Date.now()}`) if (opts?.raw) params.push('raw=1') + if (opts?.preview) params.push('preview=1') return params.length > 0 ? `${base}&${params.join('&')}` : base } @@ -109,7 +116,11 @@ export function createPublicFileContentSource( token: string, contentUrl: string ): FileContentSource { - return inlineImageSource(() => contentUrl, `/api/files/public/${token}/inline`) + return inlineImageSource( + (_key, opts) => + opts?.preview ? `${contentUrl}${contentUrl.includes('?') ? '&' : '?'}preview=1` : contentUrl, + `/api/files/public/${token}/inline` + ) } /** diff --git a/apps/sim/lib/api/contracts/public-shares.ts b/apps/sim/lib/api/contracts/public-shares.ts index a839d372ead..b87e272c142 100644 --- a/apps/sim/lib/api/contracts/public-shares.ts +++ b/apps/sim/lib/api/contracts/public-shares.ts @@ -116,11 +116,17 @@ export const getPublicFileContract = defineRouteContract({ }, }) +const publicFileContentQuerySchema = z.object({ + /** `1` => rendering, not downloading — a HEIC may be substituted with a JPEG derivative. */ + preview: z.string().nullish(), +}) + /** Binary stream of the shared file's bytes. Authorized solely by an active token. */ export const getPublicFileContentContract = defineRouteContract({ method: 'GET', path: '/api/files/public/[token]/content', params: publicFileTokenParamsSchema, + query: publicFileContentQuerySchema, response: { mode: 'binary', }, diff --git a/apps/sim/lib/api/contracts/storage-transfer.ts b/apps/sim/lib/api/contracts/storage-transfer.ts index f0b975d9aa4..92dbaf89bfa 100644 --- a/apps/sim/lib/api/contracts/storage-transfer.ts +++ b/apps/sim/lib/api/contracts/storage-transfer.ts @@ -501,6 +501,8 @@ export const fileServeParamsSchema = z.object({ export const fileServeQuerySchema = z.object({ raw: z.string().nullish(), + /** `1` => rendering, not downloading — a HEIC may be substituted with a JPEG derivative. */ + preview: z.string().nullish(), /** Content version (the file record's `updatedAt`). Present => the URL is content-immutable and may be cached indefinitely by the browser. */ v: z.string().nullish(), }) diff --git a/apps/sim/lib/copilot/chat/attachment-preview.test.ts b/apps/sim/lib/copilot/chat/attachment-preview.test.ts new file mode 100644 index 00000000000..1b4b2e997a8 --- /dev/null +++ b/apps/sim/lib/copilot/chat/attachment-preview.test.ts @@ -0,0 +1,35 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { getMothershipAttachmentPreviewUrl } from './attachment-preview' + +describe('getMothershipAttachmentPreviewUrl', () => { + it('asks for a preview derivative on images', () => { + const url = getMothershipAttachmentPreviewUrl({ + key: 'copilot/a.heic', + media_type: 'image/heic', + }) + expect(url).toContain('preview=1') + }) + + it('omits preview on videos, which have no derivative path', () => { + const url = getMothershipAttachmentPreviewUrl({ key: 'copilot/a.mp4', media_type: 'video/mp4' }) + expect(url).not.toContain('preview') + expect(url).toContain('context=mothership') + }) + + it('returns undefined for a non-renderable attachment', () => { + expect( + getMothershipAttachmentPreviewUrl({ key: 'copilot/a.pdf', media_type: 'application/pdf' }) + ).toBeUndefined() + }) + + it('encodes the key so a slashed storage key stays one path segment', () => { + const url = getMothershipAttachmentPreviewUrl({ + key: 'copilot/nested/a b.png', + media_type: 'image/png', + }) + expect(url).toContain(encodeURIComponent('copilot/nested/a b.png')) + }) +}) diff --git a/apps/sim/lib/copilot/chat/attachment-preview.ts b/apps/sim/lib/copilot/chat/attachment-preview.ts index f4a65aa0ce0..80980b3674d 100644 --- a/apps/sim/lib/copilot/chat/attachment-preview.ts +++ b/apps/sim/lib/copilot/chat/attachment-preview.ts @@ -2,8 +2,13 @@ export function getMothershipAttachmentPreviewUrl(file: { key: string media_type: string }): string | undefined { - if (!file.media_type.startsWith('image/') && !file.media_type.startsWith('video/')) { + const isImage = file.media_type.startsWith('image/') + if (!isImage && !file.media_type.startsWith('video/')) { return undefined } - return `/api/files/serve/${encodeURIComponent(file.key)}?context=mothership` + // `preview=1` only for images: this URL backs a rendered thumbnail, so the serve route + // may substitute a browser-renderable derivative for a format no browser decodes (HEIC). + // A video has no derivative path, so asking would only spend a brand sniff per request. + const preview = isImage ? '&preview=1' : '' + return `/api/files/serve/${encodeURIComponent(file.key)}?context=mothership${preview}` } diff --git a/apps/sim/lib/uploads/server/heic.test.ts b/apps/sim/lib/uploads/server/heic.test.ts index e44c350a525..c34c72f017c 100644 --- a/apps/sim/lib/uploads/server/heic.test.ts +++ b/apps/sim/lib/uploads/server/heic.test.ts @@ -2,7 +2,11 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic' +import { + isHeifContainer, + isHevcHeifContainer, + transcodeHeicToJpeg, +} from '@/lib/uploads/server/heic' /** * An ISO-BMFF `ftyp` box: 4-byte size, the `ftyp` marker, the major brand, a @@ -69,12 +73,44 @@ describe('isHeifContainer', () => { expect(isHeifContainer(truncated)).toBe(false) }) + it('caps the scan so an inflated declared box size cannot drive the loop', () => { + // The declared size is attacker-controlled; without the cap this scans the + // whole buffer, and this runs on every preview request. + const inflated = Buffer.alloc(64 * 1024) + inflated.writeUInt32BE(0xffffffff, 0) + inflated.write('ftyp', 4, 'ascii') + inflated.write('isom', 8, 'ascii') + inflated.write('heic', 60 * 1024, 'ascii') + expect(isHeifContainer(inflated)).toBe(false) + }) + it('rejects buffers too short to carry a brand', () => { expect(isHeifContainer(Buffer.alloc(0))).toBe(false) expect(isHeifContainer(ftypHeader('heic').subarray(0, 11))).toBe(false) }) }) +describe('isHevcHeifContainer', () => { + it.each(['heic', 'heix', 'heim', 'heis', 'hevc', 'hevx'])( + 'detects the %s brand, which names HEVC outright', + (brand) => { + expect(isHevcHeifContainer(ftypHeader(brand))).toBe(true) + } + ) + + it.each(['mif1', 'msf1'])('rejects the generic %s brand — the codec is unstated', (brand) => { + expect(isHevcHeifContainer(ftypHeader(brand))).toBe(false) + }) + + it.each(['avif', 'avis'])('rejects the AV1-coded %s brand, which browsers render', (brand) => { + expect(isHevcHeifContainer(ftypHeader(brand))).toBe(false) + }) + + it('detects an HEVC brand declared only among the compatible brands', () => { + expect(isHevcHeifContainer(ftypHeader('mif1', ['heic']))).toBe(true) + }) +}) + describe('transcodeHeicToJpeg', () => { it('refuses to decode above the input ceiling', async () => { // Uploads allow 100MB; without this bound a tenant could spend an unbounded diff --git a/apps/sim/lib/uploads/server/heic.ts b/apps/sim/lib/uploads/server/heic.ts index 319a9eba79b..5ebc570e24a 100644 --- a/apps/sim/lib/uploads/server/heic.ts +++ b/apps/sim/lib/uploads/server/heic.ts @@ -4,25 +4,19 @@ import { getErrorMessage } from '@sim/utils/errors' const logger = createLogger('HeicTranscode') /** - * ISO-BMFF major brands in the HEIF family. The brand occupies bytes 8-11, - * immediately after the `ftyp` box marker at 4-7. - * - * The list is deliberately broad, `avif` included. It answers "are these bytes - * worth handing to a HEIF decoder", not "which codec is inside" — the brand cannot - * answer the latter anyway, since `mif1` is generic and carries either HEVC or AV1. + * ISO-BMFF brands that name HEVC as the coded format outright. Every browser and + * every vision model rejects these, so they are exactly the set worth transcoding + * before anything else has been tried. + */ +const HEVC_HEIF_BRANDS = new Set(['heic', 'heix', 'heim', 'heis', 'hevc', 'hevx']) + +/** + * Every ISO-BMFF brand in the HEIF family. Broader than {@link HEVC_HEIF_BRANDS}: + * it answers "are these bytes worth handing to a HEIF decoder", not "which codec is + * inside". `mif1`/`msf1` are generic and carry either HEVC or AV1, and `avif`/`avis` + * are included because a decoder that already failed on them has nothing to lose. */ -const HEIF_BRANDS = new Set([ - 'heic', - 'heix', - 'heim', - 'heis', - 'hevc', - 'hevx', - 'mif1', - 'msf1', - 'avif', - 'avis', -]) +const HEIF_BRANDS = new Set([...HEVC_HEIF_BRANDS, 'mif1', 'msf1', 'avif', 'avis']) /** * Byte ceiling for a fallback decode. Uploads allow 100MB and the vision path runs @@ -36,30 +30,54 @@ const HEIF_BRANDS = new Set([ */ const MAX_TRANSCODE_INPUT_BYTES = 20 * 1024 * 1024 +/** A real `ftyp` box holds a handful of brands; anything larger is malformed or hostile. */ +const MAX_FTYP_BOX_BYTES = 512 + /** - * Whether these bytes are an ISO-BMFF container in the HEIF family. + * Whether an ISO-BMFF `ftyp` box names any of `brands`, as either the major brand + * or a compatible brand. * * Sniffed rather than read off the declared type because the common case is a * `.heic` stored as `application/octet-stream`, where the declared type says * nothing at all. */ -export function isHeifContainer(buffer: Buffer): boolean { +function declaresBrand(buffer: Buffer, brands: ReadonlySet): boolean { if (buffer.length < 12) return false if (buffer.toString('ascii', 4, 8) !== 'ftyp') return false - if (HEIF_BRANDS.has(buffer.toString('ascii', 8, 12))) return true + if (brands.has(buffer.toString('ascii', 8, 12))) return true // A standards-valid HEIF may carry a generic major brand such as `isom` and name // the HEIF brand only among the compatible brands, which follow the 4-byte // minor_version at offset 12 and run to the end of the box. A declared size of 0 // or 1 (the ISO-BMFF size escapes, which `ftyp` does not use) leaves `end` below - // the loop's start, so those simply do not scan. - const end = Math.min(buffer.readUInt32BE(0), buffer.length) + // the loop's start, so those simply do not scan. The size is attacker-controlled + // and this runs on every preview, so cap it rather than trusting the declaration. + const end = Math.min(buffer.readUInt32BE(0), buffer.length, MAX_FTYP_BOX_BYTES) for (let offset = 16; offset + 4 <= end; offset += 4) { - if (HEIF_BRANDS.has(buffer.toString('ascii', offset, offset + 4))) return true + if (brands.has(buffer.toString('ascii', offset, offset + 4))) return true } return false } +/** + * Whether these bytes are an ISO-BMFF container in the HEIF family, whatever codec + * they carry. Use where a decode has already been attempted and failed — the extra + * breadth costs nothing there, and it catches the generic `mif1` brand. + */ +export function isHeifContainer(buffer: Buffer): boolean { + return declaresBrand(buffer, HEIF_BRANDS) +} + +/** + * Whether these bytes declare HEVC-coded HEIF. Use where the decode has *not* been + * attempted yet and the answer decides whether to try: an AV1-coded HEIF (`avif`) + * renders natively everywhere, so treating it as a transcode candidate only buys a + * wasted decode and a misleading failure. + */ +export function isHevcHeifContainer(buffer: Buffer): boolean { + return declaresBrand(buffer, HEVC_HEIF_BRANDS) +} + /** * Transcode a HEVC-coded HEIF still to JPEG. * diff --git a/apps/sim/lib/uploads/server/image-derivative.test.ts b/apps/sim/lib/uploads/server/image-derivative.test.ts new file mode 100644 index 00000000000..c9c2d7746bc --- /dev/null +++ b/apps/sim/lib/uploads/server/image-derivative.test.ts @@ -0,0 +1,104 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockDownloadFile, mockUploadFile, mockTranscode } = vi.hoisted(() => ({ + mockDownloadFile: vi.fn(), + mockUploadFile: vi.fn(), + mockTranscode: vi.fn(), +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + downloadFile: mockDownloadFile, + uploadFile: mockUploadFile, +})) + +vi.mock('@/lib/uploads/server/heic', async (importOriginal) => ({ + ...(await importOriginal()), + transcodeHeicToJpeg: mockTranscode, +})) + +import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative' + +/** An ISO-BMFF `ftyp` box declaring `brand` as its major brand. */ +function ftypBytes(brand: string): Buffer { + const header = Buffer.alloc(16) + header.writeUInt32BE(16, 0) + header.write('ftyp', 4, 'ascii') + header.write(brand, 8, 'ascii') + return header +} + +/** An ISO-BMFF `ftyp` box declaring a HEVC-coded HEIF still. */ +function heifBytes(): Buffer { + return ftypBytes('heic') +} + +const JPEG = Buffer.from([0xff, 0xd8, 0xff, 0xe0]) + +describe('resolveServableImageBytes', () => { + beforeEach(() => { + vi.clearAllMocks() + mockDownloadFile.mockRejectedValue(new Error('not found')) + mockUploadFile.mockResolvedValue(undefined) + mockTranscode.mockResolvedValue(JPEG) + }) + + it('leaves non-HEIF bytes untouched', async () => { + expect(await resolveServableImageBytes(JPEG, 'workspace/ws/a.jpg')).toBeNull() + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockTranscode).not.toHaveBeenCalled() + }) + + // AVIF is a HEIF container too, but AV1-coded and rendered natively by every + // browser. Probing it would spend a storage round trip and a WebAssembly decode + // on every request to learn nothing. + it.each(['avif', 'avis'])('leaves the %s brand untouched without probing', async (brand) => { + expect(await resolveServableImageBytes(ftypBytes(brand), 'workspace/ws/a.avif')).toBeNull() + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockTranscode).not.toHaveBeenCalled() + expect(mockUploadFile).not.toHaveBeenCalled() + }) + + it('transcodes and caches on a miss', async () => { + const result = await resolveServableImageBytes(heifBytes(), 'workspace/ws/a.heic') + + expect(result).toEqual({ buffer: JPEG, contentType: 'image/jpeg' }) + expect(mockTranscode).toHaveBeenCalledOnce() + expect(mockUploadFile).toHaveBeenCalledOnce() + }) + + it('serves the cached derivative without decoding again', async () => { + const cached = Buffer.from([0xff, 0xd8, 0xca, 0xce]) + mockDownloadFile.mockResolvedValue(cached) + + const result = await resolveServableImageBytes(heifBytes(), 'workspace/ws/a.heic') + + expect(result).toEqual({ buffer: cached, contentType: 'image/jpeg' }) + expect(mockTranscode).not.toHaveBeenCalled() + }) + + it('keys the derivative by storage key, so replaced content misses the old entry', async () => { + await resolveServableImageBytes(heifBytes(), 'workspace/ws/111-a.heic') + await resolveServableImageBytes(heifBytes(), 'workspace/ws/222-a.heic') + + const [first, second] = mockUploadFile.mock.calls.map((call) => call[0].customKey) + expect(first).not.toEqual(second) + }) + + it('still serves the image when caching the derivative fails', async () => { + mockUploadFile.mockRejectedValue(new Error('s3 down')) + + const result = await resolveServableImageBytes(heifBytes(), 'workspace/ws/a.heic') + + expect(result).toEqual({ buffer: JPEG, contentType: 'image/jpeg' }) + }) + + it('falls back to the stored bytes when the decode fails', async () => { + mockTranscode.mockResolvedValue(null) + + expect(await resolveServableImageBytes(heifBytes(), 'workspace/ws/a.heic')).toBeNull() + expect(mockUploadFile).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/uploads/server/image-derivative.ts b/apps/sim/lib/uploads/server/image-derivative.ts new file mode 100644 index 00000000000..b722476bf80 --- /dev/null +++ b/apps/sim/lib/uploads/server/image-derivative.ts @@ -0,0 +1,81 @@ +import { createHash } from 'node:crypto' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { downloadFile, uploadFile } from '@/lib/uploads/core/storage-service' +import { isHevcHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic' + +const logger = createLogger('ImageDerivative') + +/** + * Keyed by the source's storage key rather than a hash of its bytes. Workspace keys + * are regenerated on every content replacement, so the key is already a content + * version — using it avoids streaming the whole original just to hash it. + */ +function derivativeKey(storageKey: string): string { + const hash = createHash('sha256').update(storageKey, 'utf-8').digest('hex') + return `image-derivative/${hash}.jpg` +} + +async function loadDerivative(storageKey: string): Promise { + try { + return await downloadFile({ key: derivativeKey(storageKey), context: 'copilot' }) + } catch { + return null + } +} + +/** + * Stores the derivative, swallowing failure. + * + * Unlike the compiled-doc store — which throws, because its serve path is load-only + * and cannot rebuild a missing artifact — a miss here is fully recoverable: the next + * read transcodes again. Failing the request would turn a cache problem into a broken + * image for bytes we have already rendered successfully. + */ +async function storeDerivative(storageKey: string, jpeg: Buffer): Promise { + try { + await uploadFile({ + file: jpeg, + fileName: 'derivative.jpg', + contentType: 'image/jpeg', + context: 'copilot', + customKey: derivativeKey(storageKey), + preserveKey: true, + }) + } catch (error) { + logger.warn('Failed to store image derivative', { + storageKey, + error: getErrorMessage(error), + }) + } +} + +/** + * A browser-renderable form of stored image bytes, or `null` when the stored bytes + * are already renderable and should be served untouched. + * + * Only HEVC-coded HEIF needs this today: no browser outside Safari decodes it, and + * serving it under `X-Content-Type-Options: nosniff` guarantees a broken image. + * AV1-coded HEIF (`.avif`) renders natively everywhere and is left alone — probing + * it would cost a decode that can only fail. The transcoded JPEG is cached, because + * a preview is re-fetched on every view and the WebAssembly decode costs roughly a + * second for a phone photo. + * + * The original always remains the stored object — this runs only for `preview=1` + * requests, so it never rewrites what a download hands back. + */ +export async function resolveServableImageBytes( + buffer: Buffer, + storageKey: string +): Promise<{ buffer: Buffer; contentType: string } | null> { + if (!isHevcHeifContainer(buffer)) return null + + const cached = await loadDerivative(storageKey) + if (cached) return { buffer: cached, contentType: 'image/jpeg' } + + const jpeg = await transcodeHeicToJpeg(buffer) + if (!jpeg) return null + + await storeDerivative(storageKey, jpeg) + return { buffer: jpeg, contentType: 'image/jpeg' } +}