From 1257e7bba787817e735a38638a6654eed70228ac Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 6 Aug 2026 15:31:30 -0700 Subject: [PATCH 1/3] feat(files): let the agent read HEIC photos iPhone photos reach the model as HEIC, which no vision model accepts - the Claude Messages API takes JPEG, PNG, GIF and WebP only - so the agent saw nothing. 75 HEIC files are already in production, 64 of them in one workspace uploaded over the last two days. sharp cannot cover this: its prebuilt libvips ships libheif with AV1 but not HEVC (sharp.format.heif.input.fileSuffix is ['.avif']), so a real iPhone photo fails with 'Security limit exceeded'. Verified against both a HEVC-coded sample (sharp fails, heic-convert decodes 2.99MB to a 3992x2992 JPEG in ~950ms) and an AV1-coded mif1 sample (sharp decodes it natively). Decoder selection is capability-based, not brand-based: sharp is always tried first and the WebAssembly decoder runs only on bytes it could not read. The container brand cannot identify the codec anyway - mif1 carries either - so choosing from it would push AV1 files down the slow path. This mirrors how PhotoPrism layers libvips over libheif. Also route the image path on the effective MIME type, since a phone upload commonly stores as application/octet-stream and would otherwise be read as a binary the model never sees, and stop reporting an undecodable image as 'too large'. --- apps/sim/lib/copilot/vfs/file-reader.ts | 82 ++++++++++++++++++------ apps/sim/lib/uploads/server/heic.test.ts | 55 ++++++++++++++++ apps/sim/lib/uploads/server/heic.ts | 72 +++++++++++++++++++++ apps/sim/package.json | 2 + bun.lock | 15 +++++ 5 files changed, 205 insertions(+), 21 deletions(-) create mode 100644 apps/sim/lib/uploads/server/heic.test.ts create mode 100644 apps/sim/lib/uploads/server/heic.ts diff --git a/apps/sim/lib/copilot/vfs/file-reader.ts b/apps/sim/lib/copilot/vfs/file-reader.ts index 30ad2848c0d..36c09de1f8f 100644 --- a/apps/sim/lib/copilot/vfs/file-reader.ts +++ b/apps/sim/lib/copilot/vfs/file-reader.ts @@ -14,7 +14,12 @@ import { recordFileRead } from '@/lib/copilot/request/metrics' import { markSpanForError } from '@/lib/copilot/request/otel' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager' -import { isImageFileType } from '@/lib/uploads/utils/file-utils' +import { + HEIC_TRANSCODE_MEDIA_TYPE, + isHeifContainer, + transcodeHeicToJpeg, +} from '@/lib/uploads/server/heic' +import { isImageFileType, resolveEffectiveMimeType } from '@/lib/uploads/utils/file-utils' // Lazy tracer (same pattern as lib/copilot/request/otel.ts). function getVfsTracer() { @@ -91,54 +96,86 @@ interface PreparedVisionImage { * dimension/quality chosen. */ async function prepareImageForVision( - buffer: Buffer, + sourceBuffer: Buffer, claimedType: string ): Promise { return getVfsTracer().startActiveSpan( TraceSpan.CopilotVfsPrepareImage, { attributes: { - [TraceAttr.CopilotVfsInputBytes]: buffer.length, + [TraceAttr.CopilotVfsInputBytes]: sourceBuffer.length, [TraceAttr.CopilotVfsInputMediaTypeClaimed]: claimedType, }, }, async (span) => { try { - const mediaType = detectImageMime(buffer, claimedType) - span.setAttribute(TraceAttr.CopilotVfsInputMediaTypeDetected, mediaType) + const detectedType = detectImageMime(sourceBuffer, claimedType) + span.setAttribute(TraceAttr.CopilotVfsInputMediaTypeDetected, detectedType) let sharpModule: SharpConstructor try { sharpModule = (await import('sharp')).default } catch (err) { logger.warn('Failed to load sharp for image preparation', { - mediaType, + mediaType: detectedType, error: toError(err).message, }) span.setAttribute(TraceAttr.CopilotVfsSharpLoadFailed, true) - const fitsWithoutSharp = buffer.length <= MAX_IMAGE_READ_BYTES + const fitsWithoutSharp = sourceBuffer.length <= MAX_IMAGE_READ_BYTES span.setAttribute( TraceAttr.CopilotVfsOutcome, fitsWithoutSharp ? 'passthrough_no_sharp' : 'rejected_no_sharp' ) - return fitsWithoutSharp ? { buffer, mediaType, resized: false } : null + return fitsWithoutSharp + ? { buffer: sourceBuffer, mediaType: detectedType, resized: false } + : null } - let metadata: Awaited['metadata']>> - try { - metadata = await sharpModule(buffer, { limitInputPixels: false }).metadata() - } catch (err) { - logger.warn('Failed to read image metadata for VFS read', { - mediaType, - error: toError(err).message, - }) + const readMetadata = (candidate: Buffer) => + sharpModule(candidate, { limitInputPixels: false }) + .metadata() + .catch((err: unknown) => { + logger.warn('Failed to read image metadata for VFS read', { + mediaType: detectedType, + error: toError(err).message, + }) + return null + }) + + /** + * sharp is always tried first — its libvips reads every format we accept + * except HEVC-coded HEIF, and it is roughly an order of magnitude faster + * than the WebAssembly decoder. Only bytes it cannot read at all are worth + * a transcode. This is the same libvips-preferred / libheif-fallback + * layering PhotoPrism uses, and it is deliberately capability-based: + * choosing the decoder from the container brand would send AV1-coded + * `mif1` files down the slow path even though sharp handles them natively. + */ + let buffer = sourceBuffer + let mediaType = detectedType + let metadata = await readMetadata(sourceBuffer) + + if (!metadata && isHeifContainer(sourceBuffer)) { + const transcoded = await transcodeHeicToJpeg(sourceBuffer) + if (transcoded) { + buffer = transcoded + mediaType = HEIC_TRANSCODE_MEDIA_TYPE + metadata = await readMetadata(transcoded) + } + } + + if (!metadata) { span.setAttribute(TraceAttr.CopilotVfsMetadataFailed, true) - const fitsWithoutSharp = buffer.length <= MAX_IMAGE_READ_BYTES + // HEIF that neither decoder could read is genuinely unreadable. Passing + // those bytes through would hand the model a format it cannot decode, + // which it then describes as empty rather than reporting as broken. + const passthroughViable = + !isHeifContainer(buffer) && buffer.length <= MAX_IMAGE_READ_BYTES span.setAttribute( TraceAttr.CopilotVfsOutcome, - fitsWithoutSharp ? 'passthrough_no_metadata' : 'rejected_no_metadata' + passthroughViable ? 'passthrough_no_metadata' : 'rejected_no_metadata' ) - return fitsWithoutSharp ? { buffer, mediaType, resized: false } : null + return passthroughViable ? { buffer, mediaType, resized: false } : null } const width = metadata.width ?? 0 @@ -300,14 +337,17 @@ export async function readFileRecord(record: WorkspaceFileRecord): Promise { try { - if (isImageFileType(record.type)) { + // Resolve against the filename: a phone upload commonly stores as + // `application/octet-stream`, and matching the raw type would route a real + // image down the binary path where the model never sees it. + if (isImageFileType(resolveEffectiveMimeType(record.type, record.name))) { span.setAttribute(TraceAttr.CopilotVfsReadPath, CopilotVfsReadPath.Image) const originalBuffer = await fetchWorkspaceFileBuffer(record) const prepared = await prepareImageForVision(originalBuffer, record.type) if (!prepared) { span.setAttribute(TraceAttr.CopilotVfsReadOutcome, CopilotVfsReadOutcome.ImageTooLarge) return { - content: `[Image too large: ${record.name} (${(record.size / 1024 / 1024).toFixed(1)}MB, limit 5MB after resize/compression)]`, + content: `[Image unavailable: ${record.name} (${(record.size / 1024 / 1024).toFixed(1)}MB). It could not be decoded, or still exceeded the 5MB vision limit after resizing.]`, totalLines: 1, } } diff --git a/apps/sim/lib/uploads/server/heic.test.ts b/apps/sim/lib/uploads/server/heic.test.ts new file mode 100644 index 00000000000..cf6fbdd5135 --- /dev/null +++ b/apps/sim/lib/uploads/server/heic.test.ts @@ -0,0 +1,55 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { isHeifContainer } from '@/lib/uploads/server/heic' + +/** An ISO-BMFF header: 4-byte box size, the `ftyp` marker, then the major brand. */ +function ftypHeader(brand: string): Buffer { + const header = Buffer.alloc(16) + header.writeUInt32BE(16, 0) + header.write('ftyp', 4, 'ascii') + header.write(brand, 8, 'ascii') + return header +} + +describe('isHeifContainer', () => { + it.each(['heic', 'heix', 'heim', 'heis', 'hevc', 'hevx', 'mif1', 'msf1'])( + 'detects the %s brand', + (brand) => { + expect(isHeifContainer(ftypHeader(brand))).toBe(true) + } + ) + + it.each(['avif', 'avis'])( + 'also claims the %s brand — the question is "is this HEIF", not "which codec"', + (brand) => { + expect(isHeifContainer(ftypHeader(brand))).toBe(true) + } + ) + + it('rejects other image formats', () => { + expect(isHeifContainer(Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0, 0, 0, 0]))).toBe( + false + ) + expect(isHeifContainer(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0, 0, 0, 0, 0, 0, 0, 0]))).toBe( + false + ) + }) + + it('rejects a RIFF container whose brand offset would otherwise collide', () => { + const webp = Buffer.alloc(16) + webp.write('RIFF', 0, 'ascii') + webp.write('WEBP', 8, 'ascii') + expect(isHeifContainer(webp)).toBe(false) + }) + + it('rejects an unknown brand in a well-formed ftyp box', () => { + expect(isHeifContainer(ftypHeader('qt '))).toBe(false) + }) + + it('rejects buffers too short to carry a brand', () => { + expect(isHeifContainer(Buffer.alloc(0))).toBe(false) + expect(isHeifContainer(ftypHeader('heic').subarray(0, 11))).toBe(false) + }) +}) diff --git a/apps/sim/lib/uploads/server/heic.ts b/apps/sim/lib/uploads/server/heic.ts new file mode 100644 index 00000000000..7dcbb764729 --- /dev/null +++ b/apps/sim/lib/uploads/server/heic.ts @@ -0,0 +1,72 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' + +const logger = createLogger('HeicTranscode') + +/** + * ISO-BMFF major brands in the HEIF family. The brand occupies bytes 8-11, + * immediately after the `ftyp` box marker at 4-7. + * + * The list is deliberately broad, `avif` included. It answers "are these bytes + * worth handing to a HEIF decoder", not "which codec is inside" — the brand cannot + * answer the latter anyway, since `mif1` is generic and carries either HEVC or AV1. + * Callers reach this only after a faster decoder has already failed. + */ +const HEIF_BRANDS = new Set([ + 'heic', + 'heix', + 'heim', + 'heis', + 'hevc', + 'hevx', + 'mif1', + 'msf1', + 'avif', + 'avis', +]) + +/** JPEG quality for the transcode, on heic-convert's 0-1 scale. */ +const TRANSCODE_QUALITY = 0.92 + +export const HEIC_TRANSCODE_MEDIA_TYPE = 'image/jpeg' + +/** + * Whether these bytes are an ISO-BMFF container in the HEIF family. + * + * Sniffed rather than read off the declared type because the common case is a + * `.heic` stored as `application/octet-stream`, where the declared type says + * nothing at all. + */ +export function isHeifContainer(buffer: Buffer): boolean { + if (buffer.length < 12) return false + if (buffer.toString('ascii', 4, 8) !== 'ftyp') return false + return HEIF_BRANDS.has(buffer.toString('ascii', 8, 12)) +} + +/** + * Transcode a HEVC-coded HEIF still to JPEG. + * + * Needed at two levels, neither of which has a workaround: no vision model accepts + * HEIC (the Claude Messages API takes JPEG, PNG, GIF, and WebP only), and sharp's + * prebuilt libvips ships libheif with AV1 support but not HEVC, so it decodes AVIF + * and rejects an iPhone photo. `heic-convert` wraps a WebAssembly build of libheif, + * which also keeps a historically CVE-prone parser inside the WASM sandbox rather + * than in-process. + * + * Returns `null` when the bytes cannot be decoded — a corrupt or truncated upload + * must degrade to "unreadable", never to a partial image the model would describe + * with false confidence. + */ +export async function transcodeHeicToJpeg(buffer: Buffer): Promise { + try { + const convert = (await import('heic-convert')).default + const jpeg = await convert({ buffer, format: 'JPEG', quality: TRANSCODE_QUALITY }) + return Buffer.from(jpeg) + } catch (error) { + logger.warn('Failed to transcode HEIC image', { + bytes: buffer.length, + error: getErrorMessage(error), + }) + return null + } +} diff --git a/apps/sim/package.json b/apps/sim/package.json index 01233e2f11c..83fb0170392 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -168,6 +168,7 @@ "google-auth-library": "10.5.0", "gray-matter": "^4.0.3", "groq-sdk": "^0.15.0", + "heic-convert": "2.1.0", "html-to-text": "^9.0.5", "http-proxy-agent": "7.0.2", "https-proxy-agent": "7.0.6", @@ -248,6 +249,7 @@ "@types/archiver": "8.0.0", "@types/busboy": "1.5.4", "@types/fluent-ffmpeg": "2.1.28", + "@types/heic-convert": "2.1.1", "@types/html-to-text": "9.0.4", "@types/js-yaml": "4.0.9", "@types/jsdom": "21.1.7", diff --git a/bun.lock b/bun.lock index f464866fee9..0132ebcfa12 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "simstudio", @@ -269,6 +270,7 @@ "google-auth-library": "10.5.0", "gray-matter": "^4.0.3", "groq-sdk": "^0.15.0", + "heic-convert": "2.1.0", "html-to-text": "^9.0.5", "http-proxy-agent": "7.0.2", "https-proxy-agent": "7.0.6", @@ -349,6 +351,7 @@ "@types/archiver": "8.0.0", "@types/busboy": "1.5.4", "@types/fluent-ffmpeg": "2.1.28", + "@types/heic-convert": "2.1.1", "@types/html-to-text": "9.0.4", "@types/js-yaml": "4.0.9", "@types/jsdom": "21.1.7", @@ -2120,6 +2123,8 @@ "@types/hast": ["@types/hast@3.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ=="], + "@types/heic-convert": ["@types/heic-convert@2.1.1", "", {}, "sha512-+s14762Nf62z9zziIs7ItvAkSUCS3ls4Z5XPT9BlVve3Q3S4DAnf6qekffMTvEWycSUF+kulkGaSN65fK6eKvg=="], + "@types/html-to-text": ["@types/html-to-text@9.0.4", "", {}, "sha512-pUY3cKH/Nm2yYrEmDlPR1mR7yszjGx4DrwPjQ702C4/D5CwHuZTgZdIdwPkRbcuhs7BAh2L5rg3CL5cbRiGTCQ=="], "@types/http-cache-semantics": ["@types/http-cache-semantics@4.2.0", "", {}, "sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q=="], @@ -3090,6 +3095,10 @@ "hastscript": ["hastscript@9.0.1", "", { "dependencies": { "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-parse-selector": "^4.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0" } }, "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w=="], + "heic-convert": ["heic-convert@2.1.0", "", { "dependencies": { "heic-decode": "^2.0.0", "jpeg-js": "^0.4.4", "pngjs": "^6.0.0" } }, "sha512-1qDuRvEHifTVAj3pFIgkqGgJIr0M3X7cxEPjEp0oG4mo8GFjq99DpCo8Eg3kg17Cy0MTjxpFdoBHOatj7ZVKtg=="], + + "heic-decode": ["heic-decode@2.1.0", "", { "dependencies": { "libheif-js": "^1.19.8" } }, "sha512-0fB3O3WMk38+PScbHLVp66jcNhsZ/ErtQ6u2lMYu/YxXgbBtl+oKOhGQHa4RpvE68k8IzbWkABzHnyAIjR758A=="], + "help-me": ["help-me@5.0.0", "", {}, "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="], "hex-rgb": ["hex-rgb@4.3.0", "", {}, "sha512-Ox1pJVrDCyGHMG9CFg1tmrRUMRPRsAWYc/PinY0XzJU4K7y7vjNoLKIQ7BR5UJMCxNN8EM1MNDmHWA/B3aZUuw=="], @@ -3244,6 +3253,8 @@ "joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="], + "jpeg-js": ["jpeg-js@0.4.4", "", {}, "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg=="], + "js-tiktoken": ["js-tiktoken@1.0.21", "", { "dependencies": { "base64-js": "^1.5.1" } }, "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g=="], "js-tokens": ["js-tokens@10.0.0", "", {}, "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q=="], @@ -3308,6 +3319,8 @@ "libbase64": ["libbase64@1.3.0", "", {}, "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg=="], + "libheif-js": ["libheif-js@1.19.8", "", {}, "sha512-vQJWusIxO7wavpON1dusciL8Go9jsIQ+EUrckauFYAiSTjcmLAsuJh3SszLpvkwPci3JcL41ek2n+LUZGFpPIQ=="], + "libmime": ["libmime@5.3.7", "", { "dependencies": { "encoding-japanese": "2.2.0", "iconv-lite": "0.6.3", "libbase64": "1.3.0", "libqp": "2.1.1" } }, "sha512-FlDb3Wtha8P01kTL3P9M+ZDNDWPKPmKHWaU/cG/lg5pfuAwdflVpZE+wm9m7pKmC5ww6s+zTxBKS1p6yl3KpSw=="], "libqp": ["libqp@2.1.1", "", {}, "sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow=="], @@ -3800,6 +3813,8 @@ "plist": ["plist@3.1.0", "", { "dependencies": { "@xmldom/xmldom": "^0.8.8", "base64-js": "^1.5.1", "xmlbuilder": "^15.1.1" } }, "sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ=="], + "pngjs": ["pngjs@6.0.0", "", {}, "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg=="], + "points-on-curve": ["points-on-curve@0.2.0", "", {}, "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A=="], "points-on-path": ["points-on-path@0.2.1", "", { "dependencies": { "path-data-parser": "0.1.0", "points-on-curve": "0.2.0" } }, "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g=="], From 618843801c357fd38a7dc17f370083fc1c16d820 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 6 Aug 2026 15:40:39 -0700 Subject: [PATCH 2/3] refactor(files): gate every vision passthrough on model-supported media types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found two passthroughs that still handed the model bytes it cannot decode. The sharp-load-failure branch returned raw HEIF, and the already-small-enough branch returned raw AVIF, TIFF, BMP or ICO — all of which isImageFileType accepts and no vision model does. Gating all three on the existing MODEL_SUPPORTED_IMAGE_MIME_TYPES subsumes the ad-hoc isHeifContainer re-sniff, and re-encoding an unsupported format falls out of the resize ladder that was already there. Also drop two constants that were pure indirection (a one-use alias for 'image/jpeg', and a quality value identical to heic-convert's default), trim the oversized comments, log successful transcodes so the ratio is visible in prod, and replace a detection test that could not fail. --- apps/sim/lib/copilot/vfs/file-reader.ts | 44 ++++++++++++------------ apps/sim/lib/uploads/server/heic.test.ts | 20 +++++++---- apps/sim/lib/uploads/server/heic.ts | 26 ++++++-------- 3 files changed, 46 insertions(+), 44 deletions(-) diff --git a/apps/sim/lib/copilot/vfs/file-reader.ts b/apps/sim/lib/copilot/vfs/file-reader.ts index 36c09de1f8f..05283aedca0 100644 --- a/apps/sim/lib/copilot/vfs/file-reader.ts +++ b/apps/sim/lib/copilot/vfs/file-reader.ts @@ -14,12 +14,12 @@ import { recordFileRead } from '@/lib/copilot/request/metrics' import { markSpanForError } from '@/lib/copilot/request/otel' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic' import { - HEIC_TRANSCODE_MEDIA_TYPE, - isHeifContainer, - transcodeHeicToJpeg, -} from '@/lib/uploads/server/heic' -import { isImageFileType, resolveEffectiveMimeType } from '@/lib/uploads/utils/file-utils' + isImageFileType, + MODEL_SUPPORTED_IMAGE_MIME_TYPES, + resolveEffectiveMimeType, +} from '@/lib/uploads/utils/file-utils' // Lazy tracer (same pattern as lib/copilot/request/otel.ts). function getVfsTracer() { @@ -121,7 +121,9 @@ async function prepareImageForVision( error: toError(err).message, }) span.setAttribute(TraceAttr.CopilotVfsSharpLoadFailed, true) - const fitsWithoutSharp = sourceBuffer.length <= MAX_IMAGE_READ_BYTES + const fitsWithoutSharp = + MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(detectedType) && + sourceBuffer.length <= MAX_IMAGE_READ_BYTES span.setAttribute( TraceAttr.CopilotVfsOutcome, fitsWithoutSharp ? 'passthrough_no_sharp' : 'rejected_no_sharp' @@ -142,15 +144,10 @@ async function prepareImageForVision( return null }) - /** - * sharp is always tried first — its libvips reads every format we accept - * except HEVC-coded HEIF, and it is roughly an order of magnitude faster - * than the WebAssembly decoder. Only bytes it cannot read at all are worth - * a transcode. This is the same libvips-preferred / libheif-fallback - * layering PhotoPrism uses, and it is deliberately capability-based: - * choosing the decoder from the container brand would send AV1-coded - * `mif1` files down the slow path even though sharp handles them natively. - */ + // sharp first: its libvips reads everything we accept except HEVC-coded + // HEIF, and it is ~10x faster than the WASM decoder. Capability-based + // rather than brand-based, so AV1-coded `mif1` — which sharp handles + // natively — does not get sent down the slow path. let buffer = sourceBuffer let mediaType = detectedType let metadata = await readMetadata(sourceBuffer) @@ -159,18 +156,17 @@ async function prepareImageForVision( const transcoded = await transcodeHeicToJpeg(sourceBuffer) if (transcoded) { buffer = transcoded - mediaType = HEIC_TRANSCODE_MEDIA_TYPE + mediaType = 'image/jpeg' metadata = await readMetadata(transcoded) } } if (!metadata) { span.setAttribute(TraceAttr.CopilotVfsMetadataFailed, true) - // HEIF that neither decoder could read is genuinely unreadable. Passing - // those bytes through would hand the model a format it cannot decode, - // which it then describes as empty rather than reporting as broken. + // Bytes the model cannot decode are worse than no image: it describes + // them as empty rather than reporting them as broken. const passthroughViable = - !isHeifContainer(buffer) && buffer.length <= MAX_IMAGE_READ_BYTES + MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mediaType) && buffer.length <= MAX_IMAGE_READ_BYTES span.setAttribute( TraceAttr.CopilotVfsOutcome, passthroughViable ? 'passthrough_no_metadata' : 'rejected_no_metadata' @@ -185,11 +181,15 @@ async function prepareImageForVision( [TraceAttr.CopilotVfsInputHeight]: height, }) - const needsResize = + // A format the model cannot decode has to be re-encoded even when it is + // already small enough — the ladder below emits JPEG or WebP, both of + // which it accepts. + const needsReencode = + !MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mediaType) || buffer.length > MAX_IMAGE_READ_BYTES || width > MAX_IMAGE_DIMENSION || height > MAX_IMAGE_DIMENSION - if (!needsResize) { + if (!needsReencode) { span.setAttributes({ [TraceAttr.CopilotVfsResized]: false, [TraceAttr.CopilotVfsOutcome]: CopilotVfsOutcome.PassthroughFitsBudget, diff --git a/apps/sim/lib/uploads/server/heic.test.ts b/apps/sim/lib/uploads/server/heic.test.ts index cf6fbdd5135..f966e7a3ab6 100644 --- a/apps/sim/lib/uploads/server/heic.test.ts +++ b/apps/sim/lib/uploads/server/heic.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { isHeifContainer } from '@/lib/uploads/server/heic' +import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic' /** An ISO-BMFF header: 4-byte box size, the `ftyp` marker, then the major brand. */ function ftypHeader(brand: string): Buffer { @@ -37,11 +37,11 @@ describe('isHeifContainer', () => { ) }) - it('rejects a RIFF container whose brand offset would otherwise collide', () => { - const webp = Buffer.alloc(16) - webp.write('RIFF', 0, 'ascii') - webp.write('WEBP', 8, 'ascii') - expect(isHeifContainer(webp)).toBe(false) + it('rejects a HEIF brand that is not behind an ftyp box', () => { + const riff = Buffer.alloc(16) + riff.write('RIFF', 0, 'ascii') + riff.write('heic', 8, 'ascii') + expect(isHeifContainer(riff)).toBe(false) }) it('rejects an unknown brand in a well-formed ftyp box', () => { @@ -53,3 +53,11 @@ describe('isHeifContainer', () => { expect(isHeifContainer(ftypHeader('heic').subarray(0, 11))).toBe(false) }) }) + +describe('transcodeHeicToJpeg', () => { + it('returns null for bytes libheif cannot decode', async () => { + // Also proves the dynamic `heic-convert` import resolves at runtime, which no + // amount of type-checking establishes for a lazily loaded WebAssembly module. + expect(await transcodeHeicToJpeg(ftypHeader('heic'))).toBeNull() + }) +}) diff --git a/apps/sim/lib/uploads/server/heic.ts b/apps/sim/lib/uploads/server/heic.ts index 7dcbb764729..4b971aa5770 100644 --- a/apps/sim/lib/uploads/server/heic.ts +++ b/apps/sim/lib/uploads/server/heic.ts @@ -10,7 +10,6 @@ const logger = createLogger('HeicTranscode') * The list is deliberately broad, `avif` included. It answers "are these bytes * worth handing to a HEIF decoder", not "which codec is inside" — the brand cannot * answer the latter anyway, since `mif1` is generic and carries either HEVC or AV1. - * Callers reach this only after a faster decoder has already failed. */ const HEIF_BRANDS = new Set([ 'heic', @@ -25,11 +24,6 @@ const HEIF_BRANDS = new Set([ 'avis', ]) -/** JPEG quality for the transcode, on heic-convert's 0-1 scale. */ -const TRANSCODE_QUALITY = 0.92 - -export const HEIC_TRANSCODE_MEDIA_TYPE = 'image/jpeg' - /** * Whether these bytes are an ISO-BMFF container in the HEIF family. * @@ -46,25 +40,25 @@ export function isHeifContainer(buffer: Buffer): boolean { /** * Transcode a HEVC-coded HEIF still to JPEG. * - * Needed at two levels, neither of which has a workaround: no vision model accepts - * HEIC (the Claude Messages API takes JPEG, PNG, GIF, and WebP only), and sharp's - * prebuilt libvips ships libheif with AV1 support but not HEVC, so it decodes AVIF - * and rejects an iPhone photo. `heic-convert` wraps a WebAssembly build of libheif, - * which also keeps a historically CVE-prone parser inside the WASM sandbox rather - * than in-process. + * Two reasons, neither with a workaround: no vision model accepts HEIC (the Claude + * Messages API takes JPEG, PNG, GIF, and WebP only), and sharp's prebuilt libvips + * ships libheif with AV1 but not HEVC — it decodes AVIF and rejects an iPhone photo. * - * Returns `null` when the bytes cannot be decoded — a corrupt or truncated upload - * must degrade to "unreadable", never to a partial image the model would describe - * with false confidence. + * Returns `null` when the bytes cannot be decoded; never a partial image. */ export async function transcodeHeicToJpeg(buffer: Buffer): Promise { try { const convert = (await import('heic-convert')).default - const jpeg = await convert({ buffer, format: 'JPEG', quality: TRANSCODE_QUALITY }) + const jpeg = await convert({ buffer, format: 'JPEG' }) + logger.info('Transcoded HEIC image', { + inputBytes: buffer.length, + outputBytes: jpeg.length, + }) return Buffer.from(jpeg) } catch (error) { logger.warn('Failed to transcode HEIC image', { bytes: buffer.length, + brand: buffer.toString('ascii', 8, 12), error: getErrorMessage(error), }) return null From d0889a898d4d58dbfaebb1731b7120d6fca44c42 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 6 Aug 2026 15:46:50 -0700 Subject: [PATCH 3/3] fix(files): read HEIF compatible brands, not just the major brand A standards-valid HEIF may carry a generic major brand such as isom and declare heic, heix or mif1 only among the compatible brands that follow the minor_version at offset 12. Reading bytes 8-11 alone classified those as non-HEIF, skipping the fallback decode and leaving a small undecodable file to reach the model as raw bytes. --- apps/sim/lib/uploads/server/heic.test.ts | 29 ++++++++++++++++++++---- apps/sim/lib/uploads/server/heic.ts | 13 ++++++++++- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/uploads/server/heic.test.ts b/apps/sim/lib/uploads/server/heic.test.ts index f966e7a3ab6..3619dfd1dfa 100644 --- a/apps/sim/lib/uploads/server/heic.test.ts +++ b/apps/sim/lib/uploads/server/heic.test.ts @@ -4,12 +4,17 @@ import { describe, expect, it } from 'vitest' import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic' -/** An ISO-BMFF header: 4-byte box size, the `ftyp` marker, then the major brand. */ -function ftypHeader(brand: string): Buffer { - const header = Buffer.alloc(16) - header.writeUInt32BE(16, 0) +/** + * An ISO-BMFF `ftyp` box: 4-byte size, the `ftyp` marker, the major brand, a + * 4-byte minor version, then any compatible brands. + */ +function ftypHeader(brand: string, compatible: string[] = []): Buffer { + const size = 16 + compatible.length * 4 + const header = Buffer.alloc(size) + header.writeUInt32BE(size, 0) header.write('ftyp', 4, 'ascii') header.write(brand, 8, 'ascii') + compatible.forEach((entry, index) => header.write(entry, 16 + index * 4, 'ascii')) return header } @@ -48,6 +53,22 @@ describe('isHeifContainer', () => { expect(isHeifContainer(ftypHeader('qt '))).toBe(false) }) + it('detects a HEIF brand declared only among the compatible brands', () => { + // Standards-valid: a generic major brand with the HEIF brand listed after it. + expect(isHeifContainer(ftypHeader('isom', ['iso2', 'heic', 'mif1']))).toBe(true) + expect(isHeifContainer(ftypHeader('mp42', ['heix']))).toBe(true) + }) + + it('rejects a box whose compatible brands are all non-HEIF', () => { + expect(isHeifContainer(ftypHeader('isom', ['iso2', 'mp41', 'mp42']))).toBe(false) + }) + + it('does not read compatible brands past the declared box size', () => { + const truncated = ftypHeader('isom', ['heic']) + truncated.writeUInt32BE(16, 0) + expect(isHeifContainer(truncated)).toBe(false) + }) + it('rejects buffers too short to carry a brand', () => { expect(isHeifContainer(Buffer.alloc(0))).toBe(false) expect(isHeifContainer(ftypHeader('heic').subarray(0, 11))).toBe(false) diff --git a/apps/sim/lib/uploads/server/heic.ts b/apps/sim/lib/uploads/server/heic.ts index 4b971aa5770..9ff537d284a 100644 --- a/apps/sim/lib/uploads/server/heic.ts +++ b/apps/sim/lib/uploads/server/heic.ts @@ -34,7 +34,18 @@ const HEIF_BRANDS = new Set([ export function isHeifContainer(buffer: Buffer): boolean { if (buffer.length < 12) return false if (buffer.toString('ascii', 4, 8) !== 'ftyp') return false - return HEIF_BRANDS.has(buffer.toString('ascii', 8, 12)) + if (HEIF_BRANDS.has(buffer.toString('ascii', 8, 12))) return true + + // A standards-valid HEIF may carry a generic major brand such as `isom` and name + // the HEIF brand only among the compatible brands, which follow the 4-byte + // minor_version at offset 12 and run to the end of the box. A declared size of 0 + // or 1 (the ISO-BMFF size escapes, which `ftyp` does not use) leaves `end` below + // the loop's start, so those simply do not scan. + const end = Math.min(buffer.readUInt32BE(0), buffer.length) + for (let offset = 16; offset + 4 <= end; offset += 4) { + if (HEIF_BRANDS.has(buffer.toString('ascii', offset, offset + 4))) return true + } + return false } /**