refactor(files): gate every vision passthrough on model-supported media types

Review found two passthroughs that still handed the model bytes it cannot
decode. The sharp-load-failure branch returned raw HEIF, and the
already-small-enough branch returned raw AVIF, TIFF, BMP or ICO — all of
which isImageFileType accepts and no vision model does.

Gating all three on the existing MODEL_SUPPORTED_IMAGE_MIME_TYPES subsumes
the ad-hoc isHeifContainer re-sniff, and re-encoding an unsupported format
falls out of the resize ladder that was already there.

Also drop two constants that were pure indirection (a one-use alias for
'image/jpeg', and a quality value identical to heic-convert's default), trim
the oversized comments, log successful transcodes so the ratio is visible in
prod, and replace a detection test that could not fail.

Author: waleedlatif1

apps/sim/lib/copilot/vfs/file-reader.ts

@@ -14,12 +14,12 @@ import { recordFileRead } from '@/lib/copilot/request/metrics'
 import { markSpanForError } from '@/lib/copilot/request/otel'
 import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
 import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
+import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic'
 import {
-  HEIC_TRANSCODE_MEDIA_TYPE,
-  isHeifContainer,
-  transcodeHeicToJpeg,
-} from '@/lib/uploads/server/heic'
-import { isImageFileType, resolveEffectiveMimeType } from '@/lib/uploads/utils/file-utils'
+  isImageFileType,
+  MODEL_SUPPORTED_IMAGE_MIME_TYPES,
+  resolveEffectiveMimeType,
+} from '@/lib/uploads/utils/file-utils'
 
 // Lazy tracer (same pattern as lib/copilot/request/otel.ts).
 function getVfsTracer() {
@@ -121,7 +121,9 @@ async function prepareImageForVision(
             error: toError(err).message,
           })
           span.setAttribute(TraceAttr.CopilotVfsSharpLoadFailed, true)
-          const fitsWithoutSharp = sourceBuffer.length <= MAX_IMAGE_READ_BYTES
+          const fitsWithoutSharp =
+            MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(detectedType) &&
+            sourceBuffer.length <= MAX_IMAGE_READ_BYTES
           span.setAttribute(
             TraceAttr.CopilotVfsOutcome,
             fitsWithoutSharp ? 'passthrough_no_sharp' : 'rejected_no_sharp'
@@ -142,15 +144,10 @@ async function prepareImageForVision(
               return null
             })
 
-        /**
-         * sharp is always tried first — its libvips reads every format we accept
-         * except HEVC-coded HEIF, and it is roughly an order of magnitude faster
-         * than the WebAssembly decoder. Only bytes it cannot read at all are worth
-         * a transcode. This is the same libvips-preferred / libheif-fallback
-         * layering PhotoPrism uses, and it is deliberately capability-based:
-         * choosing the decoder from the container brand would send AV1-coded
-         * `mif1` files down the slow path even though sharp handles them natively.
-         */
+        // sharp first: its libvips reads everything we accept except HEVC-coded
+        // HEIF, and it is ~10x faster than the WASM decoder. Capability-based
+        // rather than brand-based, so AV1-coded `mif1` — which sharp handles
+        // natively — does not get sent down the slow path.
         let buffer = sourceBuffer
         let mediaType = detectedType
         let metadata = await readMetadata(sourceBuffer)
@@ -159,18 +156,17 @@ async function prepareImageForVision(
           const transcoded = await transcodeHeicToJpeg(sourceBuffer)
           if (transcoded) {
             buffer = transcoded
-            mediaType = HEIC_TRANSCODE_MEDIA_TYPE
+            mediaType = 'image/jpeg'
             metadata = await readMetadata(transcoded)
           }
         }
 
         if (!metadata) {
           span.setAttribute(TraceAttr.CopilotVfsMetadataFailed, true)
-          // HEIF that neither decoder could read is genuinely unreadable. Passing
-          // those bytes through would hand the model a format it cannot decode,
-          // which it then describes as empty rather than reporting as broken.
+          // Bytes the model cannot decode are worse than no image: it describes
+          // them as empty rather than reporting them as broken.
           const passthroughViable =
-            !isHeifContainer(buffer) && buffer.length <= MAX_IMAGE_READ_BYTES
+            MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mediaType) && buffer.length <= MAX_IMAGE_READ_BYTES
           span.setAttribute(
             TraceAttr.CopilotVfsOutcome,
             passthroughViable ? 'passthrough_no_metadata' : 'rejected_no_metadata'
@@ -185,11 +181,15 @@ async function prepareImageForVision(
           [TraceAttr.CopilotVfsInputHeight]: height,
         })
 
-        const needsResize =
+        // A format the model cannot decode has to be re-encoded even when it is
+        // already small enough — the ladder below emits JPEG or WebP, both of
+        // which it accepts.
+        const needsReencode =
+          !MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mediaType) ||
           buffer.length > MAX_IMAGE_READ_BYTES ||
           width > MAX_IMAGE_DIMENSION ||
           height > MAX_IMAGE_DIMENSION
-        if (!needsResize) {
+        if (!needsReencode) {
           span.setAttributes({
             [TraceAttr.CopilotVfsResized]: false,
             [TraceAttr.CopilotVfsOutcome]: CopilotVfsOutcome.PassthroughFitsBudget,

apps/sim/lib/uploads/server/heic.test.ts

@@ -2,7 +2,7 @@
  * @vitest-environment node
  */
 import { describe, expect, it } from 'vitest'
-import { isHeifContainer } from '@/lib/uploads/server/heic'
+import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic'
 
 /** An ISO-BMFF header: 4-byte box size, the `ftyp` marker, then the major brand. */
 function ftypHeader(brand: string): Buffer {
@@ -37,11 +37,11 @@ describe('isHeifContainer', () => {
     )
   })
 
-  it('rejects a RIFF container whose brand offset would otherwise collide', () => {
-    const webp = Buffer.alloc(16)
-    webp.write('RIFF', 0, 'ascii')
-    webp.write('WEBP', 8, 'ascii')
-    expect(isHeifContainer(webp)).toBe(false)
+  it('rejects a HEIF brand that is not behind an ftyp box', () => {
+    const riff = Buffer.alloc(16)
+    riff.write('RIFF', 0, 'ascii')
+    riff.write('heic', 8, 'ascii')
+    expect(isHeifContainer(riff)).toBe(false)
   })
 
   it('rejects an unknown brand in a well-formed ftyp box', () => {
@@ -53,3 +53,11 @@ describe('isHeifContainer', () => {
     expect(isHeifContainer(ftypHeader('heic').subarray(0, 11))).toBe(false)
   })
 })
+
+describe('transcodeHeicToJpeg', () => {
+  it('returns null for bytes libheif cannot decode', async () => {
+    // Also proves the dynamic `heic-convert` import resolves at runtime, which no
+    // amount of type-checking establishes for a lazily loaded WebAssembly module.
+    expect(await transcodeHeicToJpeg(ftypHeader('heic'))).toBeNull()
+  })
+})

apps/sim/lib/uploads/server/heic.ts

@@ -10,7 +10,6 @@ const logger = createLogger('HeicTranscode')
  * The list is deliberately broad, `avif` included. It answers "are these bytes
  * worth handing to a HEIF decoder", not "which codec is inside" — the brand cannot
  * answer the latter anyway, since `mif1` is generic and carries either HEVC or AV1.
- * Callers reach this only after a faster decoder has already failed.
  */
 const HEIF_BRANDS = new Set([
   'heic',
@@ -25,11 +24,6 @@ const HEIF_BRANDS = new Set([
   'avis',
 ])
 
-/** JPEG quality for the transcode, on heic-convert's 0-1 scale. */
-const TRANSCODE_QUALITY = 0.92
-
-export const HEIC_TRANSCODE_MEDIA_TYPE = 'image/jpeg'
-
 /**
  * Whether these bytes are an ISO-BMFF container in the HEIF family.
  *
@@ -46,25 +40,25 @@ export function isHeifContainer(buffer: Buffer): boolean {
 /**
  * Transcode a HEVC-coded HEIF still to JPEG.
  *
- * Needed at two levels, neither of which has a workaround: no vision model accepts
- * HEIC (the Claude Messages API takes JPEG, PNG, GIF, and WebP only), and sharp's
- * prebuilt libvips ships libheif with AV1 support but not HEVC, so it decodes AVIF
- * and rejects an iPhone photo. `heic-convert` wraps a WebAssembly build of libheif,
- * which also keeps a historically CVE-prone parser inside the WASM sandbox rather
- * than in-process.
+ * Two reasons, neither with a workaround: no vision model accepts HEIC (the Claude
+ * Messages API takes JPEG, PNG, GIF, and WebP only), and sharp's prebuilt libvips
+ * ships libheif with AV1 but not HEVC — it decodes AVIF and rejects an iPhone photo.
  *
- * Returns `null` when the bytes cannot be decoded — a corrupt or truncated upload
- * must degrade to "unreadable", never to a partial image the model would describe
- * with false confidence.
+ * Returns `null` when the bytes cannot be decoded; never a partial image.
  */
 export async function transcodeHeicToJpeg(buffer: Buffer): Promise<Buffer | null> {
   try {
     const convert = (await import('heic-convert')).default
-    const jpeg = await convert({ buffer, format: 'JPEG', quality: TRANSCODE_QUALITY })
+    const jpeg = await convert({ buffer, format: 'JPEG' })
+    logger.info('Transcoded HEIC image', {
+      inputBytes: buffer.length,
+      outputBytes: jpeg.length,
+    })
     return Buffer.from(jpeg)
   } catch (error) {
     logger.warn('Failed to transcode HEIC image', {
       bytes: buffer.length,
+      brand: buffer.toString('ascii', 8, 12),
       error: getErrorMessage(error),
     })
     return null