feat(sandboxes): repair a missing image at create, where the truth is observable

Six review rounds narrowed the window between deleting a shared template and
another workspace adopting its content hash, and each fix exposed the next facet.
They all share a cause: the registry row and the provider template are two systems
with no shared transaction, so any scheme that keeps them in step is guessing.

Create is the one step that does not have to guess. It either gets a sandbox or it
does not, so a `ready` row pointing at a deleted template now corrects itself the
first time it is used, rather than needing someone to re-save the sandbox.

- `SandboxImageBuilder.isMissingImage` asks the provider to classify its own
  failure. Prebuilt-only, because a runtime provider has no image to miss
- E2B answers it off `NotFoundError`, which the SDK maps from a 404. The only
  resource a create names is the template, and the two subclasses that describe
  other calls — a missing file, an exited sandbox — are excluded. The classifier
  stays deliberately narrow: treating auth or rate-limit failures as a missing
  image would turn a provider outage into a build storm
- `repairMissingSandboxImage` invalidates the cache, rebuilds with
  `imageKnownGone` (no cooldown, since this observed the image is gone rather than
  inferring it), and returns copy telling the author to run again
- `ResolvedSandbox` carries `specHash` so the failing execution can name what to
  rebuild

This subsumes the open facets rather than adding another guard beside them: the
stale per-replica cache, an adopter left `ready` against a deleted ref, and a
rebuild that never took all end at the same place — the next run repairs itself.

Co-Authored-By: Claude <noreply@anthropic.com>

Author: icecrasher321

apps/sim/lib/execution/remote-sandbox/e2b.ts

@@ -256,6 +256,27 @@ const e2bImages: SandboxImageBuilder = {
    * endpoint directly. A non-2xx throws so the caller leaves the registry row in
    * place and retries, rather than orphaning the remote template.
    */
+  /**
+   * E2B maps a 404 from the control plane to `NotFoundError`, and the only resource
+   * a create request names is the template — so a 404 there means the ref is gone.
+   *
+   * Its two subclasses are excluded because they describe other calls entirely: a
+   * missing file inside a running sandbox, or a sandbox that has already exited.
+   * Neither is reachable from a create. Everything else — auth, rate limit,
+   * transport — is deliberately not a missing image, since rebuilding on those
+   * would turn a provider outage into a build storm.
+   */
+  async isMissingImage(error: unknown): Promise<boolean> {
+    const { FileNotFoundError, NotFoundError, SandboxNotFoundError } = await import(
+      '@e2b/code-interpreter'
+    )
+    return (
+      error instanceof NotFoundError &&
+      !(error instanceof SandboxNotFoundError) &&
+      !(error instanceof FileNotFoundError)
+    )
+  },
+
   async deleteImage(build: SandboxImageBuild): Promise<void> {
     const apiKey = env.E2B_API_KEY
     if (!apiKey) {

apps/sim/lib/execution/remote-sandbox/index.ts

@@ -6,6 +6,7 @@ import {
   provisionRuntimeDependencies,
   type ResolvedSandbox,
   RUNTIME_INSTALL_TIMEOUT_MS,
+  repairMissingSandboxImage,
   resolveWorkspaceSandbox,
 } from '@/lib/execution/remote-sandbox/resolve'
 import type {
@@ -38,6 +39,31 @@ async function createSandbox(
   return sandbox
 }
 
+/**
+ * Creates a sandbox, turning "that image is gone" into a rebuild rather than a
+ * failure the author has to resolve by hand.
+ *
+ * Create is the only step that observes whether the provider image really exists,
+ * which is why the repair hangs off it: the registry row and the remote template
+ * are two systems with no shared transaction, so keeping them in step is always
+ * best-effort, while checking at the point of use is not. Any other failure is
+ * rethrown untouched.
+ */
+async function createSelectedSandbox(
+  kind: SandboxKind,
+  options: CreateSandboxOptions,
+  selected: ResolvedSandbox | null
+): Promise<SandboxHandle> {
+  try {
+    return await createSandbox(kind, options)
+  } catch (error) {
+    if (!selected) throw error
+    const rebuilding = await repairMissingSandboxImage(selected, error)
+    if (!rebuilding) throw error
+    throw new Error(rebuilding)
+  }
+}
+
 /**
  * Materializes sandbox input files before user code runs. `content` entries are written inline;
  * `url` entries are fetched from inside the sandbox via `curl` — their bytes never pass through the
@@ -288,7 +314,11 @@ export async function executeInSandbox(
     sandboxId: req.sandboxId,
   })
 
-  const sandbox = await createSandbox(kind, { language, imageRef: selected?.imageRef })
+  const sandbox = await createSelectedSandbox(
+    kind,
+    { language, imageRef: selected?.imageRef },
+    selected
+  )
   const sandboxId = sandbox.sandboxId
 
   try {
@@ -377,7 +407,7 @@ export async function executeShellInSandbox(
     sandboxId: req.sandboxId,
   })
 
-  const sandbox = await createSandbox(kind, { imageRef: selected?.imageRef })
+  const sandbox = await createSelectedSandbox(kind, { imageRef: selected?.imageRef }, selected)
   const sandboxId = sandbox.sandboxId
 
   try {

apps/sim/lib/execution/remote-sandbox/resolve.test.ts

@@ -8,12 +8,14 @@
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import { CodeLanguage } from '@/lib/execution/languages'
 
-const { mockSelect, mockUpdate, mockProviderStrategy, mockEnsureSandboxImage } = vi.hoisted(() => ({
-  mockSelect: vi.fn(),
-  mockUpdate: vi.fn(),
-  mockProviderStrategy: { current: 'prebuilt' as 'prebuilt' | 'runtime' },
-  mockEnsureSandboxImage: vi.fn(),
-}))
+const { mockSelect, mockUpdate, mockProviderStrategy, mockEnsureSandboxImage, mockIsMissingImage } =
+  vi.hoisted(() => ({
+    mockSelect: vi.fn(),
+    mockUpdate: vi.fn(),
+    mockProviderStrategy: { current: 'prebuilt' as 'prebuilt' | 'runtime' },
+    mockEnsureSandboxImage: vi.fn(),
+    mockIsMissingImage: vi.fn(),
+  }))
 
 vi.mock('@/lib/execution/remote-sandbox/image-registry', () => ({
   ensureSandboxImage: mockEnsureSandboxImage,
@@ -57,12 +59,18 @@ vi.mock('@/lib/execution/remote-sandbox/provider', () => ({
     get dependencyStrategy() {
       return mockProviderStrategy.current
     },
+    get images() {
+      return mockProviderStrategy.current === 'prebuilt'
+        ? { isMissingImage: mockIsMissingImage }
+        : undefined
+    },
   }),
 }))
 
 import {
   invalidateSandboxResolution,
   provisionRuntimeDependencies,
+  repairMissingSandboxImage,
   resolveWorkspaceSandbox,
 } from '@/lib/execution/remote-sandbox/resolve'
 
@@ -368,3 +376,58 @@ describe('provisionRuntimeDependencies', () => {
     expect(options.timeoutMs).toBeGreaterThan(0)
   })
 })
+
+/**
+ * The registry and the provider template are two systems with no shared
+ * transaction, so every attempt to keep them in step leaves some window. Create is
+ * the one step that observes the truth, which is why the repair hangs off it.
+ */
+describe('repairMissingSandboxImage', () => {
+  const SELECTED = {
+    id: 'sbx-1',
+    name: 'bigquery-etl',
+    language: CodeLanguage.Python,
+    dependencies: ['pandas'],
+    specHash: 'hash-1',
+    strategy: 'prebuilt' as const,
+    imageRef: 'sim-sbx-abc',
+  }
+
+  it('rebuilds without a cooldown, because create observed the image is gone', async () => {
+    mockIsMissingImage.mockResolvedValue(true)
+
+    const message = await repairMissingSandboxImage(SELECTED, new Error('404'))
+
+    expect(message).toMatch(/being rebuilt/)
+    expect(mockEnsureSandboxImage).toHaveBeenCalledWith(
+      { language: 'python', dependencies: ['pandas'] },
+      'hash-1',
+      { imageKnownGone: true }
+    )
+  })
+
+  /**
+   * The classifier has to stay narrow: treating an auth or rate-limit failure as a
+   * missing image would rebuild every sandbox on a provider outage.
+   */
+  it('leaves any other provider failure alone', async () => {
+    mockIsMissingImage.mockResolvedValue(false)
+
+    const message = await repairMissingSandboxImage(SELECTED, new Error('rate limited'))
+
+    expect(message).toBeNull()
+    expect(mockEnsureSandboxImage).not.toHaveBeenCalled()
+  })
+
+  it('does nothing for a runtime-strategy sandbox, which has no image to miss', async () => {
+    mockIsMissingImage.mockResolvedValue(true)
+
+    const message = await repairMissingSandboxImage(
+      { ...SELECTED, strategy: 'runtime', imageRef: undefined },
+      new Error('404')
+    )
+
+    expect(message).toBeNull()
+    expect(mockEnsureSandboxImage).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/execution/remote-sandbox/resolve.ts

@@ -75,6 +75,8 @@ export interface ResolvedSandbox {
   name: string
   language: SandboxLanguage
   dependencies: string[]
+  /** Content address of the package set, so a failed create can rebuild it. */
+  specHash: string
   strategy: SandboxDependencyStrategy
   /** Provider image to create from. Present under the `prebuilt` strategy. */
   imageRef?: string
@@ -194,6 +196,7 @@ export async function resolveWorkspaceSandbox(args: {
     name: row.name,
     language: row.language,
     dependencies: row.dependencies ?? [],
+    specHash: row.specHash,
     envs: envsFor(row.language),
   }
 
@@ -294,7 +297,8 @@ async function readImage(providerId: string, specHash: string): Promise<CachedIm
  */
 async function scheduleImageRepair(
   spec: { language: SandboxLanguage; dependencies: string[] },
-  specHash: string
+  specHash: string,
+  options?: { imageKnownGone?: boolean }
 ): Promise<void> {
   try {
     const { ensureSandboxImage, FAILED_BUILD_RETRY_COOLDOWN_MS } = await import(
@@ -303,13 +307,52 @@ async function scheduleImageRepair(
     await ensureSandboxImage(
       { language: spec.language, dependencies: spec.dependencies },
       specHash,
-      { minFailureAgeMs: FAILED_BUILD_RETRY_COOLDOWN_MS }
+      // A create that just failed on a missing image has observed the truth, so it
+      // reclaims whatever the row says and skips the cooldown. Resolution reading a
+      // row it cannot verify only gets the rate-limited retry.
+      options?.imageKnownGone
+        ? { imageKnownGone: true }
+        : { minFailureAgeMs: FAILED_BUILD_RETRY_COOLDOWN_MS }
     )
   } catch (error) {
     logger.warn('Failed to schedule sandbox image repair', { specHash, error })
   }
 }
 
+/**
+ * Repairs a sandbox whose image turned out to be gone when the provider was asked
+ * to create from it.
+ *
+ * This is the backstop the registry cannot be: the row and the provider template
+ * are two systems with no shared transaction, so every attempt to keep them in step
+ * leaves some window — a released image adopted mid-delete, a stale cache on another
+ * replica, a rebuild that did not take. Create is the one place that observes ground
+ * truth, so a `ready` row pointing at nothing repairs itself here on first use
+ * instead of needing someone to re-save the sandbox.
+ *
+ * Returns the message to fail this execution with, or `null` when the failure was
+ * anything else and must surface unchanged.
+ */
+export async function repairMissingSandboxImage(
+  selected: ResolvedSandbox,
+  error: unknown
+): Promise<string | null> {
+  if (selected.strategy !== 'prebuilt' || !selected.imageRef) return null
+
+  const provider = resolveProvider()
+  if (!provider.images) return null
+  if (!(await provider.images.isMissingImage(error))) return null
+
+  invalidateSandboxResolution()
+  await scheduleImageRepair(selected, selected.specHash, { imageKnownGone: true })
+  logger.warn('Sandbox image was missing at create; rebuilding it', {
+    sandbox: selected.name,
+    specHash: selected.specHash,
+  })
+
+  return `Sandbox "${selected.name}" is being rebuilt because its image is no longer available. Run again in a moment.`
+}
+
 function assertLanguageMatches(sandbox: ResolvedSandbox, language?: CodeLanguage): void {
   if (!language || sandbox.language === language) return
   throw new Error(

apps/sim/lib/execution/remote-sandbox/types.ts

@@ -189,6 +189,17 @@ export interface SandboxImageBuilder {
   getBuildStatus(build: SandboxImageBuild, spec: SandboxSpec): Promise<SandboxImageBuildStatus>
   /** Removes a built image from the provider. Used by the retention sweep. */
   deleteImage(build: SandboxImageBuild): Promise<void>
+  /**
+   * Whether a failure from {@link SandboxProvider.create} means the image itself
+   * is gone, rather than anything else that can go wrong reaching the provider.
+   *
+   * Only a `prebuilt` provider can answer this, and it is what makes a dead
+   * `imageRef` self-correcting: the registry and the provider are two systems with
+   * no shared transaction, so instead of trying to keep them in step, the create
+   * that observes the truth reports it. Must stay narrow — treating an auth or
+   * rate-limit failure as a missing image would rebuild on every outage.
+   */
+  isMissingImage(error: unknown): Promise<boolean>
 }
 
 export interface SandboxProvider {