fix(extensions): preserve catalog fingerprint when bundle build fails (swamp-club#265)

design/extension.md

@@ -1173,6 +1173,24 @@ the model registry is wired up initially.
      by-name `loadSingleType`, and the extension-attach predicate)
      retain their existing failure semantics because they are not in
      the read-only steady-state loop.
+   - **Fingerprint preservation on build failure (issue #265).** When
+     `bundleWithCache` cannot regenerate a bundle (bare specifiers without
+     a project `deno.json`, or a transient build error) and falls back to
+     the cached `.js`, the `rebundleAndUpdateCatalog` caller preserves the
+     catalog's _stored_ `source_fingerprint` instead of writing the new
+     one. This keeps the file "stale" so `findStaleFiles` retries on the
+     next warm-start invocation. Without this, the new fingerprint would
+     be written alongside the old bundle content, permanently masking the
+     staleness — `findStaleFiles` would see matching fingerprints and
+     never retry. The warning log fires only on the fallback case
+     (`fromCache && newFingerprint !== catalogFingerprint`), not on
+     legitimate cache hits where the source hasn't changed.
+     `findStaleFiles` uses fingerprint comparison, not RowState, for
+     staleness decisions. `BundleBuildFailed` rows are skipped when
+     fingerprints match (source unchanged) and retried when they mismatch
+     (source changed) — warm-start and reconcile operate on orthogonal
+     axes.
+
    - If not populated (first run or DB deleted): falls back to the existing
      full-import path, then populates the catalog from the loaded registry
 

integration/source_extension_rebundle_test.ts

@@ -0,0 +1,224 @@
+// Swamp, an Automation Framework
+// Copyright (C) 2026 System Initiative, Inc.
+//
+// This file is part of Swamp.
+//
+// Swamp is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License version 3
+// as published by the Free Software Foundation, with the Swamp
+// Extension and Definition Exception (found in the "COPYING-EXCEPTION"
+// file).
+//
+// Swamp is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with Swamp.  If not, see <https://www.gnu.org/licenses/>.
+
+import { assertEquals, assertStringIncludes } from "@std/assert";
+import { join } from "@std/path";
+import { ensureDir } from "@std/fs";
+import { initializeTestRepo, runCliCommand } from "./test_helpers.ts";
+import { stringify as stringifyYaml } from "@std/yaml";
+
+const BUNDLEABLE_V1 = `
+import { z } from "npm:zod@4";
+
+export const model = {
+  type: "@user/source-rebundle-it",
+  version: "2026.01.01.1",
+  methods: {
+    run: {
+      description: "Run",
+      arguments: z.object({}),
+      execute: async () => ({ dataHandles: [], greeting: "V1" }),
+    },
+  },
+};
+`;
+
+const BARE_SPECIFIER_V2 = `
+import { z } from "zod";
+
+export const model = {
+  type: "@user/source-rebundle-it",
+  version: "2026.01.01.2",
+  methods: {
+    run: {
+      description: "Run",
+      arguments: z.object({}),
+      execute: async () => ({ dataHandles: [], greeting: "V2" }),
+    },
+    newMethod: {
+      description: "New method added in V2",
+      arguments: z.object({}),
+      execute: async () => ({ dataHandles: [], result: "NEW" }),
+    },
+  },
+};
+`;
+
+const WARNING_SUBSTRING = "source fingerprint preserved";
+
+/**
+ * Warm-start regression test for issue #265. Verifies that source-mounted
+ * extensions with bare specifiers (triggering isExpectedBundleFailure) don't
+ * permanently poison the catalog fingerprint when the bundle can't be
+ * regenerated.
+ *
+ * Three assertion groups:
+ * (a) Warning-log precision: unchanged source produces no warning.
+ * (b) Permanent-failure determinism: modified source with bare specifier
+ *     preserves catalog fingerprint and emits warning on every warm-start.
+ * (c) Bundle content: old bundle stays on disk, new methods don't appear.
+ */
+Deno.test("Source-mounted extension: fingerprint preserved when bundle build fails (#265)", async () => {
+  const repoDir = await Deno.makeTempDir({
+    prefix: "swamp_it_source_rebundle_",
+  });
+  const extDir = await Deno.makeTempDir({
+    prefix: "swamp_it_source_ext_",
+  });
+  try {
+    await initializeTestRepo(repoDir);
+
+    // Set up a source-mounted extension in a separate directory (no deno.json).
+    const modelsDir = join(extDir, "models");
+    await ensureDir(modelsDir);
+    const sourcePath = join(modelsDir, "source_rebundle_it.ts");
+
+    // Write .swamp-sources.yaml pointing to the external directory.
+    await Deno.writeTextFile(
+      join(repoDir, ".swamp-sources.yaml"),
+      stringifyYaml({
+        sources: [{ path: extDir }],
+      } as Record<string, unknown>),
+    );
+
+    // --- Step 1: Prime with bundleable V1 ---
+    await Deno.writeTextFile(sourcePath, BUNDLEABLE_V1);
+    const prime = await runCliCommand(
+      ["model", "type", "search", "source-rebundle-it", "--json"],
+      repoDir,
+    );
+    assertEquals(
+      prime.code,
+      0,
+      `Prime run failed:\nstdout=${prime.stdout}\nstderr=${prime.stderr}`,
+    );
+    assertStringIncludes(prime.stdout, "@user/source-rebundle-it");
+
+    // Verify V1 bundle was created with V1 content.
+    const bundlePath = await findBundle(repoDir, "source_rebundle_it.js");
+    const v1Bundle = await Deno.readTextFile(bundlePath);
+    assertStringIncludes(v1Bundle, "V1", "V1 bundle must contain V1 marker");
+
+    // --- Step 2 (scenario c): Unchanged source — no warning ---
+    // Use non-JSON mode so LogTape warnings reach stderr.
+    const unchanged = await runCliCommand(
+      ["model", "type", "search", "source-rebundle-it"],
+      repoDir,
+    );
+    assertEquals(unchanged.code, 0);
+    assertEquals(
+      unchanged.stderr.includes(WARNING_SUBSTRING),
+      false,
+      "No warning expected when source is unchanged (legitimate cache hit)",
+    );
+
+    // --- Step 3 (scenario b): Modify source to bare specifier + new method ---
+    await Deno.writeTextFile(sourcePath, BARE_SPECIFIER_V2);
+
+    const firstRetry = await runCliCommand(
+      ["model", "type", "search", "source-rebundle-it"],
+      repoDir,
+    );
+    assertEquals(
+      firstRetry.code,
+      0,
+      `First retry failed:\nstdout=${firstRetry.stdout}\nstderr=${firstRetry.stderr}`,
+    );
+
+    // Warning must fire: fromCache=true AND fingerprint differs.
+    assertStringIncludes(
+      firstRetry.stderr,
+      WARNING_SUBSTRING,
+      "Warning expected when bundle build fails and fingerprint is preserved",
+    );
+
+    // Bundle content must still be V1 (stale — the build failed).
+    const afterFirstRetry = await Deno.readTextFile(bundlePath);
+    assertStringIncludes(
+      afterFirstRetry,
+      "V1",
+      "Bundle must still contain V1 marker — build failed, old cache used",
+    );
+    assertEquals(
+      afterFirstRetry.includes("V2"),
+      false,
+      "V2 marker must NOT be in bundle — build failed",
+    );
+
+    // --- Step 4 (scenario b continued): Second warm-start — deterministic ---
+    const secondRetry = await runCliCommand(
+      ["model", "type", "search", "source-rebundle-it"],
+      repoDir,
+    );
+    assertEquals(
+      secondRetry.code,
+      0,
+      `Second retry failed:\nstdout=${secondRetry.stdout}\nstderr=${secondRetry.stderr}`,
+    );
+
+    // Warning must fire again — fingerprint still mismatches.
+    assertStringIncludes(
+      secondRetry.stderr,
+      WARNING_SUBSTRING,
+      "Warning expected on second retry — fingerprint still preserved",
+    );
+
+    // Bundle must still be V1.
+    const afterSecondRetry = await Deno.readTextFile(bundlePath);
+    assertStringIncludes(
+      afterSecondRetry,
+      "V1",
+      "Bundle must still contain V1 marker on second retry",
+    );
+  } finally {
+    try {
+      await Deno.remove(repoDir, { recursive: true });
+    } catch { /* EBUSY from sqlite — temp dir is ephemeral, OS reclaims */ }
+    try {
+      await Deno.remove(extDir, { recursive: true });
+    } catch { /* best-effort cleanup */ }
+  }
+});
+
+/**
+ * Locates the bundle file by walking `.swamp/bundles/` — the intermediate
+ * hash segment varies per source dir, so we search by filename.
+ */
+async function findBundle(
+  repoDir: string,
+  bundleName: string,
+): Promise<string> {
+  const bundlesRoot = join(repoDir, ".swamp", "bundles");
+  return await walkForFile(bundlesRoot, bundleName);
+}
+
+async function walkForFile(dir: string, target: string): Promise<string> {
+  for await (const entry of Deno.readDir(dir)) {
+    const path = join(dir, entry.name);
+    if (entry.isFile && entry.name === target) return path;
+    if (entry.isDirectory) {
+      try {
+        return await walkForFile(path, target);
+      } catch {
+        // Not found in this subtree — continue.
+      }
+    }
+  }
+  throw new Error(`${target} not found under ${dir}`);
+}

src/domain/datastore/user_datastore_loader.ts

@@ -31,6 +31,7 @@ import {
   uint8ArrayToBase64,
 } from "../models/bundle.ts";
 import {
+  type BundleResult,
   computeSourceFingerprint,
   createFreshnessCache,
   findStaleFiles as findStaleFilesShared,
@@ -179,12 +180,16 @@ export class UserDatastoreLoader {
           continue;
         }
 
-        const js = await this.bundleWithCache(
+        const { js, fromCache } = await this.bundleWithCache(
           absolutePath,
           file,
           denoPath,
           baseDir,
         );
+        if (fromCache) {
+          logger
+            .warn`Using cached bundle for ${file} — source may have changed but bundle could not be regenerated`;
+        }
         const module = await this.importBundle(js, file, baseDir);
 
         if (!module.datastore) {
@@ -242,7 +247,7 @@ export class UserDatastoreLoader {
     relativePath: string,
     denoPath: string,
     boundaryDir: string,
-  ): Promise<string> {
+  ): Promise<BundleResult> {
     if (this.repoDir) {
       const bundlePath = join(
         this.repoDir,
@@ -272,7 +277,7 @@ export class UserDatastoreLoader {
       // and we'd wastefully spawn Deno before falling back to the
       // cached bundle anyway.
       if (bundleExists && isExpectedBundleFailure(absolutePath, this.repoDir)) {
-        return await Deno.readTextFile(bundlePath);
+        return { js: await Deno.readTextFile(bundlePath), fromCache: true };
       }
 
       // Try to rebundle from source. If bundling fails (e.g. bare specifiers
@@ -286,7 +291,7 @@ export class UserDatastoreLoader {
         await Deno.mkdir(dirname(bundlePath), { recursive: true });
         await Deno.writeTextFile(bundlePath, js);
         logger.debug`Wrote datastore bundle cache: ${bundlePath}`;
-        return js;
+        return { js, fromCache: false };
       } catch (bundleError) {
         if (bundleExists) {
           try {
@@ -313,7 +318,7 @@ export class UserDatastoreLoader {
               logger
                 .warn`Rebundle failed for ${relativePath}, using cached bundle: ${msg}`;
             }
-            return cached;
+            return { js: cached, fromCache: true };
           } catch {
             // Cache file was removed between stat and read — treat as no cache.
           }
@@ -323,7 +328,8 @@ export class UserDatastoreLoader {
     }
 
     // No repo dir — just bundle without caching
-    return await bundleExtension(absolutePath, denoPath);
+    const js = await bundleExtension(absolutePath, denoPath);
+    return { js, fromCache: false };
   }
 
   /**
@@ -717,6 +723,7 @@ export class UserDatastoreLoader {
       typeNormalized: string;
       bundlePath: string;
       fingerprint: string;
+      fromCache: boolean;
     }
     | null
   > {
@@ -727,7 +734,7 @@ export class UserDatastoreLoader {
 
     installZodGlobal();
     const denoPath = await this.denoRuntime.ensureDeno();
-    const js = await this.bundleWithCache(
+    const { js, fromCache } = await this.bundleWithCache(
       args.absolutePath,
       args.relativePath,
       denoPath,
@@ -755,6 +762,7 @@ export class UserDatastoreLoader {
       typeNormalized: parsed.data.type.toLowerCase(),
       bundlePath,
       fingerprint,
+      fromCache,
     };
   }
 
@@ -773,7 +781,7 @@ export class UserDatastoreLoader {
       return;
     }
 
-    const js = await this.bundleWithCache(
+    const { js, fromCache } = await this.bundleWithCache(
       absolutePath,
       relativePath,
       denoPath,
@@ -793,6 +801,23 @@ export class UserDatastoreLoader {
       absolutePath,
       baseDir,
     );
+
+    // When the bundle came from cache (build failed or isExpectedBundleFailure),
+    // preserve the catalog's stored fingerprint so findStaleFiles retries on the
+    // next warm-start. Only warn on the fallback case (fingerprint actually
+    // differs), not on legitimate cache hits where source hasn't changed.
+    let effectiveFingerprint = sourceFingerprint;
+    if (fromCache) {
+      const existing = catalog.findBySourcePath(absolutePath);
+      if (existing?.source_fingerprint) {
+        if (existing.source_fingerprint !== sourceFingerprint) {
+          logger
+            .warn`Bundle could not be regenerated for ${relativePath} — source fingerprint preserved, will retry on next command`;
+        }
+        effectiveFingerprint = existing.source_fingerprint;
+      }
+    }
+
     const bundlePath = this.getDatastoreBundlePath(relativePath, baseDir);
 
     const parsed = UserDatastoreSchema.safeParse(module.datastore);
@@ -803,7 +828,7 @@ export class UserDatastoreLoader {
         kind: "datastore",
         bundlePath,
         sourceMtime,
-        sourceFingerprint,
+        sourceFingerprint: effectiveFingerprint,
       });
       throw new Error(this.formatValidationError(parsed.error));
     }
@@ -819,7 +844,7 @@ export class UserDatastoreLoader {
       description: parsed.data.description,
       extends_type: "",
       source_mtime: sourceMtime,
-      source_fingerprint: sourceFingerprint,
+      source_fingerprint: effectiveFingerprint,
     });
 
     // Also register since we already imported