feat: prefer stable id when attaching demos; warn on name-only/collision (additive)

scripts/sync-demos-to-crm.mjs

@@ -139,17 +139,27 @@
       if (!cur.website && website) patch.website = website;
       if (Object.keys(patch).length) { await r.hset(`custom_lead:${cur.id}`, patch); Object.assign(cur, patch); }
     }
     linkedLeads++;
 
-    // (2) attach the demo preview
+    // (2) attach the demo preview. Carry the stable id + the builder's canonical
+    // match key through when the manifest has them (absent = behaves as before).
+    const id = (e.id ?? "").toString().trim();
     const value = {
       previewUrl: e.link, linkedAt: new Date().toISOString(), status: normStatus(e.status ?? "ready"),
       ...(e.category ? { category: e.category } : {}),
       ...(city ? { area: e.area || city } : {}),
       ...(thumb ? { thumbnailUrl: thumb } : {}),
       ...(slug ? { slug } : {}),
+      ...(id ? { id } : {}),
+      ...(e.matchKey ? { matchKey: e.matchKey } : {}),
     };
-    await r.hset("lead_previews", { [key]: JSON.stringify(value) });
+    const json = JSON.stringify(value);
+    // ALWAYS write the back-compat name key.
+    await r.hset("lead_previews", { [key]: json });
+    // ADDITIONALLY index under the stable id (id:<id>) so the CRM can join id-first
+    // without fuzzy name matching. Same hash, distinct field; the name key above is
+    // left untouched for back-compat with already-stored links.
+    if (id) await r.hset("lead_previews", { [`id:${id}`]: json });
     previews++;
   }
 

src/app/api/crm/admin/preview-url/route.ts

@@ -31,6 +31,10 @@ interface PreviewEntry {
   claimByDate?: string;
   thumbnailUrl?: string;
   slug?: string;
+  // Stable business id (Overture id) + the builder's canonical match key, when the
+  // /websites manifest carries them. Both optional — absent posts behave as before.
+  id?: string;
+  matchKey?: string;
 }
 
 export async function POST(req: NextRequest) {
@@ -62,6 +66,8 @@ export async function POST(req: NextRequest) {
         claimByDate: e.claimByDate,
         thumbnailUrl: e.thumbnailUrl,
         slug: e.slug,
+        id: e.id,
+        matchKey: e.matchKey,
       });
       linked++;
     }

src/app/api/crm/leads/route.ts

@@ -1,5 +1,5 @@
 import { NextRequest, NextResponse } from "next/server";
-import { getCustomLeads, getAllClaims, getAllAssignments, getTerritory, getLeadPreviewObjects, previewKey, getLeadActions, getDoNotCallPhones, normalizePhone, logLeadFetch, type LeadActions } from "@/lib/db";
+import { getCustomLeads, getAllClaims, getAllAssignments, getTerritory, getLeadPreviewObjects, previewKey, idPreviewKey, getLeadActions, getDoNotCallPhones, normalizePhone, logLeadFetch, type LeadActions } from "@/lib/db";
 import { rateLimit } from "@/lib/rateLimit";
 
 // Lead source CSV. Defaults to the national deduped export, but can be pointed at
@@ -457,8 +457,30 @@ export async function GET(req: NextRequest) {
     // Do-Not-Call list — one SMEMBERS, matched on normalized phone. Leads on the
     // list are FLAGGED (doNotCall: true), never removed from the result.
     const dncSet = new Set(await getDoNotCallPhones());
+
+    // Observability for the demo↔lead join (this page only; bounded, non-fatal):
+    //  • nameOnlyMatches — demos matched by previewKey name alone (no stable id
+    //    hit). These are the fuzzy-match risk; surfaced so a drift is visible.
+    //  • collisions — two DIFFERENT leads on this page that normalize to the SAME
+    //    previewKey. That's the "wrong business gets another's demo" hazard.
+    let nameOnlyMatches = 0;
+    const keyToNames = new Map<string, Set<string>>();
+    for (const l of pageLeads) {
+      const k = previewKey(l.name);
+      if (!k) continue;
+      const set = keyToNames.get(k) ?? new Set<string>();
+      set.add(l.name);
+      keyToNames.set(k, set);
+    }
+
     const leads = pageLeads.map((l) => {
-      const pkg = previews[previewKey(l.name)];
+      // Prefer the stable id (the CSV global pool carries an `id` column); fall
+      // back to the back-compat name key only when there's no id hit. Behavior is
+      // identical to before whenever the id path misses.
+      const byId = l.id ? previews[idPreviewKey(l.id)] : undefined;
+      const byName = previews[previewKey(l.name)];
+      const pkg = byId ?? byName;
+      if (pkg && !byId && byName) nameOnlyMatches++;
       const asg = assignmentMap[l.id];
       const normPhone = normalizePhone(l.phone);
       return {
@@ -477,6 +499,23 @@ export async function GET(req: NextRequest) {
       };
     });
 
+    // LOUD, non-fatal join diagnostics (once per request, bounded summaries).
+    if (nameOnlyMatches > 0) {
+      console.warn(
+        `[previews] ${nameOnlyMatches} demo(s) matched by name only (no stable id) — see the data/outreach-links.json seam contract; prefer the id-keyed join.`
+      );
+    }
+    const collisions = [...keyToNames.entries()].filter(([, names]) => names.size > 1);
+    if (collisions.length > 0) {
+      const sample = collisions
+        .slice(0, 5)
+        .map(([k, names]) => `"${k}" ⇐ ${[...names].slice(0, 4).join(" | ")}`)
+        .join("; ");
+      console.warn(
+        `[previews] previewKey COLLISION: ${collisions.length} key(s) shared by 2+ distinct leads on this page — risk of the wrong business getting another's demo. ${sample}${collisions.length > 5 ? " …" : ""}`
+      );
+    }
+
     const counties = [...new Set(all.map((l) => l.county).filter(Boolean))].sort();
     const niches = [...new Set(all.map((l) => l.category).filter(Boolean))].sort();
 

src/lib/crm/matchKey.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+import { norm, matchKey } from "./matchKey";
+
+describe("matchKey — TIGHT join key (strips only legal-entity forms)", () => {
+  it("keeps distinguishing words; strips only the legal suffix", () => {
+    expect(matchKey("Acme Realty LLC")).toBe("acmerealty");
+    expect(matchKey("Acme Group")).toBe("acmegroup");
+    expect(matchKey("Joe's Cafe")).toBe("joescafe");
+    expect(matchKey("A & B Co.")).toBe("ab");
+  });
+
+  it("does not collide where the loose norm does", () => {
+    // norm collapses both to "acme"; matchKey keeps them distinct.
+    expect(matchKey("Acme Realty")).not.toBe(matchKey("Acme Group"));
+  });
+
+  it("is tolerant of null/undefined/empty input", () => {
+    expect(matchKey(undefined as unknown as string)).toBe("");
+    expect(matchKey("")).toBe("");
+  });
+});
+
+describe("norm — LOOSE suppression key (strips distinguishing words)", () => {
+  it("strips industry filler words AND legal suffixes ('realty', 'llc')", () => {
+    expect(norm("Acme Realty LLC")).toBe("acme");
+  });
+
+  it("collapses several filler words and punctuation together", () => {
+    // the + properties both stripped -> "harbor"
+    expect(norm("The Harbor Properties, Inc.")).toBe("harbor");
+  });
+
+  it("is tolerant of null/undefined/empty input", () => {
+    expect(norm(undefined as unknown as string)).toBe("");
+    expect(norm("")).toBe("");
+  });
+});

src/lib/crm/matchKey.ts

@@ -0,0 +1,27 @@
+// Canonical business-name normalizer — byte-identical to
+// scraper-app/contract/normalize.js and the Websites builder's
+// scripts/lib/match-key.mjs.
+//
+// TWO distinct keys, do NOT conflate them:
+//   * norm()     — LOOSE suppression/dedup key. Strips distinguishing words
+//                  (realty/group/team/...). Two different firms can collide,
+//                  so NEVER join on it.
+//   * matchKey() — TIGHT join key. Strips ONLY legal-entity forms
+//                  (llc/inc/corp/co/ltd/llp/...) and keeps every distinguishing
+//                  word. This is the demo<->lead JOIN key.
+//
+// This module is NOT yet swapped into previewKey() (doing so would orphan demo
+// links already stored under the old key); it remains a future-cutover module.
+// The live join still uses previewKey() in src/lib/db.ts; the manifest's
+// `matchKey` value is passed through unchanged by sync/preview, never recomputed
+// here with the loose key.
+export function norm(name: string): string {
+  let n = (name || "").toLowerCase().replace(/[^a-z0-9 ]+/g, " ");
+  n = n.replace(/\b(llc|inc|incorporated|co|company|group|team|realty|realtors|real estate|properties|brokerage|the)\b/g, " ");
+  return n.replace(/\s+/g, "");
+}
+export function matchKey(name: string): string {
+  let n = (name || "").toLowerCase().replace(/[^a-z0-9 ]+/g, " ");
+  n = n.replace(/\b(llc|inc|incorporated|corp|corporation|co|company|ltd|llp)\b/g, " ");
+  return n.replace(/\s+/g, "");
+}

src/lib/db.ts

@@ -576,6 +576,14 @@ export interface LeadPreview {
   claimByDate?: string;
   thumbnailUrl?: string;
   slug?: string;
+  // Stable business id (Overture id) carried through from the /websites manifest,
+  // when present. Enables an id-first join in the lead queue that doesn't depend on
+  // fuzzy name matching. Absent on legacy/name-only previews.
+  id?: string;
+  // The canonical normalized name the /websites builder matched on (its
+  // match-key.mjs output). Informational/observability only — the live join still
+  // uses previewKey() for back-compat. Absent on older manifests.
+  matchKey?: string;
 }
 
 const LEAD_PREVIEWS_KEY = "lead_previews";
@@ -584,10 +592,24 @@ const LEAD_PREVIEWS_KEY = "lead_previews";
 // endpoint, from the /websites manifest name) and the reader (the lead queue,
 // from the CSV name) run a name through this, so the two sides line up without
 // a shared id.
+//
+// DO NOT CHANGE THIS ALGORITHM. Demo links already in Redis are keyed by its
+// output; altering it would orphan every stored preview. The id-first join (see
+// idPreviewKey + the leads route) is layered ON TOP of this for new entries; this
+// remains the back-compat fallback key forever. A canonical normalizer for a
+// future cutover lives in src/lib/crm/matchKey.ts (not wired in here on purpose).
 export function previewKey(name: string): string {
   return name.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
 }
 
+// Hash field used to ALSO index a preview under its stable business id, alongside
+// the previewKey entry. The leads route prefers this when a lead has an id, and
+// falls back to previewKey when it misses — so behavior is unchanged whenever no
+// id is present on either side.
+export function idPreviewKey(id: string): string {
+  return `id:${id}`;
+}
+
 function normalizeDemoStatus(raw: string | undefined): DemoStatus {
   if (raw === "needs-review" || raw === "needs_review") return "needs_review";
   if (raw === "archived") return "archived";
@@ -608,6 +630,7 @@ export async function setLeadPreview(
   const key = previewKey(name);
   if (!key || !previewUrl) return;
   const redis = getRedis();
+  const id = (extra.id ?? "").trim();
   const value: LeadPreview = {
     previewUrl,
     linkedAt: new Date().toISOString(),
@@ -619,8 +642,20 @@ export async function setLeadPreview(
     ...(extra.claimByDate ? { claimByDate: extra.claimByDate } : {}),
     ...(extra.thumbnailUrl ? { thumbnailUrl: extra.thumbnailUrl } : {}),
     ...(extra.slug ? { slug: extra.slug } : {}),
+    // Stable id + the builder's canonical match key, when supplied. Carried inside
+    // the value JSON so both the name-keyed and id-keyed entries hold them.
+    ...(id ? { id } : {}),
+    ...(extra.matchKey ? { matchKey: extra.matchKey } : {}),
   };
-  await redis.hset(LEAD_PREVIEWS_KEY, { [key]: JSON.stringify(value) });
+  const json = JSON.stringify(value);
+  // ALWAYS write the back-compat previewKey entry (the existing join key).
+  await redis.hset(LEAD_PREVIEWS_KEY, { [key]: json });
+  // ADDITIONALLY index under the stable id when present, so the lead queue can do
+  // an id-first lookup that never collides on name. Same hash, distinct `id:<id>`
+  // field — the previewKey entry above is untouched.
+  if (id) {
+    await redis.hset(LEAD_PREVIEWS_KEY, { [idPreviewKey(id)]: json });
+  }
 }
 
 // Returns a map of normalized-name → previewUrl for enriching the lead queue.