diff --git a/scripts/sync-demos-to-crm.mjs b/scripts/sync-demos-to-crm.mjs index abd0f87..1b8f995 100644 --- a/scripts/sync-demos-to-crm.mjs +++ b/scripts/sync-demos-to-crm.mjs @@ -141,15 +141,25 @@ async function main() { } linkedLeads++; - // (2) attach the demo preview + // (2) attach the demo preview. Carry the stable id + the builder's canonical + // match key through when the manifest has them (absent = behaves as before). + const id = (e.id ?? "").toString().trim(); const value = { previewUrl: e.link, linkedAt: new Date().toISOString(), status: normStatus(e.status ?? "ready"), ...(e.category ? { category: e.category } : {}), ...(city ? { area: e.area || city } : {}), ...(thumb ? { thumbnailUrl: thumb } : {}), ...(slug ? { slug } : {}), + ...(id ? { id } : {}), + ...(e.matchKey ? { matchKey: e.matchKey } : {}), }; - await r.hset("lead_previews", { [key]: JSON.stringify(value) }); + const json = JSON.stringify(value); + // ALWAYS write the back-compat name key. + await r.hset("lead_previews", { [key]: json }); + // ADDITIONALLY index under the stable id (id:) so the CRM can join id-first + // without fuzzy name matching. Same hash, distinct field; the name key above is + // left untouched for back-compat with already-stored links. + if (id) await r.hset("lead_previews", { [`id:${id}`]: json }); previews++; } diff --git a/src/app/api/crm/admin/preview-url/route.ts b/src/app/api/crm/admin/preview-url/route.ts index f457d4e..dd04efb 100644 --- a/src/app/api/crm/admin/preview-url/route.ts +++ b/src/app/api/crm/admin/preview-url/route.ts @@ -31,6 +31,10 @@ interface PreviewEntry { claimByDate?: string; thumbnailUrl?: string; slug?: string; + // Stable business id (Overture id) + the builder's canonical match key, when the + // /websites manifest carries them. Both optional — absent posts behave as before. + id?: string; + matchKey?: string; } export async function POST(req: NextRequest) { @@ -62,6 +66,8 @@ export async function POST(req: NextRequest) { claimByDate: e.claimByDate, thumbnailUrl: e.thumbnailUrl, slug: e.slug, + id: e.id, + matchKey: e.matchKey, }); linked++; } diff --git a/src/app/api/crm/leads/route.ts b/src/app/api/crm/leads/route.ts index 771d114..0b354b6 100644 --- a/src/app/api/crm/leads/route.ts +++ b/src/app/api/crm/leads/route.ts @@ -1,5 +1,5 @@ import { NextRequest, NextResponse } from "next/server"; -import { getCustomLeads, getAllClaims, getAllAssignments, getTerritory, getLeadPreviewObjects, previewKey, getLeadActions, getDoNotCallPhones, normalizePhone, logLeadFetch, type LeadActions } from "@/lib/db"; +import { getCustomLeads, getAllClaims, getAllAssignments, getTerritory, getLeadPreviewObjects, previewKey, idPreviewKey, getLeadActions, getDoNotCallPhones, normalizePhone, logLeadFetch, type LeadActions } from "@/lib/db"; import { rateLimit } from "@/lib/rateLimit"; // Lead source CSV. Defaults to the national deduped export, but can be pointed at @@ -457,8 +457,30 @@ export async function GET(req: NextRequest) { // Do-Not-Call list — one SMEMBERS, matched on normalized phone. Leads on the // list are FLAGGED (doNotCall: true), never removed from the result. const dncSet = new Set(await getDoNotCallPhones()); + + // Observability for the demo↔lead join (this page only; bounded, non-fatal): + // • nameOnlyMatches — demos matched by previewKey name alone (no stable id + // hit). These are the fuzzy-match risk; surfaced so a drift is visible. + // • collisions — two DIFFERENT leads on this page that normalize to the SAME + // previewKey. That's the "wrong business gets another's demo" hazard. + let nameOnlyMatches = 0; + const keyToNames = new Map>(); + for (const l of pageLeads) { + const k = previewKey(l.name); + if (!k) continue; + const set = keyToNames.get(k) ?? new Set(); + set.add(l.name); + keyToNames.set(k, set); + } + const leads = pageLeads.map((l) => { - const pkg = previews[previewKey(l.name)]; + // Prefer the stable id (the CSV global pool carries an `id` column); fall + // back to the back-compat name key only when there's no id hit. Behavior is + // identical to before whenever the id path misses. + const byId = l.id ? previews[idPreviewKey(l.id)] : undefined; + const byName = previews[previewKey(l.name)]; + const pkg = byId ?? byName; + if (pkg && !byId && byName) nameOnlyMatches++; const asg = assignmentMap[l.id]; const normPhone = normalizePhone(l.phone); return { @@ -477,6 +499,23 @@ export async function GET(req: NextRequest) { }; }); + // LOUD, non-fatal join diagnostics (once per request, bounded summaries). + if (nameOnlyMatches > 0) { + console.warn( + `[previews] ${nameOnlyMatches} demo(s) matched by name only (no stable id) — see the data/outreach-links.json seam contract; prefer the id-keyed join.` + ); + } + const collisions = [...keyToNames.entries()].filter(([, names]) => names.size > 1); + if (collisions.length > 0) { + const sample = collisions + .slice(0, 5) + .map(([k, names]) => `"${k}" ⇐ ${[...names].slice(0, 4).join(" | ")}`) + .join("; "); + console.warn( + `[previews] previewKey COLLISION: ${collisions.length} key(s) shared by 2+ distinct leads on this page — risk of the wrong business getting another's demo. ${sample}${collisions.length > 5 ? " …" : ""}` + ); + } + const counties = [...new Set(all.map((l) => l.county).filter(Boolean))].sort(); const niches = [...new Set(all.map((l) => l.category).filter(Boolean))].sort(); diff --git a/src/lib/crm/matchKey.test.ts b/src/lib/crm/matchKey.test.ts new file mode 100644 index 0000000..ca2f6bc --- /dev/null +++ b/src/lib/crm/matchKey.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { norm, matchKey } from "./matchKey"; + +describe("matchKey — TIGHT join key (strips only legal-entity forms)", () => { + it("keeps distinguishing words; strips only the legal suffix", () => { + expect(matchKey("Acme Realty LLC")).toBe("acmerealty"); + expect(matchKey("Acme Group")).toBe("acmegroup"); + expect(matchKey("Joe's Cafe")).toBe("joescafe"); + expect(matchKey("A & B Co.")).toBe("ab"); + }); + + it("does not collide where the loose norm does", () => { + // norm collapses both to "acme"; matchKey keeps them distinct. + expect(matchKey("Acme Realty")).not.toBe(matchKey("Acme Group")); + }); + + it("is tolerant of null/undefined/empty input", () => { + expect(matchKey(undefined as unknown as string)).toBe(""); + expect(matchKey("")).toBe(""); + }); +}); + +describe("norm — LOOSE suppression key (strips distinguishing words)", () => { + it("strips industry filler words AND legal suffixes ('realty', 'llc')", () => { + expect(norm("Acme Realty LLC")).toBe("acme"); + }); + + it("collapses several filler words and punctuation together", () => { + // the + properties both stripped -> "harbor" + expect(norm("The Harbor Properties, Inc.")).toBe("harbor"); + }); + + it("is tolerant of null/undefined/empty input", () => { + expect(norm(undefined as unknown as string)).toBe(""); + expect(norm("")).toBe(""); + }); +}); diff --git a/src/lib/crm/matchKey.ts b/src/lib/crm/matchKey.ts new file mode 100644 index 0000000..03a7bbd --- /dev/null +++ b/src/lib/crm/matchKey.ts @@ -0,0 +1,27 @@ +// Canonical business-name normalizer — byte-identical to +// scraper-app/contract/normalize.js and the Websites builder's +// scripts/lib/match-key.mjs. +// +// TWO distinct keys, do NOT conflate them: +// * norm() — LOOSE suppression/dedup key. Strips distinguishing words +// (realty/group/team/...). Two different firms can collide, +// so NEVER join on it. +// * matchKey() — TIGHT join key. Strips ONLY legal-entity forms +// (llc/inc/corp/co/ltd/llp/...) and keeps every distinguishing +// word. This is the demo<->lead JOIN key. +// +// This module is NOT yet swapped into previewKey() (doing so would orphan demo +// links already stored under the old key); it remains a future-cutover module. +// The live join still uses previewKey() in src/lib/db.ts; the manifest's +// `matchKey` value is passed through unchanged by sync/preview, never recomputed +// here with the loose key. +export function norm(name: string): string { + let n = (name || "").toLowerCase().replace(/[^a-z0-9 ]+/g, " "); + n = n.replace(/\b(llc|inc|incorporated|co|company|group|team|realty|realtors|real estate|properties|brokerage|the)\b/g, " "); + return n.replace(/\s+/g, ""); +} +export function matchKey(name: string): string { + let n = (name || "").toLowerCase().replace(/[^a-z0-9 ]+/g, " "); + n = n.replace(/\b(llc|inc|incorporated|corp|corporation|co|company|ltd|llp)\b/g, " "); + return n.replace(/\s+/g, ""); +} diff --git a/src/lib/db.ts b/src/lib/db.ts index 84d1b06..56e52a6 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -576,6 +576,14 @@ export interface LeadPreview { claimByDate?: string; thumbnailUrl?: string; slug?: string; + // Stable business id (Overture id) carried through from the /websites manifest, + // when present. Enables an id-first join in the lead queue that doesn't depend on + // fuzzy name matching. Absent on legacy/name-only previews. + id?: string; + // The canonical normalized name the /websites builder matched on (its + // match-key.mjs output). Informational/observability only — the live join still + // uses previewKey() for back-compat. Absent on older manifests. + matchKey?: string; } const LEAD_PREVIEWS_KEY = "lead_previews"; @@ -584,10 +592,24 @@ const LEAD_PREVIEWS_KEY = "lead_previews"; // endpoint, from the /websites manifest name) and the reader (the lead queue, // from the CSV name) run a name through this, so the two sides line up without // a shared id. +// +// DO NOT CHANGE THIS ALGORITHM. Demo links already in Redis are keyed by its +// output; altering it would orphan every stored preview. The id-first join (see +// idPreviewKey + the leads route) is layered ON TOP of this for new entries; this +// remains the back-compat fallback key forever. A canonical normalizer for a +// future cutover lives in src/lib/crm/matchKey.ts (not wired in here on purpose). export function previewKey(name: string): string { return name.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim(); } +// Hash field used to ALSO index a preview under its stable business id, alongside +// the previewKey entry. The leads route prefers this when a lead has an id, and +// falls back to previewKey when it misses — so behavior is unchanged whenever no +// id is present on either side. +export function idPreviewKey(id: string): string { + return `id:${id}`; +} + function normalizeDemoStatus(raw: string | undefined): DemoStatus { if (raw === "needs-review" || raw === "needs_review") return "needs_review"; if (raw === "archived") return "archived"; @@ -608,6 +630,7 @@ export async function setLeadPreview( const key = previewKey(name); if (!key || !previewUrl) return; const redis = getRedis(); + const id = (extra.id ?? "").trim(); const value: LeadPreview = { previewUrl, linkedAt: new Date().toISOString(), @@ -619,8 +642,20 @@ export async function setLeadPreview( ...(extra.claimByDate ? { claimByDate: extra.claimByDate } : {}), ...(extra.thumbnailUrl ? { thumbnailUrl: extra.thumbnailUrl } : {}), ...(extra.slug ? { slug: extra.slug } : {}), + // Stable id + the builder's canonical match key, when supplied. Carried inside + // the value JSON so both the name-keyed and id-keyed entries hold them. + ...(id ? { id } : {}), + ...(extra.matchKey ? { matchKey: extra.matchKey } : {}), }; - await redis.hset(LEAD_PREVIEWS_KEY, { [key]: JSON.stringify(value) }); + const json = JSON.stringify(value); + // ALWAYS write the back-compat previewKey entry (the existing join key). + await redis.hset(LEAD_PREVIEWS_KEY, { [key]: json }); + // ADDITIONALLY index under the stable id when present, so the lead queue can do + // an id-first lookup that never collides on name. Same hash, distinct `id:` + // field — the previewKey entry above is untouched. + if (id) { + await redis.hset(LEAD_PREVIEWS_KEY, { [idPreviewKey(id)]: json }); + } } // Returns a map of normalized-name → previewUrl for enriching the lead queue.