feat(directory): per-host dedup + featured-pinning (re-land #53 onto main)

worker/src/directory_rank.ts

@@ -0,0 +1,74 @@
+// directory_rank.ts — pure ranking + per-host dedup for /directory.
+//
+// The directory was reading as spam: the 2h Shopify cron seeds hundreds of
+// products from a few stores, KV.list returns ~insertion order, so a single host
+// (aloyoga) flooded the page before diverse hosts appeared. Fix = rank, then cap
+// per host so no store dominates — diversity by host, not just raw count.
+// Extracted as a pure function so the dedup logic is unit-testable without Hono.
+
+export interface DirEntry {
+  url: string;
+  adapter: string;
+  ts: number;
+  title: string | null;
+  slug: string;
+  verified: boolean;
+  featured_rank: number | null;
+}
+
+export interface RankedEntry extends DirEntry {
+  host: string;
+}
+
+// Cap key = full host, lowercased, leading "www." stripped. NOTE: this does NOT
+// collapse to the registrable domain, so shop.x.com and x.com count as two hosts
+// and each gets its own per-host quota. That's intentional (sub-stores can be
+// genuinely distinct, and eTLD+1 collapsing needs a public-suffix list to handle
+// .co.uk etc.) — but it means a motivated multi-subdomain store can still split
+// its flood. Acceptable for the cron-seeded directory; revisit if abuse shows up.
+export function hostOf(u: string): string {
+  try { return new URL(u).host.replace(/^www\./, "").toLowerCase(); } catch { return u; }
+}
+
+/**
+ * Rank + diversify directory entries.
+ *   • Sort: featured first (asc rank), then newest ts.
+ *   • Cap each host to `perHost` entries (0 disables the PER-HOST cap only) so one
+ *     store can't flood. `limit` ALWAYS applies on top — perHost=0 is not "full
+ *     list", it still truncates to `limit`.
+ *   • Featured entries bypass the per-host cap (hand-picked, always shown).
+ *   • Truncate to `limit`.
+ * Returns the ranked entries + the distinct-host count OF THE RETURNED entries
+ * (counted after truncation, so the "N stores" badge matches the rows shown).
+ */
+export function rankDirectory(
+  raw: DirEntry[],
+  opts: { perHost?: number; limit?: number } = {}
+): { entries: RankedEntry[]; distinct_hosts: number } {
+  const perHost = opts.perHost ?? 3;
+  const limit = opts.limit ?? 200;
+
+  const all: RankedEntry[] = raw.map((e) => ({ ...e, host: hostOf(e.url) }));
+
+  // Featured (asc rank) first, then newest ts — so the per-host cap keeps each
+  // host's newest entries and featured always survives.
+  all.sort((a, b) => {
+    const ar = a.featured_rank ?? Number.POSITIVE_INFINITY;
+    const br = b.featured_rank ?? Number.POSITIVE_INFINITY;
+    if (ar !== br) return ar - br;
+    return b.ts - a.ts;
+  });
+
+  const perHostCount: Record<string, number> = {};
+  const deduped = perHost === 0 ? all : all.filter((e) => {
+    if (e.featured_rank != null) return true; // featured bypasses the cap
+    const n = (perHostCount[e.host] = (perHostCount[e.host] || 0) + 1);
+    return n <= perHost;
+  });
+
+  // Count distinct hosts AFTER truncation so the "N stores" badge can't claim
+  // more stores than the rows actually returned (deduped may exceed `limit`).
+  const entries = deduped.slice(0, limit);
+  const distinct_hosts = new Set(entries.map((e) => e.host)).size;
+  return { entries, distinct_hosts };
+}

worker/src/index.ts

@@ -208,11 +208,16 @@ app.get("/api/v1/stats/public", async (c) => {
 });
 
 app.get("/api/v1/directory", async (c) => {
+  const limit = Math.min(500, parseInt(c.req.query("limit") || "200", 10));
+  // Max entries shown per host so one big store (e.g. the cron's Shopify seed)
+  // can't flood the page and read as spam. ?per_host=0 disables the per-host cap
+  // only — `limit` (max 500) still applies, so this is not an "everything" escape.
+  const perHost = Math.max(0, parseInt(c.req.query("per_host") || "3", 10));
   const { slugFromUrl } = await import("./slug");
 
   // Paginate the FULL seen: set (KV.list caps at 1000/call) so the directory
-  // shows every site, not just the first 200 keys (which clustered on a few
-  // high-volume stores). The page groups these by host itself.
+  // sees every site, not just the first 1000 keys (which clustered on a few
+  // high-volume stores). rankDirectory then dedups per-host + applies `limit`.
   const seenKeys: any[] = [];
   let cursor: string | undefined;
   let pages = 0;
@@ -244,7 +249,7 @@ app.get("/api/v1/directory", async (c) => {
     })
   );
 
-  const entries = list.keys
+  const raw = list.keys
     .map((k: any) => k.metadata)
     .filter((m: any) => m && m.url)
     .map((m: any) => {
@@ -260,17 +265,15 @@ app.get("/api/v1/directory", async (c) => {
       };
     });
 
-  // Sort: featured first (asc by rank), then by ts desc.
-  entries.sort((a: any, b: any) => {
-    const ar = a.featured_rank ?? Number.POSITIVE_INFINITY;
-    const br = b.featured_rank ?? Number.POSITIVE_INFINITY;
-    if (ar !== br) return ar - br;
-    return b.ts - a.ts;
-  });
-
-  return c.json({ entries, list_complete: list.list_complete }, 200, {
-    "cache-control": "public, max-age=300, s-maxage=300",
-  });
+  // Per-host dedup + featured-first ordering lives in rankDirectory (pure,
+  // unit-tested) — it supersedes the old inline sort and adds the per-host cap.
+  const { rankDirectory } = await import("./directory_rank");
+  const { entries, distinct_hosts } = rankDirectory(raw, { perHost, limit });
+  return c.json(
+    { entries, list_complete: list.list_complete, distinct_hosts, per_host_cap: perHost || null },
+    200,
+    { "cache-control": "public, max-age=300, s-maxage=300" }
+  );
 });
 
 app.get("/directory", (c) => {

worker/test/directory_rank.test.ts

@@ -0,0 +1,90 @@
+// test/directory_rank.test.ts — per-host dedup + featured-pinning for /directory.
+import { describe, it, expect } from "vitest";
+import { rankDirectory, hostOf, type DirEntry } from "../src/directory_rank";
+
+function e(url: string, ts: number, extra: Partial<DirEntry> = {}): DirEntry {
+  return { url, adapter: "shopify", ts, title: null, slug: url, verified: false, featured_rank: null, ...extra };
+}
+
+describe("rankDirectory", () => {
+  it("caps each host so one store can't flood (the aloyoga problem)", () => {
+    const raw: DirEntry[] = [];
+    for (let i = 0; i < 50; i++) raw.push(e(`https://aloyoga.com/products/p${i}`, 1000 + i));
+    raw.push(e("https://gymshark.com/products/x", 900));
+    raw.push(e("https://allbirds.com/products/y", 800));
+    const { entries, distinct_hosts } = rankDirectory(raw, { perHost: 3, limit: 200 });
+    const aloCount = entries.filter((x) => x.host === "aloyoga.com").length;
+    expect(aloCount).toBe(3);                // capped, not 50
+    expect(distinct_hosts).toBe(3);          // alo + gymshark + allbirds
+    expect(entries.some((x) => x.host === "gymshark.com")).toBe(true);
+    expect(entries.some((x) => x.host === "allbirds.com")).toBe(true);
+  });
+
+  it("keeps each host's NEWEST entries under the cap", () => {
+    const raw = [
+      e("https://aloyoga.com/products/old", 100),
+      e("https://aloyoga.com/products/mid", 200),
+      e("https://aloyoga.com/products/new", 300),
+      e("https://aloyoga.com/products/oldest", 50),
+    ];
+    const { entries } = rankDirectory(raw, { perHost: 2 });
+    const slugs = entries.map((x) => x.url.split("/").pop());
+    expect(slugs).toEqual(["new", "mid"]); // newest two, in ts-desc order
+  });
+
+  it("featured entries bypass the cap and sort to the top", () => {
+    const raw = [
+      e("https://aloyoga.com/products/a", 1000),
+      e("https://aloyoga.com/products/b", 999),
+      e("https://aloyoga.com/products/c", 998),
+      e("https://aloyoga.com/products/featured", 1, { featured_rank: 0 }), // old but featured
+    ];
+    const { entries } = rankDirectory(raw, { perHost: 2 });
+    expect(entries[0].url.endsWith("/featured")).toBe(true); // featured pinned first
+    // 4 entries survive: the featured one (bypasses cap) + 2 capped + ... actually
+    // featured bypasses, then 2 non-featured under the cap = 3 total.
+    expect(entries.filter((x) => x.host === "aloyoga.com").length).toBe(3);
+  });
+
+  it("per_host=0 disables the PER-HOST cap only (limit still applies)", () => {
+    const raw = Array.from({ length: 10 }, (_, i) => e(`https://x.com/p${i}`, i));
+    const { entries } = rankDirectory(raw, { perHost: 0 });
+    expect(entries.length).toBe(10); // all 10 returned: cap off AND under default limit
+    // but `limit` is NOT disabled by perHost=0 — it still truncates:
+    const big = Array.from({ length: 600 }, (_, i) => e(`https://x.com/p${i}`, i));
+    expect(rankDirectory(big, { perHost: 0, limit: 200 }).entries.length).toBe(200);
+  });
+
+  it("respects the overall limit after dedup", () => {
+    const raw: DirEntry[] = [];
+    for (let h = 0; h < 20; h++) for (let i = 0; i < 5; i++) raw.push(e(`https://h${h}.com/p${i}`, h * 10 + i));
+    const { entries } = rankDirectory(raw, { perHost: 3, limit: 10 });
+    expect(entries.length).toBe(10);
+  });
+
+  it("distinct_hosts counts the RETURNED rows, not the pre-truncation set", () => {
+    // 20 hosts, 1 entry each survives the cap → deduped=20, but limit=10 returns
+    // 10 rows from 10 hosts. The badge must say 10, not 20.
+    const raw = Array.from({ length: 20 }, (_, h) => e(`https://h${h}.com/p`, h));
+    const { entries, distinct_hosts } = rankDirectory(raw, { perHost: 1, limit: 10 });
+    expect(entries.length).toBe(10);
+    expect(distinct_hosts).toBe(10); // matches rows shown, not the 20 deduped
+  });
+
+  it("subdomains are counted as distinct hosts (documented limitation)", () => {
+    // hostOf does not collapse to the registrable domain, so a multi-subdomain
+    // store gets a per-host quota per subdomain. Locks in current behavior.
+    const raw = [
+      ...Array.from({ length: 4 }, (_, i) => e(`https://x.com/p${i}`, 100 + i)),
+      ...Array.from({ length: 4 }, (_, i) => e(`https://shop.x.com/p${i}`, 200 + i)),
+    ];
+    const { entries } = rankDirectory(raw, { perHost: 3 });
+    expect(entries.length).toBe(6); // 3 from x.com + 3 from shop.x.com
+  });
+
+  it("hostOf strips www + lowercases", () => {
+    expect(hostOf("https://www.Allbirds.com/products/x")).toBe("allbirds.com");
+    expect(hostOf("https://shop.gymshark.com/p")).toBe("shop.gymshark.com");
+    expect(hostOf("not a url")).toBe("not a url");
+  });
+});