diff --git a/worker/src/directory_rank.ts b/worker/src/directory_rank.ts new file mode 100644 index 0000000..8322387 --- /dev/null +++ b/worker/src/directory_rank.ts @@ -0,0 +1,74 @@ +// directory_rank.ts — pure ranking + per-host dedup for /directory. +// +// The directory was reading as spam: the 2h Shopify cron seeds hundreds of +// products from a few stores, KV.list returns ~insertion order, so a single host +// (aloyoga) flooded the page before diverse hosts appeared. Fix = rank, then cap +// per host so no store dominates — diversity by host, not just raw count. +// Extracted as a pure function so the dedup logic is unit-testable without Hono. + +export interface DirEntry { + url: string; + adapter: string; + ts: number; + title: string | null; + slug: string; + verified: boolean; + featured_rank: number | null; +} + +export interface RankedEntry extends DirEntry { + host: string; +} + +// Cap key = full host, lowercased, leading "www." stripped. NOTE: this does NOT +// collapse to the registrable domain, so shop.x.com and x.com count as two hosts +// and each gets its own per-host quota. That's intentional (sub-stores can be +// genuinely distinct, and eTLD+1 collapsing needs a public-suffix list to handle +// .co.uk etc.) — but it means a motivated multi-subdomain store can still split +// its flood. Acceptable for the cron-seeded directory; revisit if abuse shows up. +export function hostOf(u: string): string { + try { return new URL(u).host.replace(/^www\./, "").toLowerCase(); } catch { return u; } +} + +/** + * Rank + diversify directory entries. + * • Sort: featured first (asc rank), then newest ts. + * • Cap each host to `perHost` entries (0 disables the PER-HOST cap only) so one + * store can't flood. `limit` ALWAYS applies on top — perHost=0 is not "full + * list", it still truncates to `limit`. + * • Featured entries bypass the per-host cap (hand-picked, always shown). + * • Truncate to `limit`. + * Returns the ranked entries + the distinct-host count OF THE RETURNED entries + * (counted after truncation, so the "N stores" badge matches the rows shown). + */ +export function rankDirectory( + raw: DirEntry[], + opts: { perHost?: number; limit?: number } = {} +): { entries: RankedEntry[]; distinct_hosts: number } { + const perHost = opts.perHost ?? 3; + const limit = opts.limit ?? 200; + + const all: RankedEntry[] = raw.map((e) => ({ ...e, host: hostOf(e.url) })); + + // Featured (asc rank) first, then newest ts — so the per-host cap keeps each + // host's newest entries and featured always survives. + all.sort((a, b) => { + const ar = a.featured_rank ?? Number.POSITIVE_INFINITY; + const br = b.featured_rank ?? Number.POSITIVE_INFINITY; + if (ar !== br) return ar - br; + return b.ts - a.ts; + }); + + const perHostCount: Record = {}; + const deduped = perHost === 0 ? all : all.filter((e) => { + if (e.featured_rank != null) return true; // featured bypasses the cap + const n = (perHostCount[e.host] = (perHostCount[e.host] || 0) + 1); + return n <= perHost; + }); + + // Count distinct hosts AFTER truncation so the "N stores" badge can't claim + // more stores than the rows actually returned (deduped may exceed `limit`). + const entries = deduped.slice(0, limit); + const distinct_hosts = new Set(entries.map((e) => e.host)).size; + return { entries, distinct_hosts }; +} diff --git a/worker/src/index.ts b/worker/src/index.ts index 22f5938..96b3a90 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -208,11 +208,16 @@ app.get("/api/v1/stats/public", async (c) => { }); app.get("/api/v1/directory", async (c) => { + const limit = Math.min(500, parseInt(c.req.query("limit") || "200", 10)); + // Max entries shown per host so one big store (e.g. the cron's Shopify seed) + // can't flood the page and read as spam. ?per_host=0 disables the per-host cap + // only — `limit` (max 500) still applies, so this is not an "everything" escape. + const perHost = Math.max(0, parseInt(c.req.query("per_host") || "3", 10)); const { slugFromUrl } = await import("./slug"); // Paginate the FULL seen: set (KV.list caps at 1000/call) so the directory - // shows every site, not just the first 200 keys (which clustered on a few - // high-volume stores). The page groups these by host itself. + // sees every site, not just the first 1000 keys (which clustered on a few + // high-volume stores). rankDirectory then dedups per-host + applies `limit`. const seenKeys: any[] = []; let cursor: string | undefined; let pages = 0; @@ -244,7 +249,7 @@ app.get("/api/v1/directory", async (c) => { }) ); - const entries = list.keys + const raw = list.keys .map((k: any) => k.metadata) .filter((m: any) => m && m.url) .map((m: any) => { @@ -260,17 +265,15 @@ app.get("/api/v1/directory", async (c) => { }; }); - // Sort: featured first (asc by rank), then by ts desc. - entries.sort((a: any, b: any) => { - const ar = a.featured_rank ?? Number.POSITIVE_INFINITY; - const br = b.featured_rank ?? Number.POSITIVE_INFINITY; - if (ar !== br) return ar - br; - return b.ts - a.ts; - }); - - return c.json({ entries, list_complete: list.list_complete }, 200, { - "cache-control": "public, max-age=300, s-maxage=300", - }); + // Per-host dedup + featured-first ordering lives in rankDirectory (pure, + // unit-tested) — it supersedes the old inline sort and adds the per-host cap. + const { rankDirectory } = await import("./directory_rank"); + const { entries, distinct_hosts } = rankDirectory(raw, { perHost, limit }); + return c.json( + { entries, list_complete: list.list_complete, distinct_hosts, per_host_cap: perHost || null }, + 200, + { "cache-control": "public, max-age=300, s-maxage=300" } + ); }); app.get("/directory", (c) => { diff --git a/worker/test/directory_rank.test.ts b/worker/test/directory_rank.test.ts new file mode 100644 index 0000000..de19f7a --- /dev/null +++ b/worker/test/directory_rank.test.ts @@ -0,0 +1,90 @@ +// test/directory_rank.test.ts — per-host dedup + featured-pinning for /directory. +import { describe, it, expect } from "vitest"; +import { rankDirectory, hostOf, type DirEntry } from "../src/directory_rank"; + +function e(url: string, ts: number, extra: Partial = {}): DirEntry { + return { url, adapter: "shopify", ts, title: null, slug: url, verified: false, featured_rank: null, ...extra }; +} + +describe("rankDirectory", () => { + it("caps each host so one store can't flood (the aloyoga problem)", () => { + const raw: DirEntry[] = []; + for (let i = 0; i < 50; i++) raw.push(e(`https://aloyoga.com/products/p${i}`, 1000 + i)); + raw.push(e("https://gymshark.com/products/x", 900)); + raw.push(e("https://allbirds.com/products/y", 800)); + const { entries, distinct_hosts } = rankDirectory(raw, { perHost: 3, limit: 200 }); + const aloCount = entries.filter((x) => x.host === "aloyoga.com").length; + expect(aloCount).toBe(3); // capped, not 50 + expect(distinct_hosts).toBe(3); // alo + gymshark + allbirds + expect(entries.some((x) => x.host === "gymshark.com")).toBe(true); + expect(entries.some((x) => x.host === "allbirds.com")).toBe(true); + }); + + it("keeps each host's NEWEST entries under the cap", () => { + const raw = [ + e("https://aloyoga.com/products/old", 100), + e("https://aloyoga.com/products/mid", 200), + e("https://aloyoga.com/products/new", 300), + e("https://aloyoga.com/products/oldest", 50), + ]; + const { entries } = rankDirectory(raw, { perHost: 2 }); + const slugs = entries.map((x) => x.url.split("/").pop()); + expect(slugs).toEqual(["new", "mid"]); // newest two, in ts-desc order + }); + + it("featured entries bypass the cap and sort to the top", () => { + const raw = [ + e("https://aloyoga.com/products/a", 1000), + e("https://aloyoga.com/products/b", 999), + e("https://aloyoga.com/products/c", 998), + e("https://aloyoga.com/products/featured", 1, { featured_rank: 0 }), // old but featured + ]; + const { entries } = rankDirectory(raw, { perHost: 2 }); + expect(entries[0].url.endsWith("/featured")).toBe(true); // featured pinned first + // 4 entries survive: the featured one (bypasses cap) + 2 capped + ... actually + // featured bypasses, then 2 non-featured under the cap = 3 total. + expect(entries.filter((x) => x.host === "aloyoga.com").length).toBe(3); + }); + + it("per_host=0 disables the PER-HOST cap only (limit still applies)", () => { + const raw = Array.from({ length: 10 }, (_, i) => e(`https://x.com/p${i}`, i)); + const { entries } = rankDirectory(raw, { perHost: 0 }); + expect(entries.length).toBe(10); // all 10 returned: cap off AND under default limit + // but `limit` is NOT disabled by perHost=0 — it still truncates: + const big = Array.from({ length: 600 }, (_, i) => e(`https://x.com/p${i}`, i)); + expect(rankDirectory(big, { perHost: 0, limit: 200 }).entries.length).toBe(200); + }); + + it("respects the overall limit after dedup", () => { + const raw: DirEntry[] = []; + for (let h = 0; h < 20; h++) for (let i = 0; i < 5; i++) raw.push(e(`https://h${h}.com/p${i}`, h * 10 + i)); + const { entries } = rankDirectory(raw, { perHost: 3, limit: 10 }); + expect(entries.length).toBe(10); + }); + + it("distinct_hosts counts the RETURNED rows, not the pre-truncation set", () => { + // 20 hosts, 1 entry each survives the cap → deduped=20, but limit=10 returns + // 10 rows from 10 hosts. The badge must say 10, not 20. + const raw = Array.from({ length: 20 }, (_, h) => e(`https://h${h}.com/p`, h)); + const { entries, distinct_hosts } = rankDirectory(raw, { perHost: 1, limit: 10 }); + expect(entries.length).toBe(10); + expect(distinct_hosts).toBe(10); // matches rows shown, not the 20 deduped + }); + + it("subdomains are counted as distinct hosts (documented limitation)", () => { + // hostOf does not collapse to the registrable domain, so a multi-subdomain + // store gets a per-host quota per subdomain. Locks in current behavior. + const raw = [ + ...Array.from({ length: 4 }, (_, i) => e(`https://x.com/p${i}`, 100 + i)), + ...Array.from({ length: 4 }, (_, i) => e(`https://shop.x.com/p${i}`, 200 + i)), + ]; + const { entries } = rankDirectory(raw, { perHost: 3 }); + expect(entries.length).toBe(6); // 3 from x.com + 3 from shop.x.com + }); + + it("hostOf strips www + lowercases", () => { + expect(hostOf("https://www.Allbirds.com/products/x")).toBe("allbirds.com"); + expect(hostOf("https://shop.gymshark.com/p")).toBe("shop.gymshark.com"); + expect(hostOf("not a url")).toBe("not a url"); + }); +});