fix(search): populate FTS for NSID-keyed collections

.changeset/fts-nsid-keyed-collections.md

@@ -0,0 +1,24 @@
+---
+"@atmo-dev/contrail-base": patch
+"@atmo-dev/contrail-appview": patch
+"@atmo-dev/contrail-record-host": patch
+"@atmo-dev/contrail-lexicons": patch
+---
+
+Make NSID-keyed collections work through normal ingestion, not just FTS.
+
+When a collection is keyed directly by its NSID (no short alias, `collection`
+field omitted), the value defaulted to `undefined` everywhere it was read. The
+records insert and FTS sync were patched via `resolveCollectionKey`, but the
+real ingestion entry points still skipped these collections: `getCollectionNsids`
+/ `getDiscoverableNsids` / `getDependentNsids` produced `undefined` NSIDs (so
+Jetstream never subscribed and backfill never ran), `shortNameForNsid` returned
+undefined (so `notify` rejected the URI as "collection not tracked"), and
+`validateConfig` rejected the config outright (missing `collection`, dotted key
+failing short-name validation).
+
+`CollectionConfig.collection` is now optional. `resolveConfig` normalizes an
+omitted `collection` to the map key, `validateConfig` accepts NSID-keyed entries,
+and every collection-list / lookup helper resolves the NSID as `collection ?? key`
+so the behavior is correct on both raw and resolved configs.
+</content>

packages/contrail-appview/src/core/db/records.ts

@@ -18,6 +18,7 @@ import {
   recordsTableName,
   spacesRecordsTableName,
   shortNameForNsid,
+  resolveCollectionKey,
   nsidForShortName,
   normalizeFeedTarget,
   feedTargetMaxItems,
@@ -166,7 +167,7 @@ function buildFtsStatements(
   // PostgreSQL: tsvector generated column is auto-maintained, no manual FTS sync
   if (getDialect(db).ftsStrategy === "generated-column") return [];
 
-  const short = shortNameForNsid(config, event.collection);
+  const short = resolveCollectionKey(config, event.collection);
   if (!short) return [];
   const colConfig = config.collections[short];
   if (!colConfig) return [];
@@ -513,7 +514,7 @@ export async function lookupExistingRecords(
   // Group by short name (config lookup); skip events for collections not in our config.
   const byShort = new Map<string, string[]>();
   for (const e of events) {
-    const short = config ? shortNameForNsid(config, e.collection) : e.collection;
+    const short = config ? resolveCollectionKey(config, e.collection) : e.collection;
     if (!short) continue;
     const uris = byShort.get(short) ?? [];
     uris.push(e.uri);
@@ -599,11 +600,9 @@ export async function applyEvents(
   const countTargets = new Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }>();
 
   for (const e of events) {
-    // Event's collection is an NSID. Look up the short name from config.
-    // If no config or not found, treat collection string as-is (for tests that pre-populate tables).
-    const short = config
-      ? shortNameForNsid(config, e.collection) ?? (config.collections[e.collection] ? e.collection : null)
-      : e.collection;
+    // Event's collection is an NSID. Resolve its storage key from config.
+    // If no config, treat collection string as-is (for tests that pre-populate tables).
+    const short = config ? resolveCollectionKey(config, e.collection) : e.collection;
     if (!short) {
       (config?.logger ?? console).warn(
         `[ingest] drop (unknown collection in applyEvents): ${e.operation} ${e.uri} collection=${e.collection}`

packages/contrail-appview/src/core/refresh.ts

@@ -106,7 +106,7 @@ export async function refresh(
   // included because `resolveConfig` adds them to `config.collections`.
   const nsids =
     options?.nsids ??
-    Object.values(config.collections).map((c) => c.collection);
+    Object.entries(config.collections).map(([short, c]) => c.collection ?? short);
 
   const byCollection: Record<string, CollectionStats> = {};
   for (const nsid of nsids) byCollection[nsid] = emptyStats();

packages/contrail-appview/src/core/router/collection.ts

@@ -663,7 +663,7 @@ export function registerCollectionRoutes(
       // Streaming variant — same query shape, SSE'd forever. Opted in via the
       // presence of the realtime module; no explicit method config needed.
       if (pubsub && spacesCtx) {
-        const colNsid = colConfig.collection;
+        const colNsid = colConfig.collection ?? collection;
         const relations = colConfig.relations ?? {};
         const references = colConfig.references ?? {};
         // Map child-NSID → { relName, matchField } so we can route child
@@ -1068,7 +1068,7 @@ export function registerCollectionRoutes(
         const gated = await gateSpaceAccess(c, spaceUri, "read");
         if (gated instanceof Response) return gated;
 
-        const nsid = colConfig.collection;
+        const nsid = colConfig.collection ?? collection;
         const record = await spacesCtx!.adapter.getRecord(spaceUri, nsid, did, rkey);
         if (!record) return c.json({ error: "NotFound" }, 404);
         return c.json({ record });

packages/contrail-appview/src/core/router/feed.ts

@@ -118,7 +118,7 @@ async function runFeedBackfill(
     const inserted = await backfillUser(
       db,
       actor,
-      followCfg.collection,
+      followCfg.collection ?? followShort,
       Date.now() + BACKFILL_TIMEOUT_MS,
       config,
       {

packages/contrail-appview/src/core/spaces/adapter.ts

@@ -383,7 +383,7 @@ export class HostedAdapter extends HostedAuthorityAdapter implements StorageAdap
           .bind(...params)
           .first<{ count: number }>();
         const count = Number(row?.count ?? 0);
-        if (count > 0) results.push({ collection: colConfig.collection, count });
+        if (count > 0) results.push({ collection: colConfig.collection ?? short, count });
       } catch {
         // table doesn't exist (collection added after init, or allowInSpaces toggled) — skip
       }

packages/contrail-base/src/types.ts

@@ -111,8 +111,9 @@ export function buildFeedTargetCaps(
       const colCfg = config.collections[target.collection];
       if (!colCfg) continue;
       const cap = feedTargetMaxItems(feed, target);
-      const existing = caps.get(colCfg.collection) ?? 0;
-      if (cap > existing) caps.set(colCfg.collection, cap);
+      const nsid = colCfg.collection ?? target.collection;
+      const existing = caps.get(nsid) ?? 0;
+      if (cap > existing) caps.set(nsid, cap);
     }
   }
   return caps;
@@ -125,8 +126,10 @@ export const DEFAULT_COLLECTION_METHODS: CollectionMethod[] = [
 ];
 
 export interface CollectionConfig {
-  /** Full NSID of the record type this collection indexes. */
-  collection: string;
+  /** Full NSID of the record type this collection indexes. May be omitted when
+   *  the collection's map key is itself the full NSID (an "NSID-keyed" config);
+   *  `resolveConfig` normalizes the omitted value to that key. */
+  collection?: string;
   /** Include this collection in Jetstream ingest / discovery (default true).
    *  Set false for dependent collections (auto-fetched on demand). */
   discover?: boolean;
@@ -412,14 +415,15 @@ export function resolveConfig(config: ContrailConfig): ResolvedContrailConfig {
   );
   const collections: Record<string, CollectionConfig> = {};
 
-  // Default `discover: false` for any collection whose NSID lives under the
+  // Normalize an omitted `collection` (NSID-keyed config) to the map key, then
+  // default `discover: false` for any collection whose NSID lives under the
   // `app.bsky.*` namespace, since these are external/network-wide records that
   // would otherwise blow up storage if left discoverable.
-  for (const [short, c] of Object.entries(config.collections)) {
+  for (const [short, rawC] of Object.entries(config.collections)) {
+    const c =
+      rawC.collection === undefined ? { ...rawC, collection: short } : rawC;
     collections[short] =
-      c.discover === undefined &&
-      typeof c.collection === "string" &&
-      c.collection.startsWith("app.bsky.")
+      c.discover === undefined && c.collection!.startsWith("app.bsky.")
         ? { ...c, discover: false }
         : c;
   }
@@ -473,7 +477,7 @@ function _resolveQueryableMaps(config: ContrailConfig): ResolvedMaps {
   const nsidToShort: Record<string, string> = {};
 
   for (const [short, colConfig] of Object.entries(config.collections)) {
-    nsidToShort[colConfig.collection] = short;
+    nsidToShort[colConfig.collection ?? short] = short;
 
     if (colConfig.queryable) {
       queryable[short] = colConfig.queryable;
@@ -583,16 +587,16 @@ function validateShortName(short: string): void {
 export function validateConfig(config: ContrailConfig): void {
   const shortNames = new Set<string>();
   for (const [short, colConfig] of Object.entries(config.collections)) {
-    validateShortName(short);
+    // NSID-keyed collections use the map key as the NSID (no short alias), so
+    // the key legitimately contains dots and must skip short-name validation.
+    const nsidKeyed =
+      colConfig.collection === undefined || colConfig.collection === short;
+    if (!nsidKeyed) validateShortName(short);
     if (shortNames.has(short)) {
       throw new Error(`Duplicate collection short name: ${short}`);
     }
     shortNames.add(short);
 
-    if (!colConfig.collection) {
-      throw new Error(`Collection "${short}" is missing required 'collection' field (NSID)`);
-    }
-
     for (const field of Object.keys(colConfig.queryable ?? {})) {
       validateFieldName(field);
     }
@@ -700,9 +704,10 @@ export function getCollectionShortNames(config: ContrailConfig): string[] {
 /** Alias: collection short names (same as getCollectionShortNames). */
 export const getCollectionNames = getCollectionShortNames;
 
-/** All indexed record NSIDs (what Jetstream filters on). */
+/** All indexed record NSIDs (what Jetstream filters on). For NSID-keyed
+ *  collections (omitted `collection`), the map key is the NSID. */
 export function getCollectionNsids(config: ContrailConfig): string[] {
-  return Object.values(config.collections).map((c) => c.collection);
+  return Object.entries(config.collections).map(([short, c]) => c.collection ?? short);
 }
 
 export function getDependentShortNames(config: ContrailConfig): string[] {
@@ -723,15 +728,15 @@ export const getDiscoverableCollections = getDiscoverableShortNames;
 
 /** Short names of collections the user declared with `discover !== false`, mapped to NSIDs. */
 export function getDiscoverableNsids(config: ContrailConfig): string[] {
-  return Object.values(config.collections)
-    .filter((c) => c.discover !== false)
-    .map((c) => c.collection);
+  return Object.entries(config.collections)
+    .filter(([, c]) => c.discover !== false)
+    .map(([short, c]) => c.collection ?? short);
 }
 
 export function getDependentNsids(config: ContrailConfig): string[] {
-  return Object.values(config.collections)
-    .filter((c) => c.discover === false)
-    .map((c) => c.collection);
+  return Object.entries(config.collections)
+    .filter(([, c]) => c.discover === false)
+    .map(([short, c]) => c.collection ?? short);
 }
 
 /** Short name for a record NSID, if known. */
@@ -742,17 +747,36 @@ export function shortNameForNsid(
   const resolved = (config as ResolvedContrailConfig)._resolved;
   if (resolved?.nsidToShort) return resolved.nsidToShort[nsid];
   for (const [short, c] of Object.entries(config.collections)) {
-    if (c.collection === nsid) return short;
+    if ((c.collection ?? short) === nsid) return short;
   }
   return undefined;
 }
 
-/** Full NSID for a collection short name. */
+/** The config key a collection's rows are stored under: its short alias when
+ *  one exists, otherwise the NSID itself when the config is keyed directly by
+ *  NSID. Returns null when the collection is unknown. Use this wherever you need
+ *  the storage key (records insert, FTS, existing-record lookup). Unlike
+ *  {@link shortNameForNsid}, which only reports an alias and so returns
+ *  undefined for NSID-keyed configs. */
+export function resolveCollectionKey(
+  config: ContrailConfig,
+  nsid: string
+): string | null {
+  return (
+    shortNameForNsid(config, nsid) ??
+    (config.collections[nsid] ? nsid : null)
+  );
+}
+
+/** Full NSID for a collection short name. For NSID-keyed collections (omitted
+ *  `collection`), the short name is itself the NSID. */
 export function nsidForShortName(
   config: ContrailConfig,
   short: string
 ): string | undefined {
-  return config.collections[short]?.collection;
+  const c = config.collections[short];
+  if (!c) return undefined;
+  return c.collection ?? short;
 }
 
 /** The methods a collection should expose via XRPC. */

packages/contrail-record-host/src/adapter.ts

@@ -408,7 +408,7 @@ export class HostedRecordHostAdapter implements RecordHost {
           .bind(...params)
           .first<{ count: number }>();
         const count = Number(row?.count ?? 0);
-        if (count > 0) results.push({ collection: colConfig.collection, count });
+        if (count > 0) results.push({ collection: colConfig.collection ?? short, count });
       } catch {
         // table doesn't exist (collection added after init, or allowInSpaces toggled) — skip
       }

packages/contrail-record-host/src/sync.ts

@@ -242,7 +242,7 @@ async function streamCatchup(args: {
     if (colCfg.allowInSpaces === false) continue;
     if (isClosed()) return highest;
 
-    const collectionNsid = colCfg.collection;
+    const collectionNsid = colCfg.collection ?? _short;
     const short = shortNameForNsid(config, collectionNsid);
     if (!short) continue;
     const table = spacesRecordsTableName(short);

packages/contrail/tests/notify.test.ts

@@ -1,6 +1,8 @@
 import { describe, it, expect, beforeEach, vi, afterEach } from "vitest";
 import type { Database } from "../src/core/types";
-import { applyEvents, createTestDbWithSchema, makeEvent, TEST_CONFIG } from "./helpers";
+import { resolveConfig } from "../src/core/types";
+import { applyEvents, createTestDb, createTestDbWithSchema, makeEvent, TEST_CONFIG } from "./helpers";
+import { initSchema } from "../src/core/db/schema";
 import { parseAtUri } from "../src/core/router/notify";
 import { createApp } from "../src/core/router/index";
 import { queryRecords } from "../src/core/db/records";
@@ -421,6 +423,53 @@ describe("POST notifyOfUpdate", () => {
     expect(body.deleted).toBe(0);
   });
 
+  it("ingests an NSID-keyed collection end-to-end (no short alias)", async () => {
+    // Config keyed directly by NSID, with `collection` omitted — the shape
+    // PR #59 advertised but that normal ingestion entry points used to skip.
+    const nsidConfig = resolveConfig({
+      namespace: "com.example",
+      notify: true,
+      collections: {
+        "com.example.thing": { queryable: { name: {} } },
+      },
+    });
+    const nsidDb = createTestDb();
+    await initSchema(nsidDb, nsidConfig);
+    const nsidApp = createApp(nsidDb, nsidConfig);
+
+    const did = "did:plc:nsidtest";
+    const uri = `at://${did}/com.example.thing/t1`;
+    const record = { name: "Findable Thing" };
+
+    await nsidDb
+      .prepare(
+        "INSERT OR REPLACE INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?)"
+      )
+      .bind(did, "test.handle", "https://pds.example.com", Date.now())
+      .run();
+    mockFetch({ [uri]: { value: record, cid: "bafynsid" } });
+
+    const res = await nsidApp.request("/xrpc/com.example.notifyOfUpdate", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ uri }),
+    });
+
+    expect(res.status).toBe(200);
+    const body = await res.json();
+    // The collection must NOT be rejected as "not tracked", and must ingest.
+    expect(body.errors).toBeUndefined();
+    expect(body.indexed).toBe(1);
+
+    // And the record is retrievable under the NSID-keyed collection.
+    const result = await queryRecords(nsidDb, nsidConfig, {
+      collection: "com.example.thing",
+    });
+    expect(result.records).toHaveLength(1);
+    expect(result.records[0].uri).toBe(uri);
+    expect(JSON.parse(result.records[0].record!)).toEqual(record);
+  });
+
   it("decrements counts when deleting a relation record", async () => {
     const did = "did:plc:test";
     const eventUri = `at://${did}/community.lexicon.calendar.event/evt1`;

packages/contrail/tests/resolve-collection-key.test.ts

@@ -0,0 +1,44 @@
+import { describe, it, expect } from "vitest";
+import { resolveConfig, resolveCollectionKey } from "../src/core/types";
+
+// resolveCollectionKey returns the key a collection's rows are stored under:
+// its short alias when one exists, otherwise the NSID itself when the config is
+// keyed directly by NSID. This is the resolution the records insert, FTS, and
+// lookup paths all need (storage key), distinct from shortNameForNsid which
+// only reports an alias and returns undefined for NSID-keyed configs.
+describe("resolveCollectionKey", () => {
+  it("returns the short alias for an alias-keyed collection", () => {
+    const config = resolveConfig({
+      namespace: "com.example",
+      collections: {
+        events: {
+          collection: "community.lexicon.calendar.event",
+          queryable: { name: {} },
+        },
+      },
+    });
+    expect(
+      resolveCollectionKey(config, "community.lexicon.calendar.event"),
+    ).toBe("events");
+  });
+
+  it("falls back to the NSID when the collection is keyed directly by its NSID", () => {
+    const config = resolveConfig({
+      namespace: "com.example",
+      collections: {
+        "community.lexicon.calendar.event": { queryable: { name: {} } },
+      },
+    });
+    expect(
+      resolveCollectionKey(config, "community.lexicon.calendar.event"),
+    ).toBe("community.lexicon.calendar.event");
+  });
+
+  it("returns null for a collection the config does not know", () => {
+    const config = resolveConfig({
+      namespace: "com.example",
+      collections: { "test.known": { queryable: { name: {} } } },
+    });
+    expect(resolveCollectionKey(config, "test.unknown")).toBeNull();
+  });
+});