memory: seed the categories memU actually advertises

nerve/memory/memu_bridge.py

@@ -123,6 +123,34 @@ def strip_references(t: str | None) -> str | None:
     return text
 
 
+def _memu_cat_name(name: str) -> str:
+    """The display name memU advertises and stores: ``name.strip() or "Untitled"``.
+
+    ``memu/app/memorize.py:930-938`` (prompt) and ``:663-665`` (persist).  Not
+    lowercased: lowercasing happens only at lookup.
+    """
+    return name.strip() or "Untitled"
+
+
+def _memu_cat_key(name: str) -> str:
+    """The key memU's own resolver computes: ``name.strip().lower()``.
+
+    ``memu/app/memorize.py:682``.  The name-to-id rebuild in ``_initialize_impl``
+    does NOT strip, so occupancy is judged in the rebuild's domain, not here.
+    """
+    return _memu_cat_name(name).lower()
+
+
+def _memu_cat_embed_text(name: str, description: str) -> str:
+    """memU's ``_category_embedding_text`` (``memu/app/memorize.py:670-673``).
+
+    One copy, so seeds and repairs share the space ``cosine_topk`` ranks in.
+    """
+    desc = (description or "").strip()
+    name = _memu_cat_name(name)
+    return f"{name}: {desc}" if desc else name
+
+
 class _VectorIndex:
     """Incremental brute-force vector index over memU item embeddings.
 
@@ -1564,10 +1592,10 @@ async def _initialize_impl(self) -> bool:
                        if not self.config.openai_api_key else {}),
                 },
             )
-            self._available = True
-            self._metrics.service_available = True
-            self._metrics.initialized_at = datetime.now(timezone.utc).isoformat()
-            logger.info("memU service initialized with SQLite at %s", sqlite_dsn)
+            # Availability is published at the end of this method, not here:
+            # ~290 further lines of init follow (category seeding, interceptors,
+            # vector preload), and engine.py discards initialize()'s return value,
+            # so _available is the only failure signal the running agent sees.
 
             # Per-connection pragmas (busy_timeout, synchronous=NORMAL).
             self._attach_engine_pragmas()
@@ -1791,7 +1819,8 @@ def _log_after_step(step_ctx, state):
             # recall doesn't pay the 2s JSON-parse cost.
             try:
                 self._service.database.memory_item_repo.list_items()
-                self._service.database.memory_category_repo.list_categories()
+                # Categories are already cached: _ensure_categories loads them
+                # whenever a service exists, which _initialize_impl guarantees.
 
                 # Convert cached embeddings from list[float] to numpy float32.
                 # Pydantic coerces numpy → list during model construction, so we
@@ -1851,6 +1880,11 @@ def _numpy_create_item(self, *args, **kwargs):
                 _numpy_create_item._nerve_numpy_wrapped = True  # type: ignore[attr-defined]
                 SQLiteMemoryItemRepo.create_item = _numpy_create_item
 
+            self._available = True
+            self._metrics.service_available = True
+            self._metrics.initialized_at = datetime.now(timezone.utc).isoformat()
+            logger.info("memU service initialized with SQLite at %s", sqlite_dsn)
+
             return True
 
         except ImportError:
@@ -1861,27 +1895,126 @@ def _numpy_create_item(self, *args, **kwargs):
             return False
 
     async def _ensure_categories(self) -> None:
-        """Create seed categories from config that don't already exist in the DB."""
-        if not self._service or not self.config.memory.categories:
+        """Create the categories memU advertises that don't already exist in the DB.
+
+        Seeds from ``self._service.category_configs``, the *effective* set memU
+        formats into the memorize prompt, not ``config.memory.categories``.  When
+        Nerve configures none, memU falls back to its own defaults, so the two sets
+        differ and every advertised name would be left with no row to resolve to.
+        """
+        if not self._service:
             return
 
-        existing: set[str] = set()
-        try:
-            cats = self._service.database.memory_category_repo.list_categories()
-            for _, cat in cats.items():
-                existing.add(getattr(cat, "name", ""))
-        except Exception:
-            pass
+        repo = self._service.database.memory_category_repo
+        # Unconditional, and read failures propagate: this call also warms the
+        # repo cache that _initialize_impl's name-to-ID rebuild reads, and an
+        # ``existing`` left empty by a swallowed error would re-seed every name.
+        rows = list(repo.list_categories().values())
+
+        # Group by the LOOKUP key, not the display name: the name-to-id rebuild in
+        # _initialize_impl and memU's own resolver agree on it up to stripping, so
+        # a rename that is free among display names can still collapse two live
+        # rows onto one map entry.
+        key_owners: dict[str, list[Any]] = {}
+        for cat in rows:
+            key_owners.setdefault(_memu_cat_key(getattr(cat, "name", "")), []).append(cat)
+
+        # Repair rows persisted under a non-normalized name so their rebuild key
+        # becomes the one memU computes.  Two live sources: an upgrade from the
+        # pre-fix code, which stored the raw configured name, and
+        # ``_create_category_impl``, the runtime path this change leaves alone.
+        # The row id survives the rename, so its category_items stay linked.
+        # A key owned by more than one row is left entirely alone -- renaming
+        # either would make them indistinguishable to every lookup, and merging
+        # would have to discard one row's items.  Read from the whole set, not
+        # the rows walked so far: list_categories() applies no ORDER BY.
+        repairs: list[tuple[Any, str]] = []
+        for cat in rows:
+            raw = getattr(cat, "name", "")
+            norm = _memu_cat_name(raw)
+            if norm != raw and len(key_owners[_memu_cat_key(raw)]) == 1:
+                repairs.append((cat, norm))
+
+        # Occupancy must be judged in the domain the REBUILD keys on -- cat.name.lower()
+        # at the name-to-id rebuild in _initialize_impl, which does NOT strip -- and
+        # over the names the rows will carry AFTER the repairs above.  A key that only
+        # exists as a _memu_cat_key of some padded row is not one any consumer computes,
+        # so treating it as taken suppresses the seed that would supply it.
+        renamed = {id(row): new for row, new in repairs}
+        existing = {renamed.get(id(cat), getattr(cat, "name", "")).lower() for cat in rows}
+        # Snapshot: never iterate the advertised list while seeding from it.
+        missing = [
+            c for c in list(self._service.category_configs)
+            if _memu_cat_name(c.name).lower() not in existing
+        ]
+        await self._seed_categories(missing, repairs)
+
+    async def _seed_categories(
+        self, cat_cfgs: list[Any], repairs: list[tuple[Any, str]] | None = None,
+    ) -> None:
+        """Write the planned repairs and seeds, embedding once before any write.
+
+        Seeds go through the repository rather than ``_create_category_impl``,
+        which also appends a ``CategoryConfig``: memU built ``category_configs`` /
+        ``category_config_map`` / ``_category_prompt_str`` from those entries
+        before this runs, so appending here would advertise every category twice.
+        """
+        repairs = list(repairs or ())
+        if not self._service or (not cat_cfgs and not repairs):
+            return
 
-        for cat_cfg in self.config.memory.categories:
-            if cat_cfg.name in existing:
+        # Embed before any write, in ONE batched call covering repairs and seeds
+        # alike.  A row persisted with a null embedding is never repaired by a
+        # later boot (get_or_create_category returns an existing row untouched)
+        # and both rankers skip null vectors, so a failure must propagate rather
+        # than fall back to None -- and it must not leave a partial migration behind.
+        # A repaired row is re-embedded from its NORMALIZED text for the same
+        # reason its name is normalized: the stored vector is what cosine_topk
+        # ranks, so seeds and repairs have to share one space.
+        plan: list[tuple[str, Any]] = (
+            [("repair", r) for r in repairs] + [("seed", c) for c in cat_cfgs]
+        )
+        texts: list[str] = []
+        for kind, item in plan:
+            src = item[0] if kind == "repair" else item
+            texts.append(_memu_cat_embed_text(
+                getattr(src, "name", ""), getattr(src, "description", "") or "",
+            ))
+        embeddings: list[Any] = [None] * len(plan)
+        if self._has_embeddings:
+            embeddings = list(await self._service._get_llm_client("embedding").embed(texts))
+
+        # Pair up front, so a provider returning the wrong number of vectors fails
+        # before the first write rather than part-way through the loop.
+        pairs = list(zip(plan, embeddings, strict=True))
+
+        repo = self._service.database.memory_category_repo
+        for (kind, item), embedding in pairs:
+            if kind == "repair":
+                row, name = item
+                repo.update_category(category_id=row.id, name=name, embedding=embedding)
+                logger.info("Repaired category name: %r -> %s", getattr(row, "name", ""), name)
+                await self._audit(
+                    "category_updated", "category", row.id, "bridge",
+                    {"name_before": getattr(row, "name", ""), "name_after": name},
+                )
                 continue
-            try:
-                # Already on the memU loop (called from _initialize_impl) —
-                # invoke the impl directly instead of re-submitting.
-                await self._create_category_impl(cat_cfg.name, cat_cfg.description)
-            except Exception as e:
-                logger.warning("Failed to seed category %s: %s", cat_cfg.name, e)
+            # Store memU's normalized form: it is the name the memorize prompt
+            # shows the LLM and the only one its reverse lookups can resolve.
+            name = _memu_cat_name(item.name)
+            description = item.description.strip()
+            repo.get_or_create_category(
+                name=name,
+                description=description,
+                embedding=embedding,
+                user_data={},
+            )
+            # Audit the STORED name, so target_id identifies what was written.
+            logger.info("Seeded category: %s", name)
+            await self._audit(
+                "category_created", "category", name, "bridge",
+                {"description": description},
+            )
 
     # Maximum time (seconds) for a single memorize operation before cancellation.
     # Try to load malloc_trim for returning freed arenas to the OS.