Check every indexed source for an embedding mismatch, not just repositories

backend/synsc/services/embedding_consistency.py

@@ -13,6 +13,7 @@
 
 from __future__ import annotations
 
+import logging
 from dataclasses import dataclass
 from typing import Any
 
@@ -23,26 +24,40 @@
 logger = structlog.get_logger(__name__)
 
 
+# Every table that stores its own embeddings and records which model made
+# them. A mismatch is equally meaningless in any of them, so the check cannot
+# be repository-only: a documentation corpus indexed by a different model
+# returns the same confident noise a repository would.
+SOURCE_TABLES: tuple[tuple[str, str, str], ...] = (
+    ("repositories", "repo_id", "owner || '/' || name"),
+    ("documentation_sources", "docs_id", "coalesce(display_name, url)"),
+    ("papers", "paper_id", "title"),
+    ("datasets", "dataset_id", "name"),
+)
+
+
 @dataclass(frozen=True)
 class EmbeddingMismatch:
-    """One repository whose index does not match the query-time model."""
+    """One indexed source whose vectors do not match the query-time model."""
 
     repo_id: str
     repo_name: str
     indexed_with: str
     querying_with: str
+    source_type: str = "repositories"
 
     def as_dict(self) -> dict[str, str]:
         return {
             "repo_id": self.repo_id,
             "repo_name": self.repo_name,
+            "source_type": self.source_type,
             "indexed_with": self.indexed_with,
             "querying_with": self.querying_with,
         }
 
     def message(self) -> str:
         return (
-            f"{self.repo_name or self.repo_id} was indexed with "
+            f"{self.source_type}: {self.repo_name or self.repo_id} was indexed with "
             f"'{self.indexed_with}' but is being queried with "
             f"'{self.querying_with}'. Vector scores against this repository "
             f"are not meaningful until it is re-indexed or the query-time "
@@ -55,49 +70,69 @@ def find_embedding_mismatches(
     active_model: str,
     repo_ids: list[str] | None = None,
 ) -> list[EmbeddingMismatch]:
-    """Return repositories indexed with a model other than ``active_model``.
+    """Return indexed sources built by a model other than ``active_model``.
 
-    Repositories with no recorded model are skipped rather than reported:
-    they predate the column being populated, so their state is unknown and
-    flagging them would be a guess.
+    Covers every table that stores embeddings, not just repositories: a
+    documentation corpus indexed by a different model returns exactly the same
+    confident noise, and is just as invisible.
+
+    Sources with no recorded model are skipped rather than reported. They
+    predate the column being populated, so their state is unknown and flagging
+    them would be a guess. ``repo_ids`` scopes the repository check only —
+    other source types are always checked in full, since a search that touches
+    them does not name them by repository id.
     """
     if not active_model:
         return []
 
-    params: dict[str, Any] = {"active": active_model}
-    clause = ""
-    if repo_ids:
-        placeholders = ", ".join(f":rid_{i}" for i in range(len(repo_ids)))
-        clause = f"AND repo_id IN ({placeholders})"
-        for index, repo_id in enumerate(repo_ids):
-            params[f"rid_{index}"] = repo_id
-
-    sql = text(
-        f"""
-        SELECT repo_id, owner || '/' || name AS repo_name, embedding_model
-        FROM repositories
-        WHERE embedding_model IS NOT NULL
-          AND embedding_model <> ''
-          AND embedding_model <> :active
-          {clause}
-        """
-    )
-
-    try:
-        rows = session.execute(sql, params).mappings().all()
-    except Exception as exc:  # noqa: BLE001 - a diagnostic must never break search
-        logger.warning("embedding consistency check failed", error=str(exc))
-        return []
-
-    return [
-        EmbeddingMismatch(
-            repo_id=str(row["repo_id"]),
-            repo_name=row["repo_name"] or "",
-            indexed_with=row["embedding_model"],
-            querying_with=active_model,
+    found: list[EmbeddingMismatch] = []
+    for table, id_column, name_expr in SOURCE_TABLES:
+        params: dict[str, Any] = {"active": active_model}
+        clause = ""
+        if repo_ids and table == "repositories":
+            placeholders = ", ".join(f":rid_{i}" for i in range(len(repo_ids)))
+            clause = f"AND {id_column} IN ({placeholders})"
+            for index, repo_id in enumerate(repo_ids):
+                params[f"rid_{index}"] = repo_id
+
+        sql = text(
+            f"""
+            SELECT {id_column} AS source_id, {name_expr} AS source_name,
+                   embedding_model
+            FROM {table}
+            WHERE embedding_model IS NOT NULL
+              AND embedding_model <> ''
+              AND embedding_model <> :active
+              {clause}
+            """
+        )
+        try:
+            rows = session.execute(sql, params).mappings().all()
+        except Exception as exc:  # noqa: BLE001 - a diagnostic must not break search
+            # A table can legitimately be absent on an older schema. Anything
+            # else means this check is silently not checking, which is the
+            # exact failure mode it exists to prevent, so it is logged loudly.
+            message = str(exc)
+            missing = "does not exist" in message or "UndefinedTable" in message
+            logger.log(
+                logging.DEBUG if missing else logging.ERROR,
+                "embedding consistency check could not read %s: %s",
+                table,
+                message[:160],
+            )
+            continue
+
+        found.extend(
+            EmbeddingMismatch(
+                repo_id=str(row["source_id"]),
+                repo_name=row["source_name"] or "",
+                indexed_with=row["embedding_model"],
+                querying_with=active_model,
+                source_type=table,
+            )
+            for row in rows
         )
-        for row in rows
-    ]
+    return found
 
 
 def active_embedding_model() -> str:

backend/tests/test_embedding_consistency.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 from synsc.services.embedding_consistency import (
+    SOURCE_TABLES,
     EmbeddingMismatch,
     find_embedding_mismatches,
 )
@@ -19,51 +20,90 @@ def all(self):
 
 
 class _Session:
-    def __init__(self, rows):
-        self._rows = rows
-        self.params = None
-
-    def execute(self, _statement, params):
-        self.params = params
-        return _Rows(self._rows)
-
-
-def test_reports_repository_indexed_by_another_model():
-    session = _Session([
-        {
-            "repo_id": "r1",
-            "repo_name": "etcd-io/etcd",
+    """Returns rows for one named table and nothing for the others."""
+
+    def __init__(self, rows_by_table=None):
+        self._rows_by_table = rows_by_table or {}
+        self.calls: list[tuple[str, dict]] = []
+
+    def execute(self, statement, params):
+        sql = str(statement)
+        table = next(
+            (t for t, _, _ in SOURCE_TABLES if f"FROM {t}" in sql), "unknown"
+        )
+        self.calls.append((table, params))
+        return _Rows(self._rows_by_table.get(table, []))
+
+
+def test_reports_a_repository_indexed_by_another_model():
+    session = _Session({
+        "repositories": [{
+            "source_id": "r1",
+            "source_name": "etcd-io/etcd",
             "embedding_model": "text-embedding-3-small",
-        }
-    ])
+        }]
+    })
     found = find_embedding_mismatches(session, "gemini-embedding-001")
     assert len(found) == 1
+    assert found[0].source_type == "repositories"
     assert found[0].indexed_with == "text-embedding-3-small"
-    assert found[0].querying_with == "gemini-embedding-001"
 
 
-def test_scopes_the_check_to_requested_repositories():
-    session = _Session([])
+def test_reports_a_documentation_source_too():
+    """A docs corpus in the wrong space is just as invisible as a repo."""
+    session = _Session({
+        "documentation_sources": [{
+            "source_id": "d1",
+            "source_name": "numpy docs",
+            "embedding_model": "gemini-embedding-001",
+        }]
+    })
+    found = find_embedding_mismatches(session, "text-embedding-3-small")
+    assert [m.source_type for m in found] == ["documentation_sources"]
+
+
+def test_checks_every_source_table():
+    session = _Session()
+    find_embedding_mismatches(session, "model-a")
+    assert {table for table, _ in session.calls} == {t for t, _, _ in SOURCE_TABLES}
+
+
+def test_repo_ids_scope_only_the_repository_check():
+    # Other source types are not named by repository id, so scoping them
+    # would silently skip the very sources a search still touches.
+    session = _Session()
     find_embedding_mismatches(session, "model-a", repo_ids=["r1", "r2"])
-    assert session.params["rid_0"] == "r1"
-    assert session.params["rid_1"] == "r2"
+    by_table = dict(session.calls)
+    assert by_table["repositories"]["rid_0"] == "r1"
+    assert "rid_0" not in by_table["documentation_sources"]
 
 
-def test_no_active_model_means_nothing_to_compare():
-    session = _Session([{"repo_id": "r1", "repo_name": "x", "embedding_model": "m"}])
-    assert find_embedding_mismatches(session, "") == []
+def test_a_missing_table_does_not_break_the_check():
+    class _PartlyBroken(_Session):
+        def execute(self, statement, params):
+            if "FROM papers" in str(statement):
+                raise RuntimeError('relation "papers" does not exist')
+            return super().execute(statement, params)
 
+    session = _PartlyBroken({
+        "repositories": [{
+            "source_id": "r1", "source_name": "x", "embedding_model": "other",
+        }]
+    })
+    assert len(find_embedding_mismatches(session, "model-a")) == 1
 
-def test_a_failing_check_never_breaks_search():
-    class _Broken:
-        def execute(self, *a, **k):
-            raise RuntimeError("database is down")
 
-    assert find_embedding_mismatches(_Broken(), "model-a") == []
+def test_no_active_model_means_nothing_to_compare():
+    session = _Session({"repositories": [
+        {"source_id": "r1", "source_name": "x", "embedding_model": "m"}
+    ]})
+    assert find_embedding_mismatches(session, "") == []
 
 
-def test_message_names_both_models_and_the_remedy():
-    message = EmbeddingMismatch("r1", "etcd-io/etcd", "openai-x", "gemini-y").message()
-    assert "openai-x" in message
-    assert "gemini-y" in message
+def test_message_names_the_source_type_and_both_models():
+    message = EmbeddingMismatch(
+        "d1", "numpy docs", "openai-x", "gemini-y", "documentation_sources"
+    ).message()
+    assert "documentation_sources" in message
+    assert "openai-x" in message and "gemini-y" in message
     assert "re-index" in message