diff --git a/backend/synsc/services/embedding_consistency.py b/backend/synsc/services/embedding_consistency.py index 985dbd4..2ac91b2 100644 --- a/backend/synsc/services/embedding_consistency.py +++ b/backend/synsc/services/embedding_consistency.py @@ -13,6 +13,7 @@ from __future__ import annotations +import logging from dataclasses import dataclass from typing import Any @@ -23,26 +24,40 @@ logger = structlog.get_logger(__name__) +# Every table that stores its own embeddings and records which model made +# them. A mismatch is equally meaningless in any of them, so the check cannot +# be repository-only: a documentation corpus indexed by a different model +# returns the same confident noise a repository would. +SOURCE_TABLES: tuple[tuple[str, str, str], ...] = ( + ("repositories", "repo_id", "owner || '/' || name"), + ("documentation_sources", "docs_id", "coalesce(display_name, url)"), + ("papers", "paper_id", "title"), + ("datasets", "dataset_id", "name"), +) + + @dataclass(frozen=True) class EmbeddingMismatch: - """One repository whose index does not match the query-time model.""" + """One indexed source whose vectors do not match the query-time model.""" repo_id: str repo_name: str indexed_with: str querying_with: str + source_type: str = "repositories" def as_dict(self) -> dict[str, str]: return { "repo_id": self.repo_id, "repo_name": self.repo_name, + "source_type": self.source_type, "indexed_with": self.indexed_with, "querying_with": self.querying_with, } def message(self) -> str: return ( - f"{self.repo_name or self.repo_id} was indexed with " + f"{self.source_type}: {self.repo_name or self.repo_id} was indexed with " f"'{self.indexed_with}' but is being queried with " f"'{self.querying_with}'. Vector scores against this repository " f"are not meaningful until it is re-indexed or the query-time " @@ -55,49 +70,69 @@ def find_embedding_mismatches( active_model: str, repo_ids: list[str] | None = None, ) -> list[EmbeddingMismatch]: - """Return repositories indexed with a model other than ``active_model``. + """Return indexed sources built by a model other than ``active_model``. - Repositories with no recorded model are skipped rather than reported: - they predate the column being populated, so their state is unknown and - flagging them would be a guess. + Covers every table that stores embeddings, not just repositories: a + documentation corpus indexed by a different model returns exactly the same + confident noise, and is just as invisible. + + Sources with no recorded model are skipped rather than reported. They + predate the column being populated, so their state is unknown and flagging + them would be a guess. ``repo_ids`` scopes the repository check only — + other source types are always checked in full, since a search that touches + them does not name them by repository id. """ if not active_model: return [] - params: dict[str, Any] = {"active": active_model} - clause = "" - if repo_ids: - placeholders = ", ".join(f":rid_{i}" for i in range(len(repo_ids))) - clause = f"AND repo_id IN ({placeholders})" - for index, repo_id in enumerate(repo_ids): - params[f"rid_{index}"] = repo_id - - sql = text( - f""" - SELECT repo_id, owner || '/' || name AS repo_name, embedding_model - FROM repositories - WHERE embedding_model IS NOT NULL - AND embedding_model <> '' - AND embedding_model <> :active - {clause} - """ - ) - - try: - rows = session.execute(sql, params).mappings().all() - except Exception as exc: # noqa: BLE001 - a diagnostic must never break search - logger.warning("embedding consistency check failed", error=str(exc)) - return [] - - return [ - EmbeddingMismatch( - repo_id=str(row["repo_id"]), - repo_name=row["repo_name"] or "", - indexed_with=row["embedding_model"], - querying_with=active_model, + found: list[EmbeddingMismatch] = [] + for table, id_column, name_expr in SOURCE_TABLES: + params: dict[str, Any] = {"active": active_model} + clause = "" + if repo_ids and table == "repositories": + placeholders = ", ".join(f":rid_{i}" for i in range(len(repo_ids))) + clause = f"AND {id_column} IN ({placeholders})" + for index, repo_id in enumerate(repo_ids): + params[f"rid_{index}"] = repo_id + + sql = text( + f""" + SELECT {id_column} AS source_id, {name_expr} AS source_name, + embedding_model + FROM {table} + WHERE embedding_model IS NOT NULL + AND embedding_model <> '' + AND embedding_model <> :active + {clause} + """ + ) + try: + rows = session.execute(sql, params).mappings().all() + except Exception as exc: # noqa: BLE001 - a diagnostic must not break search + # A table can legitimately be absent on an older schema. Anything + # else means this check is silently not checking, which is the + # exact failure mode it exists to prevent, so it is logged loudly. + message = str(exc) + missing = "does not exist" in message or "UndefinedTable" in message + logger.log( + logging.DEBUG if missing else logging.ERROR, + "embedding consistency check could not read %s: %s", + table, + message[:160], + ) + continue + + found.extend( + EmbeddingMismatch( + repo_id=str(row["source_id"]), + repo_name=row["source_name"] or "", + indexed_with=row["embedding_model"], + querying_with=active_model, + source_type=table, + ) + for row in rows ) - for row in rows - ] + return found def active_embedding_model() -> str: diff --git a/backend/tests/test_embedding_consistency.py b/backend/tests/test_embedding_consistency.py index ba31e5b..1b26786 100644 --- a/backend/tests/test_embedding_consistency.py +++ b/backend/tests/test_embedding_consistency.py @@ -2,6 +2,7 @@ from __future__ import annotations from synsc.services.embedding_consistency import ( + SOURCE_TABLES, EmbeddingMismatch, find_embedding_mismatches, ) @@ -19,51 +20,90 @@ def all(self): class _Session: - def __init__(self, rows): - self._rows = rows - self.params = None - - def execute(self, _statement, params): - self.params = params - return _Rows(self._rows) - - -def test_reports_repository_indexed_by_another_model(): - session = _Session([ - { - "repo_id": "r1", - "repo_name": "etcd-io/etcd", + """Returns rows for one named table and nothing for the others.""" + + def __init__(self, rows_by_table=None): + self._rows_by_table = rows_by_table or {} + self.calls: list[tuple[str, dict]] = [] + + def execute(self, statement, params): + sql = str(statement) + table = next( + (t for t, _, _ in SOURCE_TABLES if f"FROM {t}" in sql), "unknown" + ) + self.calls.append((table, params)) + return _Rows(self._rows_by_table.get(table, [])) + + +def test_reports_a_repository_indexed_by_another_model(): + session = _Session({ + "repositories": [{ + "source_id": "r1", + "source_name": "etcd-io/etcd", "embedding_model": "text-embedding-3-small", - } - ]) + }] + }) found = find_embedding_mismatches(session, "gemini-embedding-001") assert len(found) == 1 + assert found[0].source_type == "repositories" assert found[0].indexed_with == "text-embedding-3-small" - assert found[0].querying_with == "gemini-embedding-001" -def test_scopes_the_check_to_requested_repositories(): - session = _Session([]) +def test_reports_a_documentation_source_too(): + """A docs corpus in the wrong space is just as invisible as a repo.""" + session = _Session({ + "documentation_sources": [{ + "source_id": "d1", + "source_name": "numpy docs", + "embedding_model": "gemini-embedding-001", + }] + }) + found = find_embedding_mismatches(session, "text-embedding-3-small") + assert [m.source_type for m in found] == ["documentation_sources"] + + +def test_checks_every_source_table(): + session = _Session() + find_embedding_mismatches(session, "model-a") + assert {table for table, _ in session.calls} == {t for t, _, _ in SOURCE_TABLES} + + +def test_repo_ids_scope_only_the_repository_check(): + # Other source types are not named by repository id, so scoping them + # would silently skip the very sources a search still touches. + session = _Session() find_embedding_mismatches(session, "model-a", repo_ids=["r1", "r2"]) - assert session.params["rid_0"] == "r1" - assert session.params["rid_1"] == "r2" + by_table = dict(session.calls) + assert by_table["repositories"]["rid_0"] == "r1" + assert "rid_0" not in by_table["documentation_sources"] -def test_no_active_model_means_nothing_to_compare(): - session = _Session([{"repo_id": "r1", "repo_name": "x", "embedding_model": "m"}]) - assert find_embedding_mismatches(session, "") == [] +def test_a_missing_table_does_not_break_the_check(): + class _PartlyBroken(_Session): + def execute(self, statement, params): + if "FROM papers" in str(statement): + raise RuntimeError('relation "papers" does not exist') + return super().execute(statement, params) + session = _PartlyBroken({ + "repositories": [{ + "source_id": "r1", "source_name": "x", "embedding_model": "other", + }] + }) + assert len(find_embedding_mismatches(session, "model-a")) == 1 -def test_a_failing_check_never_breaks_search(): - class _Broken: - def execute(self, *a, **k): - raise RuntimeError("database is down") - assert find_embedding_mismatches(_Broken(), "model-a") == [] +def test_no_active_model_means_nothing_to_compare(): + session = _Session({"repositories": [ + {"source_id": "r1", "source_name": "x", "embedding_model": "m"} + ]}) + assert find_embedding_mismatches(session, "") == [] -def test_message_names_both_models_and_the_remedy(): - message = EmbeddingMismatch("r1", "etcd-io/etcd", "openai-x", "gemini-y").message() - assert "openai-x" in message - assert "gemini-y" in message +def test_message_names_the_source_type_and_both_models(): + message = EmbeddingMismatch( + "d1", "numpy docs", "openai-x", "gemini-y", "documentation_sources" + ).message() + assert "documentation_sources" in message + assert "openai-x" in message and "gemini-y" in message assert "re-index" in message