diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md index 59a83e69de..581d7ab7a4 100644 --- a/docs/en/guides/01-configuration.md +++ b/docs/en/guides/01-configuration.md @@ -246,6 +246,28 @@ When the embedding provider experiences consecutive transient failures (e.g. `42 | `circuit_breaker.reset_timeout` | float | Base reset timeout in seconds (default: `60`) | | `circuit_breaker.max_reset_timeout` | float | Maximum reset timeout in seconds when backing off (default: `600`) | +#### Embedding Auto Rebuild + +When the embedding dimension in the configuration changes (e.g., switching from a 1024-dimension model to a 1536-dimension model), the existing vector collection becomes incompatible. OpenViking can automatically rebuild the collection by dropping and recreating it with the new dimension. + +```json +{ + "embedding": { + "auto_rebuild": true + } +} +``` + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `auto_rebuild` | bool | When `true`, automatically drop and recreate the collection when embedding dimension mismatch is detected. When `false`, raise `EmbeddingRebuildRequiredError` and require manual intervention. | `false` | + +> **Note:** When `auto_rebuild` is enabled, the existing **vector index** will be dropped and recreated with the new dimension. The original source files (stored in RAGFS) are **not affected** — they remain intact. After rebuild, OpenViking will re-embed the files to regenerate vectors with the new model/dimension. + +If `auto_rebuild` is `false` (default) and a dimension mismatch is detected, OpenViking raises `EmbeddingRebuildRequiredError` with instructions to either: +- Set `embedding.auto_rebuild=true` to automatically rebuild the vector index +- Manually drop and recreate the vector collection + **Available Models** | Model | Dimension | Input Type | Notes | diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md index 93b32e9ed1..613d8acda5 100644 --- a/docs/zh/guides/01-configuration.md +++ b/docs/zh/guides/01-configuration.md @@ -248,6 +248,28 @@ openviking-server doctor | `circuit_breaker.reset_timeout` | float | 基础恢复等待时间（秒，默认：`60`） | | `circuit_breaker.max_reset_timeout` | float | 指数退避后的最大恢复等待时间（秒，默认：`600`） | +#### Embedding 自动重建 + +当配置中的 embedding 维度发生变化时（例如从 1024 维模型切换到 1536 维模型），现有的向量集合会变得不兼容。OpenViking 可以通过删除并用新维度重新创建集合来自动重建。 + +```json +{ + "embedding": { + "auto_rebuild": true + } +} +``` + +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `auto_rebuild` | bool | 设为 `true` 时，检测到 embedding 维度不匹配会自动删除并重建集合。设为 `false` 时，抛出 `EmbeddingRebuildRequiredError` 并需要手动处理。 | `false` | + +> **说明：** 启用 `auto_rebuild` 后，只会删除并重建**向量索引**，原始源文件（存储在 RAGFS 中）**不受影响** — 文件内容保持完整。重建后，OpenViking 会使用新的模型/维度重新嵌入文件以生成向量。 + +如果 `auto_rebuild` 为 `false`（默认值）且检测到维度不匹配，OpenViking 会抛出 `EmbeddingRebuildRequiredError`，并提示以下处理方式： +- 设置 `embedding.auto_rebuild=true` 自动重建向量索引 +- 手动删除并重新创建向量集合 + **可用模型** | 模型 | 维度 | 输入类型 | 说明 | diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py index e8789df887..dd3f770b3b 100644 --- a/openviking/storage/collection_schemas.py +++ b/openviking/storage/collection_schemas.py @@ -212,7 +212,51 @@ def _decode_collection_description( return base.strip(), payload if isinstance(payload, dict) else None -async def init_context_collection(storage) -> bool: +async def _auto_rebuild_collection( + storage, + config: "OpenVikingConfig", + existing_dim: int, + current_dim: int, + allow_recurse: bool = True, +) -> bool: + """Drop and recreate the collection when embedding dimensions mismatch. + + Raises ``EmbeddingRebuildRequiredError`` when auto-rebuild is disabled, + the backend cannot drop collections, the drop fails, or a recursive + rebuild is detected. + """ + auto_rebuild = bool(getattr(config.embedding, "auto_rebuild", False)) + if not auto_rebuild: + raise EmbeddingRebuildRequiredError( + f"Existing collection embedding dimension ({existing_dim}) does not match " + f"current configuration ({current_dim}). Vectors are incompatible; " + f"rebuild is required. Set embedding.auto_rebuild=true in ov.conf to " + f"automatically rebuild." + ) + if not hasattr(storage, "drop_collection"): + raise EmbeddingRebuildRequiredError( + "Storage backend does not support drop_collection. Manual rebuild required." + ) + if not allow_recurse: + raise EmbeddingRebuildRequiredError( + "Auto-rebuild recursion detected. Manual rebuild required." + ) + logger.warning( + "Dimension mismatch detected (existing=%d, config=%d). " + "Auto-rebuilding collection as embedding.auto_rebuild=true...", + existing_dim, + current_dim, + ) + dropped = await storage.drop_collection() + if not dropped: + raise EmbeddingRebuildRequiredError( + "Failed to drop existing collection for auto-rebuild. " + "Manual rebuild required." + ) + return await init_context_collection(storage, _allow_recurse=False) + + +async def init_context_collection(storage, _allow_recurse: bool = True) -> bool: """ Initialize the context collection with proper schema. @@ -278,6 +322,15 @@ async def init_context_collection(storage) -> bool: return False if existing_embedding_meta is None: + # Old collection without embedding metadata - check actual index dimension + actual_dimension = existing_meta.get("Dimension") + current_dimension = embedding_meta.get("dimension") + + if actual_dimension is not None and current_dimension is not None and actual_dimension != current_dimension: + return await _auto_rebuild_collection( + storage, config, actual_dimension, current_dimension, _allow_recurse, + ) + logger.warning( "Existing collection has %d vector(s) but no embedding metadata " "(created by an older version). Backfilling with current config and continuing.", @@ -337,16 +390,15 @@ async def init_context_collection(storage) -> bool: return False if dimension_changed: - raise EmbeddingRebuildRequiredError( - "Existing collection embedding dimension " - f"({existing_dimension}) does not match current configuration " - f"({current_dimension}). Vectors are incompatible; rebuild is required." + return await _auto_rebuild_collection( + storage, config, existing_dimension, current_dimension, _allow_recurse, ) raise EmbeddingRebuildRequiredError( "Existing collection embedding metadata does not match current configuration. " - "Rebuild is required before using the current embedding model, or set " - "embedding.allow_metadata_override=true to keep existing vectors when " + "Rebuild is required before using the current embedding model. " + "Set embedding.auto_rebuild=true in ov.conf to automatically rebuild, " + "or set embedding.allow_metadata_override=true to keep existing vectors when " "only provider/model changed (dimension must remain the same)." ) diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.py index 448ab0d008..4eeb4d8860 100644 --- a/openviking_cli/utils/config/embedding_config.py +++ b/openviking_cli/utils/config/embedding_config.py @@ -651,6 +651,19 @@ class EmbeddingConfig(BaseModel): "actually changed; only enable when you understand the implication." ), ) + auto_rebuild: bool = Field( + default=False, + description=( + "When true, automatically rebuild the vector index when a dimension " + "mismatch is detected between the existing collection and current config. " + "This will drop the existing collection and recreate it with the new " + "dimension, triggering a full reindex of all documents. " + "WARNING: This will delete all existing vectors and require re-embedding " + "all documents, which may consume significant API credits and time. " + "When false (default), startup will fail with an error message instructing " + "the user to manually rebuild or enable this option." + ), + ) model_config = {"extra": "forbid"} diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py index c158321e39..3ad9b2913d 100644 --- a/tests/storage/test_collection_schemas.py +++ b/tests/storage/test_collection_schemas.py @@ -2117,3 +2117,150 @@ async def _fake_to_thread(func, /, *args, **kwargs): assert isinstance(query_filter, Eq) assert query_filter.field == "account_id" assert query_filter.value == "acc1" + + +@pytest.mark.asyncio +async def test_auto_rebuild_drops_and_recreates_on_dimension_mismatch(monkeypatch): + create_calls = [] + drop_calls = [] + + class _FakeStorage: + async def create_collection(self, name, schema): + create_calls.append((name, schema)) + if len(create_calls) == 1: + return False + return True + + async def get_collection_meta(self): + return { + "Description": ( + "Unified context collection\n\n[openviking.embedding]\n" + '{"dimension": 1024, "model": "text-embedding-3-small", ' + '"model_identity": "text-embedding-3-small", "provider": "openai"}' + ) + } + + async def count(self): + return 3 + + async def drop_collection(self): + drop_calls.append(True) + return True + + async def update_collection_description(self, description): + del description + raise AssertionError("should not update mismatched non-empty collection") + + config = _DummyConfig(_DummyEmbedder()) + config.embedding.auto_rebuild = True + monkeypatch.setattr( + "openviking_cli.utils.config.get_openviking_config", + lambda: config, + ) + + created = await init_context_collection(_FakeStorage()) + + assert created is True + assert len(drop_calls) == 1 + assert len(create_calls) == 2 + + +@pytest.mark.asyncio +async def test_auto_rebuild_disabled_raises_error_on_dimension_mismatch(monkeypatch): + class _FakeStorage: + async def create_collection(self, name, schema): + del name, schema + return False + + async def get_collection_meta(self): + return { + "Description": ( + "Unified context collection\n\n[openviking.embedding]\n" + '{"dimension": 1024, "model": "text-embedding-3-small", ' + '"model_identity": "text-embedding-3-small", "provider": "openai"}' + ) + } + + async def count(self): + return 3 + + async def drop_collection(self): + raise AssertionError("drop_collection should not be called") + + config = _DummyConfig(_DummyEmbedder()) + monkeypatch.setattr( + "openviking_cli.utils.config.get_openviking_config", + lambda: config, + ) + + with pytest.raises(EmbeddingRebuildRequiredError, match="auto_rebuild=true"): + await init_context_collection(_FakeStorage()) + + +@pytest.mark.asyncio +async def test_auto_rebuild_raises_error_when_drop_fails(monkeypatch): + class _FakeStorage: + async def create_collection(self, name, schema): + del name, schema + return False + + async def get_collection_meta(self): + return { + "Description": ( + "Unified context collection\n\n[openviking.embedding]\n" + '{"dimension": 1024, "model": "text-embedding-3-small", ' + '"model_identity": "text-embedding-3-small", "provider": "openai"}' + ) + } + + async def count(self): + return 3 + + async def drop_collection(self): + return False + + config = _DummyConfig(_DummyEmbedder()) + config.embedding.auto_rebuild = True + monkeypatch.setattr( + "openviking_cli.utils.config.get_openviking_config", + lambda: config, + ) + + with pytest.raises(EmbeddingRebuildRequiredError, match="Failed to drop"): + await init_context_collection(_FakeStorage()) + + +@pytest.mark.asyncio +async def test_auto_rebuild_old_collection_without_embedding_metadata(monkeypatch): + create_calls = [] + drop_calls = [] + + class _FakeStorage: + async def create_collection(self, name, schema): + create_calls.append((name, schema)) + if len(create_calls) == 1: + return False + return True + + async def get_collection_meta(self): + return {"Description": "Unified context collection", "Dimension": 1024} + + async def count(self): + return 0 + + async def drop_collection(self): + drop_calls.append(True) + return True + + config = _DummyConfig(_DummyEmbedder()) + config.embedding.auto_rebuild = True + config.embedding.dimension = 2 + monkeypatch.setattr( + "openviking_cli.utils.config.get_openviking_config", + lambda: config, + ) + + created = await init_context_collection(_FakeStorage()) + + assert created is True + assert len(drop_calls) == 1