From d4b9032213dba7535652fee5827658989d7f1b78 Mon Sep 17 00:00:00 2001 From: Dylan Couzon Date: Fri, 17 Jul 2026 10:33:02 -0400 Subject: [PATCH 1/2] add nomic-embed-vision-v1.5 --- fastembed/image/image_embedding.py | 7 +++- fastembed/image/onnx_embedding.py | 61 ++++++++++++++++++++++++++++- tests/test_image_onnx_embeddings.py | 11 ++++++ 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/fastembed/image/image_embedding.py b/fastembed/image/image_embedding.py index a36303456..4226fdabf 100644 --- a/fastembed/image/image_embedding.py +++ b/fastembed/image/image_embedding.py @@ -4,12 +4,15 @@ from fastembed.common.types import NumpyArray, Device from fastembed.common import ImageInput, OnnxProvider from fastembed.image.image_embedding_base import ImageEmbeddingBase -from fastembed.image.onnx_embedding import OnnxImageEmbedding +from fastembed.image.onnx_embedding import OnnxImageEmbedding, NomicVisionEmbedding from fastembed.common.model_description import DenseModelDescription class ImageEmbedding(ImageEmbeddingBase): - EMBEDDINGS_REGISTRY: list[Type[ImageEmbeddingBase]] = [OnnxImageEmbedding] + EMBEDDINGS_REGISTRY: list[Type[ImageEmbeddingBase]] = [ + OnnxImageEmbedding, + NomicVisionEmbedding, + ] @classmethod def list_supported_models(cls) -> list[dict[str, Any]]: diff --git a/fastembed/image/onnx_embedding.py b/fastembed/image/onnx_embedding.py index 3e1739951..5f5eefbc8 100644 --- a/fastembed/image/onnx_embedding.py +++ b/fastembed/image/onnx_embedding.py @@ -58,12 +58,32 @@ ), ] +supported_nomic_models: list[DenseModelDescription] = [ + DenseModelDescription( + model="nomic-ai/nomic-embed-vision-v1.5", + dim=768, + description="Image embeddings, Multimodal (text&image), 2024 year", + license="apache-2.0", + size_in_GB=0.37, + sources=ModelSource(hf="nomic-ai/nomic-embed-vision-v1.5"), + model_file="onnx/model.onnx", + ), + DenseModelDescription( + model="nomic-ai/nomic-embed-vision-v1.5-Q", + dim=768, + description="Image embeddings, Multimodal (text&image), 2024 year", + license="apache-2.0", + size_in_GB=0.1, + sources=ModelSource(hf="nomic-ai/nomic-embed-vision-v1.5"), + model_file="onnx/model_quantized.onnx", + ), +] + class OnnxImageEmbedding(ImageEmbeddingBase, OnnxImageModel[NumpyArray]): def __init__( self, model_name: str, - cache_dir: str | None = None, threads: int | None = None, providers: Sequence[OnnxProvider] | None = None, @@ -215,3 +235,42 @@ def init_embedding(self, model_name: str, cache_dir: str, **kwargs: Any) -> Onnx threads=1, **kwargs, ) + + +class NomicVisionEmbedding(OnnxImageEmbedding): + @classmethod + def _list_supported_models(cls) -> list[DenseModelDescription]: + """ + Lists the supported models. + + Returns: + list[DenseModelDescription]: A list of DenseModelDescription objects containing the model information. + """ + return supported_nomic_models + + @classmethod + def _get_worker_class(cls) -> Type["ImageEmbeddingWorker[NumpyArray]"]: + return NomicVisionEmbeddingWorker + + def _post_process_onnx_output( + self, output: OnnxOutputContext, **kwargs: Any + ) -> Iterable[NumpyArray]: + # nomic-embed-vision emits last_hidden_state, which onnx_embed flattens to + # (batch, tokens * dim). Recover the token axis, take the CLS token (index 0) and normalize, + # matching the reference F.normalize(last_hidden_state[:, 0], p=2, dim=1). + dim = self.model_description.dim + assert dim is not None, "Model description is missing the embedding dim" + hidden_states = output.model_output.reshape(output.model_output.shape[0], -1, dim) + return normalize(hidden_states[:, 0]) + + +class NomicVisionEmbeddingWorker(ImageEmbeddingWorker[NumpyArray]): + def init_embedding( + self, model_name: str, cache_dir: str, **kwargs: Any + ) -> NomicVisionEmbedding: + return NomicVisionEmbedding( + model_name=model_name, + cache_dir=cache_dir, + threads=1, + **kwargs, + ) diff --git a/tests/test_image_onnx_embeddings.py b/tests/test_image_onnx_embeddings.py index 5ac5e44f1..f9fe24952 100644 --- a/tests/test_image_onnx_embeddings.py +++ b/tests/test_image_onnx_embeddings.py @@ -1,4 +1,5 @@ import os +import platform from contextlib import contextmanager from io import BytesIO @@ -25,6 +26,12 @@ "jinaai/jina-clip-v1": np.array( [-0.029, 0.0216, 0.0396, 0.0283, -0.0023, 0.0151, 0.011, -0.0235, 0.0251, -0.0343] ), + "nomic-ai/nomic-embed-vision-v1.5": np.array( + [0.0048, -0.0254, 0.0067, -0.0296, -0.0435, -0.0123, 0.0024, -0.0361, -0.0703, -0.0186] + ), + "nomic-ai/nomic-embed-vision-v1.5-Q": np.array( + [-0.0011, -0.0477, 0.0024, -0.049, -0.0458, -0.0314, 0.017, -0.0383, -0.0537, -0.021] + ), } _MODELS_TO_CACHE = ("Qdrant/clip-ViT-B-32-vision",) @@ -59,9 +66,13 @@ def get_model(model_name: str): @pytest.mark.parametrize("model_name", ["Qdrant/clip-ViT-B-32-vision"]) def test_embedding(model_cache, model_name: str) -> None: is_ci = os.getenv("CI") + is_mac = platform.system() == "Darwin" is_manual = os.getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" for model_desc in ImageEmbedding._list_supported_models(): + # quantized int8 ops diverge on macOS; canonical vector is generated on linux/amd64 (CI) + if is_mac and model_desc.model == "nomic-ai/nomic-embed-vision-v1.5-Q": + continue if not should_test_model(model_desc, model_name, is_ci, is_manual): continue From e028a79b2a8cc210ab87e730d9df78b20f1456c1 Mon Sep 17 00:00:00 2001 From: Dylan Couzon Date: Mon, 20 Jul 2026 14:09:01 -0400 Subject: [PATCH 2/2] Rename and reorganize --- fastembed/image/image_embedding.py | 5 +- fastembed/image/normalized_embedding.py | 69 +++++++++++++++++++++++++ fastembed/image/onnx_embedding.py | 60 --------------------- 3 files changed, 72 insertions(+), 62 deletions(-) create mode 100644 fastembed/image/normalized_embedding.py diff --git a/fastembed/image/image_embedding.py b/fastembed/image/image_embedding.py index 4226fdabf..0924eac0f 100644 --- a/fastembed/image/image_embedding.py +++ b/fastembed/image/image_embedding.py @@ -4,14 +4,15 @@ from fastembed.common.types import NumpyArray, Device from fastembed.common import ImageInput, OnnxProvider from fastembed.image.image_embedding_base import ImageEmbeddingBase -from fastembed.image.onnx_embedding import OnnxImageEmbedding, NomicVisionEmbedding +from fastembed.image.onnx_embedding import OnnxImageEmbedding +from fastembed.image.normalized_embedding import NormalizedEmbedding from fastembed.common.model_description import DenseModelDescription class ImageEmbedding(ImageEmbeddingBase): EMBEDDINGS_REGISTRY: list[Type[ImageEmbeddingBase]] = [ OnnxImageEmbedding, - NomicVisionEmbedding, + NormalizedEmbedding, ] @classmethod diff --git a/fastembed/image/normalized_embedding.py b/fastembed/image/normalized_embedding.py new file mode 100644 index 000000000..0b83fe774 --- /dev/null +++ b/fastembed/image/normalized_embedding.py @@ -0,0 +1,69 @@ +from typing import Any, Iterable, Type + + +from fastembed.common.types import NumpyArray +from fastembed.common.onnx_model import OnnxOutputContext +from fastembed.common.utils import normalize +from fastembed.image.onnx_embedding import OnnxImageEmbedding +from fastembed.image.onnx_image_model import ImageEmbeddingWorker +from fastembed.common.model_description import DenseModelDescription, ModelSource + +supported_normalized_models: list[DenseModelDescription] = [ + DenseModelDescription( + model="nomic-ai/nomic-embed-vision-v1.5", + dim=768, + description="Image embeddings, Multimodal (text&image), 2024 year", + license="apache-2.0", + size_in_GB=0.37, + sources=ModelSource(hf="nomic-ai/nomic-embed-vision-v1.5"), + model_file="onnx/model.onnx", + ), + DenseModelDescription( + model="nomic-ai/nomic-embed-vision-v1.5-Q", + dim=768, + description="Image embeddings, Multimodal (text&image), 2024 year", + license="apache-2.0", + size_in_GB=0.1, + sources=ModelSource(hf="nomic-ai/nomic-embed-vision-v1.5"), + model_file="onnx/model_quantized.onnx", + ), +] + + +class NormalizedEmbedding(OnnxImageEmbedding): + @classmethod + def _list_supported_models(cls) -> list[DenseModelDescription]: + """ + Lists the supported models. + + Returns: + list[DenseModelDescription]: A list of DenseModelDescription objects containing the model information. + """ + return supported_normalized_models + + @classmethod + def _get_worker_class(cls) -> Type["ImageEmbeddingWorker[NumpyArray]"]: + return NormalizedEmbeddingWorker + + def _post_process_onnx_output( + self, output: OnnxOutputContext, **kwargs: Any + ) -> Iterable[NumpyArray]: + # The model emits last_hidden_state, which onnx_embed flattens to (batch, tokens * dim). + # Recover the token axis, take the CLS token (index 0) and normalize, matching the reference + # F.normalize(last_hidden_state[:, 0], p=2, dim=1). + dim = self.model_description.dim + assert dim is not None, "Model description is missing the embedding dim" + hidden_states = output.model_output.reshape(output.model_output.shape[0], -1, dim) + return normalize(hidden_states[:, 0]) + + +class NormalizedEmbeddingWorker(ImageEmbeddingWorker[NumpyArray]): + def init_embedding( + self, model_name: str, cache_dir: str, **kwargs: Any + ) -> NormalizedEmbedding: + return NormalizedEmbedding( + model_name=model_name, + cache_dir=cache_dir, + threads=1, + **kwargs, + ) diff --git a/fastembed/image/onnx_embedding.py b/fastembed/image/onnx_embedding.py index 5f5eefbc8..38b79743a 100644 --- a/fastembed/image/onnx_embedding.py +++ b/fastembed/image/onnx_embedding.py @@ -58,27 +58,6 @@ ), ] -supported_nomic_models: list[DenseModelDescription] = [ - DenseModelDescription( - model="nomic-ai/nomic-embed-vision-v1.5", - dim=768, - description="Image embeddings, Multimodal (text&image), 2024 year", - license="apache-2.0", - size_in_GB=0.37, - sources=ModelSource(hf="nomic-ai/nomic-embed-vision-v1.5"), - model_file="onnx/model.onnx", - ), - DenseModelDescription( - model="nomic-ai/nomic-embed-vision-v1.5-Q", - dim=768, - description="Image embeddings, Multimodal (text&image), 2024 year", - license="apache-2.0", - size_in_GB=0.1, - sources=ModelSource(hf="nomic-ai/nomic-embed-vision-v1.5"), - model_file="onnx/model_quantized.onnx", - ), -] - class OnnxImageEmbedding(ImageEmbeddingBase, OnnxImageModel[NumpyArray]): def __init__( @@ -235,42 +214,3 @@ def init_embedding(self, model_name: str, cache_dir: str, **kwargs: Any) -> Onnx threads=1, **kwargs, ) - - -class NomicVisionEmbedding(OnnxImageEmbedding): - @classmethod - def _list_supported_models(cls) -> list[DenseModelDescription]: - """ - Lists the supported models. - - Returns: - list[DenseModelDescription]: A list of DenseModelDescription objects containing the model information. - """ - return supported_nomic_models - - @classmethod - def _get_worker_class(cls) -> Type["ImageEmbeddingWorker[NumpyArray]"]: - return NomicVisionEmbeddingWorker - - def _post_process_onnx_output( - self, output: OnnxOutputContext, **kwargs: Any - ) -> Iterable[NumpyArray]: - # nomic-embed-vision emits last_hidden_state, which onnx_embed flattens to - # (batch, tokens * dim). Recover the token axis, take the CLS token (index 0) and normalize, - # matching the reference F.normalize(last_hidden_state[:, 0], p=2, dim=1). - dim = self.model_description.dim - assert dim is not None, "Model description is missing the embedding dim" - hidden_states = output.model_output.reshape(output.model_output.shape[0], -1, dim) - return normalize(hidden_states[:, 0]) - - -class NomicVisionEmbeddingWorker(ImageEmbeddingWorker[NumpyArray]): - def init_embedding( - self, model_name: str, cache_dir: str, **kwargs: Any - ) -> NomicVisionEmbedding: - return NomicVisionEmbedding( - model_name=model_name, - cache_dir=cache_dir, - threads=1, - **kwargs, - )