Dev

python/python/embed_anything/_embed_anything.pyi

@@ -335,6 +335,7 @@ class ColpaliModel:
             revision: The revision of the model.
         """
 
+    @staticmethod
     def from_pretrained(model_id: str, revision: str | None = None) -> ColpaliModel:
         """
         Loads a pre-trained Colpali model from the Hugging Face model hub.
@@ -347,6 +348,7 @@ class ColpaliModel:
             A ColpaliModel object.
         """
 
+    @staticmethod
     def from_pretrained_onnx(
         model_id: str, revision: str | None = None
     ) -> ColpaliModel:
@@ -400,8 +402,8 @@ class ColbertModel:
         Initializes the ColbertModel object.
         """
 
+    @staticmethod
     def from_pretrained_onnx(
-        self,
         hf_model_id: str | None = None,
         revision: str | None = None,
         path_in_repo: str | None = None,
@@ -437,6 +439,7 @@ class Reranker:
         Initializes the Reranker object.
         """
 
+    @staticmethod
     def from_pretrained(
         model_id: str, revision: str | None = None, dtype: Dtype | None = None, path_in_repo: str | None = None
     ) -> Reranker:
@@ -590,6 +593,7 @@ class EmbeddingModel:
     Represents an embedding model.
     """
 
+    @staticmethod
     def from_pretrained_hf(
         model_id: str,
         revision: str | None = None,
@@ -617,6 +621,7 @@ class EmbeddingModel:
 
         """
 
+    @staticmethod
     def from_pretrained_cloud(
         model: WhichModel, model_id: str, api_key: str | None = None
     ) -> EmbeddingModel:
@@ -657,6 +662,7 @@ class EmbeddingModel:
         ```
         """
 
+    @staticmethod
     def from_pretrained_onnx(
         model: WhichModel,
         model_name: Optional[ONNXModel] | None = None,
@@ -830,23 +836,6 @@ class EmbeddingModel:
             A list of EmbedData objects.
         """
 
-    def embed_webpage(
-        self,
-        url: str,
-        config: TextEmbedConfig | None = None,
-        adapter: Adapter | None = None,
-    ) -> list[EmbedData]:
-        """
-        Embeds the given webpage and returns a list of EmbedData objects.
-
-        Args:
-            url: The URL of the webpage to embed.
-            config: The configuration for the embedding.
-            adapter: The adapter for the embedding.
-
-        Returns:
-            A list of EmbedData objects.
-        """
 
 class AudioDecoderModel:
     """
@@ -875,6 +864,7 @@ class AudioDecoderModel:
     model_type: str
     quantized: bool
 
+    @staticmethod
     def from_pretrained_hf(
         model_id: str | None = None,
         revision: str | None = None,
@@ -1005,5 +995,3 @@ class ONNXModel(Enum):
     SPLADEPPENV1 = "SPLADEPPENV1"
 
     SPLADEPPENV2 = "SPLADEPPENV2"
-
-    ModernBERTBase = "ModernBERTBase"

rust/src/embeddings/local/clip.rs

@@ -203,7 +203,7 @@ impl ClipEmbedder {
     ) -> anyhow::Result<Tokenizer> {
         let tokenizer = match tokenizer {
             None => {
-                let api = hf_hub::api::sync::Api::new()?;
+                let api = hf_hub::api::sync::ApiBuilder::from_env().build()?;
                 let api = match revision {
                     Some(rev) => api.repo(hf_hub::Repo::with_revision(
                         model_id,

rust/src/embeddings/local/colbert.rs

@@ -7,7 +7,7 @@ extern crate accelerate_src;
 use std::{ops::Mul, sync::RwLock};
 
 use anyhow::{Error as E, Result};
-use hf_hub::{api::sync::Api, Repo};
+use hf_hub::{api::sync::ApiBuilder, Repo};
 use ndarray::{Array2, Axis};
 use ort::{
     execution_providers::{CUDAExecutionProvider, CoreMLExecutionProvider, ExecutionProvider},
@@ -52,7 +52,7 @@ impl OrtColbertEmbedder {
         };
 
         let (_, tokenizer_filename, weights_filename, tokenizer_config_filename, data_filename) = {
-            let api = Api::new().unwrap();
+            let api = ApiBuilder::from_env().build().unwrap();
             let api = match revision {
                 Some(rev) => api.repo(Repo::with_revision(
                     hf_model_id.to_string(),

rust/src/embeddings/local/colpali.rs

@@ -43,7 +43,7 @@ pub struct ColPaliEmbedder {
 
 impl ColPaliEmbedder {
     pub fn new(model_id: &str, revision: Option<&str>) -> Result<Self, anyhow::Error> {
-        let api = hf_hub::api::sync::Api::new()?;
+        let api = hf_hub::api::sync::ApiBuilder::from_env().build()?;
         let repo: hf_hub::api::sync::ApiRepo = match revision {
             Some(rev) => api.repo(hf_hub::Repo::with_revision(
                 model_id.to_string(),

rust/src/embeddings/local/colpali_ort.rs

@@ -31,7 +31,7 @@ impl OrtColPaliEmbedder {
         revision: Option<&str>,
         path_in_repo: Option<&str>,
     ) -> Result<Self, E> {
-        let api = hf_hub::api::sync::Api::new()?;
+        let api = hf_hub::api::sync::ApiBuilder::from_env().build()?;
         let repo: hf_hub::api::sync::ApiRepo = match revision {
             Some(rev) => api.repo(hf_hub::Repo::with_revision(
                 model_id.to_string(),

rust/src/embeddings/local/colsmol.rs

@@ -21,7 +21,7 @@ pub struct ColSmolEmbedder {
 
 impl ColSmolEmbedder {
     pub fn new(model_id: &str, revision: Option<&str>) -> Result<Self, anyhow::Error> {
-        let api = hf_hub::api::sync::Api::new()?;
+        let api = hf_hub::api::sync::ApiBuilder::from_env().build()?;
         let repo: hf_hub::api::sync::ApiRepo = match revision {
             Some(rev) => api.repo(hf_hub::Repo::with_revision(
                 model_id.to_string(),

rust/src/embeddings/local/ort_bert.rs

@@ -9,7 +9,7 @@ use crate::embeddings::utils::{get_type_ids_ndarray, tokenize_batch_ndarray};
 
 use crate::Dtype;
 use anyhow::Error as E;
-use hf_hub::api::sync::Api;
+use hf_hub::api::sync::ApiBuilder;
 use hf_hub::Repo;
 use ndarray::prelude::*;
 use ort::execution_providers::{CUDAExecutionProvider, CoreMLExecutionProvider, ExecutionProvider};
@@ -62,7 +62,7 @@ impl OrtBertEmbedder {
         };
 
         let (_, tokenizer_filename, weights_filename, tokenizer_config_filename) = {
-            let api = Api::new().unwrap();
+            let api = ApiBuilder::from_env().build().unwrap();
             let api = match revision {
                 Some(rev) => api.repo(Repo::with_revision(
                     hf_model_id.to_string(),
@@ -418,7 +418,7 @@ impl OrtSparseBertEmbedder {
         };
 
         let (_, tokenizer_filename, weights_filename, tokenizer_config_filename) = {
-            let api = Api::new().unwrap();
+            let api = ApiBuilder::from_env().build().unwrap();
             let api = match revision {
                 Some(rev) => api.repo(Repo::with_revision(
                     hf_model_id.to_string(),

rust/src/embeddings/local/ort_colsmol.rs

@@ -34,7 +34,7 @@ impl OrtColSmolEmbedder {
         revision: Option<&str>,
         path_in_repo: Option<&str>,
     ) -> Result<Self, E> {
-        let api = hf_hub::api::sync::Api::new()?;
+        let api = hf_hub::api::sync::ApiBuilder::from_env().build()?;
         let repo: hf_hub::api::sync::ApiRepo = match revision {
             Some(rev) => api.repo(hf_hub::Repo::with_revision(
                 model_id.to_string(),

rust/src/embeddings/local/ort_jina.rs

@@ -6,7 +6,7 @@ use crate::embeddings::embed::EmbeddingResult;
 use crate::embeddings::utils::tokenize_batch_ndarray;
 use crate::Dtype;
 use anyhow::Error as E;
-use hf_hub::api::sync::Api;
+use hf_hub::api::sync::ApiBuilder;
 use hf_hub::Repo;
 use ndarray::prelude::*;
 use std::sync::RwLock;
@@ -63,7 +63,7 @@ impl OrtJinaEmbedder {
         };
 
         let (_, tokenizer_filename, weights_filename, tokenizer_config_filename) = {
-            let api = Api::new().unwrap();
+            let api = ApiBuilder::from_env().build().unwrap();
             let api = match revision {
                 Some(rev) => api.repo(Repo::with_revision(
                     hf_model_id.to_string(),

rust/src/file_processor/audio/audio_processor.rs

@@ -9,7 +9,7 @@ use std::path::PathBuf;
 use anyhow::{Error as E, Result};
 use candle_core::{Device, IndexOp, Tensor};
 use candle_nn::{ops::softmax, VarBuilder};
-use hf_hub::{api::sync::Api, Repo, RepoType};
+use hf_hub::{api::sync::ApiBuilder, Repo, RepoType};
 use rand::{
     distr::{weighted::WeightedIndex, Distribution},
     SeedableRng,
@@ -458,7 +458,7 @@ pub fn build_model(
         (None, None) => (default_model, default_revision),
     };
 
-    let api = Api::new()?;
+    let api = ApiBuilder::from_env().build()?;
     let repo = api.repo(Repo::with_revision(
         model_id.to_string(),
         RepoType::Model,

rust/src/models/idefics3/array_processing.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 
 use anyhow::{Error, Ok};
-use hf_hub::{api::sync::Api, Repo, RepoType};
+use hf_hub::{api::sync::ApiBuilder, Repo, RepoType};
 use image::{imageops::FilterType, DynamicImage, GenericImageView, RgbImage};
 use ndarray::{s, Array2};
 use regex::Regex;
@@ -65,7 +65,7 @@ impl Idefics3ImageProcessor {
     }
 
     pub fn from_pretrained(model_id: &str) -> Result<Self, anyhow::Error> {
-        let api = Api::new()?;
+        let api = ApiBuilder::from_env().build()?;
         let repo = api.repo(Repo::new(model_id.to_string(), RepoType::Model));
         let config_file = repo.get("preprocessor_config.json").unwrap();
         let processor: Idefics3ImageProcessor =
@@ -692,7 +692,7 @@ fn normalize(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use hf_hub::api::sync::Api;
+    use hf_hub::api::sync::ApiBuilder;
     use hf_hub::{Repo, RepoType};
     use image::RgbImage;
 
@@ -701,7 +701,7 @@ mod tests {
         let image = image::open("/home/akshay/projects/EmbedAnything/test.jpg").unwrap();
         let image_array = image.to_rgb8().into_raw();
 
-        let api = Api::new().unwrap();
+        let api = ApiBuilder::from_env().build().unwrap();
         let repo = api.repo(Repo::new(
             "onnx-community/colSmol-256M-ONNX".to_string(),
             RepoType::Model,

rust/src/models/idefics3/tensor_processing.rs

@@ -1,5 +1,5 @@
 use candle_core::{DType, Device, Tensor};
-use hf_hub::{api::sync::Api, Repo, RepoType};
+use hf_hub::{api::sync::ApiBuilder, Repo, RepoType};
 use image::{imageops::FilterType, DynamicImage, GenericImageView, RgbImage};
 use serde::Deserialize;
 use std::collections::HashMap;
@@ -503,7 +503,7 @@ impl Idefics3ImageProcessor {
     }
 
     pub fn from_pretrained(model_id: &str) -> Result<Self, anyhow::Error> {
-        let api = Api::new()?;
+        let api = ApiBuilder::from_env().build()?;
         let repo = api.repo(Repo::new(model_id.to_string(), RepoType::Model));
         let config_file = repo.get("preprocessor_config.json")?;
         let processor: Idefics3ImageProcessor =
@@ -525,7 +525,7 @@ pub struct Idefics3Processor {
 impl Idefics3Processor {
     pub fn from_pretrained(model_id: &str) -> anyhow::Result<Self> {
         let image_processor = Idefics3ImageProcessor::from_pretrained(model_id)?;
-        let api = Api::new()?;
+        let api = ApiBuilder::from_env().build()?;
         let repo = api.repo(Repo::new(model_id.to_string(), RepoType::Model));
 
         let processor_config_file = repo.get("processor_config.json")?;

rust/src/reranker/model.rs

@@ -2,7 +2,7 @@ use std::sync::RwLock;
 
 use anyhow::{Error as E, Result};
 use candle_core::{Device, Tensor};
-use hf_hub::{api::sync::Api, Repo};
+use hf_hub::{api::sync::ApiBuilder, Repo};
 use ndarray::Array2;
 use ort::{
     execution_providers::{CUDAExecutionProvider, CoreMLExecutionProvider, ExecutionProvider},
@@ -41,7 +41,7 @@ impl Reranker {
         path_in_repo: Option<&str>,
     ) -> Result<Self, E> {
         let (config_filename, tokenizer_filename, weights_filename, tokenizer_config_filename) = {
-            let api = Api::new().unwrap();
+            let api = ApiBuilder::from_env().build().unwrap();
             let api = match revision {
                 Some(rev) => api.repo(Repo::with_revision(
                     model_id.to_string(),