build(core): add FAISSBackend and migrate to lighter, reproducible manylinux build

.gitignore

@@ -1,40 +1,38 @@
-### C++ ###
 # Prerequisites
-*.d
+**/*.d
 
 # Compiled Object files
-*.slo
-*.lo
-*.o
-*.obj
+**/*.slo
+**/*.lo
+**/*.o
+**/*.obj
 
 # Precompiled Headers
-*.gch
-*.pch
+**/*.gch
+**/*.pch
 
 # Compiled Dynamic libraries
-*.so
-*.dylib
-*.dll
+**/*.so
+**/*.dylib
+**/*.dll
 
 # Fortran module files
-*.mod
-*.smod
+**/*.mod
+**/*.smod
 
 # Compiled Static libraries
-*.lai
-*.la
-*.a
-*.lib
+**/*.lai
+**/*.la
+**/*.a
+**/*.lib
 
 # Executables
-*.exe
-*.out
-*.app
+**/*.exe
+**/*.out
+**/*.app
 
 ### Python ###
-pycache/
-__pycache__/
+*pycache*/
 .pytest_cache/
 test_bindings.py
 .pyd
@@ -44,37 +42,35 @@ CMakeUserPresets.json
 CMakeFiles/
 
 bkp/
-build/
-models/
-libs/libtorch
-!libs/tokenizers-cpp
-!libs/openai-cpp
+**/build/
+
+# Ignore everything in the models folder
+models/*
+!models/hf_extract_model.py
+!models/hf_model_to_onnx.py
+!models/README.md
 
 libtorch*.zip
-libtorch-cxx11-abi-shared*
-libtorch-cxx11-abi-shared-with-deps-2.5.1+cpu.zip
+libs/libtorch
 
-models
-conan.lock
 libtorch/
 extern/*
 
-openai_api_key*
+**/*openai_api_key*
 
-package/build/
-package/dist/
+**/build/*
+**/dist/*
 packer
-package/*.egg-info
+**/*.egg-info
 
 .env
-.venv/
-.venv*/
-.vscode/
-.vs/
+.venv/*
+.venv*/*
+.vscode/*
+.vs/*
+venv
 
-tests/
 compile_commands.json
 .cache/
-*.pdf
-*.docx
-test*
+**/*.pdf
+**/*.docx

.gitmodules

@@ -1,13 +1,10 @@
 [submodule "libs/tokenizers-cpp"]
 	path = libs/tokenizers-cpp
 	url = https://github.com/mlc-ai/tokenizers-cpp.git
+
 [submodule "libs/openai-cpp"]
 	path = libs/openai-cpp
 	url = https://github.com/olrea/openai-cpp.git
-[submodule "extern/pybind11"]
-	path = extern/pybind11
-	url = ../../pybind/pybind11
-	branch = stable
-[submodule "tokenizers-cpp"]
-	path = tokenizers-cpp
-	url = https://github.com/mlc-ai/tokenizers-cpp.git
+[submodule "libs/faiss"]
+	path = libs/faiss
+	url = https://github.com/facebookresearch/faiss.git