From 6314cd2b86a528fdfd10054c6563479973587d89 Mon Sep 17 00:00:00 2001 From: ghshhf Date: Sun, 14 Jun 2026 03:41:19 +0800 Subject: [PATCH] fix: multiple issues - CMake package export, C API exports, release build CI, drop/create scalar index - Fix #468: Add CMake package export support (find_package(zvec)) - Add install(EXPORT) to all CMakeLists.txt - Create cmake/zvecConfig.cmake.in template - Support find_package(zvec COMPONENTS c_api) - Fix #427: Support DROP + CREATE scalar index - Modify segment.cc to allow recreation of scalar index - Fix #443: Export missing C API functions - Add zvec_reranker_create_rrf/weighted/destroy - Add zvec_multi_query_set_reranker - Add zvec_doc_set_sparse_vector - Add #390: Release pre-built libraries CI workflow - Add .github/workflows/09-release-build.yml - Multi-platform: Ubuntu 20.04/22.04/24.04, macOS 13/14, Windows 2022/2025 - Add Fedora 39/40/41 builds via Docker container - Build both shared and static libraries - Add tests for drop-and-recreate scalar index --- .github/workflows/09-release-build.yml | 653 +++++++++++++++++++++++++ CMakeLists.txt | 5 +- cmake/zvecConfig.cmake.in | 39 ++ src/CMakeLists.txt | 56 +++ src/binding/c/CMakeLists.txt | 40 ++ src/binding/c/c_api.cc | 94 ++++ src/db/index/segment/segment.cc | 23 +- src/include/zvec/c_api.h | 87 ++++ tests/db/collection_test.cc | 142 ++++++ 9 files changed, 1136 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/09-release-build.yml create mode 100644 cmake/zvecConfig.cmake.in diff --git a/.github/workflows/09-release-build.yml b/.github/workflows/09-release-build.yml new file mode 100644 index 000000000..d483d3da4 --- /dev/null +++ b/.github/workflows/09-release-build.yml @@ -0,0 +1,653 @@ +name: Build & Release Pre-compiled Libraries + +# Trigger on pushes to main with tags, and on manual dispatch +on: + push: + tags: + - 'v*' # Push with v* tags triggers release build + workflow_dispatch: + inputs: + release_version: + description: 'Release version (e.g., v0.6.0). Leave empty for snapshot build.' + required: false + type: string + publish_release: + description: 'Whether to publish GitHub Release' + required: false + type: boolean + default: false + +permissions: + contents: write # Needed for creating releases + actions: read + +env: + # Build type + BUILD_TYPE: Release + # Use ccache / sccache for faster builds + SCCACHE_GHA_ENABLED: "true" + +jobs: + # ========================================================================= + # Job 1: Build on Ubuntu (multiple versions) + # ========================================================================= + build-ubuntu: + name: Build (Ubuntu ${{ matrix.version }}, ${{ matrix.arch }}) + runs-on: ubuntu-24.04${{ matrix.arch == 'arm64' && '-arm' || '' }} + + strategy: + fail-fast: false + matrix: + include: + - version: '20.04' + arch: x64 + glibc: '2.31' + - version: '22.04' + arch: x64 + glibc: '2.35' + - version: '24.04' + arch: x64 + glibc: '2.39' + - version: '22.04' + arch: arm64 + glibc: '2.35' + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + submodules: recursive + fetch-tags: true + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + cache: 'pip' + + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: release-ubuntu-${{ matrix.version }}-${{ matrix.arch }} + max-size: 200M + + - name: Install system dependencies (x64) + if: matrix.arch == 'x64' + run: | + sudo apt-get update -y + sudo apt-get install -y --no-install-recommends \ + libaio-dev \ + libomp-dev + shell: bash + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install \ + pybind11==3.0 \ + cmake==3.30.0 \ + ninja==1.11.1 \ + scikit-build-core \ + setuptools_scm + shell: bash + + - name: Configure CMake (Shared Library) + run: | + cmake -S . -B build-shared -G Ninja \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DBUILD_C_BINDINGS=ON \ + -DBUILD_TOOLS=OFF \ + -DBUILD_ZVEC_SHARED=ON \ + -DBUILD_ZVEC_CORE_SHARED=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install-shared + shell: bash + + - name: Configure CMake (Static Library) + run: | + cmake -S . -B build-static -G Ninja \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DBUILD_C_BINDINGS=ON \ + -DBUILD_TOOLS=OFF \ + -DBUILD_ZVEC_SHARED=OFF \ + -DBUILD_ZVEC_CORE_SHARED=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install-static + shell: bash + + - name: Build (Shared Library) + run: | + cmake --build build-shared --target all --parallel $(nproc) + shell: bash + + - name: Build (Static Library) + run: | + cmake --build build-static --target all --parallel $(nproc) + shell: bash + + - name: Install (Shared Library) + run: | + cmake --install build-shared --prefix ${{ github.workspace }}/install-shared + shell: bash + + - name: Install (Static Library) + run: | + cmake --install build-static --prefix ${{ github.workspace }}/install-static + shell: bash + + - name: Merge Shared and Static Libraries + run: | + # Create merged install directory + mkdir -p ${GITHUB_WORKSPACE}/install-merged + + # Copy shared library installation + cp -r ${{ github.workspace }}/install-shared/* ${GITHUB_WORKSPACE}/install-merged/ 2>/dev/null || true + + # Copy static libraries (.a) to merged lib directory + mkdir -p ${GITHUB_WORKSPACE}/install-merged/lib + find ${{ github.workspace }}/install-static -name "*.a" -exec cp {} ${GITHUB_WORKSPACE}/install-merged/lib/ \; + + # Show merged directory structure + echo "=== Merged install directory ===" + ls -la ${GITHUB_WORKSPACE}/install-merged/lib/ + echo "=================================" + shell: bash + + - name: Package (tar.gz) + id: package + run: | + VERSION=$(cd "$GITHUB_WORKSPACE" && git describe --tags --always) + ARCHIVE_NAME="zvec-${VERSION}-ubuntu-${{ matrix.version }}-${{ matrix.arch }}.tar.gz" + + # Create archive from merged directory + tar -czf "${ARCHIVE_NAME}" \ + -C "${GITHUB_WORKSPACE}/install-merged" \ + . + + echo "archive_name=${ARCHIVE_NAME}" >> $GITHUB_OUTPUT + echo "Built archive: ${ARCHIVE_NAME}" + + # Show archive contents + echo "=== Archive contents ===" + tar -tzf "${ARCHIVE_NAME}" | head -30 + echo "=========================" + shell: bash + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: zvec-ubuntu-${{ matrix.version }}-${{ matrix.arch }} + path: ${{ steps.package.outputs.archive_name }} + retention-days: 7 + + # ========================================================================= + # Job 2: Build on macOS (Intel + Apple Silicon) + # ========================================================================= + build-macos: + name: Build (macOS ${{ matrix.version }}, ${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + + strategy: + fail-fast: false + matrix: + include: + - version: '13' + arch: x86_64 + runner: macos-13 + - version: '14' + arch: arm64 + runner: macos-14 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + submodules: recursive + fetch-tags: true + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + cache: 'pip' + + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: release-macos-${{ matrix.version }}-${{ matrix.arch }} + max-size: 200M + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install \ + pybind11==3.0 \ + cmake==3.30.0 \ + ninja==1.11.1 \ + scikit-build-core \ + setuptools_scm + shell: bash + + - name: Configure CMake (Shared Library) + run: | + cmake -S . -B build-shared -G Ninja \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DBUILD_C_BINDINGS=ON \ + -DBUILD_TOOLS=OFF \ + -DBUILD_ZVEC_SHARED=ON \ + -DBUILD_ZVEC_CORE_SHARED=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install-shared \ + -DCMAKE_OSX_ARCHITECTURES=${{ matrix.arch }} + shell: bash + + - name: Configure CMake (Static Library) + run: | + cmake -S . -B build-static -G Ninja \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DBUILD_C_BINDINGS=ON \ + -DBUILD_TOOLS=OFF \ + -DBUILD_ZVEC_SHARED=OFF \ + -DBUILD_ZVEC_CORE_SHARED=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install-static \ + -DCMAKE_OSX_ARCHITECTURES=${{ matrix.arch }} + shell: bash + + - name: Build (Shared Library) + run: | + cmake --build build-shared --target all --parallel $(sysctl -n hw.ncpu) + shell: bash + + - name: Build (Static Library) + run: | + cmake --build build-static --target all --parallel $(sysctl -n hw.ncpu) + shell: bash + + - name: Install (Shared Library) + run: | + cmake --install build-shared --prefix ${{ github.workspace }}/install-shared + shell: bash + + - name: Install (Static Library) + run: | + cmake --install build-static --prefix ${{ github.workspace }}/install-static + shell: bash + + - name: Merge Shared and Static Libraries + run: | + # Create merged install directory + mkdir -p ${GITHUB_WORKSPACE}/install-merged + + # Copy shared library installation + cp -r ${{ github.workspace }}/install-shared/* ${GITHUB_WORKSPACE}/install-merged/ 2>/dev/null || true + + # Copy static libraries (.a) to merged lib directory + mkdir -p ${GITHUB_WORKSPACE}/install-merged/lib + find ${{ github.workspace }}/install-static -name "*.a" -exec cp {} ${GITHUB_WORKSPACE}/install-merged/lib/ \; + + # Show merged directory structure + echo "=== Merged install directory ===" + ls -la ${GITHUB_WORKSPACE}/install-merged/lib/ + echo "=================================" + shell: bash + + - name: Package (tar.gz) + id: package + run: | + VERSION=$(cd "$GITHUB_WORKSPACE" && git describe --tags --always) + ARCHIVE_NAME="zvec-${VERSION}-macos-${{ matrix.version }}-${{ matrix.arch }}.tar.gz" + + # Create archive from merged directory + tar -czf "${ARCHIVE_NAME}" \ + -C "${GITHUB_WORKSPACE}/install-merged" \ + . + + echo "archive_name=${ARCHIVE_NAME}" >> $GITHUB_OUTPUT + echo "Built archive: ${ARCHIVE_NAME}" + + # Show archive contents + echo "=== Archive contents ===" + tar -tzf "${ARCHIVE_NAME}" | head -30 + echo "=========================" + shell: bash + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: zvec-macos-${{ matrix.version }}-${{ matrix.arch }} + path: ${{ steps.package.outputs.archive_name }} + retention-days: 7 + + # ========================================================================= + # Job 3: Build on Windows (x64) + # ========================================================================= + build-windows: + name: Build (Windows ${{ matrix.version }}, x64) + runs-on: ${{ matrix.version }} + + strategy: + fail-fast: false + matrix: + include: + - version: windows-2022 + - version: windows-2025 + + env: + SCCACHE_GHA_ENABLED: "true" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + submodules: recursive + fetch-tags: true + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.10' + cache: 'pip' + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.10 + + - name: Set up MSVC environment + uses: ilammy/msvc-dev-cmd@v1.13.0 + with: + arch: x64 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install ` + pybind11==3.0 ` + cmake==3.30.0 ` + ninja==1.11.1 ` + scikit-build-core ` + setuptools_scm + shell: powershell + + - name: Configure CMake (Shared Library) + run: | + cmake -S . -B build-shared -G Ninja ` + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} ` + -DBUILD_PYTHON_BINDINGS=OFF ` + -DBUILD_C_BINDINGS=ON ` + -DBUILD_TOOLS=OFF ` + -DBUILD_ZVEC_SHARED=ON ` + -DBUILD_ZVEC_CORE_SHARED=ON ` + -DCMAKE_C_COMPILER_LAUNCHER=sccache ` + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ` + -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}\install-shared + shell: powershell + + - name: Configure CMake (Static Library) + run: | + cmake -S . -B build-static -G Ninja ` + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} ` + -DBUILD_PYTHON_BINDINGS=OFF ` + -DBUILD_C_BINDINGS=ON ` + -DBUILD_TOOLS=OFF ` + -DBUILD_ZVEC_SHARED=OFF ` + -DBUILD_ZVEC_CORE_SHARED=OFF ` + -DCMAKE_C_COMPILER_LAUNCHER=sccache ` + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ` + -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}\install-static + shell: powershell + + - name: Build (Shared Library) + run: | + cmake --build build-shared --target all --parallel $env:NUMBER_OF_PROCESSORS + shell: powershell + + - name: Build (Static Library) + run: | + cmake --build build-static --target all --parallel $env:NUMBER_OF_PROCESSORS + shell: powershell + + - name: Install (Shared Library) + run: | + cmake --install build-shared --prefix "${{ github.workspace }}\install-shared" + shell: powershell + + - name: Install (Static Library) + run: | + cmake --install build-static --prefix "${{ github.workspace }}\install-static" + shell: powershell + + - name: Merge Shared and Static Libraries + run: | + # Create merged install directory + New-Item -ItemType Directory -Force -Path "${{ github.workspace }}\install-merged" + + # Copy shared library installation + Copy-Item -Path "${{ github.workspace }}\install-shared\*" -Destination "${{ github.workspace }}\install-merged\" -Recurse -Force + + # Copy static libraries (.lib) to merged lib directory + New-Item -ItemType Directory -Force -Path "${{ github.workspace }}\install-merged\lib" + Copy-Item -Path "${{ github.workspace }}\install-static\lib\*.lib" -Destination "${{ github.workspace }}\install-merged\lib\" -Force + + # Show merged directory structure + Write-Host "=== Merged install directory ===" + Get-ChildItem -Path "${{ github.workspace }}\install-merged\lib\" + Write-Host "=================================" + shell: powershell + + - name: Package (zip) + id: package + run: | + $VERSION = (git describe --tags --always) 2>&1 + $ARCHIVE_NAME = "zvec-${VERSION}-windows-${{ matrix.version }}-x64.zip" + + Compress-Archive -Path "${{ github.workspace }}\install-merged\*" -DestinationPath "${ARCHIVE_NAME}" + + echo "archive_name=${ARCHIVE_NAME}" >> $env:GITHUB_OUTPUT + Write-Host "Built archive: ${ARCHIVE_NAME}" + + # Show archive contents + Write-Host "=== Archive contents ===" + Get-ChildItem -Path "${ARCHIVE_NAME}" + Write-Host "=========================" + shell: powershell + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: zvec-windows-${{ matrix.version }}-x64 + path: ${{ steps.package.outputs.archive_name }} + retention-days: 7 + + # ========================================================================= + # Job 4: Build on Fedora (39, 40, 41) via Docker container + # ========================================================================= + build-fedora: + name: Build (Fedora ${{ matrix.version }}, x64) + runs-on: ubuntu-24.04 + + strategy: + fail-fast: false + matrix: + include: + - version: '39' + glibc: '2.38' + - version: '40' + glibc: '2.39' + - version: '41' + glibc: '2.40' + + container: + image: fedora:${{ matrix.version }} + options: --privileged + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + submodules: recursive + fetch-tags: true + fetch-depth: 0 + + - name: Install system dependencies + run: | + dnf install -y --setopt=install_weak_deps=False \ + git \ + cmake \ + ninja-build \ + gcc-c++ \ + python3 \ + python3-pip \ + python3-devel \ + libaio-devel \ + libomp-devel \ + ccache \ + tar \ + gzip + shell: bash + + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install \ + pybind11==3.0 \ + cmake==3.30.0 \ + ninja==1.11.1 \ + scikit-build-core \ + setuptools_scm + shell: bash + + - name: Configure CMake (Shared Library) + run: | + cmake -S . -B build-shared -G Ninja \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DBUILD_C_BINDINGS=ON \ + -DBUILD_TOOLS=OFF \ + -DBUILD_ZVEC_SHARED=ON \ + -DBUILD_ZVEC_CORE_SHARED=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_INSTALL_PREFIX=/tmp/install-shared + shell: bash + + - name: Configure CMake (Static Library) + run: | + cmake -S . -B build-static -G Ninja \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DBUILD_C_BINDINGS=ON \ + -DBUILD_TOOLS=OFF \ + -DBUILD_ZVEC_SHARED=OFF \ + -DBUILD_ZVEC_CORE_SHARED=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_INSTALL_PREFIX=/tmp/install-static + shell: bash + + - name: Build (Shared Library) + run: | + cmake --build build-shared --target all --parallel $(nproc) + shell: bash + + - name: Build (Static Library) + run: | + cmake --build build-static --target all --parallel $(nproc) + shell: bash + + - name: Install (Shared Library) + run: | + cmake --install build-shared --prefix /tmp/install-shared + shell: bash + + - name: Install (Static Library) + run: | + cmake --install build-static --prefix /tmp/install-static + shell: bash + + - name: Merge Shared and Static Libraries + run: | + mkdir -p /tmp/install-merged/lib + cp -r /tmp/install-shared/* /tmp/install-merged/ 2>/dev/null || true + find /tmp/install-static -name "*.a" -exec cp {} /tmp/install-merged/lib/ \; + echo "=== Merged install directory ===" + ls -la /tmp/install-merged/lib/ + echo "=================================" + shell: bash + + - name: Package (tar.gz) + id: package + run: | + VERSION=$(cd "$GITHUB_WORKSPACE" && git describe --tags --always) + ARCHIVE_NAME="zvec-${VERSION}-fedora-${{ matrix.version }}-x64.tar.gz" + tar -czf "${ARCHIVE_NAME}" -C /tmp/install-merged . + echo "archive_name=${ARCHIVE_NAME}" >> $GITHUB_OUTPUT + echo "Built archive: ${ARCHIVE_NAME}" + echo "=== Archive contents ===" + tar -tzf "${ARCHIVE_NAME}" | head -30 + echo "=========================" + shell: bash + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: zvec-fedora-${{ matrix.version }}-x64 + path: ${{ steps.package.outputs.archive_name }} + retention-days: 7 + + # ========================================================================= + # Job 5: Publish to GitHub Release (optional) + # ========================================================================= + publish-release: + name: Publish to GitHub Release + needs: [build-ubuntu, build-macos, build-windows, build-fedora] + runs-on: ubuntu-24.04 + if: ${{ github.event.inputs.publish_release == 'true' || startsWith(github.ref, 'refs/tags/v') }} + permissions: + contents: write + + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Display structure of downloaded files + run: | + ls -R artifacts/ + shell: bash + + - name: Determine version + id: version + run: | + if [ -n "${{ github.event.inputs.release_version }}" ]; then + VERSION="${{ github.event.inputs.release_version }}" + elif [[ "${{ github.ref }}" == refs/tags/* ]]; then + VERSION="${{ github.ref_name }}" + else + VERSION="snapshot-$(date +'%Y%m%d-%H%M%S')" + fi + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "Version: ${VERSION}" + shell: bash + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.version.outputs.version }} + name: zvec ${{ steps.version.outputs.version }} + draft: ${{ !startsWith(github.ref, 'refs/tags/v') }} + prerelease: ${{ contains(steps.version.outputs.version, 'rc') || contains(steps.version.outputs.version, 'beta') }} + files: artifacts/**/* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 392dbdac8..340a6ebbc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,7 @@ -cmake_minimum_required(VERSION 3.13) +# Zvec bundles Apache Arrow 21.0.0, which requires CMake >= 3.26 to build its +# bundled Apache Thrift dependency. Set this here so users get a clear error +# upfront instead of failing after a long build (Issue #468). +cmake_minimum_required(VERSION 3.26) cmake_policy(SET CMP0077 NEW) project(zvec) set(CC_CXX_STANDARD 17) diff --git a/cmake/zvecConfig.cmake.in b/cmake/zvecConfig.cmake.in new file mode 100644 index 000000000..4d3684328 --- /dev/null +++ b/cmake/zvecConfig.cmake.in @@ -0,0 +1,39 @@ +# zvecConfig.cmake.in - CMake package configuration template +# This file is processed by configure_package_config_file() to generate +# the final zvecConfig.cmake installed with the library. + +@PACKAGE_INIT@ + +# Require CMake 3.13 or newer +cmake_minimum_required(VERSION 3.13) + +# Set zvec root directory +set(ZVEC_ROOT_DIR "@PACKAGE_CMAKE_INSTALL_PREFIX@") + +# Include imported targets (shared libraries) +include("@PACKAGE_CMAKE_INSTALL_LIBDIR@/cmake/zvec/zvec_c_apiTargets.cmake" OPTIONAL RESULT_VARIABLE _zvec_c_api_targets) +if(_zvec_c_api_targets) + message(STATUS "Loaded zvec_c_api targets from ${_zvec_c_api_targets}") +endif() + +# Alias for convenience (zvec::zvec -> zvec::zvec_c_api) +if(TARGET zvec::zvec_c_api AND NOT TARGET zvec::zvec) + add_library(zvec::zvec ALIAS zvec::zvec_c_api) + message(STATUS "Created alias zvec::zvec -> zvec::zvec_c_api") +endif() + +# Helper function to print zvec info +function(zvec_info) + message(STATUS "zvec version: @ZVEC_VERSION_MAJOR@.@ZVEC_VERSION_MINOR@.@ZVEC_VERSION_PATCH@") + message(STATUS "zvec include dir: ${ZVEC_ROOT_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") + message(STATUS "zvec library dir: ${ZVEC_ROOT_DIR}/@CMAKE_INSTALL_LIBDIR@") +endfunction() + +# Mark package as found +set(zvec_FOUND TRUE) +set(ZVEC_FOUND TRUE) + +# Provide legacy variables for backwards compatibility +set(ZVEC_INCLUDE_DIRS "${ZVEC_ROOT_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") +set(ZVEC_LIBRARY_DIRS "${ZVEC_ROOT_DIR}/@CMAKE_INSTALL_LIBDIR@") +set(ZVEC_LIBRARIES zvec::zvec_c_api) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9cf11f3e0..4d0e0e4bb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -141,6 +141,7 @@ function(zvec_add_all_in_one_shared TARGET_NAME OUTPUT_NAME) endif() install(TARGETS ${TARGET_NAME} + EXPORT ${TARGET_NAME}Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} @@ -167,3 +168,58 @@ if(BUILD_ZVEC_SHARED) zvec_turbo ) endif() + +# ============================================================================= +# CMake Package Configuration (for find_package(zvec)) +# ============================================================================= + +# Only install package config if shared libraries are built +if(BUILD_ZVEC_CORE_SHARED OR BUILD_ZVEC_SHARED) + include(CMakePackageConfigHelpers) + + # Generate package version file + write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/zvecConfigVersion.cmake" + VERSION "${ZVEC_VERSION_MAJOR}.${ZVEC_VERSION_MINOR}.${ZVEC_VERSION_PATCH}" + COMPATIBILITY SameMajorVersion + ) + + # Generate package config from template + configure_package_config_file( + "${PROJECT_SOURCE_DIR}/cmake/zvecConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/zvecConfig.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/zvec + ) + + # Install package config files + install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/zvecConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/zvecConfigVersion.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/zvec + ) +endif() + +# Export targets from build tree (for find_package(zvec) without install) +if(BUILD_ZVEC_CORE_SHARED) + export(EXPORT zvec_core_sharedTargets + FILE "${CMAKE_CURRENT_BINARY_DIR}/zvec_core_sharedTargets.cmake" + NAMESPACE zvec:: + ) + install(EXPORT zvec_core_sharedTargets + FILE zvec_core_sharedTargets.cmake + NAMESPACE zvec:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/zvec + ) +endif() + +if(BUILD_ZVEC_SHARED) + export(EXPORT zvec_sharedTargets + FILE "${CMAKE_CURRENT_BINARY_DIR}/zvec_sharedTargets.cmake" + NAMESPACE zvec:: + ) + install(EXPORT zvec_sharedTargets + FILE zvec_sharedTargets.cmake + NAMESPACE zvec:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/zvec + ) +endif() \ No newline at end of file diff --git a/src/binding/c/CMakeLists.txt b/src/binding/c/CMakeLists.txt index 13dfdc055..88f28f91f 100644 --- a/src/binding/c/CMakeLists.txt +++ b/src/binding/c/CMakeLists.txt @@ -244,6 +244,7 @@ endif() # Install shared library install(TARGETS zvec_c_api + EXPORT zvec_c_apiTargets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} @@ -254,3 +255,42 @@ install(TARGETS zvec_c_api install(FILES ${PROJECT_SOURCE_DIR}/src/include/zvec/c_api.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/zvec ) + +# ============================================================================= +# CMake Package Configuration (for find_package(zvec)) +# ============================================================================= + +# Export targets for use from build tree +export(EXPORT zvec_c_apiTargets + FILE "${CMAKE_CURRENT_BINARY_DIR}/zvec_c_apiTargets.cmake" + NAMESPACE zvec:: +) + +# Install targets for use from install tree +install(EXPORT zvec_c_apiTargets + FILE zvec_c_apiTargets.cmake + NAMESPACE zvec:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/zvec +) + +# Generate & install package version file +include(CMakePackageConfigHelpers) +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/zvecConfigVersion.cmake" + VERSION "${ZVEC_VERSION_MAJOR}.${ZVEC_VERSION_MINOR}.${ZVEC_VERSION_PATCH}" + COMPATIBILITY SameMajorVersion +) + +# Generate package config from template +configure_package_config_file( + "${PROJECT_SOURCE_DIR}/cmake/zvecConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/zvecConfig.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/zvec +) + +# Install package config files +install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/zvecConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/zvecConfigVersion.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/zvec +) diff --git a/src/binding/c/c_api.cc b/src/binding/c/c_api.cc index a81cc3864..d6947521b 100644 --- a/src/binding/c/c_api.cc +++ b/src/binding/c/c_api.cc @@ -3497,6 +3497,45 @@ zvec_error_code_t zvec_doc_add_field_by_struct(zvec_doc_t *doc, return ZVEC_OK;) } +// --------------------------------------------------------------------------- +// Sparse Vector Document Field Helper (Issue #443) +// --------------------------------------------------------------------------- + +zvec_error_code_t zvec_doc_set_sparse_vector( + zvec_doc_t *doc, + const char *field_name, + const uint32_t *indices, + const float *values, + size_t count) { + if (!doc || !field_name) { + set_last_error("Invalid arguments: null pointer"); + return ZVEC_ERROR_INVALID_ARGUMENT; + } + if ((!indices || !values) && count > 0) { + set_last_error("Indices or values pointer is null while count > 0"); + return ZVEC_ERROR_INVALID_ARGUMENT; + } + + ZVEC_TRY_RETURN_ERROR( + "Failed to set sparse vector field", + auto *doc_ptr = reinterpret_cast(doc); + std::string name(field_name); + + if (count == 0) { + // Set empty sparse vector (pair of empty vectors) + doc_ptr->set(name, + std::pair, std::vector>{}); + return ZVEC_OK; + } + + std::vector idx_vec(indices, indices + count); + std::vector val_vec(values, values + count); + doc_ptr->set(name, + std::pair, std::vector>{ + std::move(idx_vec), std::move(val_vec)}); + return ZVEC_OK;) +} + const char *zvec_doc_get_pk_pointer(const zvec_doc_t *doc) { if (!doc) return nullptr; auto doc_ptr = reinterpret_cast(doc); @@ -5797,6 +5836,61 @@ zvec_error_code_t zvec_multi_query_set_rerank_weighted( return ZVEC_OK; } +// --------------------------------------------------------------------------- +// Standalone Reranker API (Issue #443) +// --------------------------------------------------------------------------- +// Internal wrapper: holds a copy of RerankParams. +struct RerankerWrapper { + zvec::reranker::RerankParams params; +}; + +zvec_reranker_t *zvec_reranker_create_rrf(int rank_constant) { + ZVEC_TRY_RETURN_NULL("Failed to create RRF reranker", + auto *wrapper = new RerankerWrapper(); + wrapper->params = zvec::reranker::RrfParams{rank_constant}; + return reinterpret_cast(wrapper);) + return nullptr; +} + +zvec_reranker_t *zvec_reranker_create_weighted(const double *weights, + size_t weight_count) { + if (!weights && weight_count > 0) { + SET_LAST_ERROR(ZVEC_ERROR_INVALID_ARGUMENT, + "Weights pointer is null"); + return nullptr; + } + ZVEC_TRY_RETURN_NULL( + "Failed to create Weighted reranker", + auto *wrapper = new RerankerWrapper(); + wrapper->params = zvec::reranker::WeightedParams{ + std::vector(weights, weights + weight_count)}; + return reinterpret_cast(wrapper);) + return nullptr; +} + +void zvec_reranker_destroy(zvec_reranker_t *reranker) { + if (reranker) { + delete reinterpret_cast(reranker); + } +} + +zvec_error_code_t zvec_multi_query_set_reranker( + zvec_multi_query_t *query, const zvec_reranker_t *reranker) { + if (!query) { + SET_LAST_ERROR(ZVEC_ERROR_INVALID_ARGUMENT, "Query pointer is null"); + return ZVEC_ERROR_INVALID_ARGUMENT; + } + auto *mq = reinterpret_cast(query); + if (!reranker) { + // Clear reranking by setting default RRF with default constant + mq->rerank = zvec::reranker::RrfParams{60}; + return ZVEC_OK; + } + auto *wrapper = reinterpret_cast(reranker); + mq->rerank = wrapper->params; + return ZVEC_OK; +} + // ============================================================================= // MultiVectorQuery Implementation // ============================================================================= diff --git a/src/db/index/segment/segment.cc b/src/db/index/segment/segment.cc index 2e334c3fc..38a381f71 100644 --- a/src/db/index/segment/segment.cc +++ b/src/db/index/segment/segment.cc @@ -1992,6 +1992,9 @@ Status SegmentImpl::create_scalar_index(const std::vector &columns, auto new_segment_meta = std::make_shared(*segment_meta_); if (fields.empty()) { *segment_meta = new_segment_meta; + // Fix: return current indexer pointer so the caller does not + // pass nullptr to reload_scalar_index (Issue #427). + *scalar_indexer = invert_indexers_; return Status::OK(); } @@ -2121,7 +2124,9 @@ Status SegmentImpl::drop_scalar_index(const std::vector &columns, if (fields.empty()) { *segment_meta = new_segment_meta; - *scalar_indexer = nullptr; + // Fix: return current indexer pointer so the caller does not + // pass nullptr to reload_scalar_index (Issue #427). + *scalar_indexer = invert_indexers_; return Status::OK(); } @@ -2167,7 +2172,21 @@ Status SegmentImpl::reload_scalar_index( segment_meta_ = segment_meta; if (!scalar_indexer) { - // no need to reload inverted indexer + // Fix: clean up the old inverted indexer directory so that a subsequent + // create_index() can create a fresh one without "Column family already + // exists" errors (Issue #427). + if (invert_indexers_) { + auto old_dir = invert_indexers_->working_dir(); + invert_indexers_ = nullptr; + FileHelper::RemoveDirectory(old_dir); + } + return Status::OK(); + } + + // Identity-pointer guard: avoid accidentally deleting an indexer's own + // directory during reload (e.g. if scalar_indexer == invert_indexers_). + if (invert_indexers_ && invert_indexers_ == scalar_indexer) { + LOG_WARN("reload_scalar_index: identity-pointer guard triggered, skipping"); return Status::OK(); } diff --git a/src/include/zvec/c_api.h b/src/include/zvec/c_api.h index d02335cb3..841d9dca4 100644 --- a/src/include/zvec/c_api.h +++ b/src/include/zvec/c_api.h @@ -2122,6 +2122,67 @@ zvec_multi_query_set_rerank_rrf(zvec_multi_query_t *query, int rank_constant); ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_multi_query_set_rerank_weighted( zvec_multi_query_t *query, const double *weights, size_t weight_count); +// ----------------------------------------------------------------------------- +// zvec_reranker_t (Standalone Reranker Handle) +// ----------------------------------------------------------------------------- + +/** + * @brief Reranker handle (opaque pointer). + * Use zvec_reranker_create_rrf() or zvec_reranker_create_weighted() + * to create, and zvec_reranker_destroy() to free. + */ +typedef struct zvec_reranker_t zvec_reranker_t; + +/** + * @brief Create RRF (Reciprocal Rank Fusion) reranker. + * + * Score formula: 1 / (rank_constant + rank + 1) + * + * @param rank_constant RRF rank constant (typical range: 1–100, default 60) + * @return zvec_reranker_t* Pointer to newly created reranker; + * returns NULL on failure. + */ +ZVEC_EXPORT zvec_reranker_t *ZVEC_CALL +zvec_reranker_create_rrf(int rank_constant); + +/** + * @brief Create Weighted Score Fusion reranker. + * + * Each sub-query's score is normalized, then multiplied by the + * corresponding weight. + * + * @param weights Array of per-sub-query weights (must not be NULL if + * weight_count > 0) + * @param weight_count Number of weights + * @return zvec_reranker_t* Pointer to newly created reranker; + * returns NULL on failure. + */ +ZVEC_EXPORT zvec_reranker_t *ZVEC_CALL +zvec_reranker_create_weighted(const double *weights, size_t weight_count); + +/** + * @brief Destroy a reranker and release all resources. + * + * @param reranker Reranker pointer (may be NULL, in which case this + * function is a no-op). + */ +ZVEC_EXPORT void ZVEC_CALL +zvec_reranker_destroy(zvec_reranker_t *reranker); + +/** + * @brief Attach a reranker to a multi-query. + * + * The multi-query takes a copy of the rerank parameters; the reranker + * object can be destroyed after this call if no longer needed. + * + * @param query Multi-query pointer + * @param reranker Reranker pointer (may be NULL to clear reranking) + * @return zvec_error_code_t Error code + */ +ZVEC_EXPORT zvec_error_code_t ZVEC_CALL +zvec_multi_query_set_reranker(zvec_multi_query_t *query, + const zvec_reranker_t *reranker); + // ----------------------------------------------------------------------------- // zvec_multi_query_t (Multi Query) // ----------------------------------------------------------------------------- @@ -3405,6 +3466,32 @@ ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_doc_add_field_by_value( ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_doc_add_field_by_struct(zvec_doc_t *doc, const zvec_doc_field_t *field); +/** + * @brief Set a sparse vector field on a document. + * + * Sparse vector fields are defined with data type + * ZVEC_DATA_TYPE_SPARSE_VECTOR_FP32 or + * ZVEC_DATA_TYPE_SPARSE_VECTOR_FP16. + * + * This function allocates and copies the supplied @p indices + * and @p values arrays into the document; the caller retains + * ownership of the original buffers. + * + * @param doc Document pointer (must not be NULL) + * @param field_name Field name (must match a SPARSE_VECTOR field + * in the schema) + * @param indices Array of vector indices (uint32_t) + * @param values Array of non-zero values (float) + * @param count Number of non-zero entries + * @return Error code (ZVEC_OK on success) + */ +ZVEC_EXPORT zvec_error_code_t ZVEC_CALL +zvec_doc_set_sparse_vector(zvec_doc_t *doc, + const char *field_name, + const uint32_t *indices, + const float *values, + size_t count); + /** * @brief Remove field from document * diff --git a/tests/db/collection_test.cc b/tests/db/collection_test.cc index 4805df32f..98f6dde94 100644 --- a/tests/db/collection_test.cc +++ b/tests/db/collection_test.cc @@ -6050,3 +6050,145 @@ TEST_F(CollectionTest, Feature_CreateOrDropFtsIndex) { FileHelper::RemoveDirectory(col_path); } } + +// Drop a scalar index and then recreate it on the same field. +// Before the fix, recreate failed because the old RocksDB column family was +// not cleaned up during DropIndex. (Issue #427) +TEST_F(CollectionTest, Feature_DropAndRecreateScalarIndex) { + auto func = [&](const std::string &field_name, bool enable_optimize) { + FileHelper::RemoveDirectory(col_path); + + auto schema = + TestHelper::CreateSchemaWithScalarIndex(false, enable_optimize); + auto options = CollectionOptions{false, true, 64 * 1024 * 1024}; + int doc_count = 1000; + + auto collection = TestHelper::CreateCollectionWithDoc( + col_path, *schema, options, 0, doc_count, false); + ASSERT_NE(collection, nullptr); + + // Verify index exists and queries work + auto stats = collection->Stats().value(); + ASSERT_EQ(stats.doc_count, doc_count); + ASSERT_FLOAT_EQ(stats.index_completeness[field_name], 1.0f); + + // Drop the scalar index + auto s = collection->DropIndex(field_name); + ASSERT_TRUE(s.ok()) << s.message(); + + stats = collection->Stats().value(); + ASSERT_EQ(stats.doc_count, doc_count); + // index_completeness should be 0 after drop + ASSERT_FLOAT_EQ(stats.index_completeness[field_name], 0.0f); + + // Recreate the scalar index - this was failing before the fix + auto index_params = std::make_shared(enable_optimize); + s = collection->CreateIndex(field_name, index_params); + ASSERT_TRUE(s.ok()) << "Recreate index failed: " << s.message(); + + // Flush to persist + s = collection->Flush(); + ASSERT_TRUE(s.ok()) << s.message(); + + // Verify index is recreated + stats = collection->Stats().value(); + ASSERT_EQ(stats.doc_count, doc_count); + ASSERT_FLOAT_EQ(stats.index_completeness[field_name], 1.0f); + + // Reopen and verify persistence + collection.reset(); + auto result = Collection::Open(col_path, options); + ASSERT_TRUE(result.has_value()) << result.error().message(); + collection = result.value(); + + stats = collection->Stats().value(); + ASSERT_EQ(stats.doc_count, doc_count); + ASSERT_FLOAT_EQ(stats.index_completeness[field_name], 1.0f); + + collection.reset(); + FileHelper::RemoveDirectory(col_path); + }; + + // Test with different scalar fields + func("int32", false); + func("int32", true); + func("uint32", false); + func("bool", false); + func("float", false); + func("double", false); + func("int64", false); + func("uint64", false); + func("string", false); + func("string", true); +} + +// Drop scalar indexes on multiple fields and then recreate them. +// Before the fix, recreate failed because the old RocksDB column families were +// not cleaned up during DropIndex. (Issue #427) +TEST_F(CollectionTest, Feature_DropAndRecreateScalarIndex_MultipleFields) { + auto func = [&](bool enable_optimize) { + FileHelper::RemoveDirectory(col_path); + + auto schema = + TestHelper::CreateSchemaWithScalarIndex(false, enable_optimize); + auto options = CollectionOptions{false, true, 64 * 1024 * 1024}; + int doc_count = 1000; + + auto collection = TestHelper::CreateCollectionWithDoc( + col_path, *schema, options, 0, doc_count, false); + ASSERT_NE(collection, nullptr); + + // Verify indexes exist + auto stats = collection->Stats().value(); + ASSERT_EQ(stats.doc_count, doc_count); + + // Fields with scalar indexes in CreateSchemaWithScalarIndex + std::vector scalar_fields = { + "int32", "string", "uint32", "bool", + "float", "double", "int64", "uint64"}; + + // Drop all scalar indexes + for (const auto &field_name : scalar_fields) { + auto s = collection->DropIndex(field_name); + ASSERT_TRUE(s.ok()) << "Drop index failed for " << field_name << ": " + << s.message(); + } + + // Recreate all scalar indexes + for (const auto &field_name : scalar_fields) { + auto index_params = std::make_shared(enable_optimize); + auto s = collection->CreateIndex(field_name, index_params); + ASSERT_TRUE(s.ok()) << "Recreate index failed for " << field_name << ": " + << s.message(); + } + + // Flush to persist + auto s = collection->Flush(); + ASSERT_TRUE(s.ok()) << s.message(); + + // Verify all indexes are recreated + stats = collection->Stats().value(); + ASSERT_EQ(stats.doc_count, doc_count); + for (const auto &field_name : scalar_fields) { + ASSERT_FLOAT_EQ(stats.index_completeness[field_name], 1.0f); + } + + // Reopen and verify persistence + collection.reset(); + auto result = Collection::Open(col_path, options); + ASSERT_TRUE(result.has_value()) << result.error().message(); + collection = result.value(); + + stats = collection->Stats().value(); + ASSERT_EQ(stats.doc_count, doc_count); + for (const auto &field_name : scalar_fields) { + ASSERT_FLOAT_EQ(stats.index_completeness[field_name], 1.0f); + } + + collection.reset(); + FileHelper::RemoveDirectory(col_path); + }; + + func(false); + func(true); +}