Use milvus docker container for integration test

Signed-off-by: yhmo <yihua.mo@zilliz.com>

Author: yhmo

.github/workflows/main.yaml

@@ -64,13 +64,14 @@ jobs:
         run: echo "${HOME}/.local/bin" >> $GITHUB_PATH
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Cache ccache
+      - name: Cache ccache and conan
         uses: actions/cache@v4
         with:
           path: |
             ${{ github.workspace }}/.ccache
-          key: linux-${{ matrix.os.key }}-ccache-${{ github.sha }}
-          restore-keys: linux-${{ matrix.os.key }}-ccache-
+            /github/home/.conan2
+          key: linux-${{ matrix.os.key }}-cache-${{ github.sha }}
+          restore-keys: linux-${{ matrix.os.key }}-cache-
       - name: Prepare
         run: |
           sh scripts/install_deps.sh
@@ -81,7 +82,36 @@ jobs:
       - name: Unit Testing
         run: |
           make test
-  st-ubuntu:
+  integration-test:
+    name: Integration test
+    runs-on: ubuntu-22.04
+    timeout-minutes: 75
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+      CCACHE_COMPILERCHECK: content
+      CCACHE_COMPRESS: 1
+      CCACHE_COMPRESSLEVEL: 5
+      CCACHE_MAXSIZE: 2G
+    steps:
+      - name: Env
+        run: echo "${HOME}/.local/bin" >> $GITHUB_PATH
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Cache ccache and conan
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ github.workspace }}/.ccache
+            ~/.conan2
+          key: ubuntu-2204-st-cache-${{ github.sha }}
+          restore-keys: ubuntu-2204-st-cache-
+      - name: Prepare
+        run: |
+          sh scripts/install_deps.sh
+      - name: Integration Testing
+        run: |
+          make st
+  coverage-ubuntu:
     name: Test all with coverage
     runs-on: ubuntu-${{ matrix.ubuntu }}
     timeout-minutes: 120
@@ -100,13 +130,14 @@ jobs:
         run: echo "${HOME}/.local/bin" >> $GITHUB_PATH
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Cache ccache
+      - name: Cache ccache and conan
         uses: actions/cache@v4
         with:
           path: |
             ${{ github.workspace }}/.ccache
-          key: ubuntu-${{ matrix.ubuntu }}-ccache-${{ github.sha }}
-          restore-keys: ubuntu-${{ matrix.ubuntu }}-ccache-
+            ~/.conan2
+          key: ubuntu-${{ matrix.ubuntu }}-cache-${{ github.sha }}
+          restore-keys: ubuntu-${{ matrix.ubuntu }}-cache-
       - uses: actions/setup-python@v4
         with:
           python-version: '3.8'

scripts/build.sh

@@ -138,6 +138,10 @@ if [[ "${BUILD_FROM_CONAN}" == "ON" ]]; then
 
   # Build folder layout follows Conan 2 CMakeToolchain defaults:
   #   cmake_build/build/<BuildType>/generators/conan_toolchain.cmake
+  # Workaround: CMake 4.x removed compat with cmake_minimum_required < 3.5,
+  # which breaks older Conan recipes (e.g. c-ares/1.19.1).
+  export CMAKE_POLICY_VERSION_MINIMUM=3.5
+
   ${CONAN} install .. \
     -of . \
     -s build_type=${BUILD_TYPE} \
@@ -237,13 +241,15 @@ fi
 
 if [[ "${UNIT_TEST}" == "ON" ]]; then
   make -j ${JOBS}  || exit 1
-  ./test/testing-ut || exit 1
-  ./test/testing-it || exit 1
+  # Suppress gRPC verbose logs during tests
+  GRPC_VERBOSITY=ERROR ./test/testing-ut || exit 1
+  GRPC_VERBOSITY=ERROR ./test/testing-it || exit 1
 fi
 
 if [[ "${SYS_TEST}" == "ON" ]]; then
   make -j ${JOBS}  || exit 1
-  ./test/testing-st || exit 1
+  # Suppress gRPC verbose logs during tests
+  GRPC_VERBOSITY=ERROR ./test/testing-st || exit 1
 fi
 
 if [[ "${DO_INSTALL}" == "ON" ]]; then

scripts/install_deps.sh

@@ -96,6 +96,9 @@ install_deps_for_ubuntu_common() {
                        libssl-dev iwyu lcov git python3-pip clang-format clang-tidy
     install_linux_cmake_clang_toolchain
     install_conan
+
+    echo 'Pip install docker for integration tests.'
+    pip3 install --user "docker>=7.0.0"
 }
 
 install_deps_for_ubuntu_1804() {
@@ -115,6 +118,9 @@ install_deps_for_fedora_common() {
     ${SUDO} dnf -y install gcc gcc-c++ python2 gpg wget ccache make openssl-devel which lcov git rpm-build python3-pip perl perl-core
     install_linux_cmake_clang_toolchain
     install_conan
+
+    echo 'Pip install docker for integration tests.'
+    pip3 install --user "docker>=7.0.0"
 }
 
 install_deps_for_centos_8() {

scripts/run_clang_tidy.py

@@ -22,6 +22,7 @@
 import lintutils
 from subprocess import PIPE
 import sys
+import time
 from functools import partial
 
 
@@ -113,6 +114,10 @@ def _check_all(cmd, filenames):
         '-p',
         arguments.compile_commands
     ]
+
+    print("clang-tidy: {} files to check".format(len(linted_filenames)))
+    start_time = time.time()
+
     if arguments.fix:
         cmd.append('-fix')
         results = lintutils.run_parallel(
@@ -123,3 +128,6 @@ def _check_all(cmd, filenames):
 
     else:
         _check_all(cmd, linted_filenames)
+
+    elapsed = time.time() - start_time
+    print("clang-tidy: finished in {:.1f} seconds".format(elapsed))

test/it/v1/TestCreateResourceGroup.cpp

@@ -46,23 +46,23 @@ TEST_F(MilvusMockedTest, CreateResourceGroup) {
             const auto& transfer_from = config.TransferFromGroups();
             const auto& rpc_from = rpc_config.transfer_from();
             EXPECT_EQ(rpc_from.size(), transfer_from.size());
-            for (auto i = 0; i < rpc_from.size(); i++) {
-                EXPECT_TRUE(transfer_from.find(rpc_from.at(i).resource_group()) != transfer_from.end());
+            for (const auto& item : rpc_from) {
+                EXPECT_TRUE(transfer_from.find(item.resource_group()) != transfer_from.end());
             }
 
             const auto& transfer_to = config.TransferToGroups();
             const auto& rpc_to = rpc_config.transfer_to();
             EXPECT_EQ(rpc_to.size(), transfer_to.size());
-            for (auto i = 0; i < rpc_to.size(); i++) {
-                EXPECT_TRUE(transfer_to.find(rpc_to.at(i).resource_group()) != transfer_to.end());
+            for (const auto& item : rpc_to) {
+                EXPECT_TRUE(transfer_to.find(item.resource_group()) != transfer_to.end());
             }
 
             const auto& node_filters = config.NodeFilters();
             const auto& rpc_filters = rpc_config.node_filter().node_labels();
             EXPECT_EQ(rpc_filters.size(), node_filters.size());
-            for (auto i = 0; i < rpc_filters.size(); i++) {
-                EXPECT_TRUE(node_filters.find(rpc_filters.at(i).key()) != node_filters.end());
-                EXPECT_EQ(rpc_filters.at(i).value(), node_filters.at(rpc_filters.at(i).key()));
+            for (const auto& rpc_filter : rpc_filters) {
+                EXPECT_TRUE(node_filters.find(rpc_filter.key()) != node_filters.end());
+                EXPECT_EQ(rpc_filter.value(), node_filters.at(rpc_filter.key()));
             }
 
             return ::grpc::Status{};

test/it/v1/TestGetFlushState.cpp

@@ -89,7 +89,7 @@ TEST_F(MilvusMockedTest, GetFlushState) {
 
     // first round set c1 to flushed, c2 to unflushed, progress is 2/5
     // second round set c2 to flushed, progress is 5/5
-    std::vector<milvus::Progress> progresses_expected{{(uint32_t)c1_ids.size(), segment_count},
+    std::vector<milvus::Progress> progresses_expected{{static_cast<uint32_t>(c1_ids.size()), segment_count},
                                                       {segment_count, segment_count}};
 
     EXPECT_THAT(progresses, ElementsAreArray(progresses_expected));

test/it/v1/TestHybridSearch.cpp

@@ -193,7 +193,7 @@ TEST_F(MilvusMockedTest, HybridSearch) {
 
             // ids, topk and scores
             results->mutable_topks()->Add(3);
-            for (auto id : expected_ids) {
+            for (const auto& id : expected_ids) {
                 results->mutable_ids()->mutable_str_id()->add_data(id);
             }
             for (auto score : expected_scores) {

test/it/v1/TestLoadPartitions.cpp

@@ -188,10 +188,7 @@ TEST_F(MilvusMockedTest, LoadPartitionsWithQueryStatusTimeout) {
             return ::grpc::Status{};
         });
 
-    auto started = std::chrono::steady_clock::now();
     auto status = client_->LoadPartitions(collection, partitions, 2, progress_monitor);
-    auto finished = std::chrono::steady_clock::now();
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finished - started).count();
 
     EXPECT_FALSE(status.IsOk());
     EXPECT_EQ(status.Code(), StatusCode::TIMEOUT);

test/it/v1/TestQueryIterator.cpp

@@ -74,7 +74,7 @@ TEST_F(MilvusMockedTest, QueryIterator) {
             uint64_t session_ts = 999999;
             bool is_seek = false;
             uint64_t query_limit = 0;
-            for (auto pair : params) {
+            for (const auto& pair : params) {
                 if (pair.key() == milvus::LIMIT) {
                     query_limit = std::atoi(pair.value().c_str());
                     if (query_limit == 1) {

test/it/v1/TestSearchIterator.cpp

@@ -78,7 +78,7 @@ DoSearchIterator(testing::StrictMock<milvus::MilvusMockedService>& service, milv
             }
 
             auto params = request->search_params();
-            for (auto pair : params) {
+            for (const auto& pair : params) {
                 if (pair.key() == milvus::TOPK) {
                     EXPECT_GE(std::stoul(pair.value()), batch_size);
                 }
@@ -144,6 +144,7 @@ DoSearchIterator(testing::StrictMock<milvus::MilvusMockedService>& service, milv
     }
 
     std::vector<float> vector;
+    vector.reserve(milvus::T_DIMENSION);
     for (auto i = 0; i < milvus::T_DIMENSION; i++) {
         vector.push_back(1.0);
     }