From 575da2f3a12d79da44cf6f7f6bbf7c50c2602528 Mon Sep 17 00:00:00 2001 From: Liyun Xiu Date: Thu, 23 Jul 2026 09:17:21 +0000 Subject: [PATCH 1/5] Optimize Seismic and Seismic-SQ search hot path Inter-process ablation on the 8.8M-doc corpus shows ~1.4x for float Seismic and ~1.06x for the 8-bit SQ variant from: - VisitedSet: bitset dedup with sparse per-query reset, replacing the absl::flat_hash_set on the doc-scan critical path. - prefetch_vector_head: prefetch only the leading cache lines of the next doc row, bounding outstanding software prefetches. - MADV_HUGEPAGE/COLLAPSE the corpus arrays to cut TLB/page-walk cost on the random per-doc gather. - sort_cluster_docs: sort doc ids ascending within each cluster so the per-doc gather is monotonic. Doc-offset precompute and two-lane AVX512 were within noise, so omitted. Add VisitedSet unit tests for the fragile sparse-reset: within-query dedup, full reset across new_query, two ids sharing a 64-bit word, resize clearing touched state, and word boundaries (63/64/n-1). Signed-off-by: Liyun Xiu --- nsparse/cluster/inverted_list_clusters.cpp | 18 ++++ nsparse/cluster/inverted_list_clusters.h | 3 + nsparse/seismic_index.cpp | 102 ++++++++++++------ nsparse/seismic_index.h | 13 ++- nsparse/seismic_scalar_quantized_index.cpp | 41 +++---- nsparse/seismic_scalar_quantized_index.h | 6 +- nsparse/sparse_vectors.cpp | 8 ++ nsparse/utils/distance_avx512.h | 2 +- nsparse/utils/hugepage.h | 66 ++++++++++++ nsparse/utils/prefetch.h | 25 +++++ nsparse/utils/visited_set.h | 68 ++++++++++++ tests/CMakeLists.txt | 1 + tests/visited_set_test.cpp | 118 +++++++++++++++++++++ 13 files changed, 402 insertions(+), 69 deletions(-) create mode 100644 nsparse/utils/hugepage.h create mode 100644 nsparse/utils/visited_set.h create mode 100644 tests/visited_set_test.cpp diff --git a/nsparse/cluster/inverted_list_clusters.cpp b/nsparse/cluster/inverted_list_clusters.cpp index 73ff26b..22f91ae 100644 --- a/nsparse/cluster/inverted_list_clusters.cpp +++ b/nsparse/cluster/inverted_list_clusters.cpp @@ -118,6 +118,20 @@ InvertedListClusters::InvertedListClusters( docs_.insert(docs_.end(), doc_ids.begin(), doc_ids.end()); offsets_.push_back(docs_.size()); } + sort_cluster_docs(); +} + +// Sort the doc ids within each cluster ascending. Within-cluster iteration +// order does not affect search results (the top-k heap and the visited dedup +// are order-independent), but it dictates the memory access pattern of the +// per-doc gather: indptr[doc_id] and the forward-index rows it points at are +// spread across the multi-GB corpus. Ascending doc ids turn that per-doc random +// gather into a monotonic sweep the hardware prefetcher and TLB can follow. +void InvertedListClusters::sort_cluster_docs() { + if (offsets_.size() <= 1) return; + for (size_t c = 0; c + 1 < offsets_.size(); ++c) { + std::sort(docs_.begin() + offsets_[c], docs_.begin() + offsets_[c + 1]); + } } InvertedListClusters::InvertedListClusters(const InvertedListClusters& other) = @@ -291,6 +305,10 @@ void InvertedListClusters::deserialize(IOReader* reader) { offsets_.resize(n_offsets); reader->read(offsets_.data(), sizeof(idx_t), n_offsets); } + // Order within each cluster does not affect results but makes the per-doc + // gather monotonic (see sort_cluster_docs). Applied on load so existing + // serialized indexes benefit without a rebuild. + sort_cluster_docs(); reader->read(&n_clusters_, sizeof(size_t), 1); reader->read(&element_size_, sizeof(size_t), 1); diff --git a/nsparse/cluster/inverted_list_clusters.h b/nsparse/cluster/inverted_list_clusters.h index 9b6beb9..42b194c 100644 --- a/nsparse/cluster/inverted_list_clusters.h +++ b/nsparse/cluster/inverted_list_clusters.h @@ -56,6 +56,9 @@ class InvertedListClusters : public Serializable { private: // Build the term-major (CSC) transpose from a per-cluster CSR summary. void build_transpose(const SparseVectors& summaries); + // Sort doc ids ascending within each cluster (result-order invariant) so + // the per-doc gather over the forward index is monotonic, not random. + void sort_cluster_docs(); template void score_summaries_typed(const term_t* q_idx, const T* q_val, size_t q_len, std::vector& out) const; diff --git a/nsparse/seismic_index.cpp b/nsparse/seismic_index.cpp index 332cb8d..fd977b1 100644 --- a/nsparse/seismic_index.cpp +++ b/nsparse/seismic_index.cpp @@ -16,7 +16,6 @@ #include #include -#include "absl/container/flat_hash_set.h" #include "nsparse/cluster/inverted_list_clusters.h" #include "nsparse/cluster/random_kmeans.h" #include "nsparse/exact_matcher.h" @@ -32,6 +31,7 @@ #include "nsparse/utils/prefetch.h" #include "nsparse/utils/ranker.h" #include "nsparse/utils/vector_process.h" +#include "nsparse/utils/visited_set.h" namespace nsparse { namespace { @@ -41,9 +41,10 @@ constexpr int kElementSize = U32; void query_single_inverted_list( const SparseVectors* vectors, const InvertedListClusters& cluster_invlist, const std::vector& dense, const term_t* q_idx, const float* q_val, - size_t q_len, std::vector& score_scratch, const float heap_factor, + size_t q_len, std::vector& score_scratch, + std::vector& cluster_order, const float heap_factor, const bool first_list, const SearchParameters* search_parameters, - detail::TopKHolder& heap, absl::flat_hash_set& visited) { + detail::TopKHolder& heap, detail::VisitedSet& visited) { // Skip empty clusters size_t csize = cluster_invlist.cluster_size(); if (csize == 0) { @@ -58,39 +59,39 @@ void query_single_inverted_list( cluster_invlist.score_summaries_transposed( q_idx, reinterpret_cast(q_val), q_len, score_scratch); const std::vector& summary_scores = score_scratch; - size_t num_vectors = vectors->num_vectors(); - - std::vector cluster_order = - detail::reorder_clusters(summary_scores, first_list); + const size_t n_clusters = summary_scores.size(); const auto& [indptr, indices, values] = vectors->get_all_data(); - for (const size_t& cluster_id : cluster_order) { - const auto& cluster_score = summary_scores[cluster_id]; + // Process one cluster: prune by summary score, then score its docs. + // Returns false when the early-out fires on the first (sorted) list, which + // means no later cluster can qualify either — the caller stops iterating. + auto process_cluster = [&](size_t cluster_id) -> bool { + const float cluster_score = summary_scores[cluster_id]; if (heap.full() && (cluster_score * heap_factor < heap.peek_score())) { - if (first_list) { - break; - } - continue; + // On the first list clusters are visited in descending score order, + // so once one falls below the threshold every later one does too. + return !first_list; } const auto& docs = cluster_invlist.get_docs(cluster_id); const size_t n_docs = docs.size(); - static constexpr size_t kPrefetchDist1 = 2; // vector data prefetch - static constexpr size_t kPrefetchDist2 = 4; // indptr prefetch for (size_t i = 0; i < n_docs; ++i) { - const auto& doc_id = docs[i]; - if (i + kPrefetchDist2 < n_docs) { - detail::prefetch_indptr(indptr, docs[i + kPrefetchDist2]); - } + const idx_t doc_id = docs[i]; + // Prefetch one doc ahead, only the leading lines of the upcoming + // row; the row is contiguous so the hardware streamer pulls the + // tail, while bounding outstanding software prefetches keeps the + // line-fill buffers from saturating (measured optimum ~4 lines). + static constexpr size_t kPrefetchDist1 = 1; if (i + kPrefetchDist1 < n_docs) { - const idx_t next_doc = docs[i + kPrefetchDist1]; - const idx_t next_start = indptr[next_doc]; - const size_t next_len = indptr[next_doc + 1] - next_start; - detail::prefetch_vector(indices + next_start, - values + next_start, next_len); + const idx_t nd = docs[i + kPrefetchDist1]; + const idx_t next_start = indptr[nd]; + const size_t next_len = indptr[nd + 1] - next_start; + static constexpr size_t kPrefetchHeadLines = 4; + detail::prefetch_vector_head(indices + next_start, + values + next_start, next_len, + kPrefetchHeadLines); } - auto [_, inserted] = visited.insert(doc_id); - if (!inserted) { + if (!visited.insert(static_cast(doc_id))) { continue; } if (id_selector != nullptr && !id_selector->is_member(doc_id)) { @@ -102,6 +103,29 @@ void query_single_inverted_list( indices + start, values + start, len, dense.data()); heap.add(score, doc_id); } + return true; + }; + + if (first_list) { + // Only the first list is score-ordered; sort a per-thread scratch of + // narrow (u32) cluster ids instead of allocating a size_t vector per + // list. Clusters per list stay well under 2^32. + cluster_order.resize(n_clusters); + std::iota(cluster_order.begin(), cluster_order.end(), 0U); + std::ranges::sort(cluster_order, [&](uint32_t a, uint32_t b) { + return summary_scores[a] > summary_scores[b]; + }); + for (const uint32_t cluster_id : cluster_order) { + if (!process_cluster(cluster_id)) { + break; + } + } + } else { + // Later lists are visited in natural cluster order, so iterate directly + // — no order array, iota, or sort needed. + for (size_t cluster_id = 0; cluster_id < n_clusters; ++cluster_id) { + process_cluster(cluster_id); + } } } } // namespace @@ -187,8 +211,14 @@ auto SeismicIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, #pragma omp parallel { std::vector dense(dim, 0.0F); - absl::flat_hash_set visited; - visited.reserve(static_cast(std::max(k, 1)) * 4096); + // Generation-stamped visited set over the doc-id domain: O(1) reset per + // query and a single indexed load per candidate instead of a hashed + // random probe (the doc loop is memory-bound on random gathers). + detail::VisitedSet visited(vectors_->num_vectors()); + // Per-thread scratch reused across queries: the per-cluster summary + // score buffer and the sorted cluster-order buffer (first list only). + std::vector score_scratch; + std::vector cluster_order; #pragma omp for schedule(dynamic, 64) for (idx_t query_idx = 0; query_idx < n; ++query_idx) { @@ -200,7 +230,8 @@ auto SeismicIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, detail::top_k_tokens(q_indices, q_values, len, parameters->cut); auto [distances, labels] = single_query(dense, visited, q_indices, q_values, len, cuts, k, - parameters->heap_factor, search_parameters); + parameters->heap_factor, search_parameters, + score_scratch, cluster_order); result_distances[query_idx] = std::move(distances); result_labels[query_idx] = std::move(labels); } @@ -219,11 +250,13 @@ auto SeismicIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, * @return std::pair, std::vector> */ auto SeismicIndex::single_query(std::vector& dense, - absl::flat_hash_set& visited, + detail::VisitedSet& visited, const term_t* q_indices, const float* q_values, size_t q_len, const std::vector& cuts, int k, float heap_factor, - SearchParameters* search_parameters) + SearchParameters* search_parameters, + std::vector& score_scratch, + std::vector& cluster_order) -> pair_of_score_id_vector_t { size_t num_docs = vectors_->num_vectors(); if (num_docs == 0) { @@ -234,10 +267,9 @@ auto SeismicIndex::single_query(std::vector& dense, for (size_t i = 0; i < q_len; ++i) { dense[q_indices[i]] = q_values[i]; } - visited.clear(); + visited.new_query(); detail::TopKHolder holder(k); - std::vector score_scratch; bool first_list = true; for (const auto& term : cuts) { if (term >= clustered_inverted_lists.size()) [[unlikely]] { @@ -246,8 +278,8 @@ auto SeismicIndex::single_query(std::vector& dense, const auto& cluster_invlist = clustered_inverted_lists[term]; query_single_inverted_list(vectors_.get(), cluster_invlist, dense, q_indices, q_values, q_len, score_scratch, - heap_factor, first_list, search_parameters, - holder, visited); + cluster_order, heap_factor, first_list, + search_parameters, holder, visited); first_list = false; } diff --git a/nsparse/seismic_index.h b/nsparse/seismic_index.h index c9ffc8b..6a89c85 100644 --- a/nsparse/seismic_index.h +++ b/nsparse/seismic_index.h @@ -12,13 +12,13 @@ #include #include -#include "absl/container/flat_hash_set.h" #include "nsparse/cluster/inverted_list_clusters.h" #include "nsparse/index.h" #include "nsparse/io/io.h" #include "nsparse/seismic_common.h" #include "nsparse/sparse_vectors.h" #include "nsparse/types.h" +#include "nsparse/utils/visited_set.h" namespace nsparse { @@ -67,12 +67,15 @@ class SeismicIndex : public Index, public IndexIO { // `dense` and `visited` are per-thread scratch reused across the queries a // thread handles (see search()). `dense` must be all-zero on entry and is // restored to all-zero on exit via a sparse clear over the query's own - // dims (q_indices/q_len); `visited` is cleared on entry. - auto single_query(std::vector& dense, - absl::flat_hash_set& visited, + // dims (q_indices/q_len); `visited` starts a new generation on entry. + // `score_scratch` and `cluster_order` are per-thread scratch reused across + // queries (resized in place), avoiding a per-query/per-list allocation. + auto single_query(std::vector& dense, detail::VisitedSet& visited, const term_t* q_indices, const float* q_values, size_t q_len, const std::vector& cuts, int k, - float heap_factor, SearchParameters* search_parameters) + float heap_factor, SearchParameters* search_parameters, + std::vector& score_scratch, + std::vector& cluster_order) -> pair_of_score_id_vector_t; std::unique_ptr vectors_; SeismicClusterParameters cluster_parameter_; diff --git a/nsparse/seismic_scalar_quantized_index.cpp b/nsparse/seismic_scalar_quantized_index.cpp index 63599fe..a4723b1 100644 --- a/nsparse/seismic_scalar_quantized_index.cpp +++ b/nsparse/seismic_scalar_quantized_index.cpp @@ -17,7 +17,6 @@ #include #include -#include "absl/container/flat_hash_set.h" #include "nsparse/cluster/inverted_list_clusters.h" #include "nsparse/cluster/random_kmeans.h" #include "nsparse/exact_matcher.h" @@ -34,6 +33,7 @@ #include "nsparse/utils/prefetch.h" #include "nsparse/utils/scalar_quantizer.h" #include "nsparse/utils/vector_process.h" +#include "nsparse/utils/visited_set.h" namespace nsparse { namespace { @@ -46,7 +46,7 @@ void query_single_inverted_list(const SparseVectors* vectors, float heap_factor, bool first_list, const SearchParameters* search_parameters, detail::TopKHolder& heap, - absl::flat_hash_set& visited) { + detail::VisitedSet& visited) { // Skip empty clusters size_t csize = cluster_invlist.cluster_size(); if (csize == 0) { @@ -80,31 +80,23 @@ void query_single_inverted_list(const SparseVectors* vectors, } const auto& docs = cluster_invlist.get_docs(cluster_id); const size_t n_docs = docs.size(); - // Two-stage prefetch pipeline: - // Stage 1 (distance 2): prefetch indptr[docs[i+2]] so the indptr - // lookup is cached by the time we need it next iteration. - // Stage 2 (distance 1): read indptr[docs[i+1]] (now cached from - // stage 1 issued last iteration), prefetch the actual vector data. - static constexpr size_t kPrefetchDist1 = 2; // vector data prefetch - static constexpr size_t kPrefetchDist2 = 4; // indptr prefetch for (size_t i = 0; i < n_docs; ++i) { - const auto& doc_id = docs[i]; - // Stage 1: prefetch indptr entry for doc at distance 2 - if (i + kPrefetchDist2 < n_docs) { - detail::prefetch_indptr(indptr, docs[i + kPrefetchDist2]); - } - // Stage 2: prefetch vector data for next doc (indptr should - // already be cached from stage 1 issued kPrefetchDist2 - - // kPrefetchDist1 iterations ago) + const idx_t doc_id = docs[i]; + // Prefetch one doc ahead, only the leading lines of the upcoming + // row; the row is contiguous so the hardware streamer pulls the + // tail, while bounding outstanding software prefetches keeps the + // line-fill buffers from saturating (measured optimum ~4 lines). + static constexpr size_t kPrefetchDist1 = 1; if (i + kPrefetchDist1 < n_docs) { const idx_t next_doc = docs[i + kPrefetchDist1]; const idx_t next_start = indptr[next_doc]; const size_t next_len = indptr[next_doc + 1] - next_start; - detail::prefetch_vector(indices + next_start, - values + next_start, next_len); + static constexpr size_t kPrefetchHeadLines = 4; + detail::prefetch_vector_head(indices + next_start, + values + next_start, next_len, + kPrefetchHeadLines); } - auto [_, inserted] = visited.insert(doc_id); - if (!inserted) { + if (!visited.insert(static_cast(doc_id))) { continue; } if (id_selector != nullptr && !id_selector->is_member(doc_id)) { @@ -248,8 +240,7 @@ auto SeismicScalarQuantizedIndex::search(idx_t n, const idx_t* indptr, #pragma omp parallel { std::vector dense(dense_bytes, 0); - absl::flat_hash_set visited; - visited.reserve(static_cast(std::max(k, 1)) * 4096); + detail::VisitedSet visited(vectors_->num_vectors()); #pragma omp for schedule(dynamic, 64) for (idx_t query_idx = 0; query_idx < n; ++query_idx) { @@ -280,7 +271,7 @@ auto SeismicScalarQuantizedIndex::search(idx_t n, const idx_t* indptr, } auto SeismicScalarQuantizedIndex::single_query( - std::vector& dense, absl::flat_hash_set& visited, + std::vector& dense, detail::VisitedSet& visited, const term_t* q_idx, const uint8_t* q_val_bytes, size_t q_len, size_t element_size, const std::vector& cuts, int k, float heap_factor, const ScalarQuantizer& query_sq, @@ -296,7 +287,7 @@ auto SeismicScalarQuantizedIndex::single_query( std::copy_n(q_val_bytes + i * element_size, element_size, dense.data() + static_cast(q_idx[i]) * element_size); } - visited.clear(); + visited.new_query(); detail::TopKHolder holder(k); std::vector score_scratch; diff --git a/nsparse/seismic_scalar_quantized_index.h b/nsparse/seismic_scalar_quantized_index.h index 179b226..4cf7da8 100644 --- a/nsparse/seismic_scalar_quantized_index.h +++ b/nsparse/seismic_scalar_quantized_index.h @@ -14,11 +14,11 @@ #include #include -#include "absl/container/flat_hash_set.h" #include "nsparse/cluster/inverted_list_clusters.h" #include "nsparse/index.h" #include "nsparse/seismic_index.h" #include "nsparse/utils/scalar_quantizer.h" +#include "nsparse/utils/visited_set.h" namespace nsparse { @@ -70,9 +70,9 @@ class SeismicScalarQuantizedIndex : public Index, public IndexIO { // thread handles (see search()). `dense` (a dimension-sized quantized-code // buffer, element_size bytes per dim) must be all-zero on entry and is // restored to all-zero on exit via a sparse clear over the query's own dims - // (q_idx/q_len); `visited` is cleared on entry. + // (q_idx/q_len); `visited` starts a new generation on entry. auto single_query(std::vector& dense, - absl::flat_hash_set& visited, const term_t* q_idx, + detail::VisitedSet& visited, const term_t* q_idx, const uint8_t* q_val_bytes, size_t q_len, size_t element_size, const std::vector& cuts, int k, float heap_factor, const ScalarQuantizer& query_sq, diff --git a/nsparse/sparse_vectors.cpp b/nsparse/sparse_vectors.cpp index fe232f0..5558921 100644 --- a/nsparse/sparse_vectors.cpp +++ b/nsparse/sparse_vectors.cpp @@ -15,6 +15,7 @@ #include "nsparse/io/io.h" #include "nsparse/types.h" #include "nsparse/utils/checks.h" +#include "nsparse/utils/hugepage.h" namespace nsparse { SparseVectors::SparseVectors(SparseVectorsConfig config) : config_(config) { @@ -165,6 +166,13 @@ void SparseVectors::deserialize(IOReader* io_reader) { size_t value_size = indptr_[vector_count] * element_size; values_.resize(value_size); io_reader->read(values_.data(), sizeof(uint8_t), value_size); + + // The per-doc dot product gathers randomly across indices_/values_ + // (multi-GB at scale); back them with huge pages to cut TLB/page-walk + // cost. indptr_ is also randomly indexed by doc id. + detail::advise_hugepage(indptr_); + detail::advise_hugepage(indices_); + detail::advise_hugepage(values_); } } } // namespace nsparse \ No newline at end of file diff --git a/nsparse/utils/distance_avx512.h b/nsparse/utils/distance_avx512.h index 1969e23..ae79ce9 100644 --- a/nsparse/utils/distance_avx512.h +++ b/nsparse/utils/distance_avx512.h @@ -333,7 +333,7 @@ inline float dot_product_float_dense(const term_t* indices, // Gather 16 values from dense vector using indices __m512 dense_vals = _mm512_i32gather_ps(idx, dense, sizeof(float)); - // Load 16 weights (aligned load - weights must be 64-byte aligned) + // Load 16 weights __m512 weight_vals = _mm512_loadu_ps(weights + i); // Fused multiply-add: sum += weights * dense_vals diff --git a/nsparse/utils/hugepage.h b/nsparse/utils/hugepage.h new file mode 100644 index 0000000..8077f0e --- /dev/null +++ b/nsparse/utils/hugepage.h @@ -0,0 +1,66 @@ +/** + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +#ifndef NSPARSE_HUGEPAGE_H +#define NSPARSE_HUGEPAGE_H + +#include +#include +#include + +#if defined(__linux__) +#include +#include +#endif + +namespace nsparse::detail { + +// Hint the kernel to back a large, mostly-read-only array with transparent +// huge pages. Random gathers into the multi-GB posting/summary arrays are +// dominated by TLB misses + page-table walks (2 MB pages cover 512x the span +// of 4 KB pages), so this cuts the DTLB/page-walk cost with no change to the +// data layout or results. No-op on non-Linux or for small buffers. +template +inline void advise_hugepage(const std::vector& v) { +#if defined(__linux__) && defined(MADV_HUGEPAGE) + const size_t bytes = v.size() * sizeof(T); + // 2 MB is the x86-64 huge-page size; only bother once the region spans at + // least one huge page so the alignment rounding can't consume the whole + // buffer. + static constexpr size_t kHugePage = size_t{2} << 20; + if (bytes < kHugePage) { + return; + } + const auto base = reinterpret_cast(v.data()); + // madvise() requires a page-aligned start; round the start up and the end + // down to whole huge pages so only fully-covered pages are advised. + const uintptr_t start = (base + kHugePage - 1) & ~(kHugePage - 1); + const uintptr_t end = (base + bytes) & ~(kHugePage - 1); + if (end > start) { + void* p = reinterpret_cast(start); + const size_t len = static_cast(end - start); + // Mark the region so future faults and khugepaged prefer huge pages. + ::madvise(p, len, MADV_HUGEPAGE); +#if defined(MADV_COLLAPSE) + // The arrays are already faulted in (resize + read) as 4 KB pages, so a + // plain MADV_HUGEPAGE only takes effect lazily via khugepaged. + // MADV_COLLAPSE (Linux 6.1+) collapses the existing pages into huge + // pages synchronously, giving the TLB win immediately. Best-effort: + // ignore failure (unsupported kernel, fragmentation). + ::madvise(p, len, MADV_COLLAPSE); +#endif + } +#else + (void)v; +#endif +} + +} // namespace nsparse::detail + +#endif // NSPARSE_HUGEPAGE_H diff --git a/nsparse/utils/prefetch.h b/nsparse/utils/prefetch.h index 72b8784..9b1733d 100644 --- a/nsparse/utils/prefetch.h +++ b/nsparse/utils/prefetch.h @@ -9,6 +9,7 @@ #ifndef PREFETCH_H #define PREFETCH_H +#include #include #include "nsparse/types.h" @@ -45,6 +46,30 @@ inline void prefetch_vector(const term_t* indices, const T* values, } } +// Prefetch only the leading `max_lines` cache lines of each stream. A doc row +// spans ~13 lines here; prefetching all of them for several docs ahead +// overruns the core's ~10-12 line-fill buffers (l1d_pend_miss.fb_full), which +// stalls demand loads. Because the row is contiguous, touching just the first +// line or two lets the hardware stream prefetcher pull the rest while keeping +// the number of outstanding software prefetches bounded. +template +inline void prefetch_vector_head(const term_t* indices, const T* values, + size_t len, size_t max_lines) { + static constexpr size_t kCacheLineSize = 64; // bytes + const char* indices_ptr = reinterpret_cast(indices); + const char* values_ptr = reinterpret_cast(values); + const size_t indices_bytes = + std::min(len * sizeof(term_t), max_lines * kCacheLineSize); + const size_t values_bytes = + std::min(len * sizeof(T), max_lines * kCacheLineSize); + for (size_t offset = 0; offset < indices_bytes; offset += kCacheLineSize) { + NSPARSE_PREFETCH(indices_ptr + offset, 0, 0); + } + for (size_t offset = 0; offset < values_bytes; offset += kCacheLineSize) { + NSPARSE_PREFETCH(values_ptr + offset, 0, 0); + } +} + inline void prefetch_indptr(const idx_t* indptr, idx_t doc_id) { NSPARSE_PREFETCH(&indptr[doc_id], 0, 0); } diff --git a/nsparse/utils/visited_set.h b/nsparse/utils/visited_set.h new file mode 100644 index 0000000..ff96571 --- /dev/null +++ b/nsparse/utils/visited_set.h @@ -0,0 +1,68 @@ +/** + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +#ifndef NSPARSE_VISITED_SET_H +#define NSPARSE_VISITED_SET_H + +#include +#include + +namespace nsparse::detail { + +// Membership set over a fixed doc-id domain [0, n), backed by a bit per doc. +// +// The seismic doc loop tests/inserts every candidate doc to dedupe across +// clusters. A hash set hashes and probes a random cache line per candidate and +// must be cleared each query. A full generation-stamped uint32 array (4 B/doc) +// avoids the hashing but at seismic scale (~35 MB) adds a random-access stream +// larger than L2, so it misses cache on every candidate. A bitset is 32x +// smaller (1 bit/doc, ~1.1 MB at 8.8M docs) so far fewer distinct cache lines +// are touched, while the touched-word list lets a new query clear only the +// words it actually dirtied (sparse O(visited) reset, not O(n)). +class VisitedSet { +public: + VisitedSet() = default; + explicit VisitedSet(size_t n) { resize(n); } + + void resize(size_t n) { + bits_.assign((n + 63) / 64, 0); + touched_.clear(); + } + + // Begin a new query: clear only the words dirtied by the previous query. + void new_query() { + for (const size_t w : touched_) { + bits_[w] = 0; + } + touched_.clear(); + } + + // Mark `id` visited; return true if it was newly inserted this query. + bool insert(size_t id) { + const size_t w = id >> 6; + const uint64_t mask = uint64_t{1} << (id & 63); + const uint64_t prev = bits_[w]; + if (prev & mask) { + return false; + } + if (prev == 0) { + touched_.push_back(w); + } + bits_[w] = prev | mask; + return true; + } + +private: + std::vector bits_; + std::vector touched_; +}; + +} // namespace nsparse::detail + +#endif // NSPARSE_VISITED_SET_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2d59fa7..a230255 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -29,6 +29,7 @@ set(NSPARSE_TEST_SRC seismic_scalar_quantized_index_test.cpp sparse_vectors_test.cpp vector_process_test.cpp + visited_set_test.cpp ) # Fetch GoogleTest diff --git a/tests/visited_set_test.cpp b/tests/visited_set_test.cpp new file mode 100644 index 0000000..ecb6753 --- /dev/null +++ b/tests/visited_set_test.cpp @@ -0,0 +1,118 @@ +/** + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +#include "nsparse/utils/visited_set.h" + +#include + +namespace { + +using nsparse::detail::VisitedSet; + +// (a) Within one query an id is inserted once; repeats are deduped. +TEST(VisitedSetTest, dedup_within_query) { + VisitedSet s(128); + + EXPECT_TRUE(s.insert(7)); + EXPECT_FALSE(s.insert(7)); + EXPECT_FALSE(s.insert(7)); + + EXPECT_TRUE(s.insert(70)); + EXPECT_FALSE(s.insert(70)); +} + +// (b) new_query() fully resets: every id inserted last query is new again. +TEST(VisitedSetTest, new_query_resets_all_words) { + VisitedSet s(256); + + // Dirty ids spread across several 64-bit words. + for (size_t id : {0u, 63u, 64u, 130u, 255u}) { + EXPECT_TRUE(s.insert(id)); + EXPECT_FALSE(s.insert(id)); + } + + s.new_query(); + + for (size_t id : {0u, 63u, 64u, 130u, 255u}) { + EXPECT_TRUE(s.insert(id)) << "id " << id << " not reset"; + } +} + +// (c) Two ids in the same 64-bit word both dedup within a query and both clear +// on new_query, exercising the "touched once per word" reset invariant. +TEST(VisitedSetTest, two_ids_same_word_both_clear) { + VisitedSet s(128); + + // 5 and 60 share word 0; the second insert must not re-push the word, yet + // new_query must still clear both bits. + EXPECT_TRUE(s.insert(5)); + EXPECT_TRUE(s.insert(60)); + EXPECT_FALSE(s.insert(5)); + EXPECT_FALSE(s.insert(60)); + + s.new_query(); + + EXPECT_TRUE(s.insert(5)); + EXPECT_TRUE(s.insert(60)); +} + +// (d) resize() clears touched_ so a later new_query does not iterate a stale +// word index (which after shrinking would be out of bounds on bits_). +TEST(VisitedSetTest, resize_clears_touched_and_bits) { + VisitedSet s(256); + EXPECT_TRUE(s.insert(200)); // dirties word 3 + + s.resize(64); // bits_ shrinks to a single word; touched_ must be cleared + + // If touched_ retained word 3, this would be an out-of-bounds write. + s.new_query(); + + // Bits are also cleared by resize: an id valid in the new domain is fresh. + EXPECT_TRUE(s.insert(0)); + EXPECT_FALSE(s.insert(0)); +} + +// resize() to the same size still clears prior state. +TEST(VisitedSetTest, resize_same_size_clears_bits) { + VisitedSet s(128); + EXPECT_TRUE(s.insert(42)); + + s.resize(128); + + EXPECT_TRUE(s.insert(42)); +} + +// (e) Word boundaries: bit 63 (top of word 0), bit 64 (bottom of word 1), and +// the last valid id n-1 all behave and reset independently. +TEST(VisitedSetTest, word_boundaries) { + constexpr size_t n = 130; // 3 words (192 bits); last id is 129 + VisitedSet s(n); + + EXPECT_TRUE(s.insert(63)); + EXPECT_TRUE(s.insert(64)); + EXPECT_TRUE(s.insert(n - 1)); + + // Each is independent: neighbors are not marked by these inserts. + EXPECT_TRUE(s.insert(62)); + EXPECT_TRUE(s.insert(65)); + EXPECT_TRUE(s.insert(n - 2)); + + // And each dedups. + EXPECT_FALSE(s.insert(63)); + EXPECT_FALSE(s.insert(64)); + EXPECT_FALSE(s.insert(n - 1)); + + s.new_query(); + + EXPECT_TRUE(s.insert(63)); + EXPECT_TRUE(s.insert(64)); + EXPECT_TRUE(s.insert(n - 1)); +} + +} // namespace From 4f5de18c64d5eb855850a188916d37aa946a8a39 Mon Sep 17 00:00:00 2001 From: Liyun Xiu Date: Thu, 23 Jul 2026 09:17:26 +0000 Subject: [PATCH 2/5] Validate doc-id domain at load for VisitedSet safety VisitedSet indexes by doc id with no per-candidate bounds check (a branch there would erode the dedup win), so "every doc id < num_vectors" is now a load-bearing invariant: an out-of-range id is an out-of-bounds write, not the safe insert it was with the former hash set. Add InvertedListClusters::validate_doc_ids(num_docs), an O(nnz) scan that throws std::out_of_range on a stray id, called from both SeismicIndex and SeismicScalarQuantizedIndex read_index() after load. A debug assert in VisitedSet::insert documents the invariant at the point of use. Signed-off-by: Liyun Xiu --- nsparse/cluster/inverted_list_clusters.cpp | 13 +++++++++++++ nsparse/cluster/inverted_list_clusters.h | 7 +++++++ nsparse/seismic_index.cpp | 9 +++++++++ nsparse/seismic_scalar_quantized_index.cpp | 9 +++++++++ nsparse/utils/visited_set.h | 7 +++++++ 5 files changed, 45 insertions(+) diff --git a/nsparse/cluster/inverted_list_clusters.cpp b/nsparse/cluster/inverted_list_clusters.cpp index 22f91ae..76bc690 100644 --- a/nsparse/cluster/inverted_list_clusters.cpp +++ b/nsparse/cluster/inverted_list_clusters.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -134,6 +135,18 @@ void InvertedListClusters::sort_cluster_docs() { } } +void InvertedListClusters::validate_doc_ids(size_t num_docs) const { + // docs_ is sorted per cluster but not globally, so scan the flat array. + // One O(nnz) pass at load; negligible against deserialize + the sort above. + for (const idx_t doc_id : docs_) { + if (static_cast(doc_id) >= num_docs) { + throw std::out_of_range( + "InvertedListClusters: doc id out of range for corpus size; " + "index is corrupt or does not match its vectors"); + } + } +} + InvertedListClusters::InvertedListClusters(const InvertedListClusters& other) = default; InvertedListClusters& InvertedListClusters::operator=( diff --git a/nsparse/cluster/inverted_list_clusters.h b/nsparse/cluster/inverted_list_clusters.h index 42b194c..aae7357 100644 --- a/nsparse/cluster/inverted_list_clusters.h +++ b/nsparse/cluster/inverted_list_clusters.h @@ -41,6 +41,13 @@ class InvertedListClusters : public Serializable { void serialize(IOWriter* writer) const override; void deserialize(IOReader* reader) override; + // Validate that every stored doc id is within [0, num_docs). The search + // path indexes a VisitedSet sized to num_docs by doc id with no per- + // candidate bounds check (that would cost a branch on the hot path), so an + // out-of-domain id from a corrupt or mismatched serialized index would be + // an out-of-bounds write. Called once at load to fail loudly instead. + void validate_doc_ids(size_t num_docs) const; + // Accumulate per-cluster summary scores for a query into `out` (resized to // the cluster count) using the term-major transpose. The query is given as // its sparse (term, value) pairs; `q_val_bytes` points at the query values diff --git a/nsparse/seismic_index.cpp b/nsparse/seismic_index.cpp index fd977b1..e3c57c5 100644 --- a/nsparse/seismic_index.cpp +++ b/nsparse/seismic_index.cpp @@ -319,5 +319,14 @@ void SeismicIndex::read_index(IOReader* io_reader) { SeismicInvertedListsWriter inv_list_writer({}); inv_list_writer.deserialize(io_reader); clustered_inverted_lists = std::move(inv_list_writer.release()); + // The search path indexes a VisitedSet by doc id without a per-candidate + // bounds check; validate the loaded ids fall in the corpus domain once here + // so a corrupt/mismatched index fails loudly instead of writing OOB. + if (vectors_ != nullptr) { + const size_t num_docs = vectors_->num_vectors(); + for (const auto& cluster_invlist : clustered_inverted_lists) { + cluster_invlist.validate_doc_ids(num_docs); + } + } } } // namespace nsparse \ No newline at end of file diff --git a/nsparse/seismic_scalar_quantized_index.cpp b/nsparse/seismic_scalar_quantized_index.cpp index a4723b1..e72c146 100644 --- a/nsparse/seismic_scalar_quantized_index.cpp +++ b/nsparse/seismic_scalar_quantized_index.cpp @@ -344,6 +344,15 @@ void SeismicScalarQuantizedIndex::read_index(IOReader* io_reader) { SeismicInvertedListsWriter inv_list_writer({}); inv_list_writer.deserialize(io_reader); clustered_inverted_lists = std::move(inv_list_writer.release()); + // The search path indexes a VisitedSet by doc id without a per-candidate + // bounds check; validate the loaded ids fall in the corpus domain once here + // so a corrupt/mismatched index fails loudly instead of writing OOB. + if (vectors_ != nullptr) { + const size_t num_docs = vectors_->num_vectors(); + for (const auto& cluster_invlist : clustered_inverted_lists) { + cluster_invlist.validate_doc_ids(num_docs); + } + } } void SeismicScalarQuantizedIndex::write_header(IOWriter* io_writer) { diff --git a/nsparse/utils/visited_set.h b/nsparse/utils/visited_set.h index ff96571..b34ce2b 100644 --- a/nsparse/utils/visited_set.h +++ b/nsparse/utils/visited_set.h @@ -10,6 +10,7 @@ #ifndef NSPARSE_VISITED_SET_H #define NSPARSE_VISITED_SET_H +#include #include #include @@ -44,7 +45,13 @@ class VisitedSet { } // Mark `id` visited; return true if it was newly inserted this query. + // `id` must be in the doc-id domain [0, n) this set was sized for: unlike a + // hash set, an out-of-range id here is an out-of-bounds access on bits_, not + // a safe insert. Doc ids come from the corpus so this invariant holds, but + // it is now load-bearing — assert it in debug builds (compiles out in + // release, keeping the per-candidate hot path branch-free). bool insert(size_t id) { + assert(id < (bits_.size() << 6) && "VisitedSet id out of domain"); const size_t w = id >> 6; const uint64_t mask = uint64_t{1} << (id & 63); const uint64_t prev = bits_[w]; From 7037978bff58f806ecd4afd6a7dc2a186bdace94 Mon Sep 17 00:00:00 2001 From: Liyun Xiu Date: Thu, 23 Jul 2026 07:13:18 +0000 Subject: [PATCH 3/5] Raise gtest discovery timeout to avoid flaky Windows CI failures Signed-off-by: Liyun Xiu --- tests/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a230255..d71b64c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -57,4 +57,9 @@ target_link_libraries(nsparse_test PRIVATE ) include(GoogleTest) -gtest_discover_tests(nsparse_test) +# DISCOVERY_TIMEOUT: gtest_discover_tests runs the built executable with +# --gtest_list_tests as a post-build step. On cold Windows CI runners, loading +# the exe together with its deployed DLLs (e.g. abseil_dll.dll) and enumerating +# every test case can exceed the 5s default, causing a spurious build failure. +# Bump the timeout well above that to keep discovery robust. +gtest_discover_tests(nsparse_test DISCOVERY_TIMEOUT 120) From 5614de55f42b4ac74163374b694e9172d61a1cfa Mon Sep 17 00:00:00 2001 From: Liyun Xiu Date: Tue, 28 Jul 2026 05:39:29 +0000 Subject: [PATCH 4/5] Include for size_t in visited_set.h GCC in the Linux CI container does not pull global ::size_t in via /, so every unqualified size_t use failed to compile. Signed-off-by: Liyun Xiu --- nsparse/utils/visited_set.h | 1 + 1 file changed, 1 insertion(+) diff --git a/nsparse/utils/visited_set.h b/nsparse/utils/visited_set.h index b34ce2b..d3455ec 100644 --- a/nsparse/utils/visited_set.h +++ b/nsparse/utils/visited_set.h @@ -11,6 +11,7 @@ #define NSPARSE_VISITED_SET_H #include +#include #include #include From 9ee4de58a1b12b6fa52a9b11d73a41845a9ce53f Mon Sep 17 00:00:00 2001 From: Liyun Xiu Date: Tue, 28 Jul 2026 09:20:48 +0000 Subject: [PATCH 5/5] Keep only the head-prefetch optimization Per-lever ablation on base_full (8.8M docs, 72 threads, 20 reps, Welch t-test on batch times) showed head-prefetch is the only lever with a statistically significant effect: lever k=10 k=100 head-prefetch 1.15x (t=+10.7) 1.19x (t=+21.2) VisitedSet 0.98x (t=-1.6) 1.00x (t=-0.4) hugepage 1.01x (t=+0.9) 1.00x (t=+0.1) per-cluster sort 0.98x (t=-1.6) 0.98x (t=-2.3) scratch hoisting 0.97x (t=-2.5) 1.01x (t=+0.5) All levers together gave 1.20x/1.21x, i.e. no synergy beyond prefetch alone. Drop the levers that do not pay for their complexity: the bitset VisitedSet (and its load-time doc-id validation, needed only because it indexed without bounds checks), the hugepage madvise, the per-cluster doc-id sort, and the per-thread scratch hoisting. Signed-off-by: Liyun Xiu --- nsparse/cluster/inverted_list_clusters.cpp | 31 ------ nsparse/cluster/inverted_list_clusters.h | 10 -- nsparse/seismic_index.cpp | 109 ++++++------------- nsparse/seismic_index.h | 13 +-- nsparse/seismic_scalar_quantized_index.cpp | 41 +++---- nsparse/seismic_scalar_quantized_index.h | 6 +- nsparse/sparse_vectors.cpp | 8 -- nsparse/utils/distance_avx512.h | 2 +- nsparse/utils/hugepage.h | 66 ------------ nsparse/utils/visited_set.h | 76 ------------- tests/CMakeLists.txt | 1 - tests/visited_set_test.cpp | 118 --------------------- 12 files changed, 61 insertions(+), 420 deletions(-) delete mode 100644 nsparse/utils/hugepage.h delete mode 100644 nsparse/utils/visited_set.h delete mode 100644 tests/visited_set_test.cpp diff --git a/nsparse/cluster/inverted_list_clusters.cpp b/nsparse/cluster/inverted_list_clusters.cpp index 76bc690..73ff26b 100644 --- a/nsparse/cluster/inverted_list_clusters.cpp +++ b/nsparse/cluster/inverted_list_clusters.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include @@ -119,32 +118,6 @@ InvertedListClusters::InvertedListClusters( docs_.insert(docs_.end(), doc_ids.begin(), doc_ids.end()); offsets_.push_back(docs_.size()); } - sort_cluster_docs(); -} - -// Sort the doc ids within each cluster ascending. Within-cluster iteration -// order does not affect search results (the top-k heap and the visited dedup -// are order-independent), but it dictates the memory access pattern of the -// per-doc gather: indptr[doc_id] and the forward-index rows it points at are -// spread across the multi-GB corpus. Ascending doc ids turn that per-doc random -// gather into a monotonic sweep the hardware prefetcher and TLB can follow. -void InvertedListClusters::sort_cluster_docs() { - if (offsets_.size() <= 1) return; - for (size_t c = 0; c + 1 < offsets_.size(); ++c) { - std::sort(docs_.begin() + offsets_[c], docs_.begin() + offsets_[c + 1]); - } -} - -void InvertedListClusters::validate_doc_ids(size_t num_docs) const { - // docs_ is sorted per cluster but not globally, so scan the flat array. - // One O(nnz) pass at load; negligible against deserialize + the sort above. - for (const idx_t doc_id : docs_) { - if (static_cast(doc_id) >= num_docs) { - throw std::out_of_range( - "InvertedListClusters: doc id out of range for corpus size; " - "index is corrupt or does not match its vectors"); - } - } } InvertedListClusters::InvertedListClusters(const InvertedListClusters& other) = @@ -318,10 +291,6 @@ void InvertedListClusters::deserialize(IOReader* reader) { offsets_.resize(n_offsets); reader->read(offsets_.data(), sizeof(idx_t), n_offsets); } - // Order within each cluster does not affect results but makes the per-doc - // gather monotonic (see sort_cluster_docs). Applied on load so existing - // serialized indexes benefit without a rebuild. - sort_cluster_docs(); reader->read(&n_clusters_, sizeof(size_t), 1); reader->read(&element_size_, sizeof(size_t), 1); diff --git a/nsparse/cluster/inverted_list_clusters.h b/nsparse/cluster/inverted_list_clusters.h index aae7357..9b6beb9 100644 --- a/nsparse/cluster/inverted_list_clusters.h +++ b/nsparse/cluster/inverted_list_clusters.h @@ -41,13 +41,6 @@ class InvertedListClusters : public Serializable { void serialize(IOWriter* writer) const override; void deserialize(IOReader* reader) override; - // Validate that every stored doc id is within [0, num_docs). The search - // path indexes a VisitedSet sized to num_docs by doc id with no per- - // candidate bounds check (that would cost a branch on the hot path), so an - // out-of-domain id from a corrupt or mismatched serialized index would be - // an out-of-bounds write. Called once at load to fail loudly instead. - void validate_doc_ids(size_t num_docs) const; - // Accumulate per-cluster summary scores for a query into `out` (resized to // the cluster count) using the term-major transpose. The query is given as // its sparse (term, value) pairs; `q_val_bytes` points at the query values @@ -63,9 +56,6 @@ class InvertedListClusters : public Serializable { private: // Build the term-major (CSC) transpose from a per-cluster CSR summary. void build_transpose(const SparseVectors& summaries); - // Sort doc ids ascending within each cluster (result-order invariant) so - // the per-doc gather over the forward index is monotonic, not random. - void sort_cluster_docs(); template void score_summaries_typed(const term_t* q_idx, const T* q_val, size_t q_len, std::vector& out) const; diff --git a/nsparse/seismic_index.cpp b/nsparse/seismic_index.cpp index e3c57c5..781bf14 100644 --- a/nsparse/seismic_index.cpp +++ b/nsparse/seismic_index.cpp @@ -16,6 +16,7 @@ #include #include +#include "absl/container/flat_hash_set.h" #include "nsparse/cluster/inverted_list_clusters.h" #include "nsparse/cluster/random_kmeans.h" #include "nsparse/exact_matcher.h" @@ -31,7 +32,6 @@ #include "nsparse/utils/prefetch.h" #include "nsparse/utils/ranker.h" #include "nsparse/utils/vector_process.h" -#include "nsparse/utils/visited_set.h" namespace nsparse { namespace { @@ -41,10 +41,9 @@ constexpr int kElementSize = U32; void query_single_inverted_list( const SparseVectors* vectors, const InvertedListClusters& cluster_invlist, const std::vector& dense, const term_t* q_idx, const float* q_val, - size_t q_len, std::vector& score_scratch, - std::vector& cluster_order, const float heap_factor, + size_t q_len, std::vector& score_scratch, const float heap_factor, const bool first_list, const SearchParameters* search_parameters, - detail::TopKHolder& heap, detail::VisitedSet& visited) { + detail::TopKHolder& heap, absl::flat_hash_set& visited) { // Skip empty clusters size_t csize = cluster_invlist.cluster_size(); if (csize == 0) { @@ -59,39 +58,41 @@ void query_single_inverted_list( cluster_invlist.score_summaries_transposed( q_idx, reinterpret_cast(q_val), q_len, score_scratch); const std::vector& summary_scores = score_scratch; - const size_t n_clusters = summary_scores.size(); + size_t num_vectors = vectors->num_vectors(); + + std::vector cluster_order = + detail::reorder_clusters(summary_scores, first_list); const auto& [indptr, indices, values] = vectors->get_all_data(); - // Process one cluster: prune by summary score, then score its docs. - // Returns false when the early-out fires on the first (sorted) list, which - // means no later cluster can qualify either — the caller stops iterating. - auto process_cluster = [&](size_t cluster_id) -> bool { - const float cluster_score = summary_scores[cluster_id]; + for (const size_t& cluster_id : cluster_order) { + const auto& cluster_score = summary_scores[cluster_id]; if (heap.full() && (cluster_score * heap_factor < heap.peek_score())) { - // On the first list clusters are visited in descending score order, - // so once one falls below the threshold every later one does too. - return !first_list; + if (first_list) { + break; + } + continue; } const auto& docs = cluster_invlist.get_docs(cluster_id); const size_t n_docs = docs.size(); + // Prefetch one doc ahead, only the leading lines of the upcoming row; + // the row is contiguous so the hardware streamer pulls the tail, while + // bounding outstanding software prefetches keeps the line-fill buffers + // from saturating (measured optimum ~4 lines). + static constexpr size_t kPrefetchDist = 1; + static constexpr size_t kPrefetchHeadLines = 4; for (size_t i = 0; i < n_docs; ++i) { - const idx_t doc_id = docs[i]; - // Prefetch one doc ahead, only the leading lines of the upcoming - // row; the row is contiguous so the hardware streamer pulls the - // tail, while bounding outstanding software prefetches keeps the - // line-fill buffers from saturating (measured optimum ~4 lines). - static constexpr size_t kPrefetchDist1 = 1; - if (i + kPrefetchDist1 < n_docs) { - const idx_t nd = docs[i + kPrefetchDist1]; - const idx_t next_start = indptr[nd]; - const size_t next_len = indptr[nd + 1] - next_start; - static constexpr size_t kPrefetchHeadLines = 4; + const auto& doc_id = docs[i]; + if (i + kPrefetchDist < n_docs) { + const idx_t next_doc = docs[i + kPrefetchDist]; + const idx_t next_start = indptr[next_doc]; + const size_t next_len = indptr[next_doc + 1] - next_start; detail::prefetch_vector_head(indices + next_start, values + next_start, next_len, kPrefetchHeadLines); } - if (!visited.insert(static_cast(doc_id))) { + auto [_, inserted] = visited.insert(doc_id); + if (!inserted) { continue; } if (id_selector != nullptr && !id_selector->is_member(doc_id)) { @@ -103,29 +104,6 @@ void query_single_inverted_list( indices + start, values + start, len, dense.data()); heap.add(score, doc_id); } - return true; - }; - - if (first_list) { - // Only the first list is score-ordered; sort a per-thread scratch of - // narrow (u32) cluster ids instead of allocating a size_t vector per - // list. Clusters per list stay well under 2^32. - cluster_order.resize(n_clusters); - std::iota(cluster_order.begin(), cluster_order.end(), 0U); - std::ranges::sort(cluster_order, [&](uint32_t a, uint32_t b) { - return summary_scores[a] > summary_scores[b]; - }); - for (const uint32_t cluster_id : cluster_order) { - if (!process_cluster(cluster_id)) { - break; - } - } - } else { - // Later lists are visited in natural cluster order, so iterate directly - // — no order array, iota, or sort needed. - for (size_t cluster_id = 0; cluster_id < n_clusters; ++cluster_id) { - process_cluster(cluster_id); - } } } } // namespace @@ -211,14 +189,8 @@ auto SeismicIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, #pragma omp parallel { std::vector dense(dim, 0.0F); - // Generation-stamped visited set over the doc-id domain: O(1) reset per - // query and a single indexed load per candidate instead of a hashed - // random probe (the doc loop is memory-bound on random gathers). - detail::VisitedSet visited(vectors_->num_vectors()); - // Per-thread scratch reused across queries: the per-cluster summary - // score buffer and the sorted cluster-order buffer (first list only). - std::vector score_scratch; - std::vector cluster_order; + absl::flat_hash_set visited; + visited.reserve(static_cast(std::max(k, 1)) * 4096); #pragma omp for schedule(dynamic, 64) for (idx_t query_idx = 0; query_idx < n; ++query_idx) { @@ -230,8 +202,7 @@ auto SeismicIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, detail::top_k_tokens(q_indices, q_values, len, parameters->cut); auto [distances, labels] = single_query(dense, visited, q_indices, q_values, len, cuts, k, - parameters->heap_factor, search_parameters, - score_scratch, cluster_order); + parameters->heap_factor, search_parameters); result_distances[query_idx] = std::move(distances); result_labels[query_idx] = std::move(labels); } @@ -250,13 +221,11 @@ auto SeismicIndex::search(idx_t n, const idx_t* indptr, const term_t* indices, * @return std::pair, std::vector> */ auto SeismicIndex::single_query(std::vector& dense, - detail::VisitedSet& visited, + absl::flat_hash_set& visited, const term_t* q_indices, const float* q_values, size_t q_len, const std::vector& cuts, int k, float heap_factor, - SearchParameters* search_parameters, - std::vector& score_scratch, - std::vector& cluster_order) + SearchParameters* search_parameters) -> pair_of_score_id_vector_t { size_t num_docs = vectors_->num_vectors(); if (num_docs == 0) { @@ -267,9 +236,10 @@ auto SeismicIndex::single_query(std::vector& dense, for (size_t i = 0; i < q_len; ++i) { dense[q_indices[i]] = q_values[i]; } - visited.new_query(); + visited.clear(); detail::TopKHolder holder(k); + std::vector score_scratch; bool first_list = true; for (const auto& term : cuts) { if (term >= clustered_inverted_lists.size()) [[unlikely]] { @@ -278,8 +248,8 @@ auto SeismicIndex::single_query(std::vector& dense, const auto& cluster_invlist = clustered_inverted_lists[term]; query_single_inverted_list(vectors_.get(), cluster_invlist, dense, q_indices, q_values, q_len, score_scratch, - cluster_order, heap_factor, first_list, - search_parameters, holder, visited); + heap_factor, first_list, search_parameters, + holder, visited); first_list = false; } @@ -319,14 +289,5 @@ void SeismicIndex::read_index(IOReader* io_reader) { SeismicInvertedListsWriter inv_list_writer({}); inv_list_writer.deserialize(io_reader); clustered_inverted_lists = std::move(inv_list_writer.release()); - // The search path indexes a VisitedSet by doc id without a per-candidate - // bounds check; validate the loaded ids fall in the corpus domain once here - // so a corrupt/mismatched index fails loudly instead of writing OOB. - if (vectors_ != nullptr) { - const size_t num_docs = vectors_->num_vectors(); - for (const auto& cluster_invlist : clustered_inverted_lists) { - cluster_invlist.validate_doc_ids(num_docs); - } - } } } // namespace nsparse \ No newline at end of file diff --git a/nsparse/seismic_index.h b/nsparse/seismic_index.h index 6a89c85..c9ffc8b 100644 --- a/nsparse/seismic_index.h +++ b/nsparse/seismic_index.h @@ -12,13 +12,13 @@ #include #include +#include "absl/container/flat_hash_set.h" #include "nsparse/cluster/inverted_list_clusters.h" #include "nsparse/index.h" #include "nsparse/io/io.h" #include "nsparse/seismic_common.h" #include "nsparse/sparse_vectors.h" #include "nsparse/types.h" -#include "nsparse/utils/visited_set.h" namespace nsparse { @@ -67,15 +67,12 @@ class SeismicIndex : public Index, public IndexIO { // `dense` and `visited` are per-thread scratch reused across the queries a // thread handles (see search()). `dense` must be all-zero on entry and is // restored to all-zero on exit via a sparse clear over the query's own - // dims (q_indices/q_len); `visited` starts a new generation on entry. - // `score_scratch` and `cluster_order` are per-thread scratch reused across - // queries (resized in place), avoiding a per-query/per-list allocation. - auto single_query(std::vector& dense, detail::VisitedSet& visited, + // dims (q_indices/q_len); `visited` is cleared on entry. + auto single_query(std::vector& dense, + absl::flat_hash_set& visited, const term_t* q_indices, const float* q_values, size_t q_len, const std::vector& cuts, int k, - float heap_factor, SearchParameters* search_parameters, - std::vector& score_scratch, - std::vector& cluster_order) + float heap_factor, SearchParameters* search_parameters) -> pair_of_score_id_vector_t; std::unique_ptr vectors_; SeismicClusterParameters cluster_parameter_; diff --git a/nsparse/seismic_scalar_quantized_index.cpp b/nsparse/seismic_scalar_quantized_index.cpp index e72c146..ef7c45b 100644 --- a/nsparse/seismic_scalar_quantized_index.cpp +++ b/nsparse/seismic_scalar_quantized_index.cpp @@ -17,6 +17,7 @@ #include #include +#include "absl/container/flat_hash_set.h" #include "nsparse/cluster/inverted_list_clusters.h" #include "nsparse/cluster/random_kmeans.h" #include "nsparse/exact_matcher.h" @@ -33,7 +34,6 @@ #include "nsparse/utils/prefetch.h" #include "nsparse/utils/scalar_quantizer.h" #include "nsparse/utils/vector_process.h" -#include "nsparse/utils/visited_set.h" namespace nsparse { namespace { @@ -46,7 +46,7 @@ void query_single_inverted_list(const SparseVectors* vectors, float heap_factor, bool first_list, const SearchParameters* search_parameters, detail::TopKHolder& heap, - detail::VisitedSet& visited) { + absl::flat_hash_set& visited) { // Skip empty clusters size_t csize = cluster_invlist.cluster_size(); if (csize == 0) { @@ -80,23 +80,24 @@ void query_single_inverted_list(const SparseVectors* vectors, } const auto& docs = cluster_invlist.get_docs(cluster_id); const size_t n_docs = docs.size(); + // Prefetch one doc ahead, only the leading lines of the upcoming row; + // the row is contiguous so the hardware streamer pulls the tail, while + // bounding outstanding software prefetches keeps the line-fill buffers + // from saturating (measured optimum ~4 lines). + static constexpr size_t kPrefetchDist = 1; + static constexpr size_t kPrefetchHeadLines = 4; for (size_t i = 0; i < n_docs; ++i) { - const idx_t doc_id = docs[i]; - // Prefetch one doc ahead, only the leading lines of the upcoming - // row; the row is contiguous so the hardware streamer pulls the - // tail, while bounding outstanding software prefetches keeps the - // line-fill buffers from saturating (measured optimum ~4 lines). - static constexpr size_t kPrefetchDist1 = 1; - if (i + kPrefetchDist1 < n_docs) { - const idx_t next_doc = docs[i + kPrefetchDist1]; + const auto& doc_id = docs[i]; + if (i + kPrefetchDist < n_docs) { + const idx_t next_doc = docs[i + kPrefetchDist]; const idx_t next_start = indptr[next_doc]; const size_t next_len = indptr[next_doc + 1] - next_start; - static constexpr size_t kPrefetchHeadLines = 4; detail::prefetch_vector_head(indices + next_start, values + next_start, next_len, kPrefetchHeadLines); } - if (!visited.insert(static_cast(doc_id))) { + auto [_, inserted] = visited.insert(doc_id); + if (!inserted) { continue; } if (id_selector != nullptr && !id_selector->is_member(doc_id)) { @@ -240,7 +241,8 @@ auto SeismicScalarQuantizedIndex::search(idx_t n, const idx_t* indptr, #pragma omp parallel { std::vector dense(dense_bytes, 0); - detail::VisitedSet visited(vectors_->num_vectors()); + absl::flat_hash_set visited; + visited.reserve(static_cast(std::max(k, 1)) * 4096); #pragma omp for schedule(dynamic, 64) for (idx_t query_idx = 0; query_idx < n; ++query_idx) { @@ -271,7 +273,7 @@ auto SeismicScalarQuantizedIndex::search(idx_t n, const idx_t* indptr, } auto SeismicScalarQuantizedIndex::single_query( - std::vector& dense, detail::VisitedSet& visited, + std::vector& dense, absl::flat_hash_set& visited, const term_t* q_idx, const uint8_t* q_val_bytes, size_t q_len, size_t element_size, const std::vector& cuts, int k, float heap_factor, const ScalarQuantizer& query_sq, @@ -287,7 +289,7 @@ auto SeismicScalarQuantizedIndex::single_query( std::copy_n(q_val_bytes + i * element_size, element_size, dense.data() + static_cast(q_idx[i]) * element_size); } - visited.new_query(); + visited.clear(); detail::TopKHolder holder(k); std::vector score_scratch; @@ -344,15 +346,6 @@ void SeismicScalarQuantizedIndex::read_index(IOReader* io_reader) { SeismicInvertedListsWriter inv_list_writer({}); inv_list_writer.deserialize(io_reader); clustered_inverted_lists = std::move(inv_list_writer.release()); - // The search path indexes a VisitedSet by doc id without a per-candidate - // bounds check; validate the loaded ids fall in the corpus domain once here - // so a corrupt/mismatched index fails loudly instead of writing OOB. - if (vectors_ != nullptr) { - const size_t num_docs = vectors_->num_vectors(); - for (const auto& cluster_invlist : clustered_inverted_lists) { - cluster_invlist.validate_doc_ids(num_docs); - } - } } void SeismicScalarQuantizedIndex::write_header(IOWriter* io_writer) { diff --git a/nsparse/seismic_scalar_quantized_index.h b/nsparse/seismic_scalar_quantized_index.h index 4cf7da8..179b226 100644 --- a/nsparse/seismic_scalar_quantized_index.h +++ b/nsparse/seismic_scalar_quantized_index.h @@ -14,11 +14,11 @@ #include #include +#include "absl/container/flat_hash_set.h" #include "nsparse/cluster/inverted_list_clusters.h" #include "nsparse/index.h" #include "nsparse/seismic_index.h" #include "nsparse/utils/scalar_quantizer.h" -#include "nsparse/utils/visited_set.h" namespace nsparse { @@ -70,9 +70,9 @@ class SeismicScalarQuantizedIndex : public Index, public IndexIO { // thread handles (see search()). `dense` (a dimension-sized quantized-code // buffer, element_size bytes per dim) must be all-zero on entry and is // restored to all-zero on exit via a sparse clear over the query's own dims - // (q_idx/q_len); `visited` starts a new generation on entry. + // (q_idx/q_len); `visited` is cleared on entry. auto single_query(std::vector& dense, - detail::VisitedSet& visited, const term_t* q_idx, + absl::flat_hash_set& visited, const term_t* q_idx, const uint8_t* q_val_bytes, size_t q_len, size_t element_size, const std::vector& cuts, int k, float heap_factor, const ScalarQuantizer& query_sq, diff --git a/nsparse/sparse_vectors.cpp b/nsparse/sparse_vectors.cpp index 5558921..fe232f0 100644 --- a/nsparse/sparse_vectors.cpp +++ b/nsparse/sparse_vectors.cpp @@ -15,7 +15,6 @@ #include "nsparse/io/io.h" #include "nsparse/types.h" #include "nsparse/utils/checks.h" -#include "nsparse/utils/hugepage.h" namespace nsparse { SparseVectors::SparseVectors(SparseVectorsConfig config) : config_(config) { @@ -166,13 +165,6 @@ void SparseVectors::deserialize(IOReader* io_reader) { size_t value_size = indptr_[vector_count] * element_size; values_.resize(value_size); io_reader->read(values_.data(), sizeof(uint8_t), value_size); - - // The per-doc dot product gathers randomly across indices_/values_ - // (multi-GB at scale); back them with huge pages to cut TLB/page-walk - // cost. indptr_ is also randomly indexed by doc id. - detail::advise_hugepage(indptr_); - detail::advise_hugepage(indices_); - detail::advise_hugepage(values_); } } } // namespace nsparse \ No newline at end of file diff --git a/nsparse/utils/distance_avx512.h b/nsparse/utils/distance_avx512.h index ae79ce9..1969e23 100644 --- a/nsparse/utils/distance_avx512.h +++ b/nsparse/utils/distance_avx512.h @@ -333,7 +333,7 @@ inline float dot_product_float_dense(const term_t* indices, // Gather 16 values from dense vector using indices __m512 dense_vals = _mm512_i32gather_ps(idx, dense, sizeof(float)); - // Load 16 weights + // Load 16 weights (aligned load - weights must be 64-byte aligned) __m512 weight_vals = _mm512_loadu_ps(weights + i); // Fused multiply-add: sum += weights * dense_vals diff --git a/nsparse/utils/hugepage.h b/nsparse/utils/hugepage.h deleted file mode 100644 index 8077f0e..0000000 --- a/nsparse/utils/hugepage.h +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -#ifndef NSPARSE_HUGEPAGE_H -#define NSPARSE_HUGEPAGE_H - -#include -#include -#include - -#if defined(__linux__) -#include -#include -#endif - -namespace nsparse::detail { - -// Hint the kernel to back a large, mostly-read-only array with transparent -// huge pages. Random gathers into the multi-GB posting/summary arrays are -// dominated by TLB misses + page-table walks (2 MB pages cover 512x the span -// of 4 KB pages), so this cuts the DTLB/page-walk cost with no change to the -// data layout or results. No-op on non-Linux or for small buffers. -template -inline void advise_hugepage(const std::vector& v) { -#if defined(__linux__) && defined(MADV_HUGEPAGE) - const size_t bytes = v.size() * sizeof(T); - // 2 MB is the x86-64 huge-page size; only bother once the region spans at - // least one huge page so the alignment rounding can't consume the whole - // buffer. - static constexpr size_t kHugePage = size_t{2} << 20; - if (bytes < kHugePage) { - return; - } - const auto base = reinterpret_cast(v.data()); - // madvise() requires a page-aligned start; round the start up and the end - // down to whole huge pages so only fully-covered pages are advised. - const uintptr_t start = (base + kHugePage - 1) & ~(kHugePage - 1); - const uintptr_t end = (base + bytes) & ~(kHugePage - 1); - if (end > start) { - void* p = reinterpret_cast(start); - const size_t len = static_cast(end - start); - // Mark the region so future faults and khugepaged prefer huge pages. - ::madvise(p, len, MADV_HUGEPAGE); -#if defined(MADV_COLLAPSE) - // The arrays are already faulted in (resize + read) as 4 KB pages, so a - // plain MADV_HUGEPAGE only takes effect lazily via khugepaged. - // MADV_COLLAPSE (Linux 6.1+) collapses the existing pages into huge - // pages synchronously, giving the TLB win immediately. Best-effort: - // ignore failure (unsupported kernel, fragmentation). - ::madvise(p, len, MADV_COLLAPSE); -#endif - } -#else - (void)v; -#endif -} - -} // namespace nsparse::detail - -#endif // NSPARSE_HUGEPAGE_H diff --git a/nsparse/utils/visited_set.h b/nsparse/utils/visited_set.h deleted file mode 100644 index d3455ec..0000000 --- a/nsparse/utils/visited_set.h +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -#ifndef NSPARSE_VISITED_SET_H -#define NSPARSE_VISITED_SET_H - -#include -#include -#include -#include - -namespace nsparse::detail { - -// Membership set over a fixed doc-id domain [0, n), backed by a bit per doc. -// -// The seismic doc loop tests/inserts every candidate doc to dedupe across -// clusters. A hash set hashes and probes a random cache line per candidate and -// must be cleared each query. A full generation-stamped uint32 array (4 B/doc) -// avoids the hashing but at seismic scale (~35 MB) adds a random-access stream -// larger than L2, so it misses cache on every candidate. A bitset is 32x -// smaller (1 bit/doc, ~1.1 MB at 8.8M docs) so far fewer distinct cache lines -// are touched, while the touched-word list lets a new query clear only the -// words it actually dirtied (sparse O(visited) reset, not O(n)). -class VisitedSet { -public: - VisitedSet() = default; - explicit VisitedSet(size_t n) { resize(n); } - - void resize(size_t n) { - bits_.assign((n + 63) / 64, 0); - touched_.clear(); - } - - // Begin a new query: clear only the words dirtied by the previous query. - void new_query() { - for (const size_t w : touched_) { - bits_[w] = 0; - } - touched_.clear(); - } - - // Mark `id` visited; return true if it was newly inserted this query. - // `id` must be in the doc-id domain [0, n) this set was sized for: unlike a - // hash set, an out-of-range id here is an out-of-bounds access on bits_, not - // a safe insert. Doc ids come from the corpus so this invariant holds, but - // it is now load-bearing — assert it in debug builds (compiles out in - // release, keeping the per-candidate hot path branch-free). - bool insert(size_t id) { - assert(id < (bits_.size() << 6) && "VisitedSet id out of domain"); - const size_t w = id >> 6; - const uint64_t mask = uint64_t{1} << (id & 63); - const uint64_t prev = bits_[w]; - if (prev & mask) { - return false; - } - if (prev == 0) { - touched_.push_back(w); - } - bits_[w] = prev | mask; - return true; - } - -private: - std::vector bits_; - std::vector touched_; -}; - -} // namespace nsparse::detail - -#endif // NSPARSE_VISITED_SET_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d71b64c..0275bdf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -29,7 +29,6 @@ set(NSPARSE_TEST_SRC seismic_scalar_quantized_index_test.cpp sparse_vectors_test.cpp vector_process_test.cpp - visited_set_test.cpp ) # Fetch GoogleTest diff --git a/tests/visited_set_test.cpp b/tests/visited_set_test.cpp deleted file mode 100644 index ecb6753..0000000 --- a/tests/visited_set_test.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -#include "nsparse/utils/visited_set.h" - -#include - -namespace { - -using nsparse::detail::VisitedSet; - -// (a) Within one query an id is inserted once; repeats are deduped. -TEST(VisitedSetTest, dedup_within_query) { - VisitedSet s(128); - - EXPECT_TRUE(s.insert(7)); - EXPECT_FALSE(s.insert(7)); - EXPECT_FALSE(s.insert(7)); - - EXPECT_TRUE(s.insert(70)); - EXPECT_FALSE(s.insert(70)); -} - -// (b) new_query() fully resets: every id inserted last query is new again. -TEST(VisitedSetTest, new_query_resets_all_words) { - VisitedSet s(256); - - // Dirty ids spread across several 64-bit words. - for (size_t id : {0u, 63u, 64u, 130u, 255u}) { - EXPECT_TRUE(s.insert(id)); - EXPECT_FALSE(s.insert(id)); - } - - s.new_query(); - - for (size_t id : {0u, 63u, 64u, 130u, 255u}) { - EXPECT_TRUE(s.insert(id)) << "id " << id << " not reset"; - } -} - -// (c) Two ids in the same 64-bit word both dedup within a query and both clear -// on new_query, exercising the "touched once per word" reset invariant. -TEST(VisitedSetTest, two_ids_same_word_both_clear) { - VisitedSet s(128); - - // 5 and 60 share word 0; the second insert must not re-push the word, yet - // new_query must still clear both bits. - EXPECT_TRUE(s.insert(5)); - EXPECT_TRUE(s.insert(60)); - EXPECT_FALSE(s.insert(5)); - EXPECT_FALSE(s.insert(60)); - - s.new_query(); - - EXPECT_TRUE(s.insert(5)); - EXPECT_TRUE(s.insert(60)); -} - -// (d) resize() clears touched_ so a later new_query does not iterate a stale -// word index (which after shrinking would be out of bounds on bits_). -TEST(VisitedSetTest, resize_clears_touched_and_bits) { - VisitedSet s(256); - EXPECT_TRUE(s.insert(200)); // dirties word 3 - - s.resize(64); // bits_ shrinks to a single word; touched_ must be cleared - - // If touched_ retained word 3, this would be an out-of-bounds write. - s.new_query(); - - // Bits are also cleared by resize: an id valid in the new domain is fresh. - EXPECT_TRUE(s.insert(0)); - EXPECT_FALSE(s.insert(0)); -} - -// resize() to the same size still clears prior state. -TEST(VisitedSetTest, resize_same_size_clears_bits) { - VisitedSet s(128); - EXPECT_TRUE(s.insert(42)); - - s.resize(128); - - EXPECT_TRUE(s.insert(42)); -} - -// (e) Word boundaries: bit 63 (top of word 0), bit 64 (bottom of word 1), and -// the last valid id n-1 all behave and reset independently. -TEST(VisitedSetTest, word_boundaries) { - constexpr size_t n = 130; // 3 words (192 bits); last id is 129 - VisitedSet s(n); - - EXPECT_TRUE(s.insert(63)); - EXPECT_TRUE(s.insert(64)); - EXPECT_TRUE(s.insert(n - 1)); - - // Each is independent: neighbors are not marked by these inserts. - EXPECT_TRUE(s.insert(62)); - EXPECT_TRUE(s.insert(65)); - EXPECT_TRUE(s.insert(n - 2)); - - // And each dedups. - EXPECT_FALSE(s.insert(63)); - EXPECT_FALSE(s.insert(64)); - EXPECT_FALSE(s.insert(n - 1)); - - s.new_query(); - - EXPECT_TRUE(s.insert(63)); - EXPECT_TRUE(s.insert(64)); - EXPECT_TRUE(s.insert(n - 1)); -} - -} // namespace