Seismic search optimizations

nsparse/seismic_index.cpp

@@ -75,19 +75,21 @@ void query_single_inverted_list(
         }
         const auto& docs = cluster_invlist.get_docs(cluster_id);
         const size_t n_docs = docs.size();
-        static constexpr size_t kPrefetchDist1 = 2;  // vector data prefetch
-        static constexpr size_t kPrefetchDist2 = 4;  // indptr prefetch
+        // Prefetch one doc ahead, only the leading lines of the upcoming row;
+        // the row is contiguous so the hardware streamer pulls the tail, while
+        // bounding outstanding software prefetches keeps the line-fill buffers
+        // from saturating (measured optimum ~4 lines).
+        static constexpr size_t kPrefetchDist = 1;
+        static constexpr size_t kPrefetchHeadLines = 4;
         for (size_t i = 0; i < n_docs; ++i) {
             const auto& doc_id = docs[i];
-            if (i + kPrefetchDist2 < n_docs) {
-                detail::prefetch_indptr(indptr, docs[i + kPrefetchDist2]);
-            }
-            if (i + kPrefetchDist1 < n_docs) {
-                const idx_t next_doc = docs[i + kPrefetchDist1];
+            if (i + kPrefetchDist < n_docs) {
+                const idx_t next_doc = docs[i + kPrefetchDist];
                 const idx_t next_start = indptr[next_doc];
                 const size_t next_len = indptr[next_doc + 1] - next_start;
-                detail::prefetch_vector(indices + next_start,
-                                        values + next_start, next_len);
+                detail::prefetch_vector_head(indices + next_start,
+                                             values + next_start, next_len,
+                                             kPrefetchHeadLines);
             }
             auto [_, inserted] = visited.insert(doc_id);
             if (!inserted) {

nsparse/seismic_scalar_quantized_index.cpp

@@ -80,28 +80,21 @@ void query_single_inverted_list(const SparseVectors* vectors,
         }
         const auto& docs = cluster_invlist.get_docs(cluster_id);
         const size_t n_docs = docs.size();
-        // Two-stage prefetch pipeline:
-        // Stage 1 (distance 2): prefetch indptr[docs[i+2]] so the indptr
-        //   lookup is cached by the time we need it next iteration.
-        // Stage 2 (distance 1): read indptr[docs[i+1]] (now cached from
-        //   stage 1 issued last iteration), prefetch the actual vector data.
-        static constexpr size_t kPrefetchDist1 = 2;  // vector data prefetch
-        static constexpr size_t kPrefetchDist2 = 4;  // indptr prefetch
+        // Prefetch one doc ahead, only the leading lines of the upcoming row;
+        // the row is contiguous so the hardware streamer pulls the tail, while
+        // bounding outstanding software prefetches keeps the line-fill buffers
+        // from saturating (measured optimum ~4 lines).
+        static constexpr size_t kPrefetchDist = 1;
+        static constexpr size_t kPrefetchHeadLines = 4;
         for (size_t i = 0; i < n_docs; ++i) {
             const auto& doc_id = docs[i];
-            // Stage 1: prefetch indptr entry for doc at distance 2
-            if (i + kPrefetchDist2 < n_docs) {
-                detail::prefetch_indptr(indptr, docs[i + kPrefetchDist2]);
-            }
-            // Stage 2: prefetch vector data for next doc (indptr should
-            // already be cached from stage 1 issued kPrefetchDist2 -
-            // kPrefetchDist1 iterations ago)
-            if (i + kPrefetchDist1 < n_docs) {
-                const idx_t next_doc = docs[i + kPrefetchDist1];
+            if (i + kPrefetchDist < n_docs) {
+                const idx_t next_doc = docs[i + kPrefetchDist];
                 const idx_t next_start = indptr[next_doc];
                 const size_t next_len = indptr[next_doc + 1] - next_start;
-                detail::prefetch_vector(indices + next_start,
-                                        values + next_start, next_len);
+                detail::prefetch_vector_head(indices + next_start,
+                                             values + next_start, next_len,
+                                             kPrefetchHeadLines);
             }
             auto [_, inserted] = visited.insert(doc_id);
             if (!inserted) {

nsparse/utils/prefetch.h

@@ -9,6 +9,7 @@
 
 #ifndef PREFETCH_H
 #define PREFETCH_H
+#include <algorithm>
 #include <cstddef>
 
 #include "nsparse/types.h"
@@ -45,6 +46,30 @@ inline void prefetch_vector(const term_t* indices, const T* values,
     }
 }
 
+// Prefetch only the leading `max_lines` cache lines of each stream. A doc row
+// spans ~13 lines here; prefetching all of them for several docs ahead
+// overruns the core's ~10-12 line-fill buffers (l1d_pend_miss.fb_full), which
+// stalls demand loads. Because the row is contiguous, touching just the first
+// line or two lets the hardware stream prefetcher pull the rest while keeping
+// the number of outstanding software prefetches bounded.
+template <class T>
+inline void prefetch_vector_head(const term_t* indices, const T* values,
+                                 size_t len, size_t max_lines) {
+    static constexpr size_t kCacheLineSize = 64;  // bytes
+    const char* indices_ptr = reinterpret_cast<const char*>(indices);
+    const char* values_ptr = reinterpret_cast<const char*>(values);
+    const size_t indices_bytes =
+        std::min(len * sizeof(term_t), max_lines * kCacheLineSize);
+    const size_t values_bytes =
+        std::min(len * sizeof(T), max_lines * kCacheLineSize);
+    for (size_t offset = 0; offset < indices_bytes; offset += kCacheLineSize) {
+        NSPARSE_PREFETCH(indices_ptr + offset, 0, 0);
+    }
+    for (size_t offset = 0; offset < values_bytes; offset += kCacheLineSize) {
+        NSPARSE_PREFETCH(values_ptr + offset, 0, 0);
+    }
+}
+
 inline void prefetch_indptr(const idx_t* indptr, idx_t doc_id) {
     NSPARSE_PREFETCH(&indptr[doc_id], 0, 0);
 }

tests/CMakeLists.txt

@@ -56,4 +56,9 @@ target_link_libraries(nsparse_test PRIVATE
 )
 
 include(GoogleTest)
-gtest_discover_tests(nsparse_test)
+# DISCOVERY_TIMEOUT: gtest_discover_tests runs the built executable with
+# --gtest_list_tests as a post-build step. On cold Windows CI runners, loading
+# the exe together with its deployed DLLs (e.g. abseil_dll.dll) and enumerating
+# every test case can exceed the 5s default, causing a spurious build failure.
+# Bump the timeout well above that to keep discovery robust.
+gtest_discover_tests(nsparse_test DISCOVERY_TIMEOUT 120)