diff --git a/nsparse/seismic_index.cpp b/nsparse/seismic_index.cpp index 332cb8d..781bf14 100644 --- a/nsparse/seismic_index.cpp +++ b/nsparse/seismic_index.cpp @@ -75,19 +75,21 @@ void query_single_inverted_list( } const auto& docs = cluster_invlist.get_docs(cluster_id); const size_t n_docs = docs.size(); - static constexpr size_t kPrefetchDist1 = 2; // vector data prefetch - static constexpr size_t kPrefetchDist2 = 4; // indptr prefetch + // Prefetch one doc ahead, only the leading lines of the upcoming row; + // the row is contiguous so the hardware streamer pulls the tail, while + // bounding outstanding software prefetches keeps the line-fill buffers + // from saturating (measured optimum ~4 lines). + static constexpr size_t kPrefetchDist = 1; + static constexpr size_t kPrefetchHeadLines = 4; for (size_t i = 0; i < n_docs; ++i) { const auto& doc_id = docs[i]; - if (i + kPrefetchDist2 < n_docs) { - detail::prefetch_indptr(indptr, docs[i + kPrefetchDist2]); - } - if (i + kPrefetchDist1 < n_docs) { - const idx_t next_doc = docs[i + kPrefetchDist1]; + if (i + kPrefetchDist < n_docs) { + const idx_t next_doc = docs[i + kPrefetchDist]; const idx_t next_start = indptr[next_doc]; const size_t next_len = indptr[next_doc + 1] - next_start; - detail::prefetch_vector(indices + next_start, - values + next_start, next_len); + detail::prefetch_vector_head(indices + next_start, + values + next_start, next_len, + kPrefetchHeadLines); } auto [_, inserted] = visited.insert(doc_id); if (!inserted) { diff --git a/nsparse/seismic_scalar_quantized_index.cpp b/nsparse/seismic_scalar_quantized_index.cpp index 63599fe..ef7c45b 100644 --- a/nsparse/seismic_scalar_quantized_index.cpp +++ b/nsparse/seismic_scalar_quantized_index.cpp @@ -80,28 +80,21 @@ void query_single_inverted_list(const SparseVectors* vectors, } const auto& docs = cluster_invlist.get_docs(cluster_id); const size_t n_docs = docs.size(); - // Two-stage prefetch pipeline: - // Stage 1 (distance 2): prefetch indptr[docs[i+2]] so the indptr - // lookup is cached by the time we need it next iteration. - // Stage 2 (distance 1): read indptr[docs[i+1]] (now cached from - // stage 1 issued last iteration), prefetch the actual vector data. - static constexpr size_t kPrefetchDist1 = 2; // vector data prefetch - static constexpr size_t kPrefetchDist2 = 4; // indptr prefetch + // Prefetch one doc ahead, only the leading lines of the upcoming row; + // the row is contiguous so the hardware streamer pulls the tail, while + // bounding outstanding software prefetches keeps the line-fill buffers + // from saturating (measured optimum ~4 lines). + static constexpr size_t kPrefetchDist = 1; + static constexpr size_t kPrefetchHeadLines = 4; for (size_t i = 0; i < n_docs; ++i) { const auto& doc_id = docs[i]; - // Stage 1: prefetch indptr entry for doc at distance 2 - if (i + kPrefetchDist2 < n_docs) { - detail::prefetch_indptr(indptr, docs[i + kPrefetchDist2]); - } - // Stage 2: prefetch vector data for next doc (indptr should - // already be cached from stage 1 issued kPrefetchDist2 - - // kPrefetchDist1 iterations ago) - if (i + kPrefetchDist1 < n_docs) { - const idx_t next_doc = docs[i + kPrefetchDist1]; + if (i + kPrefetchDist < n_docs) { + const idx_t next_doc = docs[i + kPrefetchDist]; const idx_t next_start = indptr[next_doc]; const size_t next_len = indptr[next_doc + 1] - next_start; - detail::prefetch_vector(indices + next_start, - values + next_start, next_len); + detail::prefetch_vector_head(indices + next_start, + values + next_start, next_len, + kPrefetchHeadLines); } auto [_, inserted] = visited.insert(doc_id); if (!inserted) { diff --git a/nsparse/utils/prefetch.h b/nsparse/utils/prefetch.h index 72b8784..9b1733d 100644 --- a/nsparse/utils/prefetch.h +++ b/nsparse/utils/prefetch.h @@ -9,6 +9,7 @@ #ifndef PREFETCH_H #define PREFETCH_H +#include #include #include "nsparse/types.h" @@ -45,6 +46,30 @@ inline void prefetch_vector(const term_t* indices, const T* values, } } +// Prefetch only the leading `max_lines` cache lines of each stream. A doc row +// spans ~13 lines here; prefetching all of them for several docs ahead +// overruns the core's ~10-12 line-fill buffers (l1d_pend_miss.fb_full), which +// stalls demand loads. Because the row is contiguous, touching just the first +// line or two lets the hardware stream prefetcher pull the rest while keeping +// the number of outstanding software prefetches bounded. +template +inline void prefetch_vector_head(const term_t* indices, const T* values, + size_t len, size_t max_lines) { + static constexpr size_t kCacheLineSize = 64; // bytes + const char* indices_ptr = reinterpret_cast(indices); + const char* values_ptr = reinterpret_cast(values); + const size_t indices_bytes = + std::min(len * sizeof(term_t), max_lines * kCacheLineSize); + const size_t values_bytes = + std::min(len * sizeof(T), max_lines * kCacheLineSize); + for (size_t offset = 0; offset < indices_bytes; offset += kCacheLineSize) { + NSPARSE_PREFETCH(indices_ptr + offset, 0, 0); + } + for (size_t offset = 0; offset < values_bytes; offset += kCacheLineSize) { + NSPARSE_PREFETCH(values_ptr + offset, 0, 0); + } +} + inline void prefetch_indptr(const idx_t* indptr, idx_t doc_id) { NSPARSE_PREFETCH(&indptr[doc_id], 0, 0); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2d59fa7..0275bdf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -56,4 +56,9 @@ target_link_libraries(nsparse_test PRIVATE ) include(GoogleTest) -gtest_discover_tests(nsparse_test) +# DISCOVERY_TIMEOUT: gtest_discover_tests runs the built executable with +# --gtest_list_tests as a post-build step. On cold Windows CI runners, loading +# the exe together with its deployed DLLs (e.g. abseil_dll.dll) and enumerating +# every test case can exceed the 5s default, causing a spurious build failure. +# Bump the timeout well above that to keep discovery robust. +gtest_discover_tests(nsparse_test DISCOVERY_TIMEOUT 120)