From dbe6ea47c1e687b2a3338711c311ec991c245ac4 Mon Sep 17 00:00:00 2001
From: Mason Sharp <mason.sharp@pgedge.com>
Date: Tue, 11 Aug 2026 17:36:36 -0700
Subject: [PATCH 01/10] Bound corpus stats staleness by growth as well as by
 time

A wall-clock TTL bounds staleness in time, but the harm is proportional:
a thousand chunks added to a million barely move the weights, the same
thousand added to two hundred change them several fold. The figures are
also written into stored vectors by the worker rather than only used for
a query, so a stale reading does not simply correct itself.

corpus_stats_cache_max_growth re-reads once uses of a cached entry reach
that percentage of N, alongside the existing TTL, whichever comes first.
It scales itself: a million-row corpus tolerates fifty thousand uses, a
two-hundred-row one ten, and re-reading a small table is cheap because
it is small. Counting uses is a proxy rather than a measurement, since
the worker does not insert chunk rows itself and a search does not grow
the corpus at all; the comment says so.

Test 24 covers both directions: a use inside the budget is served from
cache, the use that spends it re-reads and matches an uncached read, and
with the bound off the same sequence stays stale.
---
 docs/index.md                 |  1 +
 src/bm25.c                    | 39 ++++++++++++++++++++++++++-
 src/guc.c                     | 17 ++++++++++++
 src/pgedge_vectorizer.h       |  1 +
 test/expected/hybrid_test.out | 49 ++++++++++++++++++++++++++++++++++
 test/sql/hybrid_test.sql      | 50 +++++++++++++++++++++++++++++++++++
 6 files changed, 156 insertions(+), 1 deletion(-)

diff --git a/docs/index.md b/docs/index.md
index 0c59f0d..fb7c649 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -174,6 +174,7 @@ different use cases.
 | `pgedge_vectorizer.bm25_k1` | `1.2` | `0.0-3.0` | BM25 term-frequency saturation |
 | `pgedge_vectorizer.bm25_b` | `0.75` | `0.0-1.0` | BM25 document-length normalization |
 | `pgedge_vectorizer.corpus_stats_cache_ttl` | `60s` | `0-3600` | Seconds a backend reuses a chunk table's corpus size and mean document length before reading them again. Set to 0 to read them on every call |
+| `pgedge_vectorizer.corpus_stats_cache_max_growth` | `5` | `0-100` | Percent the corpus may grow before those cached figures are re-read, applied alongside the TTL so that whichever is reached first wins. Bounds staleness in proportion rather than in time, which matters most while a corpus is small and growing. Set to 0 to bound by time alone |
 
 For more information or to download Vectorizer visit:
 
diff --git a/src/bm25.c b/src/bm25.c
index 15a0736..e66ce1f 100644
--- a/src/bm25.c
+++ b/src/bm25.c
@@ -241,10 +241,43 @@ typedef struct
 	int64		total_docs;
 	float8		avg_doc_len;
 	TimestampTz	read_at;
+	int64		uses_since_read;
 } CorpusStatsEntry;
 
 static HTAB *corpus_stats_cache = NULL;
 
+/*
+ * corpus_stats_stale_by_growth — has the corpus plausibly moved too far?
+ *
+ * A wall-clock TTL bounds staleness in time, but the harm is proportional:
+ * a thousand chunks added to a million barely move the weights, while the
+ * same thousand added to two hundred change them several fold.  Worse, N is
+ * cached while doc_freq is read fresh, so a corpus that outgrows a stale N
+ * gives ln((N+1)/(df+0.5)) < 0 for a common term, and a negative score is
+ * dropped from the stored vector altogether.
+ *
+ * Each use of a cached entry is counted as one chunk that may have been
+ * added since, and the entry is re-read once that count reaches the given
+ * percentage of N.  That is a proxy, not a measurement: the worker does not
+ * insert chunk rows itself, and a search does not grow the corpus at all.
+ * It is deliberately pessimistic in the direction that matters, and it
+ * scales itself — a million-row corpus tolerates fifty thousand uses, a
+ * two-hundred-row one ten, and re-reading a two-hundred-row table is free.
+ */
+static bool
+corpus_stats_stale_by_growth(const CorpusStatsEntry *entry)
+{
+	int64		budget;
+
+	if (pgedge_vectorizer_corpus_stats_cache_max_growth <= 0)
+		return false;			/* growth bound disabled; TTL only */
+
+	budget = entry->total_docs
+		* pgedge_vectorizer_corpus_stats_cache_max_growth / 100;
+
+	return entry->uses_since_read >= Max(budget, 1);
+}
+
 /*
  * bm25_corpus_stats_uncached — read N and the mean document length.
  *
@@ -355,11 +388,14 @@ bm25_corpus_stats(const char *chunk_table, float8 *avg_doc_len)
 	now = GetCurrentTimestamp();
 	entry = hash_search(corpus_stats_cache, &relid, HASH_FIND, NULL);
 
+	/* Both bounds apply: whichever is reached first forces a re-read. */
 	if (entry != NULL &&
 		!TimestampDifferenceExceeds(entry->read_at, now,
 									pgedge_vectorizer_corpus_stats_cache_ttl
-									* 1000))
+									* 1000) &&
+		!corpus_stats_stale_by_growth(entry))
 	{
+		entry->uses_since_read++;
 		*avg_doc_len = entry->avg_doc_len;
 		return entry->total_docs;
 	}
@@ -377,6 +413,7 @@ bm25_corpus_stats(const char *chunk_table, float8 *avg_doc_len)
 	entry->total_docs = total_docs;
 	entry->avg_doc_len = *avg_doc_len;
 	entry->read_at = now;
+	entry->uses_since_read = 0;
 
 	return total_docs;
 }
diff --git a/src/guc.c b/src/guc.c
index 295933d..2177328 100644
--- a/src/guc.c
+++ b/src/guc.c
@@ -52,6 +52,7 @@ bool   pgedge_vectorizer_enable_hybrid = false;
 double pgedge_vectorizer_bm25_k1       = 1.2;
 double pgedge_vectorizer_bm25_b        = 0.75;
 int    pgedge_vectorizer_corpus_stats_cache_ttl = 60;
+int    pgedge_vectorizer_corpus_stats_cache_max_growth = 5;
 
 /*
  * Initialize all GUC variables
@@ -330,5 +331,21 @@ pgedge_vectorizer_init_guc(void)
 		GUC_UNIT_S,
 		NULL, NULL, NULL);
 
+	DefineCustomIntVariable(
+		"pgedge_vectorizer.corpus_stats_cache_max_growth",
+		"Percent the corpus may grow before cached figures are re-read",
+		"Bounds staleness in proportion rather than in time, which is how it "
+		"harms ranking: a thousand chunks added to a million barely move the "
+		"weights, the same thousand added to two hundred change them several "
+		"fold. Applied alongside corpus_stats_cache_ttl, whichever is reached "
+		"first. Set to 0 to bound by time alone.",
+		&pgedge_vectorizer_corpus_stats_cache_max_growth,
+		5,      /* default */
+		0,      /* min: 0 disables the growth bound */
+		100,    /* max */
+		PGC_USERSET,
+		0,
+		NULL, NULL, NULL);
+
 	elog(DEBUG1, "pgedge_vectorizer GUC variables initialized");
 }
diff --git a/src/pgedge_vectorizer.h b/src/pgedge_vectorizer.h
index 586bf39..5267670 100644
--- a/src/pgedge_vectorizer.h
+++ b/src/pgedge_vectorizer.h
@@ -73,6 +73,7 @@ extern bool   pgedge_vectorizer_enable_hybrid;
 extern double pgedge_vectorizer_bm25_k1;
 extern double pgedge_vectorizer_bm25_b;
 extern int    pgedge_vectorizer_corpus_stats_cache_ttl;
+extern int    pgedge_vectorizer_corpus_stats_cache_max_growth;
 
 /*
  * Chunking strategy enumeration
diff --git a/test/expected/hybrid_test.out b/test/expected/hybrid_test.out
index 5fe7af7..604217c 100644
--- a/test/expected/hybrid_test.out
+++ b/test/expected/hybrid_test.out
@@ -724,6 +724,55 @@ NOTICE:  drop cascades to 2 other objects
 DETAIL:  drop cascades to table cache_tenant_b.t_chunks
 drop cascades to table cache_tenant_b.t_chunks_idf_stats
 ---------------------------------------------------------------------------
+-- Test 24: the corpus statistics cache re-reads once the corpus has grown
+---------------------------------------------------------------------------
+-- A time bound alone lets a small, fast-growing corpus go badly stale, and
+-- the figures are written into stored vectors rather than only used for a
+-- query. corpus_stats_cache_max_growth bounds staleness in proportion instead:
+-- at 5% of twenty documents, one cached use is allowed before a re-read.
+CREATE TABLE growth_chunks (id BIGSERIAL PRIMARY KEY, content TEXT,
+                            token_count INT);
+CREATE TABLE growth_chunks_idf_stats (term TEXT PRIMARY KEY,
+                                      doc_freq INT NOT NULL);
+INSERT INTO growth_chunks (content, token_count)
+SELECT 'alpha', 10 FROM generate_series(1, 20);
+INSERT INTO growth_chunks_idf_stats VALUES ('alpha', 5);
+-- Long enough that the time bound cannot be what fires.
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
+SET pgedge_vectorizer.corpus_stats_cache_max_growth = 5;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS g1 \gset
+INSERT INTO growth_chunks (content, token_count)
+SELECT 'alpha', 900 FROM generate_series(1, 60);
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS g2 \gset
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS g3 \gset
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 0;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS gtruth \gset
+SELECT :'g1'::sparsevec = :'g2'::sparsevec AS use_within_budget_is_cached,
+       :'g1'::sparsevec <> :'g3'::sparsevec AS budget_spent_forces_reread,
+       :'g3'::sparsevec = :'gtruth'::sparsevec AS reread_matches_uncached;
+ use_within_budget_is_cached | budget_spent_forces_reread | reread_matches_uncached 
+-----------------------------+----------------------------+-------------------------
+ t                           | t                          | t
+(1 row)
+
+-- With the growth bound off, the same sequence stays on the stale figures.
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
+SET pgedge_vectorizer.corpus_stats_cache_max_growth = 0;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h1 \gset
+INSERT INTO growth_chunks (content, token_count)
+SELECT 'alpha', 900 FROM generate_series(1, 200);
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h2 \gset
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h3 \gset
+SELECT :'h1'::sparsevec = :'h3'::sparsevec AS time_bound_alone_stays_stale;
+ time_bound_alone_stays_stale 
+------------------------------
+ t
+(1 row)
+
+RESET pgedge_vectorizer.corpus_stats_cache_ttl;
+RESET pgedge_vectorizer.corpus_stats_cache_max_growth;
+DROP TABLE growth_chunks, growth_chunks_idf_stats;
+---------------------------------------------------------------------------
 -- Cleanup
 ---------------------------------------------------------------------------
 SELECT pgedge_vectorizer.disable_vectorization(
diff --git a/test/sql/hybrid_test.sql b/test/sql/hybrid_test.sql
index 4760028..48ccf75 100644
--- a/test/sql/hybrid_test.sql
+++ b/test/sql/hybrid_test.sql
@@ -558,6 +558,56 @@ RESET pgedge_vectorizer.corpus_stats_cache_ttl;
 DROP SCHEMA cache_tenant_a CASCADE;
 DROP SCHEMA cache_tenant_b CASCADE;
 
+---------------------------------------------------------------------------
+-- Test 24: the corpus statistics cache re-reads once the corpus has grown
+---------------------------------------------------------------------------
+
+-- A time bound alone lets a small, fast-growing corpus go badly stale, and
+-- the figures are written into stored vectors rather than only used for a
+-- query. corpus_stats_cache_max_growth bounds staleness in proportion instead:
+-- at 5% of twenty documents, one cached use is allowed before a re-read.
+
+CREATE TABLE growth_chunks (id BIGSERIAL PRIMARY KEY, content TEXT,
+                            token_count INT);
+CREATE TABLE growth_chunks_idf_stats (term TEXT PRIMARY KEY,
+                                      doc_freq INT NOT NULL);
+INSERT INTO growth_chunks (content, token_count)
+SELECT 'alpha', 10 FROM generate_series(1, 20);
+INSERT INTO growth_chunks_idf_stats VALUES ('alpha', 5);
+
+-- Long enough that the time bound cannot be what fires.
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
+SET pgedge_vectorizer.corpus_stats_cache_max_growth = 5;
+
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS g1 \gset
+INSERT INTO growth_chunks (content, token_count)
+SELECT 'alpha', 900 FROM generate_series(1, 60);
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS g2 \gset
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS g3 \gset
+
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 0;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS gtruth \gset
+
+SELECT :'g1'::sparsevec = :'g2'::sparsevec AS use_within_budget_is_cached,
+       :'g1'::sparsevec <> :'g3'::sparsevec AS budget_spent_forces_reread,
+       :'g3'::sparsevec = :'gtruth'::sparsevec AS reread_matches_uncached;
+
+-- With the growth bound off, the same sequence stays on the stale figures.
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
+SET pgedge_vectorizer.corpus_stats_cache_max_growth = 0;
+
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h1 \gset
+INSERT INTO growth_chunks (content, token_count)
+SELECT 'alpha', 900 FROM generate_series(1, 200);
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h2 \gset
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h3 \gset
+
+SELECT :'h1'::sparsevec = :'h3'::sparsevec AS time_bound_alone_stays_stale;
+
+RESET pgedge_vectorizer.corpus_stats_cache_ttl;
+RESET pgedge_vectorizer.corpus_stats_cache_max_growth;
+DROP TABLE growth_chunks, growth_chunks_idf_stats;
+
 ---------------------------------------------------------------------------
 -- Cleanup
 ---------------------------------------------------------------------------

From 494cab1ff0c24299798960ba8e65cb154e5c6c9d Mon Sep 17 00:00:00 2001
From: Mason Sharp <mason.sharp@pgedge.com>
Date: Tue, 11 Aug 2026 17:37:26 -0700
Subject: [PATCH 02/10] Clamp the corpus size to the document frequency it is
 compared against

A term cannot appear in more documents than exist, but total_docs may be
a cached reading while doc_freq is read fresh, so doc_freq can outrun it.
Left alone ln((N+1)/(df+0.5)) goes negative, and bm25_compute_sparse_str
keeps only scores above zero, so the term is dropped from the vector
rather than merely underweighted.

Measured with doc_freq deliberately set past the corpus: unclamped the
query vector came back empty, clamped it carries a near-zero weight,
which is what a term appearing in every document should score.

Taking the larger of the two changes nothing whenever total_docs is at
least doc_freq, which is every case a fresh reading can produce, so the
expression stays bit-identical to the SQL it replaced for all previously
reachable inputs.
---
 src/bm25.c                    | 12 +++++++++++-
 test/expected/hybrid_test.out | 25 +++++++++++++++++++++++++
 test/sql/hybrid_test.sql      | 25 +++++++++++++++++++++++++
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/src/bm25.c b/src/bm25.c
index e66ce1f..ebd4250 100644
--- a/src/bm25.c
+++ b/src/bm25.c
@@ -594,11 +594,21 @@ bm25_load_idf_stats(const char *chunk_table, BM25Term *tokens, int ntokens,
 			 * Derived, not stored: the corpus size is identical for every
 			 * term.  Expression shape matches the SQL it replaced so that
 			 * results are bit-identical.
+			 *
+			 * A term cannot appear in more documents than exist, but
+			 * total_docs may be a cached reading while doc_freq is read
+			 * fresh, so doc_freq can outrun it.  Left alone that gives a
+			 * negative weight, and a negative score is dropped from the
+			 * vector, losing the term outright.  Taking the larger changes
+			 * nothing whenever total_docs >= doc_freq, which is every case a
+			 * fresh reading can produce.
 			 */
+			int64	corpus = Max(total_docs, (int64) row.doc_freq);
+
 			entry->idf_weight =
 				(row.doc_freq <= 0)
 					? 0.0
-					: log(1.0 + ((double) total_docs - row.doc_freq + 0.5)
+					: log(1.0 + ((double) corpus - row.doc_freq + 0.5)
 								/ (row.doc_freq + 0.5));
 		}
 		/* Duplicate terms: keep the first weight encountered */
diff --git a/test/expected/hybrid_test.out b/test/expected/hybrid_test.out
index 604217c..e747ec8 100644
--- a/test/expected/hybrid_test.out
+++ b/test/expected/hybrid_test.out
@@ -773,6 +773,31 @@ RESET pgedge_vectorizer.corpus_stats_cache_ttl;
 RESET pgedge_vectorizer.corpus_stats_cache_max_growth;
 DROP TABLE growth_chunks, growth_chunks_idf_stats;
 ---------------------------------------------------------------------------
+-- Test 25: a doc_freq larger than the corpus does not lose the term
+---------------------------------------------------------------------------
+-- doc_freq is read fresh while the corpus size may be cached, so doc_freq
+-- can outrun it. Unclamped that makes ln((N+1)/(df+0.5)) negative, and a
+-- negative score is dropped, so the term vanishes from the vector rather
+-- than merely being underweighted. The weight below is near zero, which is
+-- what a term appearing in every document should score.
+CREATE TABLE clamp_chunks (id BIGSERIAL PRIMARY KEY, content TEXT,
+                           token_count INT);
+CREATE TABLE clamp_chunks_idf_stats (term TEXT PRIMARY KEY,
+                                     doc_freq INT NOT NULL);
+INSERT INTO clamp_chunks (content, token_count)
+SELECT 'alpha', 10 FROM generate_series(1, 10);
+INSERT INTO clamp_chunks_idf_stats VALUES ('alpha', 5000);
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 0;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'clamp_chunks')
+           <> '{}/65536'::sparsevec AS term_survives_df_over_n;
+ term_survives_df_over_n 
+-------------------------
+ t
+(1 row)
+
+RESET pgedge_vectorizer.corpus_stats_cache_ttl;
+DROP TABLE clamp_chunks, clamp_chunks_idf_stats;
+---------------------------------------------------------------------------
 -- Cleanup
 ---------------------------------------------------------------------------
 SELECT pgedge_vectorizer.disable_vectorization(
diff --git a/test/sql/hybrid_test.sql b/test/sql/hybrid_test.sql
index 48ccf75..057bd57 100644
--- a/test/sql/hybrid_test.sql
+++ b/test/sql/hybrid_test.sql
@@ -608,6 +608,31 @@ RESET pgedge_vectorizer.corpus_stats_cache_ttl;
 RESET pgedge_vectorizer.corpus_stats_cache_max_growth;
 DROP TABLE growth_chunks, growth_chunks_idf_stats;
 
+---------------------------------------------------------------------------
+-- Test 25: a doc_freq larger than the corpus does not lose the term
+---------------------------------------------------------------------------
+
+-- doc_freq is read fresh while the corpus size may be cached, so doc_freq
+-- can outrun it. Unclamped that makes ln((N+1)/(df+0.5)) negative, and a
+-- negative score is dropped, so the term vanishes from the vector rather
+-- than merely being underweighted. The weight below is near zero, which is
+-- what a term appearing in every document should score.
+
+CREATE TABLE clamp_chunks (id BIGSERIAL PRIMARY KEY, content TEXT,
+                           token_count INT);
+CREATE TABLE clamp_chunks_idf_stats (term TEXT PRIMARY KEY,
+                                     doc_freq INT NOT NULL);
+INSERT INTO clamp_chunks (content, token_count)
+SELECT 'alpha', 10 FROM generate_series(1, 10);
+INSERT INTO clamp_chunks_idf_stats VALUES ('alpha', 5000);
+
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 0;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'clamp_chunks')
+           <> '{}/65536'::sparsevec AS term_survives_df_over_n;
+
+RESET pgedge_vectorizer.corpus_stats_cache_ttl;
+DROP TABLE clamp_chunks, clamp_chunks_idf_stats;
+
 ---------------------------------------------------------------------------
 -- Cleanup
 ---------------------------------------------------------------------------

From 461d0e43266b507c45d61a002c66ca56ca172dae Mon Sep 17 00:00:00 2001
From: Mason Sharp <mason.sharp@pgedge.com>
Date: Tue, 11 Aug 2026 18:55:14 -0700
Subject: [PATCH 03/10] Test the growth bound through the vectors the worker
 stores

The regression tests exercise it through bm25_query_vector, whose result
is discarded at the end of the statement. The worker runs the same lookup
per queue item and writes the outcome into sparse_embedding, which is the
case the bound exists for.

Every chunk holds the same term and token_count, so avg_doc_len cannot
vary and doc_freq is fixed by hand; the corpus size is the only input
that differs, and the weights are checked against the ratio it predicts
rather than merely for being different. With the bound disabled the
first four assertions still pass and the last two fail.
---
 test/t/006_corpus_stats_growth.pl | 157 ++++++++++++++++++++++++++++++
 1 file changed, 157 insertions(+)
 create mode 100644 test/t/006_corpus_stats_growth.pl

diff --git a/test/t/006_corpus_stats_growth.pl b/test/t/006_corpus_stats_growth.pl
new file mode 100644
index 0000000..a30b057
--- /dev/null
+++ b/test/t/006_corpus_stats_growth.pl
@@ -0,0 +1,157 @@
+# Copyright (c) 2025 - 2026, pgEdge, Inc.
+#
+# Verify that the corpus statistics growth bound reaches the vectors the worker
+# stores, not merely the ones a query computes.
+#
+# The regression tests cover the bound through bm25_query_vector(), which
+# discards its result at the end of the statement. The worker runs the same
+# lookup per queue item and writes the outcome into sparse_embedding, so a
+# stale corpus size there is persisted rather than recomputed next time. That
+# is the reason the bound exists, and it is what this test exercises.
+#
+# The queue rows are marked sparse_only, so no embedding is fetched and no
+# network call is made. The provider is still resolved and initialised before
+# that check is reached, though, so it is set to ollama, which is the one
+# provider whose init needs no API key.
+#
+# Every chunk holds the same single term and the same token_count, so
+# avg_doc_len cannot vary and doc_freq is fixed by hand. The corpus size is
+# then the only input that differs between the chunks compared below, and the
+# stored weights differ if and only if it was re-read.
+
+use strict;
+use warnings;
+
+# See the comment in 001_worker_coverage.pl about loading these at compile time.
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $dbname = 'corpus_growth';
+
+my $node = PostgreSQL::Test::Cluster->new('vectorizer_corpus_growth');
+$node->init;
+$node->append_conf(
+	'postgresql.conf', qq(
+shared_preload_libraries = 'pgedge_vectorizer'
+pgedge_vectorizer.worker_poll_interval = 200
+pgedge_vectorizer.batch_size = 1
+pgedge_vectorizer.provider = 'ollama'
+pgedge_vectorizer.enable_hybrid = true
+pgedge_vectorizer.corpus_stats_cache_ttl = 3600
+pgedge_vectorizer.corpus_stats_cache_max_growth = 5
+max_worker_processes = 16
+));
+
+$node->start;
+
+$node->safe_psql('postgres', "CREATE DATABASE $dbname");
+$node->safe_psql($dbname, 'CREATE EXTENSION vector');
+$node->safe_psql($dbname, 'CREATE EXTENSION pgedge_vectorizer');
+
+# A chunk table of the shape the extension creates, built by hand so that the
+# corpus size is under this test's control rather than the chunker's.
+$node->safe_psql(
+	$dbname, q(
+CREATE TABLE chunks (
+    id               BIGSERIAL PRIMARY KEY,
+    source_id        BIGINT,
+    chunk_index      INT,
+    content          TEXT,
+    token_count      INT,
+    embedding        vector(3),
+    sparse_embedding sparsevec(65536)
+);
+CREATE TABLE chunks_idf_stats (
+    term      TEXT PRIMARY KEY,
+    doc_freq  INT NOT NULL
+);
+INSERT INTO chunks (source_id, chunk_index, content, token_count)
+SELECT g, 0, 'alpha', 10 FROM generate_series(1, 20) g;
+INSERT INTO chunks_idf_stats VALUES ('alpha', 5);
+));
+
+$node->append_conf('postgresql.conf',
+	"pgedge_vectorizer.databases = '$dbname'\n");
+$node->reload;
+
+# Wait for a chunk's sparse_embedding to be written, rather than assuming the
+# worker has got to it, so that startup timing cannot decide the result.
+sub await_sparse
+{
+	my ($id) = @_;
+	my $deadline = time() + 60;
+
+	while (time() < $deadline)
+	{
+		my $got = $node->safe_psql($dbname,
+			"SELECT sparse_embedding IS NOT NULL FROM chunks WHERE id = $id");
+
+		return 1 if $got eq 't';
+		sleep 1;
+	}
+	return 0;
+}
+
+# First item: nothing is cached, so the corpus is read here at N = 20.
+$node->safe_psql(
+	$dbname, q(
+INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, status, metadata)
+VALUES (1, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb)
+));
+
+ok(await_sparse(1), 'the worker stores a sparse vector, so the rest of this test measures something');
+
+# Grow the corpus elevenfold without queueing any of it. A time bound alone
+# would hold the figure read above for the next hour.
+$node->safe_psql(
+	$dbname, q(
+INSERT INTO chunks (source_id, chunk_index, content, token_count)
+SELECT g, 0, 'alpha', 10 FROM generate_series(100, 299) g
+));
+
+# At N = 20 the budget is one cached use, so the second item is served from
+# cache and the third must re-read.
+$node->safe_psql(
+	$dbname, q(
+INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, status, metadata)
+VALUES (2, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb),
+       (3, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb)
+));
+
+ok(await_sparse(2), 'the second queued chunk is indexed');
+ok(await_sparse(3), 'the third queued chunk is indexed');
+
+my $cached = $node->safe_psql($dbname,
+	'SELECT (SELECT sparse_embedding FROM chunks WHERE id = 1)
+          = (SELECT sparse_embedding FROM chunks WHERE id = 2)');
+
+is($cached, 't',
+	'a use inside the budget is served from the cache, so the bound is not simply disabled');
+
+my $rereadd = $node->safe_psql($dbname,
+	'SELECT (SELECT sparse_embedding FROM chunks WHERE id = 1)
+         <> (SELECT sparse_embedding FROM chunks WHERE id = 3)');
+
+is($rereadd, 't',
+	'once the budget is spent the worker re-reads the corpus, and the stored vector reflects it');
+
+# Pin down that the difference is the corpus size and nothing else: the weight
+# is idf * a constant, so the ratio of the two stored weights must be the ratio
+# of ln((N+1)/(df+0.5)) at N = 220 and at N = 20, with df = 5.
+my $ratio = $node->safe_psql(
+	$dbname, q(
+WITH w AS (
+    SELECT (SELECT (sparse_embedding::text)::jsonb IS NOT NULL FROM chunks WHERE id = 1) AS ignored,
+           (SELECT split_part(split_part(sparse_embedding::text, ':', 2), '}', 1)::float8
+              FROM chunks WHERE id = 1) AS w1,
+           (SELECT split_part(split_part(sparse_embedding::text, ':', 2), '}', 1)::float8
+              FROM chunks WHERE id = 3) AS w3
+)
+SELECT abs(w3 / w1 - ln(221.0 / 5.5) / ln(21.0 / 5.5)) < 0.01 FROM w
+));
+
+is($ratio, 't',
+	'the re-read weight is exactly what the grown corpus size gives, not merely different');
+
+done_testing();

From ce70b6ef9a07d5eee2d9135ae0f3734c5c6c04b2 Mon Sep 17 00:00:00 2001
From: Mason Sharp <mason.sharp@pgedge.com>
Date: Tue, 11 Aug 2026 20:22:42 -0700
Subject: [PATCH 04/10] Queue the two items separately so their order is
 decided

Inserted together they shared a created_at, and the claim orders by that
column alone with no tie-breaker, so which one the worker took first was
unspecified. Taken in the other order the cached use and the re-read swap
places and both assertions invert, failing against working code.

Raised by CodeRabbit on #62.
---
 test/t/006_corpus_stats_growth.pl | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/test/t/006_corpus_stats_growth.pl b/test/t/006_corpus_stats_growth.pl
index a30b057..96c7457 100644
--- a/test/t/006_corpus_stats_growth.pl
+++ b/test/t/006_corpus_stats_growth.pl
@@ -112,14 +112,25 @@ sub await_sparse
 
 # At N = 20 the budget is one cached use, so the second item is served from
 # cache and the third must re-read.
+#
+# Queued one at a time, and waited for in between, because which of the two the
+# worker takes first decides which is the cached use and which the re-read.
+# Inserted together they would share a created_at, and the claim orders by that
+# column alone, so their order would be unspecified.
 $node->safe_psql(
 	$dbname, q(
 INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, status, metadata)
-VALUES (2, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb),
-       (3, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb)
+VALUES (2, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb)
 ));
 
 ok(await_sparse(2), 'the second queued chunk is indexed');
+
+$node->safe_psql(
+	$dbname, q(
+INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, status, metadata)
+VALUES (3, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb)
+));
+
 ok(await_sparse(3), 'the third queued chunk is indexed');
 
 my $cached = $node->safe_psql($dbname,

From f1b6f43253d9219d7f70b7e72269835fb11bfadb Mon Sep 17 00:00:00 2001
From: Mason Sharp <mason.sharp@pgedge.com>
Date: Wed, 12 Aug 2026 12:48:28 -0700
Subject: [PATCH 05/10] Match the idf_stats fixture to the schema the extension
 creates

The fixture omitted updated_at, which upsert_term_idf() writes, so every
doc_freq update failed and bm25_update_idf_stats() reduced it to a
WARNING: the test passed while logging three per-item errors, and it did
so because doc_freq was frozen at its seeded value.

With the column present doc_freq moves as it does in production, which
would have broken two assertions. Each chunk now carries its own term so
the terms have independent doc_freq, restoring the stated premise that
the corpus size is the only input that differs. The weights are compared
directly, since vectors over different terms occupy different dimensions,
and a new assertion on doc_freq catches the fixture drifting again.

Raised by dpage on #60.
---
 test/t/006_corpus_stats_growth.pl | 74 +++++++++++++++++++------------
 1 file changed, 45 insertions(+), 29 deletions(-)

diff --git a/test/t/006_corpus_stats_growth.pl b/test/t/006_corpus_stats_growth.pl
index 96c7457..3fdcc59 100644
--- a/test/t/006_corpus_stats_growth.pl
+++ b/test/t/006_corpus_stats_growth.pl
@@ -14,10 +14,17 @@
 # that check is reached, though, so it is set to ollama, which is the one
 # provider whose init needs no API key.
 #
-# Every chunk holds the same single term and the same token_count, so
-# avg_doc_len cannot vary and doc_freq is fixed by hand. The corpus size is
-# then the only input that differs between the chunks compared below, and the
-# stored weights differ if and only if it was re-read.
+# Each of the three chunks carries its own single term, seeded at the same
+# doc_freq, and they all share a token_count. avg_doc_len therefore cannot
+# vary, and because the worker increments doc_freq only for the terms of the
+# chunk it just processed, one chunk's indexing cannot move another's. The
+# corpus size is the only input that differs between the weights compared
+# below, and they differ if and only if it was re-read.
+#
+# The idf_stats fixture must match the shape the extension creates, down to
+# updated_at: upsert_term_idf() writes that column, and bm25_update_idf_stats()
+# reduces any failure to a WARNING, so a fixture missing it leaves doc_freq
+# frozen and the test measuring the wrong thing while still passing.
 
 use strict;
 use warnings;
@@ -63,12 +70,14 @@
     sparse_embedding sparsevec(65536)
 );
 CREATE TABLE chunks_idf_stats (
-    term      TEXT PRIMARY KEY,
-    doc_freq  INT NOT NULL
+    term        TEXT    PRIMARY KEY,
+    doc_freq    INT     NOT NULL DEFAULT 1,
+    updated_at  TIMESTAMPTZ DEFAULT now()
 );
 INSERT INTO chunks (source_id, chunk_index, content, token_count)
 SELECT g, 0, 'alpha', 10 FROM generate_series(1, 20) g;
-INSERT INTO chunks_idf_stats VALUES ('alpha', 5);
+INSERT INTO chunks_idf_stats (term, doc_freq)
+VALUES ('alpha', 5), ('beta', 5), ('gamma', 5);
 ));
 
 $node->append_conf('postgresql.conf',
@@ -120,7 +129,7 @@ sub await_sparse
 $node->safe_psql(
 	$dbname, q(
 INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, status, metadata)
-VALUES (2, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb)
+VALUES (2, 'chunks', 'beta', 'pending', '{"sparse_only": true}'::jsonb)
 ));
 
 ok(await_sparse(2), 'the second queued chunk is indexed');
@@ -128,41 +137,48 @@ sub await_sparse
 $node->safe_psql(
 	$dbname, q(
 INSERT INTO pgedge_vectorizer.queue (chunk_id, chunk_table, content, status, metadata)
-VALUES (3, 'chunks', 'alpha', 'pending', '{"sparse_only": true}'::jsonb)
+VALUES (3, 'chunks', 'gamma', 'pending', '{"sparse_only": true}'::jsonb)
 ));
 
 ok(await_sparse(3), 'the third queued chunk is indexed');
 
+# Each chunk holds one term, so each vector holds one element, and the weight
+# is that element.  The terms differ between chunks, so the vectors cannot be
+# compared whole -- they occupy different dimensions -- and it is the weights
+# that carry the corpus size.
+my $weights = 'SELECT split_part(split_part(sparse_embedding::text, \':\', 2),
+                                 \'}\', 1)::float8 FROM chunks WHERE id = ';
+
 my $cached = $node->safe_psql($dbname,
-	'SELECT (SELECT sparse_embedding FROM chunks WHERE id = 1)
-          = (SELECT sparse_embedding FROM chunks WHERE id = 2)');
+	"SELECT abs(($weights 1) - ($weights 2)) < 1e-9");
 
 is($cached, 't',
 	'a use inside the budget is served from the cache, so the bound is not simply disabled');
 
-my $rereadd = $node->safe_psql($dbname,
-	'SELECT (SELECT sparse_embedding FROM chunks WHERE id = 1)
-         <> (SELECT sparse_embedding FROM chunks WHERE id = 3)');
+my $reread = $node->safe_psql($dbname,
+	"SELECT abs(($weights 1) - ($weights 3)) > 1e-9");
 
-is($rereadd, 't',
+is($reread, 't',
 	'once the budget is spent the worker re-reads the corpus, and the stored vector reflects it');
 
-# Pin down that the difference is the corpus size and nothing else: the weight
-# is idf * a constant, so the ratio of the two stored weights must be the ratio
-# of ln((N+1)/(df+0.5)) at N = 220 and at N = 20, with df = 5.
-my $ratio = $node->safe_psql(
-	$dbname, q(
-WITH w AS (
-    SELECT (SELECT (sparse_embedding::text)::jsonb IS NOT NULL FROM chunks WHERE id = 1) AS ignored,
-           (SELECT split_part(split_part(sparse_embedding::text, ':', 2), '}', 1)::float8
-              FROM chunks WHERE id = 1) AS w1,
-           (SELECT split_part(split_part(sparse_embedding::text, ':', 2), '}', 1)::float8
-              FROM chunks WHERE id = 3) AS w3
-)
-SELECT abs(w3 / w1 - ln(221.0 / 5.5) / ln(21.0 / 5.5)) < 0.01 FROM w
-));
+# Pin down that the difference is the corpus size and nothing else: the tf
+# factor cancels because tf and token_count match across the three, so the
+# stored weight is exactly the IDF, and the ratio of the two must be that of
+# ln((N+1)/(df+0.5)) at N = 220 and at N = 20, both at the seeded df of 5.
+my $ratio = $node->safe_psql($dbname,
+	"SELECT abs(($weights 3) / ($weights 1)
+	            - ln(221.0 / 5.5) / ln(21.0 / 5.5)) < 0.01");
 
 is($ratio, 't',
 	'the re-read weight is exactly what the grown corpus size gives, not merely different');
 
+# The premise of all of the above: doc_freq must actually be maintained, or the
+# fixture is silently wrong in a way that leaves the weights comparable by
+# accident.  The worker increments each processed chunk's own term.
+my $df = $node->safe_psql($dbname,
+	"SELECT string_agg(doc_freq::text, ',' ORDER BY term) FROM chunks_idf_stats");
+
+is($df, '6,6,6',
+	'doc_freq is incremented for each processed term, so the fixture matches what the extension writes');
+
 done_testing();

From fd958d62d08d75ba460d3aa4df490bbbeb8b4d2b Mon Sep 17 00:00:00 2001
From: Mason Sharp <mason.sharp@pgedge.com>
Date: Wed, 12 Aug 2026 13:34:24 -0700
Subject: [PATCH 06/10] Name the cache bound for what it counts

The parameter was called max_growth and documented as bounding how far
the corpus may grow, but nothing measures the corpus: the code compares
uses of the cached entry against a percentage of N. A search spends the
budget without adding anything, so a static corpus under query traffic
re-reads periodically, which is close to the opposite of what the name
promised.

Renamed to corpus_stats_cache_max_uses_pct, with the GUC description and
the docs row now saying that uses stand in for chunks added, that this is
a proxy rather than a measurement, and that reads spend it too. The
internal helper follows the same naming.

Raised by dpage on #60.
---
 docs/index.md                     |  2 +-
 src/bm25.c                        | 12 ++++++------
 src/guc.c                         | 20 +++++++++++++-------
 src/pgedge_vectorizer.h           |  2 +-
 test/expected/hybrid_test.out     | 10 +++++-----
 test/sql/hybrid_test.sql          | 10 +++++-----
 test/t/006_corpus_stats_growth.pl |  4 ++--
 7 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/docs/index.md b/docs/index.md
index fb7c649..55227e3 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -174,7 +174,7 @@ different use cases.
 | `pgedge_vectorizer.bm25_k1` | `1.2` | `0.0-3.0` | BM25 term-frequency saturation |
 | `pgedge_vectorizer.bm25_b` | `0.75` | `0.0-1.0` | BM25 document-length normalization |
 | `pgedge_vectorizer.corpus_stats_cache_ttl` | `60s` | `0-3600` | Seconds a backend reuses a chunk table's corpus size and mean document length before reading them again. Set to 0 to read them on every call |
-| `pgedge_vectorizer.corpus_stats_cache_max_growth` | `5` | `0-100` | Percent the corpus may grow before those cached figures are re-read, applied alongside the TTL so that whichever is reached first wins. Bounds staleness in proportion rather than in time, which matters most while a corpus is small and growing. Set to 0 to bound by time alone |
+| `pgedge_vectorizer.corpus_stats_cache_max_uses_pct` | `5` | `0-100` | Percent of the cached corpus size that may be spent as uses of those figures before they are re-read, applied alongside the TTL so that whichever is reached first wins. Bounds staleness in proportion rather than in time, which matters most while a corpus is small and growing. Each use counts as one chunk that may have been added since: a proxy rather than a measurement, since the figures are read once per queue item and once per search, so a read-only workload spends the budget too. Set to 0 to bound by time alone |
 
 For more information or to download Vectorizer visit:
 
diff --git a/src/bm25.c b/src/bm25.c
index ebd4250..287ce91 100644
--- a/src/bm25.c
+++ b/src/bm25.c
@@ -247,7 +247,7 @@ typedef struct
 static HTAB *corpus_stats_cache = NULL;
 
 /*
- * corpus_stats_stale_by_growth — has the corpus plausibly moved too far?
+ * corpus_stats_uses_spent — has this entry been leaned on long enough?
  *
  * A wall-clock TTL bounds staleness in time, but the harm is proportional:
  * a thousand chunks added to a million barely move the weights, while the
@@ -265,15 +265,15 @@ static HTAB *corpus_stats_cache = NULL;
  * two-hundred-row one ten, and re-reading a two-hundred-row table is free.
  */
 static bool
-corpus_stats_stale_by_growth(const CorpusStatsEntry *entry)
+corpus_stats_uses_spent(const CorpusStatsEntry *entry)
 {
 	int64		budget;
 
-	if (pgedge_vectorizer_corpus_stats_cache_max_growth <= 0)
-		return false;			/* growth bound disabled; TTL only */
+	if (pgedge_vectorizer_corpus_stats_cache_max_uses_pct <= 0)
+		return false;			/* use bound disabled; TTL only */
 
 	budget = entry->total_docs
-		* pgedge_vectorizer_corpus_stats_cache_max_growth / 100;
+		* pgedge_vectorizer_corpus_stats_cache_max_uses_pct / 100;
 
 	return entry->uses_since_read >= Max(budget, 1);
 }
@@ -393,7 +393,7 @@ bm25_corpus_stats(const char *chunk_table, float8 *avg_doc_len)
 		!TimestampDifferenceExceeds(entry->read_at, now,
 									pgedge_vectorizer_corpus_stats_cache_ttl
 									* 1000) &&
-		!corpus_stats_stale_by_growth(entry))
+		!corpus_stats_uses_spent(entry))
 	{
 		entry->uses_since_read++;
 		*avg_doc_len = entry->avg_doc_len;
diff --git a/src/guc.c b/src/guc.c
index 2177328..2a3da73 100644
--- a/src/guc.c
+++ b/src/guc.c
@@ -52,7 +52,7 @@ bool   pgedge_vectorizer_enable_hybrid = false;
 double pgedge_vectorizer_bm25_k1       = 1.2;
 double pgedge_vectorizer_bm25_b        = 0.75;
 int    pgedge_vectorizer_corpus_stats_cache_ttl = 60;
-int    pgedge_vectorizer_corpus_stats_cache_max_growth = 5;
+int    pgedge_vectorizer_corpus_stats_cache_max_uses_pct = 5;
 
 /*
  * Initialize all GUC variables
@@ -332,16 +332,22 @@ pgedge_vectorizer_init_guc(void)
 		NULL, NULL, NULL);
 
 	DefineCustomIntVariable(
-		"pgedge_vectorizer.corpus_stats_cache_max_growth",
-		"Percent the corpus may grow before cached figures are re-read",
+		"pgedge_vectorizer.corpus_stats_cache_max_uses_pct",
+		"Percent of the corpus size that may be spent as uses of the cached "
+		"figures before they are re-read",
 		"Bounds staleness in proportion rather than in time, which is how it "
 		"harms ranking: a thousand chunks added to a million barely move the "
 		"weights, the same thousand added to two hundred change them several "
-		"fold. Applied alongside corpus_stats_cache_ttl, whichever is reached "
-		"first. Set to 0 to bound by time alone.",
-		&pgedge_vectorizer_corpus_stats_cache_max_growth,
+		"fold. Each use of a cached reading counts as one chunk that may have "
+		"been added since. That is a proxy rather than a measurement of the "
+		"corpus, since the figures are read once per queue item and once per "
+		"search, so a read-only workload spends the budget too and a static "
+		"corpus is re-read periodically for no gain. Applied alongside "
+		"corpus_stats_cache_ttl, whichever is reached first. Set to 0 to "
+		"bound by time alone.",
+		&pgedge_vectorizer_corpus_stats_cache_max_uses_pct,
 		5,      /* default */
-		0,      /* min: 0 disables the growth bound */
+		0,      /* min: 0 disables the use bound */
 		100,    /* max */
 		PGC_USERSET,
 		0,
diff --git a/src/pgedge_vectorizer.h b/src/pgedge_vectorizer.h
index 5267670..9e07f31 100644
--- a/src/pgedge_vectorizer.h
+++ b/src/pgedge_vectorizer.h
@@ -73,7 +73,7 @@ extern bool   pgedge_vectorizer_enable_hybrid;
 extern double pgedge_vectorizer_bm25_k1;
 extern double pgedge_vectorizer_bm25_b;
 extern int    pgedge_vectorizer_corpus_stats_cache_ttl;
-extern int    pgedge_vectorizer_corpus_stats_cache_max_growth;
+extern int    pgedge_vectorizer_corpus_stats_cache_max_uses_pct;
 
 /*
  * Chunking strategy enumeration
diff --git a/test/expected/hybrid_test.out b/test/expected/hybrid_test.out
index e747ec8..34ce55c 100644
--- a/test/expected/hybrid_test.out
+++ b/test/expected/hybrid_test.out
@@ -728,7 +728,7 @@ drop cascades to table cache_tenant_b.t_chunks_idf_stats
 ---------------------------------------------------------------------------
 -- A time bound alone lets a small, fast-growing corpus go badly stale, and
 -- the figures are written into stored vectors rather than only used for a
--- query. corpus_stats_cache_max_growth bounds staleness in proportion instead:
+-- query. corpus_stats_cache_max_uses_pct bounds staleness in proportion instead:
 -- at 5% of twenty documents, one cached use is allowed before a re-read.
 CREATE TABLE growth_chunks (id BIGSERIAL PRIMARY KEY, content TEXT,
                             token_count INT);
@@ -739,7 +739,7 @@ SELECT 'alpha', 10 FROM generate_series(1, 20);
 INSERT INTO growth_chunks_idf_stats VALUES ('alpha', 5);
 -- Long enough that the time bound cannot be what fires.
 SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
-SET pgedge_vectorizer.corpus_stats_cache_max_growth = 5;
+SET pgedge_vectorizer.corpus_stats_cache_max_uses_pct = 5;
 SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS g1 \gset
 INSERT INTO growth_chunks (content, token_count)
 SELECT 'alpha', 900 FROM generate_series(1, 60);
@@ -755,9 +755,9 @@ SELECT :'g1'::sparsevec = :'g2'::sparsevec AS use_within_budget_is_cached,
  t                           | t                          | t
 (1 row)
 
--- With the growth bound off, the same sequence stays on the stale figures.
+-- With the use bound off, the same sequence stays on the stale figures.
 SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
-SET pgedge_vectorizer.corpus_stats_cache_max_growth = 0;
+SET pgedge_vectorizer.corpus_stats_cache_max_uses_pct = 0;
 SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h1 \gset
 INSERT INTO growth_chunks (content, token_count)
 SELECT 'alpha', 900 FROM generate_series(1, 200);
@@ -770,7 +770,7 @@ SELECT :'h1'::sparsevec = :'h3'::sparsevec AS time_bound_alone_stays_stale;
 (1 row)
 
 RESET pgedge_vectorizer.corpus_stats_cache_ttl;
-RESET pgedge_vectorizer.corpus_stats_cache_max_growth;
+RESET pgedge_vectorizer.corpus_stats_cache_max_uses_pct;
 DROP TABLE growth_chunks, growth_chunks_idf_stats;
 ---------------------------------------------------------------------------
 -- Test 25: a doc_freq larger than the corpus does not lose the term
diff --git a/test/sql/hybrid_test.sql b/test/sql/hybrid_test.sql
index 057bd57..00c412a 100644
--- a/test/sql/hybrid_test.sql
+++ b/test/sql/hybrid_test.sql
@@ -564,7 +564,7 @@ DROP SCHEMA cache_tenant_b CASCADE;
 
 -- A time bound alone lets a small, fast-growing corpus go badly stale, and
 -- the figures are written into stored vectors rather than only used for a
--- query. corpus_stats_cache_max_growth bounds staleness in proportion instead:
+-- query. corpus_stats_cache_max_uses_pct bounds staleness in proportion instead:
 -- at 5% of twenty documents, one cached use is allowed before a re-read.
 
 CREATE TABLE growth_chunks (id BIGSERIAL PRIMARY KEY, content TEXT,
@@ -577,7 +577,7 @@ INSERT INTO growth_chunks_idf_stats VALUES ('alpha', 5);
 
 -- Long enough that the time bound cannot be what fires.
 SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
-SET pgedge_vectorizer.corpus_stats_cache_max_growth = 5;
+SET pgedge_vectorizer.corpus_stats_cache_max_uses_pct = 5;
 
 SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS g1 \gset
 INSERT INTO growth_chunks (content, token_count)
@@ -592,9 +592,9 @@ SELECT :'g1'::sparsevec = :'g2'::sparsevec AS use_within_budget_is_cached,
        :'g1'::sparsevec <> :'g3'::sparsevec AS budget_spent_forces_reread,
        :'g3'::sparsevec = :'gtruth'::sparsevec AS reread_matches_uncached;
 
--- With the growth bound off, the same sequence stays on the stale figures.
+-- With the use bound off, the same sequence stays on the stale figures.
 SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
-SET pgedge_vectorizer.corpus_stats_cache_max_growth = 0;
+SET pgedge_vectorizer.corpus_stats_cache_max_uses_pct = 0;
 
 SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h1 \gset
 INSERT INTO growth_chunks (content, token_count)
@@ -605,7 +605,7 @@ SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h3 \gset
 SELECT :'h1'::sparsevec = :'h3'::sparsevec AS time_bound_alone_stays_stale;
 
 RESET pgedge_vectorizer.corpus_stats_cache_ttl;
-RESET pgedge_vectorizer.corpus_stats_cache_max_growth;
+RESET pgedge_vectorizer.corpus_stats_cache_max_uses_pct;
 DROP TABLE growth_chunks, growth_chunks_idf_stats;
 
 ---------------------------------------------------------------------------
diff --git a/test/t/006_corpus_stats_growth.pl b/test/t/006_corpus_stats_growth.pl
index 3fdcc59..044b344 100644
--- a/test/t/006_corpus_stats_growth.pl
+++ b/test/t/006_corpus_stats_growth.pl
@@ -1,6 +1,6 @@
 # Copyright (c) 2025 - 2026, pgEdge, Inc.
 #
-# Verify that the corpus statistics growth bound reaches the vectors the worker
+# Verify that the corpus statistics use bound reaches the vectors the worker
 # stores, not merely the ones a query computes.
 #
 # The regression tests cover the bound through bm25_query_vector(), which
@@ -46,7 +46,7 @@
 pgedge_vectorizer.provider = 'ollama'
 pgedge_vectorizer.enable_hybrid = true
 pgedge_vectorizer.corpus_stats_cache_ttl = 3600
-pgedge_vectorizer.corpus_stats_cache_max_growth = 5
+pgedge_vectorizer.corpus_stats_cache_max_uses_pct = 5
 max_worker_processes = 16
 ));
 

From 8da5f91cd000faf09b1dfed6668592866f639b6e Mon Sep 17 00:00:00 2001
From: Mason Sharp <mason.sharp@pgedge.com>
Date: Wed, 12 Aug 2026 13:53:10 -0700
Subject: [PATCH 07/10] Re-read the corpus when doc_freq contradicts the cached
 size

doc_freq above N proves the cached corpus size is stale, since doc_freq
is read fresh on every call. Clamping the weight left the stale entry in
place, so every later chunk in the same window was scored against the
same wrong corpus and had that persisted into sparse_embedding. Drop the
entry and read again, which repairs this document and the ones after it.

If the fresh reading is still below the largest doc_freq the statistics
are inconsistent rather than stale, and re-reading cannot help, so the
entry adopts that value: measured, twenty further calls then cause no
additional scans, where an unguarded recheck would rescan on every one.

The maximum doc_freq is taken across the whole result set before any
weight is computed, so one corpus size covers the whole document rather
than each term being scored against its own.

Raised by dpage on #60.
---
 src/bm25.c | 106 ++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 93 insertions(+), 13 deletions(-)

diff --git a/src/bm25.c b/src/bm25.c
index 287ce91..748d394 100644
--- a/src/bm25.c
+++ b/src/bm25.c
@@ -278,6 +278,61 @@ corpus_stats_uses_spent(const CorpusStatsEntry *entry)
 	return entry->uses_since_read >= Max(budget, 1);
 }
 
+static int64 bm25_corpus_stats(const char *chunk_table, float8 *avg_doc_len);
+
+/*
+ * bm25_corpus_stats_recheck — the data contradicts the cached corpus size.
+ *
+ * A term cannot appear in more documents than exist, so doc_freq above N is
+ * proof that the cached N is stale: doc_freq is read fresh on every call while
+ * N may be up to a TTL old.  Clamping the weight alone would leave the stale
+ * entry in place, so every later chunk in the same window would be scored, and
+ * have persisted into sparse_embedding, against the same wrong corpus.  Drop
+ * the entry and read again, which repairs this document and every one after it.
+ *
+ * If the fresh reading is still below max_df then the statistics are
+ * inconsistent rather than stale — a doc_freq left too high by an earlier
+ * failure, say — and re-reading cannot fix that.  Adopt max_df as this entry's
+ * corpus size in that case: the weights stay sane, and the next call sees a
+ * cached N that the data no longer contradicts, so it does not rescan to no
+ * effect.  A natural expiry re-reads the true N and the check runs again.
+ *
+ * Caller must have an active SPI connection.
+ */
+static int64
+bm25_corpus_stats_recheck(const char *chunk_table, float8 *avg_doc_len,
+						  int64 max_df)
+{
+	Oid			relid;
+	int64		total_docs;
+
+	/* Nothing is cached to be stale, so the reading was already fresh. */
+	if (pgedge_vectorizer_corpus_stats_cache_ttl <= 0 ||
+		corpus_stats_cache == NULL)
+		return max_df;
+
+	relid = RelnameGetRelid(chunk_table);
+	if (!OidIsValid(relid))
+		return max_df;
+
+	hash_search(corpus_stats_cache, &relid, HASH_REMOVE, NULL);
+
+	total_docs = bm25_corpus_stats(chunk_table, avg_doc_len);
+
+	if (max_df > total_docs)
+	{
+		CorpusStatsEntry *entry;
+
+		entry = hash_search(corpus_stats_cache, &relid, HASH_FIND, NULL);
+		if (entry != NULL)
+			entry->total_docs = max_df;
+
+		total_docs = max_df;
+	}
+
+	return total_docs;
+}
+
 /*
  * bm25_corpus_stats_uncached — read N and the mean document length.
  *
@@ -463,6 +518,9 @@ bm25_load_idf_stats(const char *chunk_table, BM25Term *tokens, int ntokens,
 	HTAB            *htab = NULL;
 	HASHCTL          ctl;
 	int64            total_docs = 1;
+	int64            max_df = 0;
+	IdfStat         *rows = NULL;
+	int              nrows = 0;
 	Datum            terms;
 	Oid              argtype = TEXTARRAYOID;
 
@@ -581,9 +639,39 @@ bm25_load_idf_stats(const char *chunk_table, BM25Term *tokens, int ntokens,
 					   &ctl,
 					   HASH_ELEM | HASH_STRINGS | HASH_CONTEXT);
 
-	for (int i = 0; i < (int) SPI_processed; i++)
+	/*
+	 * Take the rows out of SPI_tuptable before anything else runs a query.
+	 * The recheck below reads the corpus again, and SPI_execute() replaces
+	 * SPI_tuptable, so a loop still reading from it would decode that result
+	 * as (term, doc_freq).  read_idf_row() already copies the term, so the
+	 * array outlives the tuptable.
+	 *
+	 * The largest doc_freq is wanted before any weight is computed.  A term
+	 * cannot appear in more documents than exist, so a doc_freq above the
+	 * corpus size means the corpus size is wrong: it may be a cached reading
+	 * while every doc_freq here was read fresh.  Taking one maximum also
+	 * means one corpus size covers the whole document -- weighting each term
+	 * against its own would leave two terms of one chunk scored against
+	 * different corpora, which BM25 does not contemplate.
+	 */
+	nrows = (int) SPI_processed;
+	rows  = (IdfStat *) palloc(nrows * sizeof(IdfStat));
+
+	for (int i = 0; i < nrows; i++)
 	{
-		IdfStat       row   = read_idf_row(SPI_tuptable, i);
+		rows[i] = read_idf_row(SPI_tuptable, i);
+
+		if ((int64) rows[i].doc_freq > max_df)
+			max_df = (int64) rows[i].doc_freq;
+	}
+
+	if (max_df > total_docs)
+		total_docs = bm25_corpus_stats_recheck(chunk_table, avg_doc_len,
+											   max_df);
+
+	for (int i = 0; i < nrows; i++)
+	{
+		IdfStat       row   = rows[i];
 		bool          found;
 		IdfHashEntry *entry;
 
@@ -594,26 +682,18 @@ bm25_load_idf_stats(const char *chunk_table, BM25Term *tokens, int ntokens,
 			 * Derived, not stored: the corpus size is identical for every
 			 * term.  Expression shape matches the SQL it replaced so that
 			 * results are bit-identical.
-			 *
-			 * A term cannot appear in more documents than exist, but
-			 * total_docs may be a cached reading while doc_freq is read
-			 * fresh, so doc_freq can outrun it.  Left alone that gives a
-			 * negative weight, and a negative score is dropped from the
-			 * vector, losing the term outright.  Taking the larger changes
-			 * nothing whenever total_docs >= doc_freq, which is every case a
-			 * fresh reading can produce.
 			 */
-			int64	corpus = Max(total_docs, (int64) row.doc_freq);
-
 			entry->idf_weight =
 				(row.doc_freq <= 0)
 					? 0.0
-					: log(1.0 + ((double) corpus - row.doc_freq + 0.5)
+					: log(1.0 + ((double) total_docs - row.doc_freq + 0.5)
 								/ (row.doc_freq + 0.5));
 		}
 		/* Duplicate terms: keep the first weight encountered */
 	}
 
+	pfree(rows);
+
 	return htab;
 }
 

From ab30ca7065e0ae02c5eb5a67ab14186f01005643 Mon Sep 17 00:00:00 2001
From: Mason Sharp <mason.sharp@pgedge.com>
Date: Wed, 12 Aug 2026 13:59:49 -0700
Subject: [PATCH 08/10] Record the corpus statistics cache in the changelog and
 README

Neither cache parameter appeared in either: corpus_stats_cache_ttl was
missed when the cache landed, and max_uses_pct followed it. The changelog
entry covers both, since this is the natural place to catch up the first.

Raised by dpage on #60.
---
 README.md         |  2 ++
 docs/changelog.md | 27 +++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/README.md b/README.md
index a40ff56..20b311f 100644
--- a/README.md
+++ b/README.md
@@ -215,6 +215,8 @@ SELECT * FROM pgedge_vectorizer.hybrid_search_simple(
 | `pgedge_vectorizer.enable_hybrid` | bool | `false` | — | Enable BM25 sparse vectors alongside dense embeddings |
 | `pgedge_vectorizer.bm25_k1` | real | `1.2` | `0.0–3.0` | Term frequency saturation parameter |
 | `pgedge_vectorizer.bm25_b` | real | `0.75` | `0.0–1.0` | Document length normalization parameter |
+| `pgedge_vectorizer.corpus_stats_cache_ttl` | integer | `60s` | `0–3600` | Seconds a backend may reuse a chunk table's corpus size and mean document length before reading them again. Set to 0 to read them on every call |
+| `pgedge_vectorizer.corpus_stats_cache_max_uses_pct` | integer | `5` | `0–100` | Percent of the cached corpus size that may be spent as uses of those figures before they are re-read, whichever comes first with the TTL. Each use counts as one chunk that may have been added since: a proxy rather than a measurement, so searches spend it too. Set to 0 to bound by time alone |
 
 ## Configuration Parameters
 
diff --git a/docs/changelog.md b/docs/changelog.md
index 5f438de..43eaa8c 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -56,6 +56,33 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
   vectorized tables were created under a build predating those triggers.
 - `pgedge_vectorizer.vectorizers` now records the document identifier column and
   its type, so cleanup triggers can be recreated without re-detecting it.
+- BM25 now caches a chunk table's corpus size and mean document length per
+  backend rather than deriving them on every call
+  ([#53](https://github.com/pgEdge/pgedge-vectorizer/issues/53)). Both come
+  from an aggregate that cannot use an index, so it was a full scan of the
+  chunk table for every search and for every queue item the worker processed;
+  a batch of ten scanned the same table ten times under a snapshot in which
+  the answer could not have changed. Two settings bound how stale a cached
+  reading may become, and whichever is reached first triggers a re-read:
+    - `pgedge_vectorizer.corpus_stats_cache_ttl` (default 60 seconds) bounds it
+      in time. Setting it to 0 disables the cache entirely, restoring the
+      previous behaviour exactly.
+    - `pgedge_vectorizer.corpus_stats_cache_max_uses_pct` (default 5) bounds it
+      in proportion to the corpus, which is how staleness actually harms
+      ranking: a thousand chunks added to a million barely move the weights,
+      while the same thousand added to two hundred change them several fold.
+      Each use of a cached reading counts as one chunk that may have been added
+      since — a proxy rather than a measurement, so searches spend the budget
+      too. Setting it to 0 bounds by time alone.
+
+  The figures feed a ranking heuristic rather than an account that has to
+  balance, and during ingest they are a moving target in any case, so holding
+  one briefly costs a little precision in a number that was never precise. They
+  are not maintained incrementally: chunk rows are inserted and deleted from
+  many places across the SQL — chunking, content updates, `recreate_chunks()`,
+  row deletion, truncation and disabling — and counters left wrong in any of
+  them would skew ranking silently and permanently, whereas a cache that
+  expires cannot drift for longer than its bounds and recovers by itself.
 
 ### Changed
 

From bed0d4b09923622155b02cc8c50de65a6cfa2c85 Mon Sep 17 00:00:00 2001
From: Mason Sharp <mason.sharp@pgedge.com>
Date: Wed, 12 Aug 2026 14:09:21 -0700
Subject: [PATCH 09/10] Assert what the tests were only asserting halfway

h2 was computed and never used, so the block checked only that the ends
matched; it now checks all three, which is what "no number of uses moves
the figures" actually means. Test 25 claimed in a comment that the
clamped weight is near zero and asserted only that the vector was
non-empty; it now bounds the weight.

The counter's semantics are documented rather than changed: counting the
reading call would leave no cached use at all wherever the budget works
out to one, which is every corpus below 20 rows at the default 5%.

Raised by dpage on #60.
---
 src/bm25.c                    |  5 +++++
 test/expected/hybrid_test.out | 16 +++++++++++++++-
 test/sql/hybrid_test.sql      | 12 +++++++++++-
 3 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/bm25.c b/src/bm25.c
index 748d394..a4126bb 100644
--- a/src/bm25.c
+++ b/src/bm25.c
@@ -263,6 +263,11 @@ static HTAB *corpus_stats_cache = NULL;
  * It is deliberately pessimistic in the direction that matters, and it
  * scales itself — a million-row corpus tolerates fifty thousand uses, a
  * two-hundred-row one ten, and re-reading a two-hundred-row table is free.
+ *
+ * The call that performs the read is not itself counted, so one read covers
+ * budget + 1 calls.  Counting it would mean no cached use at all wherever the
+ * budget works out to one, which is every corpus below 20 rows at the default
+ * 5%, and the small corpus is the case this bound exists to keep honest.
  */
 static bool
 corpus_stats_uses_spent(const CorpusStatsEntry *entry)
diff --git a/test/expected/hybrid_test.out b/test/expected/hybrid_test.out
index 34ce55c..25adacf 100644
--- a/test/expected/hybrid_test.out
+++ b/test/expected/hybrid_test.out
@@ -763,7 +763,10 @@ INSERT INTO growth_chunks (content, token_count)
 SELECT 'alpha', 900 FROM generate_series(1, 200);
 SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h2 \gset
 SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h3 \gset
-SELECT :'h1'::sparsevec = :'h3'::sparsevec AS time_bound_alone_stays_stale;
+-- All three, not just the ends: with the bound off no number of uses moves the
+-- figures, which is what distinguishes this from the block above.
+SELECT :'h1'::sparsevec = :'h2'::sparsevec
+       AND :'h2'::sparsevec = :'h3'::sparsevec AS time_bound_alone_stays_stale;
  time_bound_alone_stays_stale 
 ------------------------------
  t
@@ -795,6 +798,17 @@ SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'clamp_chunks')
  t
 (1 row)
 
+-- And that it is underweighted rather than merely present: a term in every
+-- document carries almost no information, so bound the weight rather than
+-- restating the claim in a comment.
+SELECT split_part(split_part(
+           pgedge_vectorizer.bm25_query_vector('alpha', 'clamp_chunks')::text,
+           ':', 2), '}', 1)::float8 < 0.001 AS weight_is_near_zero;
+ weight_is_near_zero 
+---------------------
+ t
+(1 row)
+
 RESET pgedge_vectorizer.corpus_stats_cache_ttl;
 DROP TABLE clamp_chunks, clamp_chunks_idf_stats;
 ---------------------------------------------------------------------------
diff --git a/test/sql/hybrid_test.sql b/test/sql/hybrid_test.sql
index 00c412a..f1b5159 100644
--- a/test/sql/hybrid_test.sql
+++ b/test/sql/hybrid_test.sql
@@ -602,7 +602,10 @@ SELECT 'alpha', 900 FROM generate_series(1, 200);
 SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h2 \gset
 SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'growth_chunks') AS h3 \gset
 
-SELECT :'h1'::sparsevec = :'h3'::sparsevec AS time_bound_alone_stays_stale;
+-- All three, not just the ends: with the bound off no number of uses moves the
+-- figures, which is what distinguishes this from the block above.
+SELECT :'h1'::sparsevec = :'h2'::sparsevec
+       AND :'h2'::sparsevec = :'h3'::sparsevec AS time_bound_alone_stays_stale;
 
 RESET pgedge_vectorizer.corpus_stats_cache_ttl;
 RESET pgedge_vectorizer.corpus_stats_cache_max_uses_pct;
@@ -630,6 +633,13 @@ SET pgedge_vectorizer.corpus_stats_cache_ttl = 0;
 SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'clamp_chunks')
            <> '{}/65536'::sparsevec AS term_survives_df_over_n;
 
+-- And that it is underweighted rather than merely present: a term in every
+-- document carries almost no information, so bound the weight rather than
+-- restating the claim in a comment.
+SELECT split_part(split_part(
+           pgedge_vectorizer.bm25_query_vector('alpha', 'clamp_chunks')::text,
+           ':', 2), '}', 1)::float8 < 0.001 AS weight_is_near_zero;
+
 RESET pgedge_vectorizer.corpus_stats_cache_ttl;
 DROP TABLE clamp_chunks, clamp_chunks_idf_stats;
 

From 05865b1315e0d4a8ea90677ee60de7ea5932b7f3 Mon Sep 17 00:00:00 2001
From: Mason Sharp <mason.sharp@pgedge.com>
Date: Wed, 12 Aug 2026 14:18:47 -0700
Subject: [PATCH 10/10] Cover the corpus re-read with a test

The re-read had no automated test: it was verified by hand, which is the
same gap that let the fixture mismatch through earlier on this branch.
Test 26 checks that a doc_freq above the cached size produces the weights
of a fresh read rather than a clamped one, and that the entry stays
repaired once doc_freq drops back below it. Both assertions return false
against a clamp-only build, so they discriminate on the re-read itself.

Also note why the re-read is not skipped when the reading was already
fresh: it costs one scan, only for inconsistent statistics, and only
until max_df is adopted, after which nothing contradicts the entry.
---
 src/bm25.c                    |  7 +++++
 test/expected/hybrid_test.out | 49 ++++++++++++++++++++++++++++++++++
 test/sql/hybrid_test.sql      | 50 +++++++++++++++++++++++++++++++++++
 3 files changed, 106 insertions(+)

diff --git a/src/bm25.c b/src/bm25.c
index a4126bb..4f3b596 100644
--- a/src/bm25.c
+++ b/src/bm25.c
@@ -302,6 +302,13 @@ static int64 bm25_corpus_stats(const char *chunk_table, float8 *avg_doc_len);
  * cached N that the data no longer contradicts, so it does not rescan to no
  * effect.  A natural expiry re-reads the true N and the check runs again.
  *
+ * The reading passed in may already have been fresh, in which case the read
+ * below is redundant.  That is left alone deliberately: it costs one extra
+ * scan, only for a table whose statistics are inconsistent, and only until
+ * max_df is adopted below, after which nothing contradicts the entry and the
+ * recheck stops firing.  Reporting cache hits back to the caller to avoid it
+ * would thread a flag through for a case that resolves itself.
+ *
  * Caller must have an active SPI connection.
  */
 static int64
diff --git a/test/expected/hybrid_test.out b/test/expected/hybrid_test.out
index 25adacf..b331888 100644
--- a/test/expected/hybrid_test.out
+++ b/test/expected/hybrid_test.out
@@ -812,6 +812,55 @@ SELECT split_part(split_part(
 RESET pgedge_vectorizer.corpus_stats_cache_ttl;
 DROP TABLE clamp_chunks, clamp_chunks_idf_stats;
 ---------------------------------------------------------------------------
+-- Test 26: a doc_freq above the cached corpus size re-reads it
+---------------------------------------------------------------------------
+-- doc_freq cannot exceed the corpus, so doc_freq above N is proof that the
+-- cached N is stale rather than merely inconvenient. Clamping the weight alone
+-- would leave the stale entry in place, so every later chunk in the window
+-- would be scored against the same wrong corpus and have it persisted. The use
+-- bound is disabled here so that only the re-read can refresh the entry.
+CREATE TABLE reread_chunks (id BIGSERIAL PRIMARY KEY, content TEXT,
+                            token_count INT);
+CREATE TABLE reread_chunks_idf_stats (term TEXT PRIMARY KEY,
+                                      doc_freq INT NOT NULL);
+INSERT INTO reread_chunks (content, token_count)
+SELECT 'alpha', 10 FROM generate_series(1, 20);
+INSERT INTO reread_chunks_idf_stats VALUES ('alpha', 5);
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
+SET pgedge_vectorizer.corpus_stats_cache_max_uses_pct = 0;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'reread_chunks') AS warm \gset
+-- The corpus grows and doc_freq passes the cached 20, staying under the truth.
+INSERT INTO reread_chunks (content, token_count)
+SELECT 'alpha', 10 FROM generate_series(1, 200);
+UPDATE reread_chunks_idf_stats SET doc_freq = 100 WHERE term = 'alpha';
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'reread_chunks') AS r1 \gset
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 0;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'reread_chunks') AS fresh \gset
+-- Merely clamping would score against doc_freq, 100, rather than the real 220.
+SELECT :'r1'::sparsevec = :'fresh'::sparsevec AS reread_matches_a_fresh_read;
+ reread_matches_a_fresh_read 
+-----------------------------
+ t
+(1 row)
+
+-- And the entry was repaired, not patched for one call: with doc_freq back
+-- below N nothing triggers a re-read, so this can only match if the cached
+-- corpus size is now the grown one.
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
+UPDATE reread_chunks_idf_stats SET doc_freq = 5 WHERE term = 'alpha';
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'reread_chunks') AS r2 \gset
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 0;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'reread_chunks') AS fresh2 \gset
+SELECT :'r2'::sparsevec = :'fresh2'::sparsevec AS later_calls_use_the_refreshed_size;
+ later_calls_use_the_refreshed_size 
+------------------------------------
+ t
+(1 row)
+
+RESET pgedge_vectorizer.corpus_stats_cache_ttl;
+RESET pgedge_vectorizer.corpus_stats_cache_max_uses_pct;
+DROP TABLE reread_chunks, reread_chunks_idf_stats;
+---------------------------------------------------------------------------
 -- Cleanup
 ---------------------------------------------------------------------------
 SELECT pgedge_vectorizer.disable_vectorization(
diff --git a/test/sql/hybrid_test.sql b/test/sql/hybrid_test.sql
index f1b5159..7554c20 100644
--- a/test/sql/hybrid_test.sql
+++ b/test/sql/hybrid_test.sql
@@ -643,6 +643,56 @@ SELECT split_part(split_part(
 RESET pgedge_vectorizer.corpus_stats_cache_ttl;
 DROP TABLE clamp_chunks, clamp_chunks_idf_stats;
 
+---------------------------------------------------------------------------
+-- Test 26: a doc_freq above the cached corpus size re-reads it
+---------------------------------------------------------------------------
+
+-- doc_freq cannot exceed the corpus, so doc_freq above N is proof that the
+-- cached N is stale rather than merely inconvenient. Clamping the weight alone
+-- would leave the stale entry in place, so every later chunk in the window
+-- would be scored against the same wrong corpus and have it persisted. The use
+-- bound is disabled here so that only the re-read can refresh the entry.
+
+CREATE TABLE reread_chunks (id BIGSERIAL PRIMARY KEY, content TEXT,
+                            token_count INT);
+CREATE TABLE reread_chunks_idf_stats (term TEXT PRIMARY KEY,
+                                      doc_freq INT NOT NULL);
+INSERT INTO reread_chunks (content, token_count)
+SELECT 'alpha', 10 FROM generate_series(1, 20);
+INSERT INTO reread_chunks_idf_stats VALUES ('alpha', 5);
+
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
+SET pgedge_vectorizer.corpus_stats_cache_max_uses_pct = 0;
+
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'reread_chunks') AS warm \gset
+
+-- The corpus grows and doc_freq passes the cached 20, staying under the truth.
+INSERT INTO reread_chunks (content, token_count)
+SELECT 'alpha', 10 FROM generate_series(1, 200);
+UPDATE reread_chunks_idf_stats SET doc_freq = 100 WHERE term = 'alpha';
+
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'reread_chunks') AS r1 \gset
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 0;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'reread_chunks') AS fresh \gset
+
+-- Merely clamping would score against doc_freq, 100, rather than the real 220.
+SELECT :'r1'::sparsevec = :'fresh'::sparsevec AS reread_matches_a_fresh_read;
+
+-- And the entry was repaired, not patched for one call: with doc_freq back
+-- below N nothing triggers a re-read, so this can only match if the cached
+-- corpus size is now the grown one.
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 3600;
+UPDATE reread_chunks_idf_stats SET doc_freq = 5 WHERE term = 'alpha';
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'reread_chunks') AS r2 \gset
+SET pgedge_vectorizer.corpus_stats_cache_ttl = 0;
+SELECT pgedge_vectorizer.bm25_query_vector('alpha', 'reread_chunks') AS fresh2 \gset
+
+SELECT :'r2'::sparsevec = :'fresh2'::sparsevec AS later_calls_use_the_refreshed_size;
+
+RESET pgedge_vectorizer.corpus_stats_cache_ttl;
+RESET pgedge_vectorizer.corpus_stats_cache_max_uses_pct;
+DROP TABLE reread_chunks, reread_chunks_idf_stats;
+
 ---------------------------------------------------------------------------
 -- Cleanup
 ---------------------------------------------------------------------------