fix(helpers): OFFSET/_dynRange answer #REF! (NaN) past sheet bounds — the mid-pass OOM root

CHANGELOG.md

@@ -18,6 +18,20 @@
   input cells). **A cluster member is never size-skipped** (`MAX_SHEET_SIZE_MB` silently
   dropped the monster sheets from the cluster → partial-cluster wrong fixed point; regression
   red pre-fix with `clustersTotal=0`). New `EVAL_CLUSTER_TIMEOUT_MS` (default 60min).
+- **OFFSET/_dynRange #REF! bounds (the mid-pass OOM root)** — the instrumented canonical run
+  revealed the crash class: heap steady at 6.5GB through pass 4, death mid-pass-5 needing
+  >13GB in ONE spike. `_offsetAddr`/`_offset` had no upper bounds, so a displacement (or
+  OFFSET height) fed by a poisoned/oscillating cell (~1e7-scale) materialized a value-sized
+  rectangle — an allocation proportional to the VALUE, fatal at any heap cap (hence identical
+  ~56-min deaths under 12GB and 20GB). Past Excel's sheet bounds (1,048,576 × 16,384) the
+  helpers now answer honest NaN, exactly where Excel shows #REF! (scalar OFFSET previously
+  returned a SILENT 0 there). Negative-controlled `test-offset-ref-bounds.mjs` (7 asserts,
+  red pre-fix with the predicted silent values). Also instrumented: cluster telemetry now
+  names the max-delta cell. **Canonical A-1 verdict so far (honest): NOT a 2-pass story —
+  `GPP Promote!C465` is Excel's own #DIV/0! (0/0, absent from GT) so a persistent non-finite
+  is CORRECT, and the finite surface oscillates at e7 scale (1.5e7→3.0e7→2.0e7 by pass 4);
+  the probe's "2 passes, converged" was a stride-20 sampling illusion. Oscillation-cell
+  diagnosis = next session.**
 - **cluster child per-pass telemetry + full crash capture** — the canonical A-1 cluster
   child died at ~56 min under BOTH a 12GB and a 20GB heap (cap-independent death time),
   and the 200-char error truncation discarded the V8 GC dump that says which space

eval/per-sheet-eval.mjs

@@ -593,7 +593,7 @@ try {
       // let an empty scan read maxDelta=0 and falsely converge on pass 0.
       if (_sampleKeys.length === 0) break; // converged stays false (honest)
     }
-    let maxDelta = 0, anyNonFinite = false;
+    let maxDelta = 0, anyNonFinite = false, _maxCell = '';
     for (let i = 0; i < _sampleKeys.length; i++) {
       const _cur = v[_sampleKeys[i]];
       if (typeof _cur !== 'number') continue;
@@ -606,14 +606,14 @@ try {
       if (!Number.isFinite(_cur)) { anyNonFinite = true; _nonFinite = _sampleKeys[i]; _before[i] = _cur; continue; }
       const _b = _before[i];
       // A cell going undefined -> number is a change, not convergence.
-      if (typeof _b !== 'number') { maxDelta = Infinity; _before[i] = _cur; continue; }
+      if (typeof _b !== 'number') { if (maxDelta !== Infinity) { maxDelta = Infinity; _maxCell = _sampleKeys[i]; } _before[i] = _cur; continue; }
       const d = Math.abs(_cur - _b);
-      if (d > maxDelta) maxDelta = d;
+      if (d > maxDelta) { maxDelta = d; _maxCell = _sampleKeys[i]; }
       _before[i] = _cur;
     }
     {
       const _mu = process.memoryUsage();
-      _tel('pass ' + _iters + ': ' + ((Date.now() - _tp) / 60000).toFixed(1) + 'min, maxDelta=' + (Number.isFinite(maxDelta) ? maxDelta.toExponential(2) : String(maxDelta)) + ', nonFinite=' + (anyNonFinite ? ('YES @' + _nonFinite) : 'no') + ', heapUsed=' + (_mu.heapUsed / 1073741824).toFixed(1) + 'GB, rss=' + (_mu.rss / 1073741824).toFixed(1) + 'GB, sample=' + _sampleKeys.length);
+      _tel('pass ' + _iters + ': ' + ((Date.now() - _tp) / 60000).toFixed(1) + 'min, maxDelta=' + (Number.isFinite(maxDelta) ? maxDelta.toExponential(2) : String(maxDelta)) + ' @' + _maxCell + ', nonFinite=' + (anyNonFinite ? ('YES @' + _nonFinite) : 'no') + ', heapUsed=' + (_mu.heapUsed / 1073741824).toFixed(1) + 'GB, rss=' + (_mu.rss / 1073741824).toFixed(1) + 'GB, sample=' + _sampleKeys.length);
     }
     if (anyNonFinite) {
       // Keep iterating so a TRANSIENT cold-0 can warm. Bound the churn (#61): stop fast

package.json

@@ -47,7 +47,7 @@
     "test:onboarding": "node tests/cli/test-onboarding.mjs",
     "test:golden": "node tests/cli/test-golden-master.mjs",
     "golden": "node eval/golden-master.mjs",
-    "test": "node tests/lib/test-lib.mjs && node tests/lib/test-scope-plan.mjs && node tests/lib/test-cone-emit.mjs && node tests/lib/test-lite-harness.mjs && node tests/lib/test-lite-tier0.mjs && node tests/lib/test-lite-tier0-generic.mjs && node tests/lib/test-driver-scope.mjs && node tests/lib/test-tier-recommender.mjs && node tests/lib/test-lite-evaluators.mjs && node tests/lib/test-lite-surrogate.mjs && node tests/lib/test-lite-provenance.mjs && node tests/lib/test-artifact-hash.mjs && node tests/cli/test-colrow-ref.mjs && node pipelines/rust/tests/test-date-axis-sumifs.mjs && node pipelines/rust/tests/test-shared-formula-anchors.mjs && node pipelines/rust/tests/test-structure-fidelity.mjs &&node pipelines/rust/tests/test-iferror-infinity.mjs && node pipelines/rust/tests/test-circular-honesty.mjs && node pipelines/rust/tests/test-cluster-transient-div0.mjs && node pipelines/rust/tests/test-div-nan-propagation.mjs && node pipelines/rust/tests/test-cluster-divergent-cap.mjs && node tests/cli/test-cli.mjs && node tests/cli/test-manifest-improvements.mjs && node tests/cli/test-manifest-maps.mjs && node tests/cli/test-model-type.mjs && node tests/cli/test-dedupe-inputs.mjs && node tests/cli/test-refine-label-index.mjs && node tests/cli/test-init-shared-gt.mjs && node tests/cli/test-per-sheet-eval.mjs && node pipelines/rust/tests/test-per-sheet-eval-intracycle.mjs && node pipelines/rust/tests/test-per-sheet-eval-lockstep.mjs && node pipelines/rust/tests/test-per-sheet-eval-cluster-size.mjs && node pipelines/rust/tests/test-row-chunked-modules.mjs && node pipelines/rust/tests/test-per-sheet-eval-exit-honesty.mjs && node tests/cli/test-ai-interface.mjs && node tests/cli/test-e2e4-fixes.mjs && node tests/cli/test-ship-ready.mjs && node tests/cli/use-case-suite.mjs && node tests/cli/test-onboarding.mjs && node tests/cli/test-lite-e2e.mjs && node tests/cli/test-lite-byrequest-disclosure.mjs",
+    "test": "node tests/lib/test-lib.mjs && node tests/lib/test-scope-plan.mjs && node tests/lib/test-cone-emit.mjs && node tests/lib/test-lite-harness.mjs && node tests/lib/test-lite-tier0.mjs && node tests/lib/test-lite-tier0-generic.mjs && node tests/lib/test-driver-scope.mjs && node tests/lib/test-tier-recommender.mjs && node tests/lib/test-lite-evaluators.mjs && node tests/lib/test-lite-surrogate.mjs && node tests/lib/test-lite-provenance.mjs && node tests/lib/test-artifact-hash.mjs && node tests/cli/test-colrow-ref.mjs && node pipelines/rust/tests/test-date-axis-sumifs.mjs && node pipelines/rust/tests/test-shared-formula-anchors.mjs && node pipelines/rust/tests/test-structure-fidelity.mjs &&node pipelines/rust/tests/test-iferror-infinity.mjs && node pipelines/rust/tests/test-circular-honesty.mjs && node pipelines/rust/tests/test-cluster-transient-div0.mjs && node pipelines/rust/tests/test-div-nan-propagation.mjs && node pipelines/rust/tests/test-cluster-divergent-cap.mjs && node tests/cli/test-cli.mjs && node tests/cli/test-manifest-improvements.mjs && node tests/cli/test-manifest-maps.mjs && node tests/cli/test-model-type.mjs && node tests/cli/test-dedupe-inputs.mjs && node tests/cli/test-refine-label-index.mjs && node tests/cli/test-init-shared-gt.mjs && node tests/cli/test-per-sheet-eval.mjs && node pipelines/rust/tests/test-per-sheet-eval-intracycle.mjs && node pipelines/rust/tests/test-per-sheet-eval-lockstep.mjs && node pipelines/rust/tests/test-per-sheet-eval-cluster-size.mjs && node pipelines/rust/tests/test-row-chunked-modules.mjs && node pipelines/rust/tests/test-per-sheet-eval-exit-honesty.mjs && node pipelines/rust/tests/test-offset-ref-bounds.mjs && node tests/cli/test-ai-interface.mjs && node tests/cli/test-e2e4-fixes.mjs && node tests/cli/test-ship-ready.mjs && node tests/cli/use-case-suite.mjs && node tests/cli/test-onboarding.mjs && node tests/cli/test-lite-e2e.mjs && node tests/cli/test-lite-byrequest-disclosure.mjs",
     "bench": "node benchmarks/bench.mjs"
   },
   "devDependencies": {}

pipelines/rust/src/chunked_emitter.rs

@@ -2035,13 +2035,22 @@ function _yearfrac(start, end, basis) {
   return ((B.y - A.y) * 360 + (B.m - A.m) * 30 + (d2 - d1)) / 360;
 }
 
+// Excel's hard sheet bounds. An address computed past them is #REF! in Excel —
+// and, critically, a displacement that comes from a poisoned/diverging cell can
+// be ~1e7+: materializing that rectangle is a multi-GB allocation that fatally
+// OOMs the engine MID-PASS at any heap size (observed: the A-1 canonical eval
+// died at ~56 min under both 12GB and 20GB caps while steady at 6.5GB heap).
+// Out-of-bounds => honest NaN, exactly where Excel shows #REF!.
+const _XL_MAX_ROW = 1048576, _XL_MAX_COL = 16384;
+
 function _offsetAddr(refAddr, rows, cols) {
   // Pure address arithmetic for a computed range endpoint (no dereference).
   const m = String(refAddr).match(/^(.*)!([A-Z]+)([0-9]+)$/);
   if (!m) return refAddr;
   const col = _colNum(m[2]) + Math.trunc(+cols || 0);
   const row = +m[3] + Math.trunc(+rows || 0);
   if (col < 1 || row < 1) return refAddr; // Excel #REF!; clamp to base (finite, sweep-visible)
+  if (col > _XL_MAX_COL || row > _XL_MAX_ROW) return null; // Excel #REF! — _dynRange maps to NaN
   return m[1] + '!' + _numToCol(col) + row;
 }
 
@@ -2051,6 +2060,7 @@ function _dynRange(ctx, corners) {
   // ctx.range(). Issue #66.
   let sheet = null, c1 = Infinity, r1 = Infinity, c2 = -1, r2 = -1;
   for (const addr of corners) {
+    if (addr === null) return [NaN]; // #REF! endpoint (out of sheet bounds) — poison the aggregate honestly
     const m = String(addr).match(/^(.*)!([A-Z]+)([0-9]+)$/);
     if (!m) return [];
     if (sheet === null) sheet = m[1];
@@ -2084,6 +2094,10 @@ function _offset(ctx, refAddr, rowOffset, colOffset, height, width) {
   const newCol = _colNum(col) + (+colOffset || 0);
   const h = +height || 1;
   const w = +width || 1;
+  // Target rectangle past Excel's sheet bounds is #REF! — and a poisoned
+  // offset/height of ~1e7 would otherwise allocate a value-sized array (the
+  // mid-pass OOM class). Honest NaN, like Excel's error.
+  if (newRow < 1 || newCol < 1 || newRow + h - 1 > _XL_MAX_ROW || newCol + w - 1 > _XL_MAX_COL) return NaN;
   if (h === 1 && w === 1) {
     return ctx.get(`${sheet}!${_numToCol(newCol)}${newRow}`);
   }

pipelines/rust/tests/test-offset-ref-bounds.mjs

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+/**
+ * Regression: OFFSET / computed-endpoint helpers must answer #REF! (honest NaN)
+ * past Excel's sheet bounds (1,048,576 rows x 16,384 cols) — never materialize
+ * a value-sized rectangle.
+ *
+ * A displacement (or height/width) that comes from a poisoned/diverging cell can
+ * be ~1e7+. Pre-fix, _offsetAddr happily produced "Sheet!A20000001", _dynRange
+ * spanned the rectangle, and ctx.range() allocated it: a multi-GB allocation
+ * that fatally OOMed the A-1 canonical eval MID-PASS at ~56 min under BOTH a
+ * 12GB and a 20GB heap (cap-independent death — the alloc is proportional to
+ * the VALUE, not the model). Excel's own answer past the bounds is #REF!.
+ *
+ * Pre-fix RED states this test discriminates:
+ *   - scalar OFFSET out of bounds returned a SILENT 0 (missing-cell read)
+ *   - SUM over an out-of-bounds computed-endpoint range returned a confident
+ *     finite number after a multi-million-row materialization
+ * Post-fix: all three out-of-bounds shapes are NaN; in-bounds shapes unchanged.
+ *
+ * Needs the rust-parser binary. Skips (exit 0) if it isn't built.
+ *
+ * Usage: node pipelines/rust/tests/test-offset-ref-bounds.mjs
+ */
+
+import XLSX from 'xlsx';
+import { writeFileSync, existsSync, mkdtempSync, rmSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+import { tmpdir } from 'os';
+import { execFileSync } from 'child_process';
+
+const __dir = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(__dir, '..', '..', '..');
+const exe = process.platform === 'win32' ? '.exe' : '';
+const PARSER = [
+  join(ROOT, 'pipelines/rust/target/release', `rust-parser${exe}`),
+  join(ROOT, 'pipelines/rust/target/debug', `rust-parser${exe}`),
+].find(existsSync);
+
+if (!PARSER) {
+  console.log('SKIP: rust-parser not built (cd pipelines/rust && cargo build --release)');
+  process.exit(0);
+}
+
+let passed = 0, failed = 0;
+const assert = (c, m) => { if (c) { passed++; } else { failed++; console.error(`  FAIL: ${m}`); } };
+const n = (v, f) => (f ? { t: 'n', v, f } : { t: 'n', v });
+
+console.log('Testing: OFFSET/_dynRange answer #REF! (NaN) past sheet bounds, never a value-sized allocation');
+
+const S = {
+  '!ref': 'A1:H3',
+  A1: n(1), A2: n(2), A3: n(3),
+  B1: n(2000000),                                         // poisoned-scale displacement (> max row)
+  C1: n(0, 'SUM(A1:OFFSET(A1,B1,0))'),                    // computed endpoint past bounds -> NaN
+  D1: n(0, 'OFFSET(A1,1500000,0)'),                       // scalar OFFSET past bounds -> NaN (was silent 0)
+  E1: n(2, 'OFFSET(A1,1,0)'),                             // in-bounds scalar (regression)
+  F1: n(6, 'SUM(A1:OFFSET(A1,2,0))'),                     // in-bounds computed endpoint (regression)
+  G1: n(3, 'SUM(OFFSET(A1,0,0,2,1))'),                    // in-bounds multi-cell OFFSET (regression)
+  H1: n(0, 'SUM(OFFSET(A1,0,0,2000000,1))'),              // poisoned HEIGHT -> NaN (was a 2M-row loop)
+};
+
+const tmp = mkdtempSync(join(tmpdir(), 'refb-'));
+try {
+  writeFileSync(join(tmp, 'm.xlsx'), XLSX.write({ SheetNames: ['S'], Sheets: { S } }, { type: 'buffer', bookType: 'xlsx' }));
+  execFileSync(PARSER, [join(tmp, 'm.xlsx'), join(tmp, 'out'), '--chunked'], { encoding: 'utf-8', stdio: 'pipe' });
+  const eng = await import(pathToFileURL(join(tmp, 'out', 'chunked', 'engine.js')).href);
+  const t0 = Date.now();
+  const { values } = eng.run();
+  const secs = (Date.now() - t0) / 1000;
+
+  assert(Number.isNaN(values['S!C1']), `out-of-bounds computed-endpoint SUM is NaN (got ${values['S!C1']})`);
+  assert(Number.isNaN(values['S!D1']), `out-of-bounds scalar OFFSET is NaN (got ${values['S!D1']} — pre-fix: silent 0)`);
+  assert(Number.isNaN(values['S!H1']), `out-of-bounds OFFSET height is NaN (got ${values['S!H1']})`);
+  assert(values['S!E1'] === 2, `in-bounds scalar OFFSET unchanged (got ${values['S!E1']})`);
+  assert(values['S!F1'] === 6, `in-bounds computed-endpoint SUM unchanged (got ${values['S!F1']})`);
+  assert(values['S!G1'] === 3, `in-bounds multi-cell OFFSET unchanged (got ${values['S!G1']})`);
+  assert(secs < 5, `no value-sized materialization (run took ${secs.toFixed(1)}s — pre-fix the 2M-row spans dominate)`);
+} finally {
+  rmSync(tmp, { recursive: true, force: true });
+}
+
+console.log('');
+console.log(`Results: ${passed} passed, ${failed} failed, ${passed + failed} total`);
+process.exit(failed > 0 ? 1 : 0);