diff --git a/CHANGELOG.md b/CHANGELOG.md index 28d1620..0a1cc16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,20 @@ input cells). **A cluster member is never size-skipped** (`MAX_SHEET_SIZE_MB` silently dropped the monster sheets from the cluster → partial-cluster wrong fixed point; regression red pre-fix with `clustersTotal=0`). New `EVAL_CLUSTER_TIMEOUT_MS` (default 60min). +- **OFFSET/_dynRange #REF! bounds (the mid-pass OOM root)** — the instrumented canonical run + revealed the crash class: heap steady at 6.5GB through pass 4, death mid-pass-5 needing + >13GB in ONE spike. `_offsetAddr`/`_offset` had no upper bounds, so a displacement (or + OFFSET height) fed by a poisoned/oscillating cell (~1e7-scale) materialized a value-sized + rectangle — an allocation proportional to the VALUE, fatal at any heap cap (hence identical + ~56-min deaths under 12GB and 20GB). Past Excel's sheet bounds (1,048,576 × 16,384) the + helpers now answer honest NaN, exactly where Excel shows #REF! (scalar OFFSET previously + returned a SILENT 0 there). Negative-controlled `test-offset-ref-bounds.mjs` (7 asserts, + red pre-fix with the predicted silent values). Also instrumented: cluster telemetry now + names the max-delta cell. **Canonical A-1 verdict so far (honest): NOT a 2-pass story — + `GPP Promote!C465` is Excel's own #DIV/0! (0/0, absent from GT) so a persistent non-finite + is CORRECT, and the finite surface oscillates at e7 scale (1.5e7→3.0e7→2.0e7 by pass 4); + the probe's "2 passes, converged" was a stride-20 sampling illusion. Oscillation-cell + diagnosis = next session.** - **cluster child per-pass telemetry + full crash capture** — the canonical A-1 cluster child died at ~56 min under BOTH a 12GB and a 20GB heap (cap-independent death time), and the 200-char error truncation discarded the V8 GC dump that says which space diff --git a/eval/per-sheet-eval.mjs b/eval/per-sheet-eval.mjs index a8f5c27..fc726c6 100644 --- a/eval/per-sheet-eval.mjs +++ b/eval/per-sheet-eval.mjs @@ -593,7 +593,7 @@ try { // let an empty scan read maxDelta=0 and falsely converge on pass 0. if (_sampleKeys.length === 0) break; // converged stays false (honest) } - let maxDelta = 0, anyNonFinite = false; + let maxDelta = 0, anyNonFinite = false, _maxCell = ''; for (let i = 0; i < _sampleKeys.length; i++) { const _cur = v[_sampleKeys[i]]; if (typeof _cur !== 'number') continue; @@ -606,14 +606,14 @@ try { if (!Number.isFinite(_cur)) { anyNonFinite = true; _nonFinite = _sampleKeys[i]; _before[i] = _cur; continue; } const _b = _before[i]; // A cell going undefined -> number is a change, not convergence. - if (typeof _b !== 'number') { maxDelta = Infinity; _before[i] = _cur; continue; } + if (typeof _b !== 'number') { if (maxDelta !== Infinity) { maxDelta = Infinity; _maxCell = _sampleKeys[i]; } _before[i] = _cur; continue; } const d = Math.abs(_cur - _b); - if (d > maxDelta) maxDelta = d; + if (d > maxDelta) { maxDelta = d; _maxCell = _sampleKeys[i]; } _before[i] = _cur; } { const _mu = process.memoryUsage(); - _tel('pass ' + _iters + ': ' + ((Date.now() - _tp) / 60000).toFixed(1) + 'min, maxDelta=' + (Number.isFinite(maxDelta) ? maxDelta.toExponential(2) : String(maxDelta)) + ', nonFinite=' + (anyNonFinite ? ('YES @' + _nonFinite) : 'no') + ', heapUsed=' + (_mu.heapUsed / 1073741824).toFixed(1) + 'GB, rss=' + (_mu.rss / 1073741824).toFixed(1) + 'GB, sample=' + _sampleKeys.length); + _tel('pass ' + _iters + ': ' + ((Date.now() - _tp) / 60000).toFixed(1) + 'min, maxDelta=' + (Number.isFinite(maxDelta) ? maxDelta.toExponential(2) : String(maxDelta)) + ' @' + _maxCell + ', nonFinite=' + (anyNonFinite ? ('YES @' + _nonFinite) : 'no') + ', heapUsed=' + (_mu.heapUsed / 1073741824).toFixed(1) + 'GB, rss=' + (_mu.rss / 1073741824).toFixed(1) + 'GB, sample=' + _sampleKeys.length); } if (anyNonFinite) { // Keep iterating so a TRANSIENT cold-0 can warm. Bound the churn (#61): stop fast diff --git a/package.json b/package.json index 3b87541..8d2c71c 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "test:onboarding": "node tests/cli/test-onboarding.mjs", "test:golden": "node tests/cli/test-golden-master.mjs", "golden": "node eval/golden-master.mjs", - "test": "node tests/lib/test-lib.mjs && node tests/lib/test-scope-plan.mjs && node tests/lib/test-cone-emit.mjs && node tests/lib/test-lite-harness.mjs && node tests/lib/test-lite-tier0.mjs && node tests/lib/test-lite-tier0-generic.mjs && node tests/lib/test-driver-scope.mjs && node tests/lib/test-tier-recommender.mjs && node tests/lib/test-lite-evaluators.mjs && node tests/lib/test-lite-surrogate.mjs && node tests/lib/test-lite-provenance.mjs && node tests/lib/test-artifact-hash.mjs && node tests/cli/test-colrow-ref.mjs && node pipelines/rust/tests/test-date-axis-sumifs.mjs && node pipelines/rust/tests/test-shared-formula-anchors.mjs && node pipelines/rust/tests/test-structure-fidelity.mjs &&node pipelines/rust/tests/test-iferror-infinity.mjs && node pipelines/rust/tests/test-circular-honesty.mjs && node pipelines/rust/tests/test-cluster-transient-div0.mjs && node pipelines/rust/tests/test-div-nan-propagation.mjs && node pipelines/rust/tests/test-cluster-divergent-cap.mjs && node tests/cli/test-cli.mjs && node tests/cli/test-manifest-improvements.mjs && node tests/cli/test-manifest-maps.mjs && node tests/cli/test-model-type.mjs && node tests/cli/test-dedupe-inputs.mjs && node tests/cli/test-refine-label-index.mjs && node tests/cli/test-init-shared-gt.mjs && node tests/cli/test-per-sheet-eval.mjs && node pipelines/rust/tests/test-per-sheet-eval-intracycle.mjs && node pipelines/rust/tests/test-per-sheet-eval-lockstep.mjs && node pipelines/rust/tests/test-per-sheet-eval-cluster-size.mjs && node pipelines/rust/tests/test-row-chunked-modules.mjs && node pipelines/rust/tests/test-per-sheet-eval-exit-honesty.mjs && node tests/cli/test-ai-interface.mjs && node tests/cli/test-e2e4-fixes.mjs && node tests/cli/test-ship-ready.mjs && node tests/cli/use-case-suite.mjs && node tests/cli/test-onboarding.mjs && node tests/cli/test-lite-e2e.mjs && node tests/cli/test-lite-byrequest-disclosure.mjs", + "test": "node tests/lib/test-lib.mjs && node tests/lib/test-scope-plan.mjs && node tests/lib/test-cone-emit.mjs && node tests/lib/test-lite-harness.mjs && node tests/lib/test-lite-tier0.mjs && node tests/lib/test-lite-tier0-generic.mjs && node tests/lib/test-driver-scope.mjs && node tests/lib/test-tier-recommender.mjs && node tests/lib/test-lite-evaluators.mjs && node tests/lib/test-lite-surrogate.mjs && node tests/lib/test-lite-provenance.mjs && node tests/lib/test-artifact-hash.mjs && node tests/cli/test-colrow-ref.mjs && node pipelines/rust/tests/test-date-axis-sumifs.mjs && node pipelines/rust/tests/test-shared-formula-anchors.mjs && node pipelines/rust/tests/test-structure-fidelity.mjs &&node pipelines/rust/tests/test-iferror-infinity.mjs && node pipelines/rust/tests/test-circular-honesty.mjs && node pipelines/rust/tests/test-cluster-transient-div0.mjs && node pipelines/rust/tests/test-div-nan-propagation.mjs && node pipelines/rust/tests/test-cluster-divergent-cap.mjs && node tests/cli/test-cli.mjs && node tests/cli/test-manifest-improvements.mjs && node tests/cli/test-manifest-maps.mjs && node tests/cli/test-model-type.mjs && node tests/cli/test-dedupe-inputs.mjs && node tests/cli/test-refine-label-index.mjs && node tests/cli/test-init-shared-gt.mjs && node tests/cli/test-per-sheet-eval.mjs && node pipelines/rust/tests/test-per-sheet-eval-intracycle.mjs && node pipelines/rust/tests/test-per-sheet-eval-lockstep.mjs && node pipelines/rust/tests/test-per-sheet-eval-cluster-size.mjs && node pipelines/rust/tests/test-row-chunked-modules.mjs && node pipelines/rust/tests/test-per-sheet-eval-exit-honesty.mjs && node pipelines/rust/tests/test-offset-ref-bounds.mjs && node tests/cli/test-ai-interface.mjs && node tests/cli/test-e2e4-fixes.mjs && node tests/cli/test-ship-ready.mjs && node tests/cli/use-case-suite.mjs && node tests/cli/test-onboarding.mjs && node tests/cli/test-lite-e2e.mjs && node tests/cli/test-lite-byrequest-disclosure.mjs", "bench": "node benchmarks/bench.mjs" }, "devDependencies": {} diff --git a/pipelines/rust/src/chunked_emitter.rs b/pipelines/rust/src/chunked_emitter.rs index 0ffac42..d3956d6 100644 --- a/pipelines/rust/src/chunked_emitter.rs +++ b/pipelines/rust/src/chunked_emitter.rs @@ -2035,6 +2035,14 @@ function _yearfrac(start, end, basis) { return ((B.y - A.y) * 360 + (B.m - A.m) * 30 + (d2 - d1)) / 360; } +// Excel's hard sheet bounds. An address computed past them is #REF! in Excel — +// and, critically, a displacement that comes from a poisoned/diverging cell can +// be ~1e7+: materializing that rectangle is a multi-GB allocation that fatally +// OOMs the engine MID-PASS at any heap size (observed: the A-1 canonical eval +// died at ~56 min under both 12GB and 20GB caps while steady at 6.5GB heap). +// Out-of-bounds => honest NaN, exactly where Excel shows #REF!. +const _XL_MAX_ROW = 1048576, _XL_MAX_COL = 16384; + function _offsetAddr(refAddr, rows, cols) { // Pure address arithmetic for a computed range endpoint (no dereference). const m = String(refAddr).match(/^(.*)!([A-Z]+)([0-9]+)$/); @@ -2042,6 +2050,7 @@ function _offsetAddr(refAddr, rows, cols) { const col = _colNum(m[2]) + Math.trunc(+cols || 0); const row = +m[3] + Math.trunc(+rows || 0); if (col < 1 || row < 1) return refAddr; // Excel #REF!; clamp to base (finite, sweep-visible) + if (col > _XL_MAX_COL || row > _XL_MAX_ROW) return null; // Excel #REF! — _dynRange maps to NaN return m[1] + '!' + _numToCol(col) + row; } @@ -2051,6 +2060,7 @@ function _dynRange(ctx, corners) { // ctx.range(). Issue #66. let sheet = null, c1 = Infinity, r1 = Infinity, c2 = -1, r2 = -1; for (const addr of corners) { + if (addr === null) return [NaN]; // #REF! endpoint (out of sheet bounds) — poison the aggregate honestly const m = String(addr).match(/^(.*)!([A-Z]+)([0-9]+)$/); if (!m) return []; if (sheet === null) sheet = m[1]; @@ -2084,6 +2094,10 @@ function _offset(ctx, refAddr, rowOffset, colOffset, height, width) { const newCol = _colNum(col) + (+colOffset || 0); const h = +height || 1; const w = +width || 1; + // Target rectangle past Excel's sheet bounds is #REF! — and a poisoned + // offset/height of ~1e7 would otherwise allocate a value-sized array (the + // mid-pass OOM class). Honest NaN, like Excel's error. + if (newRow < 1 || newCol < 1 || newRow + h - 1 > _XL_MAX_ROW || newCol + w - 1 > _XL_MAX_COL) return NaN; if (h === 1 && w === 1) { return ctx.get(`${sheet}!${_numToCol(newCol)}${newRow}`); } diff --git a/pipelines/rust/tests/test-offset-ref-bounds.mjs b/pipelines/rust/tests/test-offset-ref-bounds.mjs new file mode 100644 index 0000000..6c03e4f --- /dev/null +++ b/pipelines/rust/tests/test-offset-ref-bounds.mjs @@ -0,0 +1,85 @@ +#!/usr/bin/env node +/** + * Regression: OFFSET / computed-endpoint helpers must answer #REF! (honest NaN) + * past Excel's sheet bounds (1,048,576 rows x 16,384 cols) — never materialize + * a value-sized rectangle. + * + * A displacement (or height/width) that comes from a poisoned/diverging cell can + * be ~1e7+. Pre-fix, _offsetAddr happily produced "Sheet!A20000001", _dynRange + * spanned the rectangle, and ctx.range() allocated it: a multi-GB allocation + * that fatally OOMed the A-1 canonical eval MID-PASS at ~56 min under BOTH a + * 12GB and a 20GB heap (cap-independent death — the alloc is proportional to + * the VALUE, not the model). Excel's own answer past the bounds is #REF!. + * + * Pre-fix RED states this test discriminates: + * - scalar OFFSET out of bounds returned a SILENT 0 (missing-cell read) + * - SUM over an out-of-bounds computed-endpoint range returned a confident + * finite number after a multi-million-row materialization + * Post-fix: all three out-of-bounds shapes are NaN; in-bounds shapes unchanged. + * + * Needs the rust-parser binary. Skips (exit 0) if it isn't built. + * + * Usage: node pipelines/rust/tests/test-offset-ref-bounds.mjs + */ + +import XLSX from 'xlsx'; +import { writeFileSync, existsSync, mkdtempSync, rmSync } from 'fs'; +import { join, dirname } from 'path'; +import { fileURLToPath, pathToFileURL } from 'url'; +import { tmpdir } from 'os'; +import { execFileSync } from 'child_process'; + +const __dir = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(__dir, '..', '..', '..'); +const exe = process.platform === 'win32' ? '.exe' : ''; +const PARSER = [ + join(ROOT, 'pipelines/rust/target/release', `rust-parser${exe}`), + join(ROOT, 'pipelines/rust/target/debug', `rust-parser${exe}`), +].find(existsSync); + +if (!PARSER) { + console.log('SKIP: rust-parser not built (cd pipelines/rust && cargo build --release)'); + process.exit(0); +} + +let passed = 0, failed = 0; +const assert = (c, m) => { if (c) { passed++; } else { failed++; console.error(` FAIL: ${m}`); } }; +const n = (v, f) => (f ? { t: 'n', v, f } : { t: 'n', v }); + +console.log('Testing: OFFSET/_dynRange answer #REF! (NaN) past sheet bounds, never a value-sized allocation'); + +const S = { + '!ref': 'A1:H3', + A1: n(1), A2: n(2), A3: n(3), + B1: n(2000000), // poisoned-scale displacement (> max row) + C1: n(0, 'SUM(A1:OFFSET(A1,B1,0))'), // computed endpoint past bounds -> NaN + D1: n(0, 'OFFSET(A1,1500000,0)'), // scalar OFFSET past bounds -> NaN (was silent 0) + E1: n(2, 'OFFSET(A1,1,0)'), // in-bounds scalar (regression) + F1: n(6, 'SUM(A1:OFFSET(A1,2,0))'), // in-bounds computed endpoint (regression) + G1: n(3, 'SUM(OFFSET(A1,0,0,2,1))'), // in-bounds multi-cell OFFSET (regression) + H1: n(0, 'SUM(OFFSET(A1,0,0,2000000,1))'), // poisoned HEIGHT -> NaN (was a 2M-row loop) +}; + +const tmp = mkdtempSync(join(tmpdir(), 'refb-')); +try { + writeFileSync(join(tmp, 'm.xlsx'), XLSX.write({ SheetNames: ['S'], Sheets: { S } }, { type: 'buffer', bookType: 'xlsx' })); + execFileSync(PARSER, [join(tmp, 'm.xlsx'), join(tmp, 'out'), '--chunked'], { encoding: 'utf-8', stdio: 'pipe' }); + const eng = await import(pathToFileURL(join(tmp, 'out', 'chunked', 'engine.js')).href); + const t0 = Date.now(); + const { values } = eng.run(); + const secs = (Date.now() - t0) / 1000; + + assert(Number.isNaN(values['S!C1']), `out-of-bounds computed-endpoint SUM is NaN (got ${values['S!C1']})`); + assert(Number.isNaN(values['S!D1']), `out-of-bounds scalar OFFSET is NaN (got ${values['S!D1']} — pre-fix: silent 0)`); + assert(Number.isNaN(values['S!H1']), `out-of-bounds OFFSET height is NaN (got ${values['S!H1']})`); + assert(values['S!E1'] === 2, `in-bounds scalar OFFSET unchanged (got ${values['S!E1']})`); + assert(values['S!F1'] === 6, `in-bounds computed-endpoint SUM unchanged (got ${values['S!F1']})`); + assert(values['S!G1'] === 3, `in-bounds multi-cell OFFSET unchanged (got ${values['S!G1']})`); + assert(secs < 5, `no value-sized materialization (run took ${secs.toFixed(1)}s — pre-fix the 2M-row spans dominate)`); +} finally { + rmSync(tmp, { recursive: true, force: true }); +} + +console.log(''); +console.log(`Results: ${passed} passed, ${failed} failed, ${passed + failed} total`); +process.exit(failed > 0 ? 1 : 0);