Adaptive decimation by default; fork the pre-3.2 uniform decimator into decimate-uniform/

README.md

@@ -110,8 +110,10 @@ Actions execute in the order specified and can be repeated. Any action may appea
                                           opacity, scale_*, f_dc_* use transformed values
                                           (linear opacity 0-1, linear scale, linear color 0-1).
                                           Append _raw for raw PLY values (e.g. opacity_raw).
--d, --decimate         <n|n%>           Simplify to n Gaussians via merge-based decimation
-                                          Use n% to keep a percentage of Gaussians.
+-d, --decimate         <n|n%>           Simplify, allocating removal by local error (adaptive)
+                                          Use n% for a percentage. --decimate-uniform removes at a uniform
+                                          rate instead (the pre-3.2 algorithm): lower memory, and better on
+                                          uniformly-sized Gaussians.
                                           Memory-bounded and streaming: scales to scenes of 100M+
                                           Gaussians. Must be the final action, and the output must
                                           be .ply (write a decimated PLY first, then convert in a

src/cli/index.ts

@@ -1,4 +1,5 @@
 import { lstat, mkdir, readFile as pathReadFile, unlink } from 'node:fs/promises';
+import { totalmem } from 'node:os';
 import { basename, dirname, join, resolve } from 'node:path';
 import process, { exit } from 'node:process';
 import { parseArgs } from 'node:util';
@@ -15,6 +16,7 @@ import {
     DataTable,
     dataTableToChunkSource,
     decimateSource,
+    decimateSourceUniform,
     fmtBytes,
     fmtCount,
     fmtTime,
@@ -66,6 +68,7 @@ interface CliOptions extends LibOptions {
     listGpus: boolean;
     deviceIdx: number;  // -1 = auto, -2 = CPU, 0+ = GPU index
     scratchDir: string | undefined;  // decimation spill location (default: output directory)
+    memoryBudgetBytes: number;  // decimation residency policy ceiling (not an allocation, not user-facing)
 }
 
 const fileExists = async (filename: string) => {
@@ -123,6 +126,13 @@ const resolveInput = (arg: string): ResolvedInput => {
 // never dispatched as a data operation).
 type CliAction = ProcessAction | { kind: 'lod'; value: number };
 
+// `--decimate` and `--decimate-uniform` both produce a decimate action, so
+// which decimator to run rides on the action itself rather than on global
+// options — that way it always describes the action actually executed, with no
+// dependence on flag ordering or on the "exactly one decimate action" check.
+// The extra field is stripped before actions reach the library.
+type CliDecimate = Extract<ProcessAction, { kind: 'decimate' }> & { uniform: boolean };
+
 // Strip the CLI-only lod tags, narrowing back to dispatchable actions.
 const stripLodTags = (actions: CliAction[]): ProcessAction[] => {
     return actions.filter((a): a is ProcessAction => a.kind !== 'lod');
@@ -188,6 +198,7 @@ const cliOptionsConfig = {
     'filter-box': { type: 'string', short: 'B', multiple: true },
     'filter-sphere': { type: 'string', short: 'S', multiple: true },
     'decimate': { type: 'string', short: 'd', multiple: true },
+    'decimate-uniform': { type: 'string', multiple: true },
     'filter-cluster': { type: 'string', short: 'C', multiple: true },
     'filter-floaters': { type: 'string', short: 'F', multiple: true },
     params: { type: 'string', short: 'p', multiple: true },
@@ -517,6 +528,10 @@ const parseArguments = async () => {
         listGpus: v['list-gpus'],
         deviceIdx,
         scratchDir: v['scratch-dir'],
+        // Residency policy ceiling for decimation (not an upfront allocation).
+        // Half the machine's RAM, capped at 48 GiB — derived here because the
+        // library is node-free and cannot read os.totalmem() itself.
+        memoryBudgetBytes: Math.min(48 * 2 ** 30, Math.floor(totalmem() / 2)),
         lodSelect: v['select-lod'].split(',').filter(v => !!v).map(parseInteger),
         viewerSettingsJson: viewerSettingsPath && await readJsonFile(viewerSettingsPath),
         unbundled: v.unbundled,
@@ -687,7 +702,8 @@ const parseArguments = async () => {
                         kind: 'mortonOrder'
                     });
                     break;
-                case 'decimate': {
+                case 'decimate':
+                case 'decimate-uniform': {
                     const value = t.value.trim();
                     let count: number | null = null;
                     let percent: number | null = null;
@@ -706,11 +722,13 @@ const parseArguments = async () => {
                         }
                     }
 
-                    current.processActions.push({
+                    const decimate: CliDecimate = {
                         kind: 'decimate',
                         count,
-                        percent
-                    });
+                        percent,
+                        uniform: t.name === 'decimate-uniform'
+                    };
+                    current.processActions.push(decimate);
                     break;
                 }
                 case 'filter-cluster': {
@@ -787,7 +805,11 @@ ACTIONS (executed in order; can be repeated)
     -S, --filter-sphere    <x,y,z,radius>   Remove Gaussians outside sphere
     -V, --filter-value     <name,cmp,value> Keep Gaussians where <name> <cmp> <value>;
                                               cmp ∈ {lt,lte,gt,gte,eq,neq}
-    -d, --decimate         <n|n%>           Simplify to n (or n%) Gaussians via merge-based decimation.
+    -d, --decimate         <n|n%>           Simplify, allocating removal by local error (adaptive; default).
+                                              Much better on mixed-scale content such as skies.
+        --decimate-uniform <n|n%>           Simplify at a uniform rate everywhere (the pre-3.2 algorithm).
+                                              Lower memory, and better at depth on uniformly-sized
+                                              Gaussians: uniform texture, single objects, snow.
                                               Must be the final action, and the output must be .ply
         --scratch-dir      <path>           Directory for decimation spill files (deep targets on huge
                                               scenes). Default: the output file's directory
@@ -1160,7 +1182,7 @@ const main = async () => {
             }
         }
         const decimateAction = decimateIdx.length === 1 ?
-            singleSceneActions[decimateIdx[0]] as Extract<ProcessAction, { kind: 'decimate' }> :
+            singleSceneActions[decimateIdx[0]] as CliDecimate :
             null;
 
         if (
@@ -1243,16 +1265,25 @@ const main = async () => {
                 if (keepCount < 1) {
                     failExit(`--decimate target resolves to ${keepCount} gaussians; must keep at least 1`);
                 }
-                combined = await decimateSource(combined, pool, {
-                    targetCount: keepCount,
-                    createDevice: deviceCreator,
-                    spill: {
-                        writeFs: new NodeFileSystem(),
-                        readFs: new NodeReadFileSystem(),
-                        scratchDir: options.scratchDir ?? dirname(outputFilename),
-                        remove: path => unlink(path)
-                    }
-                });
+                const spill = {
+                    writeFs: new NodeFileSystem(),
+                    readFs: new NodeReadFileSystem(),
+                    scratchDir: options.scratchDir ?? dirname(outputFilename),
+                    remove: (path: string) => unlink(path)
+                };
+                combined = decimateAction.uniform ?
+                    await decimateSourceUniform(combined, pool, {
+                        targetCount: keepCount,
+                        createDevice: deviceCreator,
+                        memoryBudgetBytes: options.memoryBudgetBytes,
+                        spill
+                    }) :
+                    await decimateSource(combined, pool, {
+                        targetCount: keepCount,
+                        createDevice: deviceCreator,
+                        memoryBudgetBytes: options.memoryBudgetBytes,
+                        spill
+                    });
             }
 
             logger.info(`${fmtCount(combined.meta.numGaussians)} gaussians · ${combined.meta.shBands} SH bands`);

src/lib/decimate-uniform/README.md

@@ -0,0 +1,91 @@
+# decimate-uniform — the pre-3.2 decimator
+
+The decimator that shipped up to 3.1.x, reached through `--decimate-uniform` /
+`decimateSourceUniform()`. `src/lib/decimate/` holds the adaptive one
+(`--decimate` / `decimateSource()`).
+
+The names describe how each allocates removal, not a ranking. Both are
+supported and neither is a fallback for the other: they win on different
+content, and the choice is the user's.
+
+- **uniform** — KL-style pairwise cost with a full-SH colour term, uniform 50%
+  matching per level, so every region loses the same fraction. Lower memory,
+  and measurably better at depth on scenes of uniformly-sized Gaussians:
+  uniform texture, single objects, snow. See the `old` column in
+  `scenes/DECIMATION-RESULTS.md` (leads at L3–L6 on `crop-snow` and `fr-snow`).
+- **adaptive** — field-L2 cost with the scale-free colour term and re-costed
+  selection, so removal follows local error and redundant regions collapse
+  deeper than distinct ones. Large wins on mixed-scale content, skies
+  especially (+9 to +11 dB on `fr-sky`), at higher memory cost.
+
+## The contract
+
+This directory is **bit-for-bit output-compatible with the 3.1.6 binary**. That
+is its value: a decimation you can reproduce exactly against a known-good
+reference, and the baseline every quality comparison in
+`scenes/DECIMATION-RESULTS.md` is measured against.
+
+Every file is a copy of its `src/lib/decimate/` counterpart at the last 3.1.x
+commit. You can prove it, per file:
+
+```bash
+git diff main:src/lib/decimate/select.ts src/lib/decimate-uniform/select.ts
+```
+
+Empty output means the file is untouched. Deviations are limited to these, all
+mechanical:
+
+- **Import paths.** `../gpu/gpu-edge-cost` → `./gpu-edge-cost`,
+  `../gpu/gpu-knn` → `./gpu-knn`, `./moment-match` →
+  `../decimate/moment-match`.
+- **`gpu-knn.ts`** consumes the current `FlatKdTree` (interleaved
+  `nodePositions` / `nodeChildren`) and so drops the packing loops that built
+  that same layout internally. `buildFlatKdTree` is verified structurally
+  identical to the 3.1.x `KdTree.flatten()` at every size, so the uploaded
+  bytes are unchanged.
+
+## Shared dependencies
+
+Only two, both deliberate:
+
+- `../decimate/moment-match.ts` — has no diff against 3.1.x, and its
+  `mergeGroups` worker handler is shared. Duplicating it would mean a
+  duplicate worker task for no benefit.
+- `../spatial/kd-tree.ts` — `KdTree`'s build and query paths are unchanged
+  from 3.1.x, and it is shared with k-means.
+
+Otherwise this directory imports nothing from `../decimate/`, so work on the
+adaptive path cannot change uniform output.
+
+## Changing things here
+
+Changes are fine — bug fixes, performance work, new capability — but they are
+output changes to a path whose selling point is reproducibility, so they need
+to be deliberate rather than incidental. Before landing one:
+
+- Re-run the whole-scene comparison if you expect output to be unchanged. That
+  is what `tools/decimate-parity.mjs` is for — it chains halvings through a
+  reference binary's `--decimate` and this tree's `--decimate-uniform`, compares
+  byte for byte, reports PSNR for both, and exits non-zero on any mismatch:
+
+  ```bash
+  node tools/decimate-parity.mjs sky --ref splat-transform
+  node tools/decimate-parity.mjs snow --ref splat-transform
+  ```
+
+  Equivalence was last verified against 3.1.6 on both study scenes, `fr-sky`
+  (5.81M, 3 SH bands, multi-block) and `fr-snow` (26.1M, DC only, 13 blocks),
+  six chained halvings each: every level byte-identical, PSNR matching the
+  published `old` columns exactly.
+- If output *should* change, re-baseline `scenes/DECIMATION-RESULTS.md` — the
+  `old` column is this path, and the study's conclusions are stated relative
+  to it. That document is local-only (`scenes/` is gitignored).
+- Repin the digest in `test/decimate-uniform-parity.test.mjs`, which is the
+  in-suite tripwire for accidental drift.
+
+## If it is ever retired
+
+Nothing in `src/lib/decimate/` refers to this directory, so: `rm -rf` it, drop
+`decimateSourceUniform` from `src/lib/index.ts`, drop `--decimate-uniform`
+from the CLI, and delete `test/decimate-uniform-parity.test.mjs` and
+`tools/decimate-parity.mjs`.

src/lib/decimate-uniform/block-producer.ts

@@ -0,0 +1,94 @@
+import { type ChunkData, type ChunkLayer, type ChunkSource, type ChunkSourceMetadata, type ReadRequest } from '../chunk';
+
+/**
+ * One output chunk of the merge stream. The views hold exactly `count`
+ * records at the layer strides and alias the generator's rolling scratch —
+ * valid only until the generator's next `next()`; the consumer's read copies
+ * them out (yielding views instead of slices avoids a full extra copy of
+ * every output byte).
+ */
+type ChunkPayload = {
+    count: number;
+    position: Float32Array;
+    geometric: Float32Array;
+    color: Float32Array;
+    other?: Uint32Array;
+};
+
+/**
+ * A single-sequential-pass {@link ChunkSource} over an async generator of
+ * chunk payloads — how the decimation merge stream feeds the PLY writer
+ * (or `compact` / `writePlyStreaming` for intermediate generations) without
+ * ever materializing the output.
+ *
+ * Contract: chunk reads must arrive in order (0, 1, 2, …), each at most
+ * once; gather reads are not supported. Anything else throws — decimate
+ * output supports exactly one sequential pass.
+ *
+ * @param meta - Exact output metadata (counts are known before streaming).
+ * @param produce - Factory for the payload generator (invoked lazily on first read).
+ * @returns The stream-once source.
+ */
+const createBlockProducerSource = (
+    meta: ChunkSourceMetadata,
+    produce: () => AsyncGenerator<ChunkPayload>
+): ChunkSource => {
+    let generator: AsyncGenerator<ChunkPayload> | null = null;
+    let nextChunk = 0;
+    let done = false;
+
+    const read = async (request: ReadRequest): Promise<void> => {
+        if ('indices' in request) {
+            throw new Error('decimate output supports a single sequential pass (gather reads are not available)');
+        }
+        if ((request.lod ?? 0) !== 0) {
+            throw new Error(`decimate output has a single LOD (requested lod ${request.lod})`);
+        }
+        if (request.chunkIndex !== nextChunk) {
+            throw new Error(
+                `decimate output supports a single sequential pass (expected chunk ${nextChunk}, got ${request.chunkIndex})`
+            );
+        }
+        if (done) {
+            throw new Error('decimate output exhausted');
+        }
+        generator ??= produce();
+        const { value, done: exhausted } = await generator.next();
+        if (exhausted || !value) {
+            done = true;
+            throw new Error(`decimate output ended early at chunk ${request.chunkIndex}`);
+        }
+        const payload = value;
+        const expected = Math.min(meta.chunkSize, meta.numGaussians - request.chunkIndex * meta.chunkSize);
+        if (payload.count !== expected) {
+            throw new Error(`decimate output chunk ${request.chunkIndex}: expected ${expected} rows, produced ${payload.count}`);
+        }
+
+        const fill = (cd: ChunkData | undefined, layer: ChunkLayer): void => {
+            if (!cd) return;
+            const src = payload[layer as 'position' | 'geometric' | 'color' | 'other'];
+            if (!src) {
+                throw new Error(`decimate output has no '${layer}' layer`);
+            }
+            const bytes = payload.count * cd.stride;
+            new Uint8Array(cd.data, 0, bytes).set(new Uint8Array(src.buffer, src.byteOffset, bytes));
+        };
+        fill(request.position, 'position');
+        fill(request.geometric, 'geometric');
+        fill(request.color, 'color');
+        fill(request.other, 'other');
+
+        nextChunk++;
+        if (nextChunk >= (meta.numChunks[0] ?? 0)) done = true;
+    };
+
+    const close = async (): Promise<void> => {
+        done = true;
+        await generator?.return?.(undefined as never);
+        generator = null;
+    };
+
+    return { meta, read, close };
+};
+
+export { createBlockProducerSource, type ChunkPayload };