From b9ae5ec958940bbc1bd0c4ac7a4d2f28b024d4ca Mon Sep 17 00:00:00 2001 From: atimics Date: Wed, 12 Aug 2026 09:30:21 -0700 Subject: [PATCH 01/16] feat: add qwen ilxyr acceptance experiment --- .github/workflows/ci.yml | 21 ++ assimilation/README.md | 114 ++++--- assimilation/assimilate.bat | 25 +- assimilation/assimilate.sh | 10 +- assimilation/install.bat | 25 +- assimilation/install.py | 8 +- assimilation/install.sh | 21 +- assimilation/run.py | 33 +- experiments/qwen35_acceptance/README.md | 32 ++ experiments/qwen35_acceptance/contract.py | 28 ++ experiments/qwen35_acceptance/generate.py | 264 +++++++++++++++ experiments/qwen35_acceptance/run.py | 306 ++++++++++++++++++ .../io_and_interop/holographic_galvapack.py | 49 ++- .../io_and_interop/holographic_gdnruntime.py | 143 ++++---- .../io_and_interop/holographic_measure.py | 98 ++++-- .../io_and_interop/holographic_unicron.py | 129 +++++--- tests/test_qwen_acceptance.py | 155 +++++++++ tools/build_mini_qwen.py | 96 +++++- 18 files changed, 1292 insertions(+), 265 deletions(-) create mode 100644 experiments/qwen35_acceptance/README.md create mode 100644 experiments/qwen35_acceptance/contract.py create mode 100644 experiments/qwen35_acceptance/generate.py create mode 100644 experiments/qwen35_acceptance/run.py create mode 100644 tests/test_qwen_acceptance.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 795c577..c94a3c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,27 @@ on: workflow_dispatch: # a "Run workflow" button in the Actions tab -> full suite on demand jobs: + qwen-experiment-contract: + if: (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) || github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + PYTHONHASHSEED: "0" + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + steps: + - name: Check out the code + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install Qwen contract dependencies + run: pip install numpy pytest + - name: Run the portable Qwen and ilxyr contract gates + run: pytest -q tests/test_qwen_acceptance.py + pytest: # SHARDED, because the unscoped path is the whole suite in one 20-minute box and it started timing out. # `timeout-minutes` is PER JOB, so four shard jobs give the fast path 4x the budget -- the same reasoning diff --git a/assimilation/README.md b/assimilation/README.md index e01198c..e26e0f2 100644 --- a/assimilation/README.md +++ b/assimilation/README.md @@ -1,49 +1,65 @@ -# assimilation/ -- Unicron vs Qwen3.5-0.8B, start to finish - -Three commands, run from the repo root. A private venv appears at -`assimilation/.venv` on first run; your system Python is never touched and no -Hugging Face account or token is ever needed (the weights are public and the -download is anonymous by construction). - -Linux / macOS: - - ./assimilation/assimilate.sh --eval # 1. download + assimilate + MEASURE - ./assimilation/chat.sh --both # 2. same prompt to both models, side by side - ./assimilation/chat.sh # 3. just talk to the assimilated one - -Windows (same flags, same behaviour): - - assimilation\assimilate.bat --eval - assimilation\chat.bat --both - assimilation\chat.bat - -Layout after a run: - - assimilation/work/original/ the untouched download - assimilation/work/assimilated/ the Unicron output (same tensor names/shapes, - loads exactly like the original) + per-shard - *.unicron_report.json rank reports - + *.lecore.safetensors -- the FACTORED form: - each filtered layer as its thin (U,V) pair. - This is the model's true information size - (2x smaller on the rehearsal subject; the - dense file stays full-shape only because - transformers/llama.cpp demand the original - architecture). Loads via leCore's - unicron_reconstruct; a transformers shim - that RUNS the factored form is the planned - next step. - -What "assimilate" does and why: see `holographic_unicron.assimilate_model` -- -Marchenko-Pastur filtering keeps each projection's learned spectral outliers and -drops the still-random bulk; embeddings/norms are policy-skipped; layers whose -outliers carry <1% of energy are guarded (random != useless, measured). - -The honesty contract: `--eval` prints perplexity before vs after. Until that (or -your own harness) has run, the assimilated model is an UNVERIFIED claim -- the -report says so in as many words. A bad delta is a result worth keeping, not a -failed run. - -Nothing here touches the leCore engine's dependencies: torch/transformers live -only in this folder's venv, as the measurement-and-runtime instrument. The -engine that rewrites the weights remains NumPy + stdlib. +# Qwen3.5 integration status + +Qwen loading, tokenizer/config parsing, the text-only NumPy runtime, and +diagnostics are supported integration work. The two checkpoint-changing paths +remain experiments and are opt-in. + +## Safe default: download the untouched checkpoint + +Linux/macOS: + +```bash +./assimilation/assimilate.sh +``` + +Windows: + +```bat +assimilation\assimilate.bat +``` + +This downloads public `Qwen/Qwen3.5-0.8B` weights into +`assimilation/work/original` and stops. It does not edit tensors. + +## Experimental layer-prepending installer + +The newer design prepends blank layers and installs leCore facilities into the +new layers, reserved recurrent directions, and tokenizer-unused rows. It does +not spectrally filter the original tensors. It is gated until a complete real +Qwen acceptance run succeeds: + +```bash +./assimilation/install.sh --experimental assimilation/work/original assimilation/work/installed +``` + +The acknowledgement is intentional. The last recorded full-model attempt +reached checkpoint emission but exhausted memory during verification; fixture +success is not a substitute for a real checkpoint result. + +Generate the preregistered ilxyr acceptance project with +`experiments/qwen35_acceptance/generate.py`. Its runner checks reference logits, +thousands of paired token positions with block-bootstrap confidence, peak +memory, disk reload, official text generation, and official image-input +execution. + +## Research-only spectral control + +The original Unicron spectral experiment is retained for reproduction and +negative controls only: + +```bash +./assimilation/assimilate.sh --research-spectral --eval --no-imbue +``` + +It is disabled by default because the real Qwen study did not demonstrate a +benefit: 18 of 265 eligible tensors changed, the assimilated checkpoint +regressed, repair reverted most changes, and the remaining apparent gain was +inside an underpowered confidence interval. Do not describe this path as a +proven optimization or compression result. + +## Scope boundary + +leCore's runtime executes the text stack. Vision and MTP tensors pass through +unchanged, but that is not a vision test. Any checkpoint intended for ordinary +use must also pass the official Transformers image-text smoke test in the ilxyr +acceptance contract. diff --git a/assimilation/assimilate.bat b/assimilation/assimilate.bat index faf5960..bb1149f 100644 --- a/assimilation/assimilate.bat +++ b/assimilation/assimilate.bat @@ -1,16 +1,11 @@ @echo off REM ============================================================================ -REM assimilation\assimilate.bat -- download Qwen3.5-0.8B, run Unicron's pass, -REM and (optionally) measure the result, on Windows. One command. +REM assimilation\assimilate.bat -- download Qwen3.5-0.8B. Spectral filtering +REM is retained only as an explicit research control. REM -REM assimilate.bat download, assimilate, REPAIR, -REM and build the imbued Galvatron -REM assimilate.bat --ban "words to forbid" ...with a ward baked in -REM assimilate.bat --doc mydata.txt ...grounded in YOUR data -REM assimilate.bat --refactor 0.01 ...decomposed and rebuilt -REM smaller inside a +1% budget -REM assimilate.bat --eval ...and measure before/after -REM assimilate.bat --no-imbue weights only, no Galvatron +REM assimilate.bat download untouched weights +REM assimilate.bat --research-spectral --eval +REM run the research control REM assimilate.bat --model Qwen/Qwen3.5-2B other sizes REM REM Everything installs into a private virtual environment at @@ -18,10 +13,8 @@ REM assimilation\.venv (created on first run) -- your system Python is never REM touched, and NO Hugging Face account or token is needed: the weights are REM public and the download is anonymous by construction. REM -REM When it finishes: -REM assimilation\run_galvatron.bat chat with the Galvatron -REM assimilation\galvatron.bat work\assimilated --prove --doc lecore -REM (chat.bat is the OLD torch harness and is no longer the way in.) +REM See assimilation\README.md before opting into spectral filtering or the +REM separate experimental layer-prepending installer. REM ============================================================================ setlocal title Unicron assimilation @@ -73,8 +66,8 @@ REM --- run -------------------------------------------------------------------- "%VPY%" assimilation\run.py --workdir assimilation\work %* if not errorlevel 1 ( echo. - echo Next: assimilation\run_galvatron.bat ^(chat with it^) - echo assimilation\galvatron.bat assimilation\work\assimilated --prove --doc lecore + echo Untouched weights: assimilation\work\original + echo See assimilation\README.md for explicitly gated experiment paths. ) if errorlevel 1 ( echo. diff --git a/assimilation/assimilate.sh b/assimilation/assimilate.sh index fa42b15..9056262 100644 --- a/assimilation/assimilate.sh +++ b/assimilation/assimilate.sh @@ -1,10 +1,10 @@ #!/bin/sh # ============================================================================ -# assimilation/assimilate.sh -- download Qwen3.5-0.8B, run Unicron's pass, -# and (optionally) measure the result. One command, self-contained. +# assimilation/assimilate.sh -- download Qwen3.5-0.8B. Spectral filtering is +# retained only as an explicit research control. # -# ./assimilation/assimilate.sh # download + assimilate -# ./assimilation/assimilate.sh --eval # ...and measure perplexity +# ./assimilation/assimilate.sh # download untouched weights +# ./assimilation/assimilate.sh --research-spectral --eval # ./assimilation/assimilate.sh --model Qwen/Qwen3.5-2B # other sizes # # Everything installs into a private virtual environment at @@ -12,7 +12,7 @@ # touched, and NO Hugging Face account or token is needed: the weights are # public and the download is anonymous by construction (token=False). # -# When it finishes, chat with the result: ./assimilation/chat.sh +# See assimilation/README.md before running either experimental path. # ============================================================================ set -e cd "$(dirname "$0")/.." # repo root, same convention as serve.sh diff --git a/assimilation/install.bat b/assimilation/install.bat index 87e373d..c494e4c 100644 --- a/assimilation/install.bat +++ b/assimilation/install.bat @@ -2,9 +2,9 @@ REM ============================================================ REM install.bat -- Unicron makes Galvatron. One step, no pipeline. REM -REM install.bat work\original -> work\galvatron -REM install.bat MODEL_DIR MODEL_DIR -> work\galvatron -REM install.bat MODEL_DIR OUT_DIR wherever you like +REM install.bat --experimental use defaults +REM install.bat --experimental MODEL_DIR choose source +REM install.bat --experimental MODEL_DIR OUT_DIR choose both REM REM This REPLACES assimilate -> repair -> imbue. That path edited 18 of 265 REM tensors, repair reverted 12 of them as harmful, and what survived sat @@ -30,22 +30,9 @@ set PYTHONHASHSEED=0 set "VPY=assimilation\.venv\Scripts\python.exe" if not exist "%VPY%" set "VPY=python" -set "SRC=%~1" -set "DST=%~2" -if "%SRC%"=="" set "SRC=work\original" -if "%DST%"=="" set "DST=work\galvatron" -REM NO EXISTENCE CHECK HERE ON PURPOSE. This script cd'd to the repo root, so -REM testing "%SRC%" tests the WRONG directory for any relative path -- it would -REM reject a path that is perfectly correct from where the user is standing. -REM install.py resolves it properly (caller's cwd, then repo, then work\) and -REM prints every place it looked if it truly cannot find one. -echo %SRC% -> %DST% -echo. -if "%~2"=="" ( - "%VPY%" assimilation\install.py "%SRC%" "%DST%" %2 %3 %4 %5 %6 %7 -) else ( - "%VPY%" assimilation\install.py %* -) +REM Do not pre-parse paths here. install.py owns defaults and resolves relative +REM paths against GALVATRON_CWD before the repo root. +"%VPY%" assimilation\install.py %* if errorlevel 1 ( echo. echo [!] FAILED -- the error is printed above this line. diff --git a/assimilation/install.py b/assimilation/install.py index b07ec1a..50886f3 100644 --- a/assimilation/install.py +++ b/assimilation/install.py @@ -1,6 +1,6 @@ """install.py -- put leCore into a real model, in one pass, and verify it. - python assimilation/install.py MODEL_DIR OUT_DIR [--doc FILE] [--registers N] + python assimilation/install.py --experimental MODEL_DIR OUT_DIR [--doc FILE] THIS REPLACES assimilate -> repair -> imbue. That pipeline changed 18 of 265 tensors, repair reverted 12 of them as harmful, and the surviving difference sat @@ -70,6 +70,9 @@ def _free_rows(model_dir, n_vocab, need): def main(): ap = argparse.ArgumentParser() + ap.add_argument("--experimental", action="store_true", + help="acknowledge that the layer-prepending installer has " + "not yet completed the full Qwen3.5-0.8B acceptance run") ap.add_argument("model_dir", nargs="?", default="work/original", help="the model to assimilate (default: work/original, " "resolved from where you are standing)") @@ -90,6 +93,9 @@ def main(): help="use an accelerator if one is present (default auto)") ap.add_argument("--prepend", type=int, default=2) a = ap.parse_args() + if not a.experimental: + ap.error("the layer-prepending installer is experimental; pass " + "--experimental after reviewing assimilation/README.md") from holographic.io_and_interop.holographic_gdnruntime import ( GDNRuntime, load_runtime, load_weights_dir) diff --git a/assimilation/install.sh b/assimilation/install.sh index 35f2329..1e1071d 100644 --- a/assimilation/install.sh +++ b/assimilation/install.sh @@ -1,20 +1,13 @@ #!/bin/sh # Unicron makes Galvatron. One step, no pipeline. -# ./install.sh work/original -> work/galvatron -# ./install.sh MODEL_DIR MODEL_DIR -> work/galvatron -# ./install.sh MODEL_DIR OUT_DIR wherever you like +# ./install.sh --experimental use defaults +# ./install.sh --experimental MODEL_DIR choose source +# ./install.sh --experimental MODEL_DIR OUT_DIR choose both # Optional: --doc FILE --registers N --passages N # keep the caller's directory so relative paths still mean what they say export GALVATRON_CWD="$PWD" cd "$(dirname "$0")/.." || exit 1 -SRC="${1:-work/original}" -case "$SRC" in --*) SRC="work/original";; *) [ $# -gt 0 ] && shift;; esac -DST="${1:-work/galvatron}" -case "$DST" in --*) DST="work/galvatron";; *) [ $# -gt 0 ] && shift;; esac -if [ ! -d "$SRC" ]; then - echo " [!] no model found at $SRC" - echo " run assimilate.sh first, or pass the path: ./install.sh /path/to/model" - exit 1 -fi -echo " $SRC -> $DST" -PYTHONHASHSEED=0 python3 assimilation/install.py "$SRC" "$DST" "$@" +# Do not pre-parse paths here. The launcher changes directory, while install.py +# deliberately resolves paths against GALVATRON_CWD first; checking them after +# cd was the Unix-only bug that rejected valid caller-relative model paths. +PYTHONHASHSEED=0 exec python3 assimilation/install.py "$@" diff --git a/assimilation/run.py b/assimilation/run.py index dc37b75..6b7b01d 100644 --- a/assimilation/run.py +++ b/assimilation/run.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 -"""Download Qwen3.5-0.8B, assimilate it with Unicron, and (optionally) MEASURE. +"""Download Qwen3.5-0.8B and optionally run the research-only spectral study. Run this on your own machine (needs internet access to huggingface.co): - python3 tools/run_qwen_assimilation.py # download + assimilate - python3 tools/run_qwen_assimilation.py --eval # ...and measure perplexity - python3 tools/run_qwen_assimilation.py --model Qwen/Qwen3.5-2B # other sizes work too + python3 assimilation/run.py # download only + python3 assimilation/run.py --research-spectral --eval # filter + measure + python3 assimilation/run.py --model Qwen/Qwen3.5-2B # other sizes work too Requirements: pip install numpy huggingface_hub # download + assimilate (always) pip install torch transformers # only for --eval -What happens, in order: +With --research-spectral, what happens in order: 1. DOWNLOAD the safetensors shard(s) from huggingface.co (resumable; skips files already present in --workdir). 2. ASSIMILATE each shard: Marchenko-Pastur filter per projection (keep learned @@ -27,6 +27,9 @@ output is an UNVERIFIED claim -- the report says so in as many words. Honesty notes baked in: + * Spectral filtering is a research control, not a supported optimization. The + first real Qwen run changed 18/265 tensors, regressed perplexity, and repair + reverted most changes; no statistically powered benefit has been observed. * No accuracy is promised. The spectral cut is principled (Staats/Thamm/Rosenow measured accuracy surviving it on their networks), but Qwen3.5's hybrid DeltaNet layers are new territory -- that is exactly why step 4 exists. @@ -286,10 +289,14 @@ def evaluate(orig_dir, out_dir): # -------------------------------------------------------------------------- main -def main(): - ap = argparse.ArgumentParser(description="download + assimilate + measure") +def main(argv=None): + ap = argparse.ArgumentParser(description="download Qwen; optionally run a research-only spectral experiment") ap.add_argument("--model", default="Qwen/Qwen3.5-0.8B") ap.add_argument("--workdir", default="qwen_assimilation") + ap.add_argument("--research-spectral", action="store_true", + help="opt in to the spectral filtering/repair/imbue study. " + "Off by default because no statistically powered Qwen " + "benefit has been demonstrated") ap.add_argument("--eval", action="store_true", help="measure perplexity before vs after (needs torch+transformers)") ap.add_argument("--imbue", nargs="?", const="galvatron", default="galvatron", @@ -355,10 +362,20 @@ def main(): print("Build a bundle first:") print(" galvatron.bat MODEL_DIR --imbue work/galvatron") raise SystemExit(2) - args = ap.parse_args() + args = ap.parse_args(argv) os.makedirs(args.workdir, exist_ok=True) orig_dir, shards = download(args.model, args.workdir) + if not args.research_spectral: + print("\n[research-only] spectral filtering is disabled by default.") + print("The real Qwen run did not demonstrate a statistically powered " + "benefit, so the untouched checkpoint remains the output:") + print(" %s" % os.path.abspath(orig_dir)) + print("To reproduce the negative/control experiment explicitly, add " + "--research-spectral. To test the new layer-prepending design, " + "use install.sh --experimental or generate the ilxyr acceptance " + "project in experiments/qwen35_acceptance.") + return 0 out_dir = assimilate(orig_dir, shards, os.path.join(args.workdir, "assimilated"), force=args.force) # IMBUE BY DEFAULT. Assimilation alone yields a checkpoint that has LOST diff --git a/experiments/qwen35_acceptance/README.md b/experiments/qwen35_acceptance/README.md new file mode 100644 index 0000000..b10a68a --- /dev/null +++ b/experiments/qwen35_acceptance/README.md @@ -0,0 +1,32 @@ +# Qwen3.5 acceptance experiment + +This directory turns the open Qwen integration questions into a frozen ilxyr +project. It does not claim that the full run has passed. It generates the +hypothesis, methodology contributions, experiment contract, two explicit model +forecasts, funding record, and ordered ilxyr commands for one real checkpoint. + +The contract requires: + +- exact tokenizer IDs and reference-logit parity before installation; +- at least 4,096 paired evaluation positions with a moving-block bootstrap; +- peak resident/GPU memory reporting; +- reload of the emitted checkpoint from disk; +- official Transformers text generation and image-input smoke tests; and +- spectral filtering to remain disabled. + +Generate a project from the leCore checkout: + +```bash +python experiments/qwen35_acceptance/generate.py \ + /absolute/path/to/Qwen3.5-0.8B \ + /absolute/path/to/evaluation-corpus.txt \ + /absolute/path/to/ilxyr-project +``` + +`project.json` records the exact model/corpus digests and the ordered commands +for `~/develop/ilxyr/target/debug/ilxyr`. Review the forecasts and frozen +thresholds before contributing them to an ilxyr workspace. ilxyr executes the +runner without a shell and records its strict `metrics`/`source` envelope. + +The layer-prepending installer is deliberately invoked with `--experimental`. +An accepted ilxyr outcome is the evidence needed before removing that flag. diff --git a/experiments/qwen35_acceptance/contract.py b/experiments/qwen35_acceptance/contract.py new file mode 100644 index 0000000..c580c84 --- /dev/null +++ b/experiments/qwen35_acceptance/contract.py @@ -0,0 +1,28 @@ +"""Frozen metric contract shared by the Qwen acceptance runner and generator.""" + +METRIC_SPECS = [ + {"name": "acceptance_pass", "unit": "boolean", "description": "All mandatory Qwen installation acceptance gates passed, encoded as 0 or 1."}, + {"name": "source_clean", "unit": "boolean", "description": "The tracked leCore checkout was clean when the run began, encoded as 0 or 1."}, + {"name": "spectral_filtering_enabled", "unit": "boolean", "description": "Whether research-only spectral filtering was used; the acceptance path requires 0."}, + {"name": "experimental_installer_used", "unit": "boolean", "description": "The explicitly acknowledged layer-prepending installer ran, encoded as 0 or 1."}, + {"name": "tokenizer_parity_pass", "unit": "boolean", "description": "leCore and the official Transformers tokenizer produced identical reference token IDs."}, + {"name": "reference_logit_parity_pass", "unit": "boolean", "description": "Pre-install leCore logits matched the official Transformers text model within the frozen tolerance."}, + {"name": "reference_logit_relative_error", "unit": "ratio", "description": "Maximum absolute pre-install logit error divided by the maximum absolute reference logit."}, + {"name": "eval_tokens", "unit": "tokens", "description": "Paired token positions included in the streamed evaluation."}, + {"name": "original_perplexity", "unit": "perplexity", "description": "Original checkpoint perplexity on the frozen corpus and chunking procedure."}, + {"name": "installed_perplexity", "unit": "perplexity", "description": "Installed checkpoint perplexity on the same token positions."}, + {"name": "perplexity_delta_pct", "unit": "percent", "description": "Installed minus original perplexity as a percentage of original."}, + {"name": "paired_ci_lo_nats", "unit": "nats_per_token", "description": "Lower 95 percent paired moving-block-bootstrap bound for installed minus original NLL."}, + {"name": "paired_ci_hi_nats", "unit": "nats_per_token", "description": "Upper 95 percent paired moving-block-bootstrap bound for installed minus original NLL."}, + {"name": "statistical_gate_pass", "unit": "boolean", "description": "The paired upper confidence bound stayed within the preregistered maximum regression."}, + {"name": "paired_block_length", "unit": "tokens", "description": "Moving-block length inferred from autocorrelation in paired token loss differences."}, + {"name": "paired_effective_tokens", "unit": "tokens", "description": "Effective paired sample size after serial-correlation adjustment."}, + {"name": "peak_rss_mb", "unit": "megabytes", "description": "Peak resident memory observed across the runner and installer child process."}, + {"name": "peak_gpu_mb", "unit": "megabytes", "description": "Peak accelerator allocation reported by PyTorch, or 0 when no accelerator was used."}, + {"name": "emitted_checkpoint_mb", "unit": "megabytes", "description": "Total safetensors size of the emitted installed checkpoint."}, + {"name": "reload_pass", "unit": "boolean", "description": "The emitted checkpoint reloaded from disk and produced finite logits."}, + {"name": "text_generation_pass", "unit": "boolean", "description": "The emitted checkpoint generated text through the official Transformers model."}, + {"name": "vision_smoke_pass", "unit": "boolean", "description": "The emitted checkpoint accepted a synthetic image through the official Qwen vision-language processor and generated a token."}, +] + +METRIC_NAMES = tuple(item["name"] for item in METRIC_SPECS) diff --git a/experiments/qwen35_acceptance/generate.py b/experiments/qwen35_acceptance/generate.py new file mode 100644 index 0000000..c4c28a5 --- /dev/null +++ b/experiments/qwen35_acceptance/generate.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Generate a complete ilxyr project for the Qwen3.5 acceptance experiment.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path + + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[1] +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +from contract import METRIC_NAMES, METRIC_SPECS # noqa: E402 + + +MODEL_REF = "model://openai/codex/gpt-5/2026-08-12/qwen-acceptance-design" + + +def sha256_file(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def model_manifest(model_dir): + names = ["config.json", "tokenizer.json", "vocab.json", "merges.txt"] + paths = [model_dir / name for name in names if (model_dir / name).is_file()] + paths += sorted(model_dir.glob("*.safetensors")) + if not any(path.suffix == ".safetensors" for path in paths): + raise ValueError("no safetensors checkpoint in %s" % model_dir) + records = [{"path": path.name, "sha256": sha256_file(path), + "bytes": path.stat().st_size} for path in paths] + canonical = json.dumps(records, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(canonical).hexdigest(), records + + +def actor(role): + return {"id": "model://codex/lecore-qwen/%s" % role, + "kind": "model", "model_ref": MODEL_REF + "/" + role} + + +def contribution(identifier, stage, role, title, body, inputs, claims, confidence): + return { + "schema": "ilxyr.contribution.v1", + "id": identifier, + "stage": stage, + "actor": actor(role), + "title": title, + "body": body, + "input_refs": inputs, + "claims": claims, + "confidence": confidence, + } + + +def write_json(path, payload): + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8") + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("model_dir", type=Path) + ap.add_argument("corpus", type=Path) + ap.add_argument("out_dir", type=Path, + help="directory that will receive the ilxyr project JSON") + ap.add_argument("--installed-dir", type=Path, + help="empty destination for the experiment output checkpoint") + ap.add_argument("--python", type=Path, default=Path(sys.executable)) + ap.add_argument("--min-tokens", type=int, default=4096) + ap.add_argument("--timeout-seconds", type=int, default=21600) + ap.add_argument("--compute-credits", type=int, default=100) + ap.add_argument("--ilxyr-cli", type=Path, + default=Path.home() / "develop" / "ilxyr" / "target" / "debug" / "ilxyr") + args = ap.parse_args(argv) + + model_dir = args.model_dir.expanduser().resolve() + corpus = args.corpus.expanduser().resolve() + out_dir = args.out_dir.expanduser().resolve() + installed_dir = ((args.installed_dir.expanduser().resolve()) + if args.installed_dir else out_dir / "installed-checkpoint") + python = args.python.expanduser().resolve() + if not model_dir.is_dir() or not corpus.is_file() or not python.is_file(): + ap.error("model_dir, corpus, and --python must exist") + if int(args.min_tokens) < 1000: + ap.error("--min-tokens must be at least 1000") + + model_digest, model_files = model_manifest(model_dir) + corpus_digest = sha256_file(corpus) + source_commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip() + stem = "lecore.qwen35.install.%s.%s.v1" % ( + model_digest[:12], source_commit[:12]) + ids = { + "hypothesis": stem + ".hypothesis", + "foundation": stem + ".foundation", + "engineering": stem + ".engineering-review", + "design": stem + ".experiment-design", + "experiment": stem + ".acceptance", + } + + hypothesis = contribution( + ids["hypothesis"], "hypothesis", "research-director", + "Qwen3.5 can carry a reloadable leCore layer-prepending installation", + "The experimental layer-prepending installer can emit an ordinary Qwen3.5 checkpoint while preserving reference behavior within a preregistered paired confidence bound and retaining official text and vision-language execution.", + [], + ["Pre-install leCore logits match the official reference implementation.", + "The installed checkpoint reloads and remains within the paired regression budget.", + "Official Transformers text generation and vision input both remain operational."], + 0.35) + foundation = contribution( + ids["foundation"], "mathematical_foundation", "statistical-reviewer", + "Paired moving-block inference is required for token-loss comparisons", + "Per-token losses are serially correlated. The acceptance decision therefore uses paired installed-minus-original token NLLs, estimates an autocorrelation-aware moving-block length, and requires the upper 95 percent confidence bound to stay within a one-percent perplexity regression budget. Point estimates and independent-token bootstrap intervals are not acceptance evidence.", + [ids["hypothesis"]], + ["At least the preregistered minimum number of paired token positions is measured.", + "The statistical gate is decided by the paired block interval rather than the point perplexity delta."], + 0.95) + engineering = contribution( + ids["engineering"], "engineering_review", "engineering-reviewer", + "Qwen acceptance runner and provenance boundary", + "The shell-free runner uses absolute paths, records the exact leCore commit and checker hashes, keeps spectral filtering disabled, invokes the layer-prepending path only with its experimental acknowledgement, records peak memory, reloads the emitted artifact, and exercises the official Transformers text and image-text interfaces.", + [ids["hypothesis"], ids["foundation"]], + ["The emitted stdout is exactly the ilxyr metrics/source envelope.", + "Installer logs and a human-readable metrics artifact are retained beside the output checkpoint."], + 0.85) + design = contribution( + ids["design"], "experiment_design", "experiment-designer", + "One-shot Qwen3.5 installation acceptance run", + "Execute once against the content-bound public checkpoint and corpus. Do not tune thresholds or replace the corpus after admission. Resolve accepted only when source cleanliness, tokenizer parity, reference-logit parity, the paired statistical gate, disk reload, official text generation, and official vision smoke all pass. A cleanly executed no-go is preserved as rejected evidence.", + [ids["hypothesis"], ids["foundation"], ids["engineering"]], + ["Accepted and rejected are exhaustive for a valid metrics envelope.", + "Runtime or dependency failure resolves separately as execution_failure."], + 0.95) + + experiment = { + "schema": "ilxyr.experiment.v1", + "id": ids["experiment"], + "title": "Qwen3.5 leCore layer-prepending acceptance", + "hypothesis": hypothesis["body"], + "rationale": "This converts the unresolved Qwen integration claims into one frozen, replayable, statistically gated run. Spectral filtering is a separate research control and is forbidden in this contract.", + "proposer": actor("research-director"), + "lineage": { + "hypothesis": ids["hypothesis"], + "mathematical_foundation": ids["foundation"], + "engineering_review": ids["engineering"], + "experiment_design": ids["design"], + }, + "baseline": "baseline://Qwen/Qwen3.5-0.8B/%s" % model_digest, + "datasets": ["dataset://lecore/qwen-acceptance/%s" % corpus_digest], + "models": ["weight://Qwen/Qwen3.5-0.8B/%s" % model_digest], + "metrics": METRIC_SPECS, + "seeds": [0], + "outcome_contract": { + "primary_metric": "acceptance_pass", + "success_outcome": "accepted", + "outcomes": [ + {"id": "accepted", "description": "Every frozen Qwen acceptance gate passed.", + "predicate": {"kind": "metric", "metric": "acceptance_pass", "operator": "gte", "threshold": 1}}, + {"id": "rejected", "description": "The run completed but at least one acceptance gate failed.", + "predicate": {"kind": "metric", "metric": "acceptance_pass", "operator": "lt", "threshold": 1}}, + {"id": "execution_failure", "description": "The runner failed or did not emit its exact metric contract.", + "predicate": {"kind": "execution_failure"}}, + ], + }, + "execution": { + "executor": "local-command", + "program": str(python), + "args": [str(HERE / "run.py"), str(model_dir), str(installed_dir), + str(corpus), "--min-tokens", str(int(args.min_tokens))], + "timeout_seconds": int(args.timeout_seconds), + "max_cost_credits": int(args.compute_credits), + "network": "open", + }, + "funding": {"required_compute_credits": int(args.compute_credits), + "minimum_forecasters": 2, "minimum_total_stake": 10}, + "security": {"weight_class": "public", "code_policy": "arbitrary", + "export_policy": "artifacts"}, + "evidence_authority": { + "level": "corpus_proxy", + "scope": {"seeds": [0], + "eval_set": "dataset://lecore/qwen-acceptance/%s" % corpus_digest, + "coverage": 1.0}, + # ilxyr artifact refs name objects already present in its local + # content-addressed store; raw file SHA-256 values are not object + # refs. Start empty on a fresh workspace. compile adds the four + # contribution artifacts, while the model/corpus digests stay + # frozen in their handles and the runner attests checker files. + "provenance": {"artifact_hashes": [], + "model_lineage": "model://Qwen/Qwen3.5-0.8B/%s" % model_digest, + "checker": "checker://lecore/qwen35-acceptance/v1"}, + }, + "expected_outputs": (["metrics.%s" % name for name in METRIC_NAMES] + + ["resolved_outcome", "forecast_settlements"]), + } + + forecasts = [ + {"schema": "ilxyr.forecast.v1", "id": stem + ".forecast.empirical", + "experiment_id": ids["experiment"], "forecaster": actor("forecaster-empirical"), + "probabilities": {"accepted": 0.20, "rejected": 0.65, "execution_failure": 0.15}, + "stake": 5, + "rationale": "Structural rehearsal passes, but no complete post-memory-fix real-Qwen install and powered acceptance run has succeeded yet."}, + {"schema": "ilxyr.forecast.v1", "id": stem + ".forecast-mechanistic", + "experiment_id": ids["experiment"], "forecaster": actor("forecaster-mechanistic"), + "probabilities": {"accepted": 0.15, "rejected": 0.55, "execution_failure": 0.30}, + "stake": 5, + "rationale": "The blank prepend has a strong identity construction, while full-checkpoint memory pressure and official multimodal reload remain material execution risks."}, + ] + funding = { + "schema": "ilxyr.funding.v1", "id": stem + ".funding", + "experiment_id": ids["experiment"], + "funder": {"id": "service://lecore/qwen-experiment-generator", "kind": "service"}, + "compute_credits": int(args.compute_credits), + "rationale": "Reserve one complete public-weight Qwen acceptance run under the frozen contract.", + } + + out_dir.mkdir(parents=True, exist_ok=True) + files = { + "hypothesis.json": hypothesis, + "foundation.json": foundation, + "engineering-review.json": engineering, + "experiment-design.json": design, + "experiment.json": experiment, + "forecast-empirical.json": forecasts[0], + "forecast-mechanistic.json": forecasts[1], + "funding.json": funding, + } + for name, payload in files.items(): + write_json(out_dir / name, payload) + + cli = str(args.ilxyr_cli.expanduser().resolve()) + order = ["hypothesis.json", "foundation.json", "engineering-review.json", + "experiment-design.json"] + commands = ([[cli, "contribute", "", str(out_dir / name)] for name in order] + + [[cli, "compile", "", str(out_dir / "experiment.json")], + [cli, "forecast", "", str(out_dir / "forecast-empirical.json")], + [cli, "forecast", "", str(out_dir / "forecast-mechanistic.json")], + [cli, "fund", "", str(out_dir / "funding.json")], + [cli, "admit", "", ids["experiment"]], + [cli, "run", "", ids["experiment"], "--execute"], + [cli, "status", "", ids["experiment"]], + [cli, "verify", ""]]) + write_json(out_dir / "project.json", { + "schema": "lecore.ilxyr-project.v1", "experiment_id": ids["experiment"], + "source_commit": source_commit, "model_digest": model_digest, + "model_files": model_files, "corpus_sha256": corpus_digest, + "installed_dir": str(installed_dir), "commands": commands, + }) + print(json.dumps({"project_dir": str(out_dir), + "experiment_id": ids["experiment"], + "project": str(out_dir / "project.json")}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/qwen35_acceptance/run.py b/experiments/qwen35_acceptance/run.py new file mode 100644 index 0000000..8b3ee71 --- /dev/null +++ b/experiments/qwen35_acceptance/run.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Run the full Qwen3.5 installation acceptance contract for ilxyr.""" + +from __future__ import annotations + +import argparse +import gc +import hashlib +import json +import math +import os +import platform +import subprocess +import sys +from pathlib import Path + +import numpy as np + + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[1] +for path in (str(HERE), str(REPO)): + if path not in sys.path: + sys.path.insert(0, path) + +from contract import METRIC_NAMES # noqa: E402 + + +def log(message): + print(message, file=sys.stderr, flush=True) + + +def sha256_file(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def peak_rss_mb(who): + try: + import resource + if isinstance(who, str): + who = getattr(resource, who) + raw = float(resource.getrusage(who).ru_maxrss) + return raw / (1024.0 * 1024.0) if platform.system() == "Darwin" else raw / 1024.0 + except Exception: + return 0.0 + + +def token_losses(logits, targets): + logits = np.asarray(logits, np.float64) + targets = np.asarray(targets, np.int64) + maximum = logits.max(axis=-1, keepdims=True) + lse = np.log(np.exp(logits - maximum).sum(axis=-1)) + maximum.ravel() + return lse - logits[np.arange(len(targets)), targets] + + +def streamed_measure(runtime, token_ids, chunk_size=128, resamples=800): + """Measure thousands of tokens without materializing all vocabulary logits.""" + from holographic.io_and_interop.holographic_measure import summarize_nll + + ids = list(map(int, token_ids)) + if len(ids) < 2: + raise ValueError("streamed measurement needs at least two token ids") + state = None + losses = [] + for start in range(0, len(ids), int(chunk_size)): + chunk = ids[start:start + int(chunk_size)] + if state is None: + if len(chunk) < 2: + continue + logits, state = runtime.forward(chunk, collect_state=True) + losses.append(token_losses(logits[:-1], chunk[1:])) + else: + boundary_logits = np.asarray(state.logits) + logits, state = runtime.forward(chunk, collect_state=True, resume=state) + prediction = np.concatenate([boundary_logits[None, :], logits[:-1]], axis=0) + losses.append(token_losses(prediction, chunk)) + log(" measured %d/%d input tokens" % (min(start + len(chunk), len(ids)), len(ids))) + return summarize_nll(np.concatenate(losses), resamples=resamples, seed=0) + + +def load_reference(model_dir): + import torch + try: + from transformers import AutoModelForCausalLM + return AutoModelForCausalLM.from_pretrained( + model_dir, torch_dtype=torch.float32, trust_remote_code=True).eval() + except Exception as first: + try: + from transformers import Qwen3_5ForConditionalGeneration + return Qwen3_5ForConditionalGeneration.from_pretrained( + model_dir, torch_dtype=torch.float32).eval() + except Exception: + raise RuntimeError("official Transformers text model did not load") from first + + +def reference_parity(model_dir, token_ids, tolerance): + import torch + from transformers import AutoTokenizer + from holographic.io_and_interop.holographic_bpe import BPE + from holographic.io_and_interop.holographic_gdnruntime import load_runtime + + prompt = "The holographic engine binds and bundles hypervectors." + official = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) + ref_ids = list(official.encode(prompt, add_special_tokens=False)) + own_ids = BPE.from_dir(model_dir).encode(prompt) + tokenizer_pass = ref_ids == own_ids and len(ref_ids) >= 2 + probe = ref_ids[: min(24, len(ref_ids))] if tokenizer_pass else list(token_ids[:24]) + runtime, _ = load_runtime(model_dir) + ours = np.asarray(runtime.forward(probe), np.float64) + model = load_reference(model_dir) + with torch.no_grad(): + reference = model(input_ids=torch.tensor([probe], dtype=torch.long)).logits[0] + reference = reference.detach().float().cpu().numpy() + rel = float(np.max(np.abs(ours - reference)) / + max(float(np.max(np.abs(reference))), 1e-12)) + del model, runtime, ours, reference + gc.collect() + return float(tokenizer_pass), rel, float(rel <= float(tolerance)) + + +def run_installer(model_dir, installed_dir, corpus, transcript): + command = [ + sys.executable, + str(REPO / "assimilation" / "install.py"), + "--experimental", + str(model_dir), + str(installed_dir), + "--doc", + str(corpus), + "--device", + "cpu", + ] + log("running experimental installer; transcript: %s" % transcript) + transcript.parent.mkdir(parents=True, exist_ok=True) + with transcript.open("w", encoding="utf-8") as handle: + completed = subprocess.run(command, cwd=REPO, stdout=handle, + stderr=subprocess.STDOUT, check=False) + if completed.returncode != 0: + tail = transcript.read_text(encoding="utf-8", errors="replace")[-4000:] + log(tail) + raise RuntimeError("experimental installer exited %d" % completed.returncode) + + +def official_output_smokes(installed_dir): + import torch + from PIL import Image + from transformers import AutoProcessor + try: + from transformers import AutoModelForImageTextToText + model = AutoModelForImageTextToText.from_pretrained( + installed_dir, torch_dtype=torch.float32, trust_remote_code=True).eval() + except Exception: + from transformers import Qwen3_5ForConditionalGeneration + model = Qwen3_5ForConditionalGeneration.from_pretrained( + installed_dir, torch_dtype=torch.float32).eval() + processor = AutoProcessor.from_pretrained(installed_dir, trust_remote_code=True) + + text_inputs = processor(text="Explain what a checksum protects.", return_tensors="pt") + with torch.no_grad(): + text_out = model.generate(**text_inputs, max_new_tokens=4) + text_pass = int(text_out.shape[-1] > text_inputs["input_ids"].shape[-1]) + + image = Image.new("RGB", (32, 32), color=(32, 96, 160)) + messages = [{"role": "user", "content": [ + {"type": "image", "image": image}, + {"type": "text", "text": "Name the dominant color."}, + ]}] + vision_inputs = processor.apply_chat_template( + messages, tokenize=True, add_generation_prompt=True, + return_dict=True, return_tensors="pt") + with torch.no_grad(): + vision_out = model.generate(**vision_inputs, max_new_tokens=1) + vision_pass = int(vision_out.shape[-1] > vision_inputs["input_ids"].shape[-1]) + peak_gpu = (float(torch.cuda.max_memory_allocated()) / 1e6 + if torch.cuda.is_available() else 0.0) + del model, processor, text_inputs, text_out, vision_inputs, vision_out + gc.collect() + return float(text_pass), float(vision_pass), peak_gpu + + +def source_snapshot(): + repository = subprocess.check_output( + ["git", "remote", "get-url", "origin"], cwd=REPO, text=True).strip() + commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip() + dirty = bool(subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=no"], + cwd=REPO, text=True).strip()) + paths = [ + HERE / "run.py", + HERE / "contract.py", + REPO / "assimilation" / "install.py", + REPO / "holographic" / "io_and_interop" / "holographic_measure.py", + REPO / "holographic" / "io_and_interop" / "holographic_gdnruntime.py", + ] + return float(not dirty), { + "repository": repository, + "commit": commit, + "artifacts": [{"path": str(path.relative_to(REPO)), + "sha256": sha256_file(path)} for path in paths], + } + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("model_dir", type=Path) + ap.add_argument("installed_dir", type=Path) + ap.add_argument("corpus", type=Path) + ap.add_argument("--min-tokens", type=int, default=4096) + ap.add_argument("--chunk-size", type=int, default=128) + ap.add_argument("--max-regression", type=float, default=0.01) + ap.add_argument("--logit-tolerance", type=float, default=1e-3) + args = ap.parse_args(argv) + + model_dir = args.model_dir.resolve() + installed_dir = args.installed_dir.resolve() + corpus = args.corpus.resolve() + if installed_dir.exists() and any(installed_dir.iterdir()): + raise SystemExit("installed_dir must be absent or empty: %s" % installed_dir) + if int(args.min_tokens) < 1000: + raise SystemExit("--min-tokens must be at least 1000 for an acceptance run") + + from holographic.io_and_interop.holographic_bpe import BPE + from holographic.io_and_interop.holographic_gdnruntime import load_runtime + from holographic.io_and_interop.holographic_measure import better_than + + text = corpus.read_text(encoding="utf-8", errors="replace") + token_ids = BPE.from_dir(model_dir).encode(text) + needed = int(args.min_tokens) + 1 + if len(token_ids) < needed: + raise SystemExit("corpus produced %d tokens; acceptance requires at least %d" + % (len(token_ids), needed)) + token_ids = token_ids[:needed] + + source_clean, source = source_snapshot() + tok_pass, ref_error, ref_pass = reference_parity( + model_dir, token_ids, args.logit_tolerance) + + original, _ = load_runtime(model_dir) + before = streamed_measure(original, token_ids, chunk_size=args.chunk_size) + del original + gc.collect() + + artifact_dir = installed_dir.parent / (installed_dir.name + ".acceptance") + run_installer(model_dir, installed_dir, corpus, artifact_dir / "install.log") + child_peak = peak_rss_mb("RUSAGE_CHILDREN") + + installed, _ = load_runtime(installed_dir) + reload_logits = np.asarray(installed.forward(token_ids[:16])) + reload_pass = float(np.isfinite(reload_logits).all()) + after = streamed_measure(installed, token_ids, chunk_size=args.chunk_size) + del installed, reload_logits + gc.collect() + + comparison = better_than(after, before, resamples=1200, seed=0) + regression_limit = math.log1p(float(args.max_regression)) + statistical_pass = float(comparison["ci_hi_nats"] <= regression_limit) + text_pass, vision_pass, peak_gpu = official_output_smokes(installed_dir) + + checkpoint_mb = sum(path.stat().st_size for path in + installed_dir.glob("*.safetensors")) / 1e6 + peak_rss = max(peak_rss_mb("RUSAGE_SELF"), child_peak) + required = [source_clean, tok_pass, ref_pass, statistical_pass, + reload_pass, text_pass, vision_pass] + metrics = { + "acceptance_pass": float(all(v >= 1.0 for v in required)), + "source_clean": source_clean, + "spectral_filtering_enabled": 0.0, + "experimental_installer_used": 1.0, + "tokenizer_parity_pass": tok_pass, + "reference_logit_parity_pass": ref_pass, + "reference_logit_relative_error": ref_error, + "eval_tokens": float(comparison["n_tokens"]), + "original_perplexity": float(before["perplexity"]), + "installed_perplexity": float(after["perplexity"]), + "perplexity_delta_pct": float(comparison["delta_pct"]), + "paired_ci_lo_nats": float(comparison["ci_lo_nats"]), + "paired_ci_hi_nats": float(comparison["ci_hi_nats"]), + "statistical_gate_pass": statistical_pass, + "paired_block_length": float(comparison["block"]), + "paired_effective_tokens": float(comparison["effective_n"]), + "peak_rss_mb": float(peak_rss), + "peak_gpu_mb": float(peak_gpu), + "emitted_checkpoint_mb": float(checkpoint_mb), + "reload_pass": reload_pass, + "text_generation_pass": text_pass, + "vision_smoke_pass": vision_pass, + } + if tuple(metrics) != METRIC_NAMES: + raise RuntimeError("runner metric order/contract drift: %r != %r" + % (tuple(metrics), METRIC_NAMES)) + if not all(math.isfinite(value) for value in metrics.values()): + raise RuntimeError("non-finite acceptance metric") + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "metrics.json").write_text( + json.dumps({"metrics": metrics, "source": source}, indent=2, sort_keys=True)) + print(json.dumps({"metrics": metrics, "source": source}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/holographic/io_and_interop/holographic_galvapack.py b/holographic/io_and_interop/holographic_galvapack.py index 385228f..561e6d5 100644 --- a/holographic/io_and_interop/holographic_galvapack.py +++ b/holographic/io_and_interop/holographic_galvapack.py @@ -988,44 +988,57 @@ def repair_regressions(orig_dir, assim_dir, eval_tokens, out_dir=None, Shard-by-shard assimilation cannot evaluate anything -- a partial shard will not run -- so its filter is applied blind and the damage only becomes visible after assembly (+1.79% perplexity, measured on a real Qwen3.5-0.8B). - This pass fixes that after the fact: for every tensor the transform CHANGED, - it tries walking back toward the original (alpha=0 is a full revert) and - keeps whichever blend actually scores best on the probe. - - The result cannot be worse than EITHER input on those tokens: the original - is always one of the candidates. Wherever the filtering genuinely denoised, - the gain is kept; wherever it damaged, it is undone. + This pass tests every tensor the transform CHANGED by walking back toward + the original (alpha=0 is a full revert). A blend is kept only when its + paired moving-block interval says BETTER; a lower point estimate alone is + not evidence. The final report calls the result acceptable relative to the + original only when that same paired test does not say WORSE. """ from holographic.io_and_interop.holographic_gdnruntime import ( - load_weights_dir, load_runtime, GDNRuntime) + load_weights_dir, config_from_json, GDNRuntime) + from holographic.io_and_interop.holographic_measure import measure, better_than orig = load_weights_dir(orig_dir) - rt, cfg = load_runtime(assim_dir) cur = load_weights_dir(assim_dir) + cfg = config_from_json(os.path.join(assim_dir, "config.json"), weights=cur) changed = [k for k, v in cur.items() if k in orig and getattr(v, "ndim", 0) == 2 and np.asarray(v).shape == np.asarray(orig[k]).shape and not np.array_equal(np.asarray(v), np.asarray(orig[k]))] - ppl_assim = GDNRuntime(cur, rt.cfg).perplexity(eval_tokens) - ppl_orig = GDNRuntime(orig, rt.cfg).perplexity(eval_tokens) + measured_assim = measure(GDNRuntime(cur, cfg), eval_tokens) + measured_orig = measure(GDNRuntime(orig, cfg), eval_tokens) + ppl_assim = measured_assim["perplexity"] + ppl_orig = measured_orig["perplexity"] report = {"changed": len(changed), "reverted": 0, "kept": 0, "blended": 0, "perplexity_original": ppl_orig, "perplexity_assimilated": ppl_assim, "choices": []} ppl_cur = ppl_assim + measured_cur = measured_assim for i, name in enumerate(changed): - a_orig = np.asarray(orig[name], np.float64) - a_new = np.asarray(cur[name], np.float64) + source_dtype = np.asarray(cur[name]).dtype + work_dtype = np.float64 if source_dtype == np.float64 else np.float32 + a_orig = np.asarray(orig[name], dtype=work_dtype) + a_new = np.asarray(cur[name], dtype=work_dtype) dt = np.asarray(cur[name]).dtype best_alpha, best_ppl, best_w = 1.0, ppl_cur, None + best_measure = measured_cur for alpha in strengths: # alpha=0 -> full revert cand = ((1.0 - alpha) * a_orig + alpha * a_new).astype(dt) trial = dict(cur) trial[name] = cand - p = GDNRuntime(trial, rt.cfg).perplexity(eval_tokens) - if p < best_ppl - 1e-12: - best_alpha, best_ppl, best_w = alpha, p, cand + trial_measure = measure(GDNRuntime(trial, cfg), eval_tokens) + decision = better_than(trial_measure, best_measure) + # Point-estimate selection manufactured gains on the original + # 161-token probe. Only a paired block-bootstrap BETTER verdict is + # allowed to change the checkpoint now. + if decision["verdict"] == "BETTER": + best_alpha = alpha + best_ppl = trial_measure["perplexity"] + best_w = cand + best_measure = trial_measure if best_w is not None: cur[name] = best_w ppl_cur = best_ppl + measured_cur = best_measure report["choices"].append((name, round(best_alpha, 3))) if best_alpha == 0.0: report["reverted"] += 1 @@ -1036,7 +1049,9 @@ def repair_regressions(orig_dir, assim_dir, eval_tokens, out_dir=None, if progress: progress(i, name, ppl_cur) report["perplexity_repaired"] = ppl_cur - report["beats_original"] = bool(ppl_cur <= ppl_orig + 1e-9) + vs_original = better_than(measured_cur, measured_orig) + report["comparison_to_original"] = vs_original + report["beats_original"] = vs_original["verdict"] != "WORSE" report["gain_vs_assimilated"] = ppl_assim - ppl_cur report["gain_vs_original"] = ppl_orig - ppl_cur if out_dir: diff --git a/holographic/io_and_interop/holographic_gdnruntime.py b/holographic/io_and_interop/holographic_gdnruntime.py index 2ff4765..ac50abc 100644 --- a/holographic/io_and_interop/holographic_gdnruntime.py +++ b/holographic/io_and_interop/holographic_gdnruntime.py @@ -48,23 +48,39 @@ # ------------------------------------------------------------------- primitives +def _weight_array(value): + """Keep checkpoint weights at their useful compute precision. + + BF16 is decoded by the safetensors loader to float32. Converting every such + tensor to float64 doubled resident memory without adding source precision. + Float16 is promoted to float32 for NumPy kernels; existing float32/float64 + arrays remain zero-copy. + """ + value = np.asarray(value) + if value.dtype.kind == "f" and value.dtype.itemsize >= 4: + return value + return value.astype(np.float32) + + def _rmsnorm(x, w, eps): """Qwen3Next RMSNorm is ZERO-CENTERED: y = norm(x) * (1 + w), weight init 0. Field-caught: plain `* w` matched nothing (rel err 1.0) -- the norm is where the first full-model divergence lived, masked earlier by a standalone mixer test that bypassed the norm. NOTE the asymmetry: the GATED norm below keeps plain `* w` (its weight init is ones) -- reference has both conventions.""" - x32 = x.astype(np.float64) + dtype = np.float64 if np.asarray(x).dtype == np.float64 else np.float32 + x32 = np.asarray(x, dtype=dtype) v = np.mean(x32 * x32, axis=-1, keepdims=True) return (x32 / np.sqrt(v + eps)) * (1.0 + w) def _rmsnorm_gated(x, w, gate, eps): """Norm BEFORE gate; gate goes through SiLU (reference Qwen3NextRMSNormGated).""" - x32 = x.astype(np.float64) + dtype = np.float64 if np.asarray(x).dtype == np.float64 else np.float32 + x32 = np.asarray(x, dtype=dtype) v = np.mean(x32 * x32, axis=-1, keepdims=True) y = (x32 / np.sqrt(v + eps)) * w - g = gate.astype(np.float64) + g = np.asarray(gate, dtype=dtype) return y * (g / (1.0 + np.exp(-g))) @@ -85,10 +101,10 @@ def _causal_conv_silu(x, w): Left-pad K-1 zeros: output[t] sees inputs t-K+1..t only.""" S, C = x.shape K = w.shape[-1] - xp = np.concatenate([np.zeros((K - 1, C)), x], axis=0) + xp = np.concatenate([np.zeros((K - 1, C), dtype=x.dtype), x], axis=0) # WHY spelled out: conv weight index order is w[:, 0, k] multiplying input at # offset t-(K-1)+k -- easy to flip silently. Verified against torch. - out = np.zeros((S, C)) + out = np.zeros((S, C), dtype=x.dtype) for k in range(K): out += xp[k:k + S] * w[:, 0, k][None, :] return _silu(out) @@ -111,7 +127,9 @@ def _kmeans(X, nc, iters=8, seed=0): def _rope_tables(dim, positions, theta): - inv = 1.0 / (theta ** (np.arange(0, dim, 2, dtype=np.float64) / dim)) + dtype = np.float64 if np.asarray(positions).dtype == np.float64 else np.float32 + inv = 1.0 / (np.asarray(theta, dtype=dtype) ** + (np.arange(0, dim, 2, dtype=dtype) / dim)) ang = np.outer(positions, inv) # (S, dim/2) emb = np.concatenate([ang, ang], axis=-1) # (S, dim) -- non-interleaved return np.cos(emb), np.sin(emb) @@ -191,8 +209,11 @@ def __init__(self, weights, cfg): self.w = weights emb_key = next(k for k in (self.root + "embed_tokens.weight", "model.embed_tokens.weight") if k in weights) - self.embed = np.asarray(weights[emb_key], np.float64) - self.lm_head = np.asarray(weights["lm_head.weight"], np.float64) \ + self.embed_key = emb_key + self.embed = _weight_array(weights[emb_key]) + self.compute_dtype = (np.float64 if self.embed.dtype == np.float64 + else np.float32) + self.lm_head = _weight_array(weights["lm_head.weight"]) \ if "lm_head.weight" in weights else self.embed # tied (the 0.8B case) def _g_opt(self, layer, name): @@ -205,10 +226,10 @@ def _g_opt(self, layer, name): key = self.root + "layers.%d.%s" % (layer, name) if key not in self.w: return None - return np.asarray(self.w[key], np.float64) + return _weight_array(self.w[key]) def _g(self, layer, name): - return np.asarray(self.w[self.root + "layers.%d.%s" % (layer, name)], np.float64) + return _weight_array(self.w[self.root + "layers.%d.%s" % (layer, name)]) def load_factors(self, factors): """Attach low-rank factors produced by refactor.decompose so the forward @@ -222,8 +243,7 @@ def load_factors(self, factors): rest = parts[1] layer = int(rest.split(".")[0]) name = rest.split(".", 1)[1] - self._factors[(layer, name)] = (np.asarray(A, np.float64), - np.asarray(B, np.float64)) + self._factors[(layer, name)] = (_weight_array(A), _weight_array(B)) return len(self._factors) def _has(self, layer, name): @@ -306,7 +326,8 @@ def _gdn(self, layer, x, collect=None, init=None): # continue the causal conv with the CARRIED window as left context, # instead of the zero padding a fresh sequence gets -- otherwise the # first tokens of every chunk are computed as if the stream restarted - pre = np.concatenate([np.asarray(init["conv"], np.float64), mixed_pre]) + pre = np.concatenate([np.asarray(init["conv"], dtype=mixed_pre.dtype), + mixed_pre]) mixed = _causal_conv_silu(pre, cw)[-S:] else: mixed = _causal_conv_silu(mixed_pre, cw) @@ -323,9 +344,10 @@ def _gdn(self, layer, x, collect=None, init=None): k = np.repeat(k, r, axis=1) q = _l2norm(q) * (dk ** -0.5) k = _l2norm(k) - St = np.zeros((Vh, dk, dv)) if (init is None or "S" not in init) \ - else np.array(init["S"], np.float64, copy=True) - out = np.zeros((S, Vh, dv)) + St = np.zeros((Vh, dk, dv), dtype=x.dtype) \ + if (init is None or "S" not in init) \ + else np.array(init["S"], dtype=x.dtype, copy=True) + out = np.zeros((S, Vh, dv), dtype=x.dtype) for t in range(S): St = St * np.exp(g[t])[:, None, None] kv = np.einsum("hkv,hk->hv", St, k[t]) @@ -339,7 +361,8 @@ def _gdn(self, layer, x, collect=None, init=None): if collect is not None: K = self._g(layer, "linear_attn.conv1d.weight").shape[-1] # the L1 line the step path will slide: last K-1 PRE-conv rows - pad = np.concatenate([np.zeros((K - 1, mixed_pre.shape[1])), mixed_pre]) + pad = np.concatenate([np.zeros((K - 1, mixed_pre.shape[1]), + dtype=mixed_pre.dtype), mixed_pre]) collect["conv"] = pad[-(K - 1):].copy() collect["S"] = St return y @@ -376,8 +399,8 @@ def _attn(self, layer, x, positions, collect=None, init=None): n_past = 0 if init is not None and "k" in init: n_past = int(np.asarray(init["k"]).shape[0]) - k = np.concatenate([np.asarray(init["k"], np.float64), k], axis=0) - v = np.concatenate([np.asarray(init["v"], np.float64), v], axis=0) + k = np.concatenate([np.asarray(init["k"], dtype=k.dtype), k], axis=0) + v = np.concatenate([np.asarray(init["v"], dtype=v.dtype), v], axis=0) if collect is not None: collect["k"], collect["v"] = k.copy(), v.copy() rep = H // Hkv @@ -515,7 +538,7 @@ def _attn(self, layer, x, positions, collect=None, init=None): # 0.667 -> 0.698 (r=8) at the tight setting and 0.858 -> 0.871 # (r=4) at the loose one, for r x tiny screen-scoring cost. acc = max(1, int(scr.get("accumulators", 1))) - cent = np.zeros((nblk, acc, H, hd)) + cent = np.zeros((nblk, acc, H, hd), dtype=k.dtype) for i in range(nblk): seg = k[i * blk:(i + 1) * blk] for j, kv in enumerate(seg): @@ -587,13 +610,14 @@ def to_device(self, on=True): self._dev = None return {"device": "cpu", "resident": 0, "why": "no accelerator available -- running on NumPy"} - moved = 0 - for k in list(self.w): - try: - self.w[k] = _to(np.asarray(self.w[k])) - moved += 1 - except Exception: - pass + # The default file-backed store is deliberately read-only. Build a + # resident mapping instead of assigning into it (the old loop silently + # moved zero tensors once lazy loading became the default). + resident = {k: _to(_weight_array(self.w[k])) for k in self.w} + self.w = resident + self.embed = resident[self.embed_key] + self.lm_head = resident.get("lm_head.weight", self.embed) + moved = len(resident) self._dev = xp return {"device": "gpu", "resident": moved, "why": "weights resident"} @@ -651,7 +675,7 @@ def forward(self, token_ids, hooks=None, collect_state=False, step_hooks=None, ids = np.asarray(token_ids, np.int64) h = self.embed[ids] past = int(getattr(resume, "pos", 0) or 0) if resume is not None else 0 - positions = np.arange(past, past + len(ids), dtype=np.float64) + positions = np.arange(past, past + len(ids), dtype=self.compute_dtype) st = InferenceState() if collect_state else None # LAYER SCHEDULE: which layers run, in what order, how many times. # Owning the forward pass makes depth up-scaling (SOLAR/Goliath-style @@ -686,10 +710,10 @@ def forward(self, token_ids, hooks=None, collect_state=False, step_hooks=None, if fn is not None: d = fn(h) if d is not None: - h = h + np.asarray(d, np.float64) + h = h + np.asarray(d, dtype=self.compute_dtype) nk = next(k for k in (self.root + "norm.weight", "model.norm.weight") if k in self.w) - h = _rmsnorm(h, np.asarray(self.w[nk], np.float64), c["rms_eps"]) + h = _rmsnorm(h, _weight_array(self.w[nk]), c["rms_eps"]) logits = h @ self.lm_head.T if collect_state: # ADVANCE FROM WHERE WE RESUMED, not from zero. A state carried out @@ -744,7 +768,7 @@ def _gdn_step(self, layer, x, st): mixed = np.concatenate([q.ravel(), k.ravel(), v.ravel()]) w = self._g(layer, "linear_attn.conv1d.weight") K = w.shape[-1] - win = st.setdefault("conv", np.zeros((K - 1, mixed.size))) + win = st.setdefault("conv", np.zeros((K - 1, mixed.size), dtype=mixed.dtype)) xw = np.concatenate([win, mixed[None, :]], axis=0) # (K, C) conv = _silu(np.sum(xw * w[:, 0, :].T, axis=0)) st["conv"] = xw[1:] # slide the L1 line @@ -757,7 +781,7 @@ def _gdn_step(self, layer, x, st): if r > 1: q = np.repeat(q, r, axis=0); k = np.repeat(k, r, axis=0) q = _l2norm(q) * (dk ** -0.5); k = _l2norm(k) - S = st.setdefault("S", np.zeros((Vh, dk, dv))) + S = st.setdefault("S", np.zeros((Vh, dk, dv), dtype=x.dtype)) S = S * np.exp(g)[:, None, None] kv = np.einsum("hkv,hk->hv", S, k) delta = (v - kv) * beta[:, None] @@ -791,7 +815,8 @@ def _attn_step(self, layer, x, st, pos): if _kn is not None: k = _rmsnorm(k, _kn, eps) rd = int(hd * c.get("partial_rotary_factor", 1.0)) - cos, sin = _rope_tables(rd, np.array([float(pos)]), c["rope_theta"]) + cos, sin = _rope_tables(rd, np.array([float(pos)], dtype=self.compute_dtype), + c["rope_theta"]) q2, k2 = _apply_rope(q[None], k[None], cos, sin) q, k = q2[0], k2[0] ks = np.concatenate([st["k"], k[None]], axis=0) if "k" in st else k[None] @@ -825,11 +850,11 @@ def step(self, token_id, state, hooks=None): if fn is not None: d = fn(h[None, :]) if d is not None: - h = h + np.asarray(d, np.float64).reshape(-1) + h = h + np.asarray(d, dtype=self.compute_dtype).reshape(-1) state.pos += 1 nk = next(k for k in (self.root + "norm.weight", "model.norm.weight") if k in self.w) - h = _rmsnorm(h, np.asarray(self.w[nk], np.float64), c["rms_eps"]) + h = _rmsnorm(h, _weight_array(self.w[nk]), c["rms_eps"]) state.logits = h @ self.lm_head.T return state.logits, state @@ -874,7 +899,7 @@ def extend(self, tokens, state, hooks=None): ids = np.asarray(tokens, np.int64) S = len(ids) h = self.embed[ids] - positions = np.arange(state.pos, state.pos + S, dtype=np.float64) + positions = np.arange(state.pos, state.pos + S, dtype=self.compute_dtype) for L in range(c["n_layers"]): hn = _rmsnorm(h, self._g(L, "input_layernorm.weight"), c["rms_eps"]) if self._is_gdn(L): @@ -890,11 +915,11 @@ def extend(self, tokens, state, hooks=None): if fn is not None: d = fn(h) if d is not None: - h = h + np.asarray(d, np.float64) + h = h + np.asarray(d, dtype=self.compute_dtype) state.pos += S nk = next(k for k in (self.root + "norm.weight", "model.norm.weight") if k in self.w) - h = _rmsnorm(h, np.asarray(self.w[nk], np.float64), c["rms_eps"]) + h = _rmsnorm(h, _weight_array(self.w[nk]), c["rms_eps"]) logits = h @ self.lm_head.T state.logits = logits[-1] return logits, state @@ -910,8 +935,8 @@ def forward_embeds(self, embeds, hooks=None, step_hooks=None): c = self.cfg hooks = hooks or {} step_hooks = step_hooks or {} - h = np.asarray(embeds, np.float64) - positions = np.arange(h.shape[0], dtype=np.float64) + h = np.asarray(embeds, dtype=self.compute_dtype) + positions = np.arange(h.shape[0], dtype=self.compute_dtype) sched = c.get("layer_schedule") or list(range(c["n_layers"])) for step_i, L in enumerate(sched): hn = _rmsnorm(h, self._g(L, "input_layernorm.weight"), c["rms_eps"]) @@ -926,10 +951,10 @@ def forward_embeds(self, embeds, hooks=None, step_hooks=None): if fn is not None: d = fn(h) if d is not None: - h = h + np.asarray(d, np.float64) + h = h + np.asarray(d, dtype=self.compute_dtype) nk = next(k for k in (self.root + "norm.weight", "model.norm.weight") if k in self.w) - h = _rmsnorm(h, np.asarray(self.w[nk], np.float64), c["rms_eps"]) + h = _rmsnorm(h, _weight_array(self.w[nk]), c["rms_eps"]) return h @ self.lm_head.T def token_nll(self, token_ids, hooks=None): @@ -1139,10 +1164,13 @@ def g(name): def load_runtime(model_dir, lazy=False, max_cached=8): - """THE ONE-LINER: a model directory -> a running GDNRuntime. Reads every - shard from the safetensors index (or the single file), parses config.json, - validates it against the weights, and returns the runtime. lazy=True holds - the weights as middle-out codes and decodes per tensor on demand.""" + """THE ONE-LINER: a model directory -> a running GDNRuntime. + + Shards are file-backed and tensors decode on demand. ``lazy=True`` retains + the older, explicitly lossy middle-out residency experiment; it first + materializes the mapped store because compression itself must inspect every + tensor. The default path is exact and no longer eagerly copies every shard. + """ from holographic.io_and_interop import holographic_unicron as U # A GALVATRON BUNDLE CARRIES ITS CONFIG IN galvatron.json, NOT config.json. # Every tool that loads a model went through here, so the fallback belongs @@ -1159,9 +1187,8 @@ def load_runtime(model_dir, lazy=False, max_cached=8): files = load_weight_files(model_dir) if not files: raise ValueError("no .safetensors files in %s" % model_dir) - weights = {} - for f in files: - weights.update(U.load_safetensors(os.path.join(model_dir, f))) + paths = [os.path.join(model_dir, f) for f in files] + weights = U.SafetensorWeights(paths, max_cached=max_cached) if os.path.exists(cfg_path): cfg = config_from_json(cfg_path, weights=weights) else: @@ -1178,7 +1205,7 @@ def load_runtime(model_dir, lazy=False, max_cached=8): raise ValueError("galvatron.json in %r has no usable config block" % model_dir) if lazy: - weights = U.LazyWeights(weights, max_cached=max_cached) + weights = U.LazyWeights(dict(weights.items()), max_cached=max_cached) rt = GDNRuntime(weights, cfg) _resolve_ambiguous_layout(rt, model_dir) _sanity_check(rt, model_dir) @@ -1255,12 +1282,20 @@ def load_weight_files(model_dir): return files -def load_weights_dir(model_dir): - """All weights from a model directory, sharded or single-file.""" +def load_weights_dir(model_dir, lazy=False, max_cached=8): + """All weights from a model directory, sharded or single-file. + + Transformation callers receive the historical writable dict. Inspection + and runtime callers can request the exact file-backed mapping with + ``lazy=True``. + """ from holographic.io_and_interop import holographic_unicron as U + files = [os.path.join(model_dir, f) for f in load_weight_files(model_dir)] + if lazy: + return U.SafetensorWeights(files, max_cached=max_cached) weights = {} - for f in load_weight_files(model_dir): - weights.update(U.load_safetensors(os.path.join(model_dir, f))) + for f in files: + weights.update(U.load_safetensors(f)) return weights diff --git a/holographic/io_and_interop/holographic_measure.py b/holographic/io_and_interop/holographic_measure.py index 9c3022b..0414e80 100644 --- a/holographic/io_and_interop/holographic_measure.py +++ b/holographic/io_and_interop/holographic_measure.py @@ -27,6 +27,70 @@ import numpy as np +def _block_shape(values): + """Estimate a moving-block bootstrap shape from serial correlation. + + Language-model token losses are a sequence, not independent draws. Use the + same rule for one-model intervals and paired model deltas so a comparison + cannot become more confident merely by going through ``better_than``. + """ + values = np.asarray(values, np.float64).reshape(-1) + n = len(values) + if n < 2: + return 1.0, 1 + x = values - values.mean() + denom = float(x @ x) + if not np.isfinite(denom) or denom <= 0: + return 1.0, 1 + ac = [] + for lag in range(1, min(64, n)): + rho = float((x[:-lag] @ x[lag:]) / denom) + if not np.isfinite(rho) or rho <= 0: + break + ac.append(rho) + tau = max(1.0, 1.0 + 2.0 * sum(ac)) + return tau, max(1, min(n, int(round(2.0 * tau)))) + + +def _bootstrap_means(values, rng, resamples, block): + """Moving-block bootstrap means, truncated back to the original length.""" + values = np.asarray(values, np.float64).reshape(-1) + n = len(values) + if block <= 1 or n <= 2 * block: + return np.asarray([ + rng.choice(values, n, replace=True).mean() + for _ in range(int(resamples)) + ]) + n_blocks = int(np.ceil(n / float(block))) + out = np.empty(int(resamples), np.float64) + for i in range(int(resamples)): + starts = rng.integers(0, n - block + 1, n_blocks) + sample = np.concatenate([values[s:s + block] for s in starts])[:n] + out[i] = sample.mean() + return out + + +def summarize_nll(nll, resamples=200, alpha=0.05, seed=0): + """Build the standard measurement record from precomputed token NLLs. + + Long-model evaluators can stream logits in bounded chunks and pass the NLLs + here instead of materializing ``tokens x vocabulary`` for the whole corpus. + """ + nll = np.asarray(nll, np.float64).reshape(-1) + if len(nll) < 1: + raise ValueError("summarize_nll needs at least one token loss") + rng = np.random.default_rng(int(seed)) + tau, block = _block_shape(nll) + boots = np.exp(_bootstrap_means(nll, rng, resamples, block)) + lo = float(np.percentile(boots, 100 * alpha / 2)) + hi = float(np.percentile(boots, 100 * (1 - alpha / 2))) + ppl = float(np.exp(nll.mean())) + return {"perplexity": ppl, "lo": lo, "hi": hi, "n_tokens": len(nll), + "nll": nll, "autocorr_time": float(tau), "block": int(block), + "effective_n": int(len(nll) / max(tau, 1.0)), + "half_width_pct": 100.0 * (hi - lo) / 2.0 / max(ppl, 1e-9)} + + def measure(runtime, token_ids, resamples=200, alpha=0.05, seed=0): """Perplexity AND its uncertainty, from the per-token likelihoods. @@ -51,35 +115,13 @@ def measure(runtime, token_ids, resamples=200, alpha=0.05, seed=0): # TOO NARROW (half-width 10.5% against 15.2% at block 32), and every # confidence interval this arc quoted was overconfident by that much. # The block length is derived from the measured tau rather than picked. - rng = np.random.default_rng(int(seed)) - x = nll - nll.mean() - denom = float(x @ x) or 1.0 - ac = [float((x[:-k] @ x[k:]) / denom) for k in range(1, min(16, len(x)))] - tau = 1.0 + 2.0 * sum(a for a in ac if a > 0) - block = max(1, int(round(2.0 * tau))) - n = len(nll) - if block <= 1 or n <= 2 * block: - boots = np.array([np.exp(rng.choice(nll, n, replace=True).mean()) - for _ in range(int(resamples))]) - else: - k = max(1, n // block) - boots = np.empty(int(resamples)) - for i in range(int(resamples)): - starts = rng.integers(0, n - block, k) - boots[i] = np.exp(np.concatenate( - [nll[s:s + block] for s in starts]).mean()) - lo = float(np.percentile(boots, 100 * alpha / 2)) - hi = float(np.percentile(boots, 100 * (1 - alpha / 2))) - ppl = float(np.exp(nll.mean())) - return {"perplexity": ppl, "lo": lo, "hi": hi, "n_tokens": len(nll), - "nll": nll, "autocorr_time": float(tau), "block": int(block), - "effective_n": int(len(nll) / max(tau, 1.0)), "half_width_pct": 100.0 * (hi - lo) / 2.0 / max(ppl, 1e-9)} + return summarize_nll(nll, resamples=resamples, alpha=alpha, seed=seed) def better_than(a, b, alpha=0.05, seed=0, resamples=400): """Is model A better than model B, or is the difference undecidable? - PAIRED bootstrap over the same positions -- the two models saw the same + PAIRED MOVING-BLOCK bootstrap over the same positions -- the two models saw the same tokens, so the difference per position is the statistic, and pairing removes the probe-choice variance that swamps everything otherwise. This is why a paired test can call a 2% difference while the unpaired intervals overlap by @@ -90,8 +132,8 @@ def better_than(a, b, alpha=0.05, seed=0, resamples=400): % (len(na), len(nb))) d = na - nb rng = np.random.default_rng(int(seed)) - boots = np.array([rng.choice(d, len(d), replace=True).mean() - for _ in range(int(resamples))]) + tau, block = _block_shape(d) + boots = _bootstrap_means(d, rng, resamples, block) lo = float(np.percentile(boots, 100 * alpha / 2)) hi = float(np.percentile(boots, 100 * (1 - alpha / 2))) pct = 100.0 * (a["perplexity"] - b["perplexity"]) / max(b["perplexity"], 1e-9) @@ -107,7 +149,9 @@ def better_than(a, b, alpha=0.05, seed=0, resamples=400): else: verdict = "WORSE" return {"verdict": verdict, "delta_pct": pct, "ci_lo_nats": lo, - "ci_hi_nats": hi, "n_tokens": len(d)} + "ci_hi_nats": hi, "n_tokens": len(d), + "autocorr_time": float(tau), "block": int(block), + "effective_n": int(len(d) / max(tau, 1.0))} def tokens_needed(reference, effect_pct, alpha=0.05): diff --git a/holographic/io_and_interop/holographic_unicron.py b/holographic/io_and_interop/holographic_unicron.py index ce0f472..11b333b 100644 --- a/holographic/io_and_interop/holographic_unicron.py +++ b/holographic/io_and_interop/holographic_unicron.py @@ -40,6 +40,8 @@ import struct import hashlib import zipfile +from collections import OrderedDict +from collections.abc import Mapping import zlib import tempfile @@ -68,7 +70,79 @@ def _decode_bf16(raw_u16): return u32.view(np.float32) -def load_safetensors(path, return_dtypes=False): +class SafetensorWeights(Mapping): + """Read-only, file-backed mapping over one or more safetensors shards. + + Native NumPy dtypes remain mmap views. BF16 has no NumPy dtype, so only the + tensor currently requested is decoded to float32 and held in a bounded LRU. + This is the lazy checkpoint store the runtime needs; mapping a file and then + copying every tensor, as the old loader did, is still an eager load. + """ + + def __init__(self, paths, max_cached=8): + if isinstance(paths, (str, os.PathLike)): + paths = [paths] + self._entries = OrderedDict() + self._cache = OrderedDict() + self._max_cached = max(0, int(max_cached)) + self.stats = {"hits": 0, "misses": 0, "decoded_bytes": 0} + for path in paths: + path = os.fspath(path) + with open(path, "rb") as fh: + raw = fh.read(8) + if len(raw) != 8: + raise ValueError("truncated safetensors header in %s" % path) + (hdr_len,) = struct.unpack(" self._max_cached: + self._cache.popitem(last=False) + return value + + +def load_safetensors(path, return_dtypes=False, lazy=False, max_cached=8): """Parse a .safetensors file into {name: ndarray} with stdlib + NumPy only. return_dtypes=True additionally returns {name: on-disk dtype string}, so a caller can hand it back to save_safetensors and keep the file size honest. @@ -77,44 +151,16 @@ def load_safetensors(path, return_dtypes=False): next N bytes = JSON mapping tensor name -> {dtype, shape, data_offsets}; the rest = the concatenated raw tensor bytes the offsets index into. bf16 tensors are decoded losslessly to float32 (see _decode_bf16).""" - # MEMORY-MAP THE PAYLOAD, DO NOT READ IT. `blob = f.read()` pulls the whole - # checkpoint into RAM before a single tensor is touched, which is precisely - # the anti-pattern safetensors was designed to avoid -- the format exists so - # the OS can page bytes in on demand rather than duplicating the file. - # Field-caught on a real 2.1 GB model: the install finished, the file wrote - # correctly, and reading it back for VERIFICATION died with MemoryError - # while the installed and original copies were still held. - # np.memmap is numpy-only, needs no dependency, and gives the same zero-copy - # behaviour the safetensors library gets from mmap. - with open(path, "rb") as f: - (hdr_len,) = struct.unpack("=1 the probe subspace leaks bulk energy and the top singular values bias low.""" - W = np.asarray(W, np.float64) + source = np.asarray(W) + dtype = np.float64 if source.dtype == np.float64 else np.float32 + W = np.asarray(source, dtype=dtype) m, n = W.shape k = min(k, min(m, n)) rng = np.random.default_rng(seed) - Q = np.linalg.qr(W @ rng.standard_normal((n, min(k + oversample, n))))[0] + Q = np.linalg.qr(W @ rng.standard_normal( + (n, min(k + oversample, n)), dtype=dtype))[0] for _ in range(power): Q = np.linalg.qr(W @ (W.T @ Q))[0] U_s, sv, Vt = np.linalg.svd(Q.T @ W, full_matrices=False) diff --git a/tests/test_qwen_acceptance.py b/tests/test_qwen_acceptance.py new file mode 100644 index 0000000..896561a --- /dev/null +++ b/tests/test_qwen_acceptance.py @@ -0,0 +1,155 @@ +"""Portable gates for the Qwen3.5/ilxyr experiment contract.""" + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import numpy as np + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_script(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_paired_gate_uses_blocks_for_correlated_losses(): + from holographic.io_and_interop.holographic_measure import better_than + + # Ten long correlated regions look like 500 independent observations to an + # IID bootstrap. They are only about ten independent pieces of evidence. + delta = np.repeat([-0.12] * 4 + [0.06] * 6, 50) + baseline = np.full(len(delta), 3.0) + a = {"nll": baseline + delta, + "perplexity": float(np.exp(np.mean(baseline + delta)))} + b = {"nll": baseline, "perplexity": float(np.exp(np.mean(baseline)))} + verdict = better_than(a, b, resamples=2000, seed=7) + + assert verdict["block"] > 1 + assert verdict["effective_n"] < len(delta) // 4 + assert verdict["ci_lo_nats"] <= 0 <= verdict["ci_hi_nats"] + assert verdict["verdict"] == "INDISTINGUISHABLE" + + +def test_randomized_svd_preserves_float32(): + from holographic.io_and_interop.holographic_unicron import rsvd + + matrix = np.random.default_rng(0).standard_normal((64, 48)).astype(np.float32) + u, singular, vt = rsvd(matrix, 8, seed=0, power=1) + assert u.dtype == singular.dtype == vt.dtype == np.float32 + assert np.isfinite(u).all() and np.isfinite(singular).all() and np.isfinite(vt).all() + + +def test_file_backed_safetensors_decodes_only_on_access(tmp_path): + from holographic.io_and_interop.holographic_unicron import ( + SafetensorWeights, save_safetensors) + + path = tmp_path / "weights.safetensors" + tensors = { + "f32": np.arange(24, dtype=np.float32).reshape(6, 4), + "bf16": np.linspace(-1, 1, 20, dtype=np.float32).reshape(5, 4), + } + save_safetensors(path, tensors, dtypes={"f32": "F32", "bf16": "BF16"}) + store = SafetensorWeights(path, max_cached=1) + + assert store.stats == {"hits": 0, "misses": 0, "decoded_bytes": 0} + assert isinstance(store["f32"], np.memmap) + assert store.stats["decoded_bytes"] == 0 + decoded = store["bf16"] + assert decoded.dtype == np.float32 + assert store.stats["decoded_bytes"] == decoded.nbytes + + +def build_fixture(tmp_path): + output = tmp_path / "mini-qwen" + command = [sys.executable, str(ROOT / "tools" / "build_mini_qwen.py"), + str(output), "--layers", "4", "--vocab", "512"] + completed = subprocess.run(command, cwd=tmp_path, capture_output=True, + text=True, check=False) + assert completed.returncode == 0, completed.stderr + return output, json.loads(completed.stdout) + + +def test_portable_qwen_fixture_loads_from_arbitrary_cwd(tmp_path): + from holographic.io_and_interop.holographic_bpe import BPE + from holographic.io_and_interop.holographic_gdnruntime import load_runtime + from holographic.io_and_interop.holographic_unicron import SafetensorWeights + + output, report = build_fixture(tmp_path) + tokenizer = BPE.from_dir(output) + ids = tokenizer.encode("portable Qwen fixture") + runtime, config = load_runtime(output) + + assert report["layers"] == config["n_layers"] == 4 + assert ids and tokenizer.decode(ids) == "portable Qwen fixture" + assert isinstance(runtime.w, SafetensorWeights) + assert runtime.embed.dtype == np.float32 + logits = runtime.forward(ids[:12]) + assert logits.shape[-1] == 512 and np.isfinite(logits).all() + + +def test_streamed_measure_matches_whole_forward(tmp_path): + from holographic.io_and_interop.holographic_bpe import BPE + from holographic.io_and_interop.holographic_gdnruntime import load_runtime + from holographic.io_and_interop.holographic_measure import measure + + output, _ = build_fixture(tmp_path) + ids = BPE.from_dir(output).encode("streamed evaluation stays paired " * 6) + whole_runtime, _ = load_runtime(output) + whole = measure(whole_runtime, ids, resamples=50) + runner = load_script("qwen_acceptance_runner", + ROOT / "experiments" / "qwen35_acceptance" / "run.py") + streamed_runtime, _ = load_runtime(output) + streamed = runner.streamed_measure(streamed_runtime, ids, chunk_size=11, + resamples=50) + + assert len(whole["nll"]) == len(streamed["nll"]) + assert np.allclose(whole["nll"], streamed["nll"], rtol=2e-5, atol=2e-5) + + +def test_generator_emits_complete_ilxyr_contract(tmp_path): + output, _ = build_fixture(tmp_path) + corpus = tmp_path / "corpus.txt" + corpus.write_text("Evidence must be paired, replayable, and recorded. " * 400) + project = tmp_path / "project" + command = [sys.executable, + str(ROOT / "experiments" / "qwen35_acceptance" / "generate.py"), + str(output), str(corpus), str(project), "--min-tokens", "1000"] + completed = subprocess.run(command, cwd=tmp_path, capture_output=True, + text=True, check=False) + assert completed.returncode == 0, completed.stderr + + contract = load_script("qwen_acceptance_contract", + ROOT / "experiments" / "qwen35_acceptance" / "contract.py") + experiment = json.loads((project / "experiment.json").read_text()) + manifest = json.loads((project / "project.json").read_text()) + assert [metric["name"] for metric in experiment["metrics"]] == list(contract.METRIC_NAMES) + assert experiment["execution"]["program"] == str(Path(sys.executable).resolve()) + assert "--research-spectral" not in experiment["execution"]["args"] + assert len(manifest["commands"]) == 12 + for name in ("hypothesis.json", "foundation.json", "engineering-review.json", + "experiment-design.json", "forecast-empirical.json", + "forecast-mechanistic.json", "funding.json"): + assert (project / name).is_file() + + +def test_risky_entrypoints_are_explicitly_opt_in(tmp_path, capsys): + installer = subprocess.run( + [sys.executable, str(ROOT / "assimilation" / "install.py")], + cwd=tmp_path, capture_output=True, text=True, check=False) + assert installer.returncode == 2 + assert "--experimental" in installer.stderr + + driver = load_script("qwen_assimilation_driver", ROOT / "assimilation" / "run.py") + driver.download = lambda _model, _work: (str(tmp_path), ["fixture.safetensors"]) + driver.assimilate = lambda *_a, **_k: (_ for _ in ()).throw( + AssertionError("spectral path ran without opt-in")) + assert driver.main(["--workdir", str(tmp_path)]) == 0 + assert "spectral filtering is disabled by default" in capsys.readouterr().out + assert "[ ! -d" not in (ROOT / "assimilation" / "install.sh").read_text() diff --git a/tools/build_mini_qwen.py b/tools/build_mini_qwen.py index 0df7b27..2e307d9 100644 --- a/tools/build_mini_qwen.py +++ b/tools/build_mini_qwen.py @@ -14,19 +14,63 @@ vocabulary, and bf16 on disk. Structure faithful, dimensions tiny. """ +import argparse import json import os +import sys import numpy as np +# Runnable by path from any directory, including an ilxyr executor workspace. +_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if _REPO not in sys.path: + sys.path.insert(0, _REPO) -def build(out_dir, shrink=8, vocab=2048, added=26, seed=0, layers=None, - real_config="/mnt/user-data/uploads/config.json"): + +def official_shape_config(): + """The public Qwen3.5-0.8B text shape, embedded for offline fixtures.""" + layer_types = ["linear_attention" if i % 4 != 3 else "full_attention" + for i in range(24)] + return { + "architectures": ["Qwen3_5ForConditionalGeneration"], + "model_type": "qwen3_5", + "tie_word_embeddings": True, + "text_config": { + "model_type": "qwen3_5_text", + "vocab_size": 248320, + "hidden_size": 1024, + "intermediate_size": 3584, + "num_hidden_layers": 24, + "num_attention_heads": 8, + "num_key_value_heads": 2, + "head_dim": 256, + "linear_num_value_heads": 16, + "linear_num_key_heads": 16, + "linear_key_head_dim": 128, + "linear_value_head_dim": 128, + "linear_conv_kernel_dim": 4, + "layer_types": layer_types, + "attn_output_gate": True, + "rms_norm_eps": 1e-6, + "rope_parameters": { + "rope_theta": 10000000.0, + "partial_rotary_factor": 0.25, + }, + }, + "vision_config": {"model_type": "qwen3_5_vision"}, + } + + +def build(out_dir, shrink=8, vocab=512, added=26, seed=0, layers=None, + real_config=None): """Write a miniature but structurally faithful Qwen3.5 checkpoint.""" from holographic.io_and_interop.holographic_unicron import save_safetensors - with open(real_config) as f: - real = json.load(f) + if real_config: + with open(real_config) as f: + real = json.load(f) + else: + real = official_shape_config() tc = dict(real["text_config"]) H = tc["hidden_size"] // shrink I = tc["intermediate_size"] // shrink @@ -105,23 +149,45 @@ def r(*shape): with open(os.path.join(out_dir, "config.json"), "w") as f: json.dump(cfg, f, indent=2) - # a tokenizer whose ADDED TOKENS sit above the plain vocab, like the real - # one -- this is what made "free rows" a dangerous over-count - plain = vocab - added + # A byte-complete tokenizer makes the fixture usable on arbitrary prose. + # Added tokens occupy 256..281, exactly the boundary reserved_rows must see; + # the remaining padded rows are genuinely free installation space. + if int(vocab) < 256 + int(added): + raise ValueError("vocab must be at least %d for byte tokens + added tokens" + % (256 + int(added))) + from holographic.io_and_interop.holographic_bpe import _byte_encoder + base = {_byte_encoder()[b]: b for b in range(256)} + added_tokens = [{"id": 256 + j, "content": "" % j} + for j in range(int(added))] with open(os.path.join(out_dir, "vocab.json"), "w") as f: - json.dump({"tok%d" % i: i for i in range(plain - 30)}, f) + json.dump(base, f) + with open(os.path.join(out_dir, "merges.txt"), "w") as f: + f.write("#version: 0.2\n") with open(os.path.join(out_dir, "tokenizer.json"), "w") as f: - json.dump({"model": {"vocab": {"tok%d" % i: i - for i in range(plain - 30)}}, - "added_tokens": [{"id": plain - 30 + j, - "content": "" % j} - for j in range(30)]}, f) + json.dump({"model": {"type": "BPE", "vocab": base, "merges": []}, + "added_tokens": added_tokens}, f) return {"dir": out_dir, "hidden": H, "layers": len(types), "vocab": vocab, "tensors": len(w), "megabytes": round(os.path.getsize( os.path.join(out_dir, "model.safetensors")) / 1e6, 2)} +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("out_dir") + ap.add_argument("--config", dest="real_config", + help="optional real config.json; the official 0.8B shape is embedded") + ap.add_argument("--shrink", type=int, default=8) + ap.add_argument("--vocab", type=int, default=512) + ap.add_argument("--added", type=int, default=26) + ap.add_argument("--layers", type=int, default=4, + help="keep the first N layers of the 3-linear/1-full pattern") + ap.add_argument("--seed", type=int, default=0) + args = ap.parse_args(argv) + print(json.dumps(build(args.out_dir, shrink=args.shrink, vocab=args.vocab, + added=args.added, seed=args.seed, layers=args.layers, + real_config=args.real_config), sort_keys=True)) + + if __name__ == "__main__": - import sys - print(build(sys.argv[1] if len(sys.argv) > 1 else "/tmp/mini_qwen")) + main() From e4be27bad0587336d9d154ebbfadc905dfd6fe6b Mon Sep 17 00:00:00 2001 From: docs-bot Date: Wed, 12 Aug 2026 16:31:09 +0000 Subject: [PATCH 02/16] docs: refresh generated docs (REFERENCE, CAPABILITIES, API_QUICKREF, PIPELINE_MAP, FACULTY_MAP, DOC_MAP) [skip ci] --- REFERENCE.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index 2a885b2..f0d8f8a 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -1,7 +1,7 @@ # leCore -- Code Reference *Auto-generated by `docgen.py` -- do not edit by hand; edit the module docstrings instead and re-run it.* -*702 modules, 245,989 lines of engine code.* +*702 modules, 246,132 lines of engine code.* > **New here? Read this first.** leCore represents *everything* -- memory, geometry, physics, rendering -- as > points in one very high-dimensional space (hypervectors), and combines them with a tiny algebra: **bind** @@ -315,14 +315,14 @@ | [`holographic_galvabundle.py`](#holographic-galvabundle) | GALVABUNDLE -- the model IS the engine. One directory that contains leCore, | 418 | | [`holographic_galvacache.py`](#holographic-galvacache) | GALVACACHE -- stop recomputing the same answer inside the model. | 263 | | [`holographic_galvadistill.py`](#holographic-galvadistill) | GALVADISTILL -- teach the weights to do what the residents do. | 219 | -| [`holographic_galvapack.py`](#holographic-galvapack) | GALVAPACK -- ship a Galvatron as a package, wear a normal model's clothes. | 1529 | +| [`holographic_galvapack.py`](#holographic-galvapack) | GALVAPACK -- ship a Galvatron as a package, wear a normal model's clothes. | 1544 | | [`holographic_galvaport.py`](#holographic-galvaport) | GALVAPORT -- carry as much of a Galvatron as a traditional runtime can hold. | 234 | | [`holographic_galvatron.py`](#holographic-galvatron) | GALVATRON -- leCore faculties living INSIDE a model's forward pass. | 443 | | [`holographic_gameshard.py`](#holographic-gameshard) | Authoritative game shard: a deterministic fixed-timestep world tick for building games on leCore. | 797 | | [`holographic_gas.py`](#holographic-gas) | holographic_gas.py -- T1: the GAS STATE. Pressure, volume, temperature and density tied by the ideal gas law. | 176 | | [`holographic_gatherattn.py`](#holographic-gatherattn) | GATHERATTN -- bank the routing saving instead of measuring it. | 227 | | [`holographic_gbuffer.py`](#holographic-gbuffer) | holographic_gbuffer.py -- the primary-visibility G-buffer for SDF scenes, and a clean render helper | 398 | -| [`holographic_gdnruntime.py`](#holographic-gdnruntime) | GDN RUNTIME -- a NumPy forward pass for Gated-DeltaNet hybrid language models | 1522 | +| [`holographic_gdnruntime.py`](#holographic-gdnruntime) | GDN RUNTIME -- a NumPy forward pass for Gated-DeltaNet hybrid language models | 1557 | | [`holographic_gemrender.py`](#holographic-gemrender) | SPECIMEN RENDERING -- one call from an SDF to a finished, denoised, graded image. | 388 | | [`holographic_generate.py`](#holographic-generate) | Procedural generation, four modalities, driving decoders the engine already has. | 249 | | [`holographic_generation.py`](#holographic-generation) | A context-conditioned word generator: the honest answer to 'why isn't the | 186 | @@ -422,7 +422,7 @@ | [`holographic_materialio.py`](#holographic-materialio) | PBR material interchange (G2-IO): the standard-format bridge for materials, and a VSA-native carrier. | 307 | | [`holographic_matlib.py`](#holographic-matlib) | holographic_matlib.py -- a comprehensive RENDER material library: plain diffuse -> a fractal planet. | 801 | | [`holographic_meaning_predict.py`](#holographic-meaning-predict) | Generation with structure: predict a next-MEANING vector and settle it, rather | 198 | -| [`holographic_measure.py`](#holographic-measure) | MEASURE -- perplexity with error bars, and decisions that respect them. | 195 | +| [`holographic_measure.py`](#holographic-measure) | MEASURE -- perplexity with error bars, and decisions that respect them. | 239 | | [`holographic_measure.py`](#holographic-measure) | The variance harness: every headline number gets a mean, a spread, and a confidence | 187 | | [`holographic_memory.py`](#holographic-memory) | MEMORY -- the Galvatron's own store, built on leCore's holographic database. | 241 | | [`holographic_memoryhome.py`](#holographic-memoryhome) | holographic_memoryhome.py -- the MEMORY home (consolidation backlog H6): keep the hot working set where the CP | 165 | @@ -697,7 +697,7 @@ | [`holographic_tucker.py`](#holographic-tucker) | holographic_tucker.py -- multi-way tensor compression: Tucker (HOSVD) and Tensor-Train, with a rank gate. | 654 | | [`holographic_twolayer.py`](#holographic-twolayer) | Smooth/sharp two-layer representation -- store each component in the basis it is cheap in. | 109 | | [`holographic_typed.py`](#holographic-typed) | B7 keystone -- ONE typed holographic structure. | 152 | -| [`holographic_unicron.py`](#holographic-unicron) | UNICRON -- consume trained models and read their weights informatively. | 2227 | +| [`holographic_unicron.py`](#holographic-unicron) | UNICRON -- consume trained models and read their weights informatively. | 2276 | | [`holographic_unified.py`](#holographic-unified) | One model over one holographic space. | 486 | | [`holographic_unified_p01_read.py`](#holographic-unified-p01-read) | Part 01 of UnifiedMind's faculty surface -- 97 methods, read .. assemble_pipeline. | 1480 | | [`holographic_unified_p02_fit_deterministic.py`](#holographic-unified-p02-fit-deterministic) | Part 02 of UnifiedMind's faculty surface -- 71 methods, fit_deterministic .. generate. | 1389 | @@ -10583,9 +10583,9 @@ - `class InferenceState` -- The model's MENTAL STATE as an explicit, holdable object -- the demoscene - `class GDNRuntime` -- Weights dict + config -> callable model. Tensor names follow the HF layout - `def config_from_json(cfg_json, weights)` -- Turn a Hugging Face config.json into a GDNRuntime config -- and VALIDATE -- `def load_runtime(model_dir, lazy, max_cached)` -- THE ONE-LINER: a model directory -> a running GDNRuntime. Reads every +- `def load_runtime(model_dir, lazy, max_cached)` -- THE ONE-LINER: a model directory -> a running GDNRuntime. - `def load_weight_files(model_dir)` -- Every weight shard in a model directory, in load order. -- `def load_weights_dir(model_dir)` -- All weights from a model directory, sharded or single-file. +- `def load_weights_dir(model_dir, lazy, max_cached)` -- All weights from a model directory, sharded or single-file. ### holographic_gemrender.py @@ -14856,6 +14856,7 @@ **Public API:** +- `def summarize_nll(nll, resamples, alpha, seed)` -- Build the standard measurement record from precomputed token NLLs. - `def measure(runtime, token_ids, resamples, alpha, seed)` -- Perplexity AND its uncertainty, from the per-token likelihoods. - `def better_than(a, b, alpha, seed, resamples)` -- Is model A better than model B, or is the difference undecidable? - `def tokens_needed(reference, effect_pct, alpha)` -- How many tokens would be needed to RESOLVE an effect of this size. @@ -28606,7 +28607,8 @@ **Public API:** -- `def load_safetensors(path, return_dtypes)` -- Parse a .safetensors file into {name: ndarray} with stdlib + NumPy only. +- `class SafetensorWeights` -- Read-only, file-backed mapping over one or more safetensors shards. +- `def load_safetensors(path, return_dtypes, lazy, max_cached)` -- Parse a .safetensors file into {name: ndarray} with stdlib + NumPy only. - `def save_safetensors(path, tensors, dtypes)` -- Write {name: ndarray} as .safetensors. `dtypes` maps name -> safetensors - `def load_model(path)` -- Front door: .safetensors or .npz -> {name: ndarray}. torch pickle files are - `def spectral_report(W, spacing)` -- Random-matrix readout of one weight matrix. Returns a plain dict. From 9b1faed3031501b7b040c04d851e29362123c228 Mon Sep 17 00:00:00 2001 From: atimics Date: Wed, 12 Aug 2026 10:58:58 -0700 Subject: [PATCH 03/16] fix wiring audit module detection --- tests/test_wiring_p8_p9_p10.py | 32 ++++++++++++++++++++++++++++ tools/wiring_report.py | 38 +++++++++++++++++++++++++++------- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/tests/test_wiring_p8_p9_p10.py b/tests/test_wiring_p8_p9_p10.py index 613dbf2..4978872 100644 --- a/tests/test_wiring_p8_p9_p10.py +++ b/tests/test_wiring_p8_p9_p10.py @@ -175,6 +175,38 @@ def test_a_parse_failure_is_recorded_loudly_and_not_swallowed(): del wr.PARSE_FAILURES[before:] # leave the module-level list as we found it +def test_from_package_import_records_the_imported_module(): + """A real ``from package import module`` door must count as a module edge.""" + import sys + + REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + sys.path.insert(0, os.path.join(REPO, "tools")) + import wiring_report as wr + + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "caller.py") + open(path, "w").write( + "from holographic.io_and_interop import holographic_vsaroles as roles\n") + imported = wr._imports(path) + assert "io_and_interop" in imported + assert "holographic_vsaroles" in imported + + +def test_live_wiring_report_sees_vsa_roles_and_ablation_doors(): + """Pin the two doors that exposed import-form and duplicate-basename bugs.""" + import sys + + REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + sys.path.insert(0, os.path.join(REPO, "tools")) + import wiring_report as wr + + dark, catalog_only, _kept = wr.analyse(REPO) + assert "holographic_vsaroles" not in dark + assert "holographic_ablate" not in dark + assert "holographic_vsaroles" not in catalog_only + assert "holographic_ablate" not in catalog_only + + def test_the_live_tree_parses_so_the_dark_report_can_be_trusted(): """The condition under which every other assertion in this file means anything.""" import os diff --git a/tools/wiring_report.py b/tools/wiring_report.py index eae3f5b..a7008ab 100644 --- a/tools/wiring_report.py +++ b/tools/wiring_report.py @@ -101,12 +101,29 @@ def _imports(path): elif isinstance(node, ast.ImportFrom): if node.module: names.add(node.module.split(".")[-1]) + # ``from package import module`` names BOTH the package and the + # imported child module. Counting only node.module made a real + # UnifiedMind door such as + # ``from holographic.io_and_interop import holographic_vsaroles`` + # invisible to this audit. Engine modules use the + # ``holographic_*`` prefix, which distinguishes them from imported + # functions and classes without needing the module table here. + for alias in node.names: + imported = alias.name.split(".")[-1] + if imported.startswith("holographic_"): + names.add(imported) return names def analyse(root="."): """Returns (dark, catalog_only, kept_negative): each a sorted list of module basenames.""" - modules = {} # basename -> path, engine modules only + # Several historical modules share a basename in different packages. A + # basename->single-path dict made the audit depend on os.walk insertion + # order: one checkout saw misc/holographic_measure (the real ablation + # caller), another saw io_and_interop/holographic_measure and called + # holographic_ablate dark. Aggregate every path for the coarse basename + # graph the report intentionally uses. + modules = {} # basename -> [paths], engine modules only for dotted, rel in discover_modules(root).items(): rel_slash = rel.replace("\\", "/") base = dotted.split(".")[-1] @@ -114,21 +131,26 @@ def analyse(root="."): continue if rel_slash.endswith("__init__.py"): # a package, not a module -- nothing "imports" it by name continue - modules[base] = os.path.join(root, rel) + modules.setdefault(base, []).append(os.path.join(root, rel)) importers = {b: set() for b in modules} - for base, path in modules.items(): - for target in _imports(path): - if target in importers and target != base: - importers[target].add(base) + for base, paths in modules.items(): + for path in paths: + for target in _imports(path): + if target in importers and target != base: + importers[target].add(base) dark, catalog_only, kept_negative = [], [], [] for base in sorted(modules): if base in EXEMPT: continue who = importers[base] - doc = _module_docstring(modules[base]).upper() - if any(mark in doc for mark in _NEGATIVE_MARKERS): + docs = [_module_docstring(path).upper() for path in modules[base]] + # A shared basename is a documented dead end only when EVERY module + # behind it says so. One negative sibling must not hide another live, + # unwired capability. + if docs and all(any(mark in doc for mark in _NEGATIVE_MARKERS) + for doc in docs): if not who: kept_negative.append(base) # documented dead end -- fine, but keep it visible continue From 9eefc8fe64f2839993f47490408c6a91146f8be2 Mon Sep 17 00:00:00 2001 From: atimics Date: Wed, 12 Aug 2026 11:15:02 -0700 Subject: [PATCH 04/16] refactor checkpoint IO out of unicron --- .../holographic_checkpointio.py | 298 ++++++++++++++++ .../io_and_interop/holographic_unicron.py | 317 +----------------- 2 files changed, 302 insertions(+), 313 deletions(-) create mode 100644 holographic/io_and_interop/holographic_checkpointio.py diff --git a/holographic/io_and_interop/holographic_checkpointio.py b/holographic/io_and_interop/holographic_checkpointio.py new file mode 100644 index 0000000..480948b --- /dev/null +++ b/holographic/io_and_interop/holographic_checkpointio.py @@ -0,0 +1,298 @@ +"""Safe, NumPy-only checkpoint I/O used by UNICRON and model runtimes. + +This module owns the file-format boundary so spectral analysis and model +transformation do not have to share a growing monolith with binary parsers. +Safetensors payloads are memory-mapped, BF16 values are decoded on demand, and +pickle-based torch checkpoints remain deliberately unsupported. +""" + +import json +import os +import struct +import zipfile +from collections import OrderedDict +from collections.abc import Mapping + +import numpy as np + + +# safetensors dtype strings -> numpy dtype used to read raw bytes. +# BF16 has no NumPy dtype: read as uint16, shift into a float32. +_ST_DTYPES = { + "F64": np.float64, "F32": np.float32, "F16": np.float16, + "I64": np.int64, "I32": np.int32, "I16": np.int16, "I8": np.int8, + "U8": np.uint8, "BOOL": np.bool_, +} + + +def _decode_bf16(raw_u16): + """Decode bfloat16 exactly by placing its bits in an IEEE float32.""" + u32 = raw_u16.astype(np.uint32) << 16 + return u32.view(np.float32) + + +class SafetensorWeights(Mapping): + """Read-only, file-backed mapping over one or more safetensors shards. + + Native NumPy dtypes remain mmap views. BF16 has no NumPy dtype, so only the + tensor currently requested is decoded to float32 and held in a bounded LRU. + """ + + def __init__(self, paths, max_cached=8): + if isinstance(paths, (str, os.PathLike)): + paths = [paths] + self._entries = OrderedDict() + self._cache = OrderedDict() + self._max_cached = max(0, int(max_cached)) + self.stats = {"hits": 0, "misses": 0, "decoded_bytes": 0} + for path in paths: + path = os.fspath(path) + with open(path, "rb") as fh: + raw = fh.read(8) + if len(raw) != 8: + raise ValueError("truncated safetensors header in %s" % path) + (hdr_len,) = struct.unpack(" self._max_cached: + self._cache.popitem(last=False) + return value + + +def load_safetensors(path, return_dtypes=False, lazy=False, max_cached=8): + """Parse safetensors into an eager dict or a lazy file-backed mapping.""" + store = SafetensorWeights(path, max_cached=max_cached) + if lazy: + if return_dtypes: + return store, store.dtypes + return store + out = {name: np.array(store[name], copy=True) for name in store} + if return_dtypes: + return out, store.dtypes + return out + + +def _encode_bf16(f32): + """Encode float32 as BF16 with round-to-nearest-even.""" + u32 = np.ascontiguousarray(f32, np.float32).view(np.uint32) + return ((u32 + 0x7FFF + ((u32 >> 16) & 1)) >> 16).astype(np.uint16) + + +def save_safetensors(path, tensors, dtypes=None): + """Write tensors in safetensors format while preserving declared dtypes.""" + inv = {v: k for k, v in _ST_DTYPES.items()} + dtypes = dtypes or {} + header, blobs, off = {}, [], 0 + for name in sorted(tensors): + arr = np.ascontiguousarray(tensors[name]) + want = dtypes.get(name) + if want == "BF16": + raw = _encode_bf16(arr).tobytes() + dt = "BF16" + elif want is not None and want in _ST_DTYPES: + arr = arr.astype(_ST_DTYPES[want]) + raw = arr.tobytes() + dt = want + else: + dt = inv.get(arr.dtype.type) + if dt is None: + raise ValueError("unsupported dtype for save: %r" % (arr.dtype,)) + raw = arr.tobytes() + header[name] = {"dtype": dt, "shape": list(arr.shape), + "data_offsets": [off, off + len(raw)]} + blobs.append(raw) + off += len(raw) + encoded_header = json.dumps(header, sort_keys=True).encode("utf-8") + with open(path, "wb") as fh: + fh.write(struct.pack(" (numpy dtype used to read raw bytes, post-decode) -# bf16 has no numpy dtype: read as uint16, shift into the high half of a float32. -_ST_DTYPES = { - "F64": np.float64, "F32": np.float32, "F16": np.float16, - "I64": np.int64, "I32": np.int32, "I16": np.int16, "I8": np.int8, - "U8": np.uint8, "BOOL": np.bool_, -} - - -def _decode_bf16(raw_u16): - """bfloat16 -> float32 exactly: bf16 IS the top 16 bits of an IEEE float32, - so a left shift into a uint32 reinterpreted as float32 is a lossless decode.""" - u32 = raw_u16.astype(np.uint32) << 16 - return u32.view(np.float32) - - -class SafetensorWeights(Mapping): - """Read-only, file-backed mapping over one or more safetensors shards. - - Native NumPy dtypes remain mmap views. BF16 has no NumPy dtype, so only the - tensor currently requested is decoded to float32 and held in a bounded LRU. - This is the lazy checkpoint store the runtime needs; mapping a file and then - copying every tensor, as the old loader did, is still an eager load. - """ - - def __init__(self, paths, max_cached=8): - if isinstance(paths, (str, os.PathLike)): - paths = [paths] - self._entries = OrderedDict() - self._cache = OrderedDict() - self._max_cached = max(0, int(max_cached)) - self.stats = {"hits": 0, "misses": 0, "decoded_bytes": 0} - for path in paths: - path = os.fspath(path) - with open(path, "rb") as fh: - raw = fh.read(8) - if len(raw) != 8: - raise ValueError("truncated safetensors header in %s" % path) - (hdr_len,) = struct.unpack(" self._max_cached: - self._cache.popitem(last=False) - return value - - -def load_safetensors(path, return_dtypes=False, lazy=False, max_cached=8): - """Parse a .safetensors file into {name: ndarray} with stdlib + NumPy only. - return_dtypes=True additionally returns {name: on-disk dtype string}, so a - caller can hand it back to save_safetensors and keep the file size honest. - - Format: first 8 bytes = little-endian uint64 length N of the JSON header; - next N bytes = JSON mapping tensor name -> {dtype, shape, data_offsets}; - the rest = the concatenated raw tensor bytes the offsets index into. - bf16 tensors are decoded losslessly to float32 (see _decode_bf16).""" - store = SafetensorWeights(path, max_cached=max_cached) - if lazy: - if return_dtypes: - return store, store.dtypes - return store - # The compatibility path remains a writable, eager dict for transformation - # callers. Runtime loading uses SafetensorWeights directly below. - out = {name: np.array(store[name], copy=True) for name in store} - if return_dtypes: - return out, store.dtypes - return out - - -def _encode_bf16(f32): - """float32 -> bfloat16 raw uint16, round-to-nearest-EVEN on the dropped 16 bits - (plain truncation biases every value toward zero; RNE is what hardware does). - Values already representable in bf16 round-trip exactly through decode.""" - u32 = np.ascontiguousarray(f32, np.float32).view(np.uint32) - return ((u32 + 0x7FFF + ((u32 >> 16) & 1)) >> 16).astype(np.uint16) - - -def save_safetensors(path, tensors, dtypes=None): - """Write {name: ndarray} as .safetensors. `dtypes` maps name -> safetensors - dtype string ("BF16", "F16", "F32", ...) to OVERRIDE the array's own dtype on - disk -- the round-trip fidelity fix: our loader decodes BF16 to float32 - losslessly, so without this override a load->save cycle silently DOUBLES the - file (measured live on Qwen3.5: 2x size, kept negative). Exists so the - selftest can round-trip WITHOUT any external model file.""" - inv = {v: k for k, v in _ST_DTYPES.items()} - dtypes = dtypes or {} - header, blobs, off = {}, [], 0 - for name in sorted(tensors): # sorted: byte-deterministic output - arr = np.ascontiguousarray(tensors[name]) - want = dtypes.get(name) - if want == "BF16": - raw = _encode_bf16(arr).tobytes() - dt = "BF16" - elif want is not None and want in _ST_DTYPES: - arr = arr.astype(_ST_DTYPES[want]) - raw = arr.tobytes() - dt = want - else: - dt = inv.get(arr.dtype.type) - if dt is None: - raise ValueError("unsupported dtype for save: %r" % (arr.dtype,)) - raw = arr.tobytes() - header[name] = {"dtype": dt, "shape": list(arr.shape), - "data_offsets": [off, off + len(raw)]} - blobs.append(raw) - off += len(raw) - hj = json.dumps(header, sort_keys=True).encode("utf-8") - with open(path, "wb") as f: - f.write(struct.pack(" {name: ndarray}. torch pickle files are - refused on purpose (arbitrary-code-execution surface; see module negatives).""" - p = str(path) - if p.endswith(".safetensors"): - return load_safetensors(p) - if p.endswith(".gguf"): - return load_gguf(p) - if p.endswith(".npz"): - with np.load(p) as z: - return {k: z[k] for k in z.files} - if p.endswith((".pt", ".bin", ".pth")) or zipfile.is_zipfile(p): - raise ValueError("torch pickle checkpoints are refused (unpickling is an " - "ACE surface); convert to .safetensors or .npz first") - raise ValueError("unknown model format: %s" % p) +from holographic.io_and_interop.holographic_checkpointio import ( + SafetensorWeights, _ST_DTYPES, _decode_bf16, load_gguf, load_model, + load_safetensors, save_gguf, save_safetensors, +) # --------------------------------------------------------------------------- spectra @@ -392,140 +217,6 @@ def compare_models(analysis_a, analysis_b): -# ------------------------------------------------------------------------------ gguf - -# ggml tensor type ids we DEQUANTIZE (llama.cpp convention). Everything else is -# refused BY NAME so the caller knows exactly which quant to convert upstream -- -# implementing every k-quant here would be a maintenance tax with no RMT payoff -# (the spectrum of a heavily quantized matrix is the quantizer's, not training's). -_GGML_F32, _GGML_F16, _GGML_Q8_0, _GGML_BF16 = 0, 1, 8, 30 -_GGUF_MAGIC = 0x46554747 # "GGUF" little-endian - -def _gguf_read_str(f): - """GGUF string: u64 length + raw utf-8 bytes (no terminator).""" - (n,) = struct.unpack(" Date: Wed, 12 Aug 2026 18:15:57 +0000 Subject: [PATCH 05/16] docs: refresh generated docs (REFERENCE, CAPABILITIES, API_QUICKREF, PIPELINE_MAP, FACULTY_MAP, DOC_MAP) [skip ci] --- REFERENCE.md | 31 ++++++++++++++++++++++--------- docs/DOC_MAP.md | 4 ++-- docs/PIPELINE_MAP.md | 2 +- pipelines.json | 2 +- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index f0d8f8a..164a4cd 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -1,7 +1,7 @@ # leCore -- Code Reference *Auto-generated by `docgen.py` -- do not edit by hand; edit the module docstrings instead and re-run it.* -*702 modules, 246,132 lines of engine code.* +*703 modules, 246,122 lines of engine code.* > **New here? Read this first.** leCore represents *everything* -- memory, geometry, physics, rendering -- as > points in one very high-dimensional space (hypervectors), and combines them with a tiny algebra: **bind** @@ -90,7 +90,7 @@ | [`holographic_splatprune.py`](#holographic-splatprune) | Splat prune / merge + a quality-budget LOD chain (holographic_splatprune). | 187 | | [`holographic_splatsharpen.py`](#holographic-splatsharpen) | C4 probe (cross-cutting: XDATA-3 negative-lobe sharpening -> splat/archive reconstruction). KEPT NEGATIVE. | 87 | -### Core & standalone (656) +### Core & standalone (657) | module | what it is | lines | |---|---|---| @@ -171,6 +171,7 @@ | [`holographic_chaos.py`](#holographic-chaos) | Nonlinear dynamics -- learning a chaotic flow the linear propagator structurally cannot. | 205 | | [`holographic_chart.py`](#holographic-chart) | Nonlinear manifold charts: a faithful low-D coordinate chart of a CURVED hypervector manifold | 238 | | [`holographic_checkerboard.py`](#holographic-checkerboard) | holographic_checkerboard.py -- CHECKERBOARD / SPARSE RENDERING (inverse-rendering IR13). | 152 | +| [`holographic_checkpointio.py`](#holographic-checkpointio) | Safe, NumPy-only checkpoint I/O used by UNICRON and model runtimes. | 299 | | [`holographic_chunkcodebook.py`](#holographic-chunkcodebook) | LEARNED CHUNK CODEBOOKS by iterated pair promotion (backlog item R1; R3's "one codebook family"). | 268 | | [`holographic_classifier.py`](#holographic-classifier) | holographic_classifier.py -- gradient-free, substrate-native classification (Family 2). | 131 | | [`holographic_clifford.py`](#holographic-clifford) | Geometric (Clifford) algebra Cl(3,0) as a PARALLEL binding mode (EXP-9). | 189 | @@ -697,7 +698,7 @@ | [`holographic_tucker.py`](#holographic-tucker) | holographic_tucker.py -- multi-way tensor compression: Tucker (HOSVD) and Tensor-Train, with a rank gate. | 654 | | [`holographic_twolayer.py`](#holographic-twolayer) | Smooth/sharp two-layer representation -- store each component in the basis it is cheap in. | 109 | | [`holographic_typed.py`](#holographic-typed) | B7 keystone -- ONE typed holographic structure. | 152 | -| [`holographic_unicron.py`](#holographic-unicron) | UNICRON -- consume trained models and read their weights informatively. | 2276 | +| [`holographic_unicron.py`](#holographic-unicron) | UNICRON -- consume trained models and read their weights informatively. | 1967 | | [`holographic_unified.py`](#holographic-unified) | One model over one holographic space. | 486 | | [`holographic_unified_p01_read.py`](#holographic-unified-p01-read) | Part 01 of UnifiedMind's faculty surface -- 97 methods, read .. assemble_pipeline. | 1480 | | [`holographic_unified_p02_fit_deterministic.py`](#holographic-unified-p02-fit-deterministic) | Part 02 of UnifiedMind's faculty surface -- 71 methods, fit_deterministic .. generate. | 1389 | @@ -3745,6 +3746,24 @@ - `def reconstruct_checkerboard(image, mask)` -- Fill the unshaded (mask False) pixels from their four cross-neighbours, which in a 2x2 checkerboard are all - `def render_checkerboard(sdf, camera, width, height, parity, light_dir, base_color, ambient)` -- Shade ONLY the checkerboard-masked pixels (~half the rays traced), then reconstruct the rest. Returns +### holographic_checkpointio.py + +> Safe, NumPy-only checkpoint I/O used by UNICRON and model runtimes. +> +> This module owns the file-format boundary so spectral analysis and model +> transformation do not have to share a growing monolith with binary parsers. +> Safetensors payloads are memory-mapped, BF16 values are decoded on demand, and +> pickle-based torch checkpoints remain deliberately unsupported. + +**Public API:** + +- `class SafetensorWeights` -- Read-only, file-backed mapping over one or more safetensors shards. +- `def load_safetensors(path, return_dtypes, lazy, max_cached)` -- Parse safetensors into an eager dict or a lazy file-backed mapping. +- `def save_safetensors(path, tensors, dtypes)` -- Write tensors in safetensors format while preserving declared dtypes. +- `def load_gguf(path)` -- Parse F32/F16/BF16/Q8_0 tensors from a GGUF v2+ file. +- `def save_gguf(path, tensors, quant)` -- Write a minimal GGUF v3 file with F32 and optional Q8_0 tensors. +- `def load_model(path)` -- Load safe model formats and refuse pickle-based torch checkpoints. + ### holographic_chunkcodebook.py > LEARNED CHUNK CODEBOOKS by iterated pair promotion (backlog item R1; R3's "one codebook family"). @@ -28607,17 +28626,11 @@ **Public API:** -- `class SafetensorWeights` -- Read-only, file-backed mapping over one or more safetensors shards. -- `def load_safetensors(path, return_dtypes, lazy, max_cached)` -- Parse a .safetensors file into {name: ndarray} with stdlib + NumPy only. -- `def save_safetensors(path, tensors, dtypes)` -- Write {name: ndarray} as .safetensors. `dtypes` maps name -> safetensors -- `def load_model(path)` -- Front door: .safetensors or .npz -> {name: ndarray}. torch pickle files are - `def spectral_report(W, spacing)` -- Random-matrix readout of one weight matrix. Returns a plain dict. - `def analyze_model(tensors, min_dim, spacing)` -- Run spectral_report over every >=2D tensor (matrices; higher-rank tensors are - `def fingerprint(analysis, dim)` -- One hypervector for a whole model: bundle over layers of - `def cosine(a, b)` -- Real part of the normalized Hermitian inner product -- the FHRR similarity. - `def compare_models(analysis_a, analysis_b)` -- Matched-layer metric deltas (b - a) plus fingerprint cosine. The distillation -- `def load_gguf(path)` -- Parse a GGUF file (llama.cpp models) into {name: ndarray} -- stdlib+NumPy. -- `def save_gguf(path, tensors, quant)` -- Minimal GGUF v3 writer (F32, or Q8_0 for names listed in `quant`). Exists for - `def subspace_overlap(A, B, k, side)` -- How much do two matrices' top-k singular SUBSPACES agree? Returns principal-angle - `def vector_localization(W, k)` -- WHERE does the learned information live? Porter-Thomas test on singular vectors. - `def rmt_filter(W, keep, mode)` -- RMT-guided weight filtering: keep the spectral OUTLIERS (learned signal), diff --git a/docs/DOC_MAP.md b/docs/DOC_MAP.md index 2862634..1c6f163 100644 --- a/docs/DOC_MAP.md +++ b/docs/DOC_MAP.md @@ -30,14 +30,14 @@ The generators it runs, read from that list at generation time so this page cann - `pipelinemap.py` -> `docs/PIPELINE_MAP.md`, `pipelines.json` - `tools/unifiers.py --write` -> `docs/UNIFIERS.md` -## Family layout (701 modules) +## Family layout (702 modules) ```mermaid graph LR H[holographic/] H --> misc["misc (150)"] H --> mesh["mesh_and_geometry (99)"] - H --> io_a["io_and_interop (82)"] + H --> io_a["io_and_interop (83)"] H --> agen["agents_and_reasoning (81)"] H --> rend["rendering (65)"] H --> samp["sampling_and_signal (57)"] diff --git a/docs/PIPELINE_MAP.md b/docs/PIPELINE_MAP.md index bd7a19f..e6e8705 100644 --- a/docs/PIPELINE_MAP.md +++ b/docs/PIPELINE_MAP.md @@ -2,7 +2,7 @@ *The workflow graph, auto-derived by `pipelinemap.py` from the catalog's `consumes`/`produces` tags. Nodes are io-kinds; an edge means some capability turns the source kind into the target kind. This is a VIEW of the live tags -- to change it, tag capabilities, not this file.* -> **Coverage: 110 of 3148 capabilities carry io-kind tags (3%).** The graph below is that tagged subset. Untagged capabilities are real but do not yet declare a typed edge -- backfilling tags grows the map. +> **Coverage: 110 of 3149 capabilities carry io-kind tags (3%).** The graph below is that tagged subset. Untagged capabilities are real but do not yet declare a typed edge -- backfilling tags grows the map. ```mermaid graph LR diff --git a/pipelines.json b/pipelines.json index 176355a..761b183 100644 --- a/pipelines.json +++ b/pipelines.json @@ -157,7 +157,7 @@ "coverage": { "percent": 3, "tagged": 110, - "total": 3148 + "total": 3149 }, "edges": [ { From 539a6335566785e66bac98a407a5a529ff034a63 Mon Sep 17 00:00:00 2001 From: atimics Date: Wed, 12 Aug 2026 11:29:02 -0700 Subject: [PATCH 06/16] fix deterministic quick reference resolution --- API_QUICKREF.md | 80 ++++++++++++++++++++-------------- apiquickref.py | 45 ++++++++++++++----- tests/test_build_ergonomics.py | 25 +++++++++++ 3 files changed, 106 insertions(+), 44 deletions(-) diff --git a/API_QUICKREF.md b/API_QUICKREF.md index 92e1f3c..f0a334f 100644 --- a/API_QUICKREF.md +++ b/API_QUICKREF.md @@ -17,10 +17,10 @@ - `remove_tag(self, handle, key, _record=True)` -- Remove one tag from an object. - `clear_override(self, handle, prop, _record=True)` -- Remove one render override from an object, so it FALLS BACK to the scene default. - `select(self, handles)` -- Set the current selection (a set of handles) and fire a 'select' event. - - `get(self, handle)` -- + - `get(self, handle)` - `set_parent(self, child, parent, _record=True)` -- Parent `child` under `parent` (both handles; parent=None for top level). - `parent_of(self, handle)` -- The parent handle of an object (None if top level). - - `children_of(self, handle)` -- + - `children_of(self, handle)` - `begin_group(self, label='Edit')` -- Open a transaction: every mutation until end_group() coalesces into ONE undo step labelled `label`. - `end_group(self)` -- Close the current transaction; commit the accumulated changes as a single step (nothing if empty). - `group(self, label='Edit')` -- `with scene.group("Move wheels"): ...` -- everything inside becomes one undo step. @@ -28,8 +28,8 @@ - `redo(self)` -- Redo the last undone step by restoring its AFTER snapshots. - `history(self)` -- The undo stack's step labels, oldest first. - `redo_history(self)` -- The redo stack's step labels (most-recently-undone last). - - `can_undo(self)` -- - - `can_redo(self)` -- + - `can_undo(self)` + - `can_redo(self)` - `scene_info(scene, verbose=True)` -- WHAT IS IN THIS SCENE -- the first call to make, before adding to it or rendering it. ### `holographic_modifier` @@ -37,13 +37,13 @@ - **class `Modifier`** -- One entry in the stack: a named operation with parameters, applied non-destructively to the previous result. - **class `ModifierStack`** -- A per-object modifier stack: a base payload + an ordered list of modifiers, evaluated non-destructively, re-evaluated O(change) (only downstream of a change), with stable handles and validation. - - `handles(self)` -- - - `names(self)` -- + - `handles(self)` + - `names(self)` - `add(self, name, op, params=None, specs=None, muted=False)` -- Append a modifier to the top of the stack; returns its stable handle. - `insert(self, index, name, op, params=None, specs=None, muted=False)` -- Insert a modifier at `index` (everything from there down must recompute). - - `remove(self, handle)` -- + - `remove(self, handle)` - `move(self, handle, to_index)` -- Reorder a modifier (a real modeling operation -- bevel-then-subdivide differs from the reverse). - - `set_muted(self, handle, muted)` -- + - `set_muted(self, handle, muted)` - `set_param(self, handle, **params)` -- Change a modifier's parameters -- the common case a dependency graph optimises. - `evaluate(self)` -- Fold the stack over the base, non-destructively. - `validate(self)` -- Well-formedness (like recipeops.validate): every op callable, and every declared param within its min/max. @@ -58,20 +58,20 @@ - `as_eval(sdf)` -- Return a plain callable `P:(M,D) -> distances:(M,)` for ANY of the engine's three ways of naming an SDF: * a node object with `.eval(P)` -- what `sphere()`/`box()`/`parse_dsl()` build * a bare callable -- what `collide`, `emitter` and every ad-hoc lambda pass around * a DSL STRING, e.g. - `sdf_normal(sdf, P, eps=0.001)` -- The surface normal at points P:(M,3) = the normalised gradient of the SDF, by central differences (6 vectorised evals). - **class `SDF`** -- A node in a signed-distance expression tree: `kind`, scalar `params`, and child SDFs. - - `eval(self, P)` -- - - `union(self, other)` -- - - `intersect(self, other)` -- - - `subtract(self, other)` -- - - `smooth_union(self, other, k=0.3)` -- - - `fillet_union(self, other, r=0.1)` -- - - `translate(self, t)` -- - - `scale(self, s)` -- - - `rotate(self, axis, angle)` -- - - `repeat(self, period)` -- - - `rounded(self, r)` -- - - `onion(self, thickness)` -- - - `displace(self, amount, freq)` -- - - `twist(self, k)` -- + - `eval(self, P)` + - `union(self, other)` + - `intersect(self, other)` + - `subtract(self, other)` + - `smooth_union(self, other, k=0.3)` + - `fillet_union(self, other, r=0.1)` + - `translate(self, t)` + - `scale(self, s)` + - `rotate(self, axis, angle)` + - `repeat(self, period)` + - `rounded(self, r)` + - `onion(self, thickness)` + - `displace(self, amount, freq)` + - `twist(self, k)` - `elongate(self, hx=0.0, hy=0.0, hz=0.0)` -- Stretch this shape by pulling it apart along the axes by half-extents (`hx`,`hy`,`hz`) -- iq's opElongate. - `mirror(self, axis=0, plane=0.0)` -- Fold space across a plane on one axis (kaleidoscopic symmetry from abs()). - `fold(self, plane=0.0)` -- Mirror all three axes about `plane` -- map the world into one octant (an 8-fold kaleidoscope). @@ -116,10 +116,10 @@ *The explicit polygon mesh kernel (FWD-1): the substrate every explicit-geometry operator mutates.* - **class `Mesh`** -- An explicit polygon mesh: positions + faces, with optional per-vertex attributes. - - `n_vertices(self)` -- - - `n_faces(self)` -- + - `n_vertices(self)` + - `n_faces(self)` - `edges(self)` -- The set of UNDIRECTED edges as frozensets {vi, vj}. - - `n_edges(self)` -- + - `n_edges(self)` - `half_edges(self)` -- Build (and cache) the half-edge table. - `vertex_faces(self, v)` -- The faces incident to vertex `v`, as a sorted list of face indices (deterministic). - `vertex_neighbours(self, v)` -- The 1-ring of vertex `v`: vertices sharing an edge with it, as a sorted list (deterministic). @@ -141,11 +141,25 @@ ## Transforms ### `holographic_transform` -*TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding.* - -- `analyse(weights, cfg)` -- Recover the block structure and per-layer memory from the weights. -- `plan(weights, cfg, target_tokens=4096, kv_rank=64, grow_gain=0.0)` -- Decide what to do to each layer, from the analysis rather than by rule. -- `apply_plan(weights, cfg, the_plan, progress=None)` -- Carry out the growth actions. +*holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G).* + +- `translation(t)` -- A 4x4 translation matrix from a 3-vector. +- `scaling(s)` -- A 4x4 scale matrix. +- `rotation_axis_angle(axis, angle)` -- A 4x4 rotation of `angle` radians about `axis` (Rodrigues' formula). +- `compose(*mats)` -- Matrix product M0 @ M1 @ ... +- `decompose(M)` -- Split a 4x4 affine transform into (translate (3,), rotation quaternion (4,), scale (3,)). +- `compose_trs(translate, quat, scale)` -- Build a 4x4 from translate (3,), a rotation quaternion (4,), and scale (3,) -- the inverse of decompose. +- `quat_normalize(q)` +- `quat_mul(a, b)` -- The Hamilton product a*b: the rotation "apply b, then a". +- `quat_from_axis_angle(axis, angle)` -- A quaternion for a rotation of `angle` radians about `axis`. +- `quat_to_axis_angle(q)` -- Recover (axis, angle) from a quaternion. +- `quat_to_matrix(q)` -- The 3x3 rotation matrix for a quaternion. +- `quat_from_matrix(R)` -- The quaternion for a 3x3 rotation matrix (Shepperd's method: branch on the largest diagonal term for numerical stability -- a naive formula loses precision when the trace is near zero). +- `quat_from_euler(rx, ry, rz)` -- A quaternion from euler angles applied X then Y then Z (R = Rz @ Ry @ Rx). +- `quat_to_euler(q)` -- Recover euler angles (rx, ry, rz) from a quaternion, inverting R = Rz @ Ry @ Rx. +- `quat_slerp(a, b, t)` -- Spherical linear interpolation between two rotations -- constant angular speed, the smooth in-between an animation wants. +- `quat_rotate(q, v)` -- Rotate a 3-vector by a quaternion. +- `look_at(eye, target, up=(0.0, 1.0, 0.0))` -- An OpenGL view matrix for a camera at `eye` looking at `target` (the engine's convention: the camera looks down -z, y is up). ## Camera @@ -179,7 +193,7 @@ - `load_png(path, mode='rgb01')` -- Read a PNG file back into an array -- the exact inverse of `save_png`, so a render survives a round trip. - `save_image(path, rgb01, level=6, filters=True)` -- Save an (H,W,3) [0,1] image, routed by extension: .png uses the stdlib encoder (deterministic, zero-dependency, always available); anything else (.jpg, .webp, .bmp, ...) uses Pillow when installed and otherwise refuses with the install command -- the same opt-in contract as every accelerator (`pip install pillow`, or the `images` extra). - `load_hdr(path, exposure=1.0)` -- Read a Radiance .hdr / .pic (RGBE) file -> (H,W,3) float32 of LINEAR radiance, UNBOUNDED. -- `save_gif(path, frames, fps=12.0, loop=0, palette='fixed', dither=False)` -- +- `save_gif(path, frames, fps=12.0, loop=0, palette='fixed', dither=False)` - `save_png(path, rgb01, level=6, filters=True)` -- Write an (H,W,3) image in [0,1] to a PNG file. - `frame_delta_tiles(prev, curr, tile=32, thresh=0.001)` -- The pixel-streaming primitive: split two frames into `tile`x`tile` blocks and return only the tiles that CHANGED, as a list of (row, col, tile_pixels). - `fit_camera(mesh, direction=(1.0, 0.75, 1.1), up=(0.0, 1.0, 0.0), fov_deg=50.0, aspect=1.0, margin=1.06)` -- Solve for the camera that FRAMES a mesh: the closest eye along `direction` that keeps every vertex inside the frustum, with the target chosen so the subject is CENTRED. @@ -205,7 +219,7 @@ - `build_pipeline(cfg, registry=None)` -- Config -> ordered, validated Pipeline. - **class `Pipeline`** -- An ordered list of stages. - `plan(self)` -- Dry run / EXPLAIN: every active stage, WHY it is here, and what it NEEDS and PRODUCES -- without rendering. - - `stage_names(self)` -- + - `stage_names(self)` - `run(self, scene=None, seed=0, prev_frame=None, renderer=None)` -- Execute the pipeline: build the shared FrameState and thread it through every stage in order. - `lower_to_program(self, machine)` -- Phase 6: LOWER the pipeline to a machine PROGRAM -- one APPLY instruction per stage, in order, then HALT. - `run_on_vm(self, machine=None, scene=None, seed=0, prev_frame=None, renderer=None)` -- Phase 6: RUN the pipeline ON the VM instead of a Python for-loop. @@ -229,7 +243,7 @@ - **class `CancelToken`** -- A cooperative cancel flag. - `cancel(self)` -- Request cancellation. - `reset(self)` -- Clear the flag so the token can be reused for the next operation. - - `cancelled(self)` -- + - `cancelled(self)` - `should_stop(self)` -- The cooperative check a loop calls between chunks. - `run_cancellable(iterable, token, on_step=None)` -- Iterate `iterable`, yielding items until `token` is cancelled -- a thin helper for wrapping any step loop (a sim advancing frames, an iterative solver) in cancellation. diff --git a/apiquickref.py b/apiquickref.py index dec17c4..bb12012 100644 --- a/apiquickref.py +++ b/apiquickref.py @@ -28,9 +28,14 @@ CURATED = [ ("Scene authoring", ["holographic_scene_doc", "holographic_modifier"]), ("Geometry / SDF", ["holographic_sdf", "holographic_sdfscene", "holographic_mesh"]), - ("Transforms", ["holographic_transform"]), + # Qualify this basename because both misc/ (app transforms) and + # io_and_interop/ (model-weight transforms) provide holographic_transform. + # os.walk order differs by filesystem, so an unqualified duplicate made + # generated docs alternate between the two modules on clean CI checkouts. + ("Transforms", ["misc/holographic_transform"]), ("Camera", ["holographic_camera"]), - ("Rendering", ["holographic_render", "holographic_pipeline", "holographic_session", "holographic_cancel"]), + ("Rendering", ["holographic_render", "holographic_pipeline", + "scene_and_pipeline/holographic_session", "holographic_cancel"]), ("Export / LOD", ["holographic_lod", "holographic_gltf"]), ] @@ -93,26 +98,43 @@ def module_entries(path): lines = [] for node in tree.body: if isinstance(node, ast.FunctionDef) and not node.name.startswith("_"): - lines.append("- `%s` -- %s" % (signature(node), first_sentence(node))) + doc = first_sentence(node) + lines.append("- `%s`%s" % (signature(node), " -- " + doc if doc else "")) elif isinstance(node, ast.ClassDef) and not node.name.startswith("_"): - lines.append("- **class `%s`** -- %s" % (node.name, first_sentence(node))) + doc = first_sentence(node) + lines.append("- **class `%s`**%s" % (node.name, " -- " + doc if doc else "")) for sub in node.body: if isinstance(sub, ast.FunctionDef) and not sub.name.startswith("_"): - lines.append(" - `%s` -- %s" % (signature(sub), first_sentence(sub))) + doc = first_sentence(sub) + lines.append(" - `%s`%s" % (signature(sub), " -- " + doc if doc else "")) return mod_doc, lines def _resolve_module_path(root, mod): """Find a curated module's actual file, whether it's flat at the repo root (old layout) or nested under - holographic// (current layout).""" - flat = os.path.join(root, mod + ".py") + holographic// (current layout). A family-qualified name resolves + directly; an ambiguous basename fails loudly instead of depending on + filesystem traversal order.""" + parts = mod.replace("\\", "/").split("/") + name = parts[-1] + flat = os.path.join(root, name + ".py") if os.path.exists(flat): return flat holo_root = os.path.join(root, "holographic") if os.path.isdir(holo_root): + if len(parts) > 1: + return os.path.join(holo_root, *parts) + ".py" + matches = [] for dirpath, _, filenames in os.walk(holo_root): - if mod + ".py" in filenames: - return os.path.join(dirpath, mod + ".py") + if name + ".py" in filenames: + matches.append(os.path.join(dirpath, name + ".py")) + matches.sort() + if len(matches) > 1: + raise ValueError("ambiguous curated module %r; qualify it as family/module: %s" + % (name, ", ".join(os.path.relpath(p, holo_root) + for p in matches))) + if matches: + return matches[0] return flat # doesn't exist either way; the not-found branch below handles it @@ -134,12 +156,13 @@ def generate(root="."): out.append("") for mod in modules: path = _resolve_module_path(root, mod) + display_mod = mod.replace("\\", "/").rsplit("/", 1)[-1] if not os.path.exists(path): - out.append("### `%s` -- *(module not found)*" % mod) + out.append("### `%s` -- *(module not found)*" % display_mod) out.append("") continue mod_doc, lines = module_entries(path) - out.append("### `%s`" % mod) + out.append("### `%s`" % display_mod) if mod_doc: out.append("*%s*" % mod_doc) out.append("") diff --git a/tests/test_build_ergonomics.py b/tests/test_build_ergonomics.py index be7ce41..b9aca47 100644 --- a/tests/test_build_ergonomics.py +++ b/tests/test_build_ergonomics.py @@ -111,6 +111,8 @@ def test_apiquickref_signature_and_generate(): "def do_thing(a, b=2, *args, key=None, **kw):\n" ' """Do the thing. Details."""\n' " return a\n" + "def undocumented():\n" + " return None\n" "class Widget:\n" ' """A widget."""\n' " def poke(self, hard=True):\n" @@ -128,3 +130,26 @@ def test_apiquickref_signature_and_generate(): assert "Do the thing." in text and "Details." not in text.split("Do the thing.")[1][:5] # first sentence assert "**class `Widget`**" in text and "poke(self, hard=True)" in text assert "_hidden" not in text # privates are excluded + assert "- `undocumented()`\n" in text and "undocumented() --" not in text + + +def test_apiquickref_duplicate_basenames_require_family_qualification(): + """Clean-checkout directory order must not choose which same-named module is documented.""" + sys.path.insert(0, _REPO_ROOT) + import apiquickref + + root = tempfile.mkdtemp() + for family in ("alpha", "zeta"): + directory = os.path.join(root, "holographic", family) + os.makedirs(directory) + with open(os.path.join(directory, "holographic_same.py"), "w") as f: + f.write('"""%s implementation."""\n' % family) + + try: + apiquickref._resolve_module_path(root, "holographic_same") + assert False, "duplicate basename should require an explicit family" + except ValueError as exc: + assert "family/module" in str(exc) + + resolved = apiquickref._resolve_module_path(root, "zeta/holographic_same") + assert resolved == os.path.join(root, "holographic", "zeta", "holographic_same.py") From 444bb89e1ac19ca3ac600acc0005490e8279e661 Mon Sep 17 00:00:00 2001 From: docs-bot Date: Wed, 12 Aug 2026 18:30:03 +0000 Subject: [PATCH 07/16] docs: refresh generated docs (REFERENCE, CAPABILITIES, API_QUICKREF, PIPELINE_MAP, FACULTY_MAP, DOC_MAP) [skip ci] --- REFERENCE.md | 278 +++++++++++++++++++++++++-------------------------- 1 file changed, 139 insertions(+), 139 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index 164a4cd..e06473d 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -76,8 +76,8 @@ | module | what it is | lines | |---|---|---| | [`holographic_session.py`](#holographic-session) | holographic_session.py -- ONE render session that ties the disconnected rendering threads together. | 227 | -| [`holographic_session.py`](#holographic-session) | SESSION STORE -- contexts that outlive the process. | 261 | | [`holographic_session.py`](#holographic-session) | SESSION -- never compute the same conversation prefix twice. | 253 | +| [`holographic_session.py`](#holographic-session) | SESSION STORE -- contexts that outlive the process. | 261 | ### `splat*` family (6) @@ -423,8 +423,8 @@ | [`holographic_materialio.py`](#holographic-materialio) | PBR material interchange (G2-IO): the standard-format bridge for materials, and a VSA-native carrier. | 307 | | [`holographic_matlib.py`](#holographic-matlib) | holographic_matlib.py -- a comprehensive RENDER material library: plain diffuse -> a fractal planet. | 801 | | [`holographic_meaning_predict.py`](#holographic-meaning-predict) | Generation with structure: predict a next-MEANING vector and settle it, rather | 198 | -| [`holographic_measure.py`](#holographic-measure) | MEASURE -- perplexity with error bars, and decisions that respect them. | 239 | | [`holographic_measure.py`](#holographic-measure) | The variance harness: every headline number gets a mean, a spread, and a confidence | 187 | +| [`holographic_measure.py`](#holographic-measure) | MEASURE -- perplexity with error bars, and decisions that respect them. | 239 | | [`holographic_memory.py`](#holographic-memory) | MEMORY -- the Galvatron's own store, built on leCore's holographic database. | 241 | | [`holographic_memoryhome.py`](#holographic-memoryhome) | holographic_memoryhome.py -- the MEMORY home (consolidation backlog H6): keep the hot working set where the CP | 165 | | [`holographic_memsearch.py`](#holographic-memsearch) | MEMSEARCH -- searchable memory that lives in the weights and runs in the pass. | 212 | @@ -577,8 +577,8 @@ | [`holographic_rolling.py`](#holographic-rolling) | holographic_rolling.py -- the CAUSAL rolling / streaming statistics kit: trailing mean, std, min, max, | 378 | | [`holographic_roomacoustic.py`](#holographic-roomacoustic) | holographic_roomacoustic.py -- A6: GEOMETRIC ROOM ACOUSTICS. How a room echoes -- reflections and reverb. | 161 | | [`holographic_route.py`](#holographic-route) | Representation routing (ARCH-7): route each operation to the representation that supports it -- the policy lay | 183 | -| [`holographic_router.py`](#holographic-router) | Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. | 173 | | [`holographic_router.py`](#holographic-router) | ROUTER -- the model DECIDING, inside one forward pass. | 229 | +| [`holographic_router.py`](#holographic-router) | Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. | 173 | | [`holographic_sampler.py`](#holographic-sampler) | holographic_sampler.py -- the SAMPLER: a placeable read-probe (modeling-app backlog, the capstone item). | 223 | | [`holographic_sampling.py`](#holographic-sampling) | Blue-noise / Poisson-disk point sampling -- the EXCLUSION principle, done right. | 134 | | [`holographic_samplinghome.py`](#holographic-samplinghome) | holographic_samplinghome.py -- the SAMPLING home (consolidation backlog R4): one place for the Monte-Carlo | 127 | @@ -683,8 +683,8 @@ | [`holographic_toolclient.py`](#holographic-toolclient) | holographic_toolclient.py -- call another node the same way leCore is called. | 98 | | [`holographic_topology.py`](#holographic-topology) | Principled topology by persistent homology (EXP-7). | 288 | | [`holographic_transfer.py`](#holographic-transfer) | holographic_transfer.py -- KERNEL SCATTER / GATHER: the ONE bundle/readout under every particle<->grid transfe | 224 | -| [`holographic_transform.py`](#holographic-transform) | TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding. | 189 | | [`holographic_transform.py`](#holographic-transform) | holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). | 297 | +| [`holographic_transform.py`](#holographic-transform) | TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding. | 189 | | [`holographic_transform_space.py`](#holographic-transform-space) | holographic_transform_space.py -- the TRANSFORM + SPACE model behind a gizmo. A gizmo is a UI; the backend it | 176 | | [`holographic_transformbank.py`](#holographic-transformbank) | holographic_transformbank.py -- a prebuilt map of hypervector transforms, and what it can and cannot hold. | 358 | | [`holographic_transformhome.py`](#holographic-transformhome) | holographic_transformhome.py -- the TRANSFORM home (consolidation backlog H5): one facade over "move / rotate | 223 | @@ -14848,6 +14848,39 @@ ### holographic_measure.py +> The variance harness: every headline number gets a mean, a spread, and a confidence +> interval across seeds -- so a lucky-seed point estimate can't pass as a real result. +> +> WHY this exists +> --------------- +> This whole engine is built on RANDOM vectors. Atoms are random, the RP-tree's +> hyperplanes are random, the reservoir is random, train/test splits are shuffled. A +> single-seed score is therefore a sample from a distribution, and reporting it alone hides +> how wide that distribution is. For an engine whose entire pitch is "measured, not +> promised," reporting a number without its noise is the sharpest blind spot -- so this +> points that same discipline at the numbers themselves. +> +> measure(run_once, seeds) runs a scored experiment once per seed and returns the mean, the +> sample standard deviation, and a 95% percentile-bootstrap confidence interval (no +> distributional assumptions). assert_robust(stats, floor) passes only if the LOWER CI +> bound clears the floor -- which is what stops a single fortunate seed from passing a +> test the typical seed would fail. report() formats "mean +/- std (95% CI [lo, hi], n)". +> +> USE REAL DATA. The point of the harness is to characterise the real distribution of a +> real claim; running it on a toy makes the spread meaningless. The measurements wired +> through it here all run on real corpora (Gutenberg Alice, UDHR, Reuters, Brown). + +**Public API:** + +- `def time_call(fn, repeats, warmup)` -- Time `fn()` honestly: warm up first, then take the MEDIAN of `repeats` timings in microseconds. +- `def measure(run_once, seeds, n_boot, boot_seed)` -- Run a scored experiment across seeds; return mean, std, and a 95% bootstrap CI. +- `def fdr_gate(rows, alpha)` -- P9 -- false-discovery control across a whole ABLATION TABLE, from the measurement home. +- `def assert_robust(stats, floor)` -- Pass only if the LOWER CI bound clears the floor -- not just the mean. This is +- `def is_fragile(stats, margin_floor)` -- A claim is FRAGILE if its spread is large relative to how far its mean sits above +- `def report(name, stats, floor)` -- Format a stats dict as 'name: mean +/- std (95% CI [lo, hi], n)', with a + +### holographic_measure.py + > MEASURE -- perplexity with error bars, and decisions that respect them. > > Moose asked what assimilation is actually doing. The answer, from his own run: @@ -14880,39 +14913,6 @@ - `def better_than(a, b, alpha, seed, resamples)` -- Is model A better than model B, or is the difference undecidable? - `def tokens_needed(reference, effect_pct, alpha)` -- How many tokens would be needed to RESOLVE an effect of this size. -### holographic_measure.py - -> The variance harness: every headline number gets a mean, a spread, and a confidence -> interval across seeds -- so a lucky-seed point estimate can't pass as a real result. -> -> WHY this exists -> --------------- -> This whole engine is built on RANDOM vectors. Atoms are random, the RP-tree's -> hyperplanes are random, the reservoir is random, train/test splits are shuffled. A -> single-seed score is therefore a sample from a distribution, and reporting it alone hides -> how wide that distribution is. For an engine whose entire pitch is "measured, not -> promised," reporting a number without its noise is the sharpest blind spot -- so this -> points that same discipline at the numbers themselves. -> -> measure(run_once, seeds) runs a scored experiment once per seed and returns the mean, the -> sample standard deviation, and a 95% percentile-bootstrap confidence interval (no -> distributional assumptions). assert_robust(stats, floor) passes only if the LOWER CI -> bound clears the floor -- which is what stops a single fortunate seed from passing a -> test the typical seed would fail. report() formats "mean +/- std (95% CI [lo, hi], n)". -> -> USE REAL DATA. The point of the harness is to characterise the real distribution of a -> real claim; running it on a toy makes the spread meaningless. The measurements wired -> through it here all run on real corpora (Gutenberg Alice, UDHR, Reuters, Brown). - -**Public API:** - -- `def time_call(fn, repeats, warmup)` -- Time `fn()` honestly: warm up first, then take the MEDIAN of `repeats` timings in microseconds. -- `def measure(run_once, seeds, n_boot, boot_seed)` -- Run a scored experiment across seeds; return mean, std, and a 95% bootstrap CI. -- `def fdr_gate(rows, alpha)` -- P9 -- false-discovery control across a whole ABLATION TABLE, from the measurement home. -- `def assert_robust(stats, floor)` -- Pass only if the LOWER CI bound clears the floor -- not just the mean. This is -- `def is_fragile(stats, margin_floor)` -- A claim is FRAGILE if its spread is large relative to how far its mean sits above -- `def report(name, stats, floor)` -- Format a stats dict as 'name: mean +/- std (95% CI [lo, hi], n)', with a - ### holographic_memory.py > MEMORY -- the Galvatron's own store, built on leCore's holographic database. @@ -22656,33 +22656,6 @@ ### holographic_router.py -> Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. -> -> WHY THIS EXISTS (measured, backlog N9/N28): -> The catalog's find_capability scores by shared content words. That is deterministic and needs no model, -> but it has no notion of MEANING: "squish a big array down for storage" shares no token with -> `holographic_coldstore`, and "airspeed velocity of an unladen swallow" confidently matched a physics -> module on the single word "velocity". Measured on the 12-ask suite: token overlap ~2/12 top-1, median -> rank 13 of 503; the nomic embedding router hit 7/12 top-1, median rank 1. -> -> WHAT SHIPS (and what does NOT): -> A 96 KB index -- 503 module vectors at 64d q8, plus the ABTT correction (mu, pc) baked in -- extracted -> from the build cache by tools/semantic/export_index.py. NO model ships. So this router can only score a -> query it already has a VECTOR for: -> * queries embedded at build time and cached (the exam's asks, an app's fixed vocabulary) -> * a caller that supplies its own query vector (an app that ran the encoder itself) -> A brand-new free-text query with no vector and no model present CANNOT be embedded here -- and this -> router says so and returns None, so the caller falls back to the token router rather than guessing. That -> honesty is the whole point: silence beats a confident wrong route. -> -> DETERMINISM: q8 dequant + fixed ABTT transform + argsort with a name tie-break. No RNG, no model. - -**Public API:** - -- `class EmbeddingRouter` -- Loads the shipped q8 index (any dim; 128d is the measured champion) and routes a query VECTOR (or a cached query text) to modules. - -### holographic_router.py - > ROUTER -- the model DECIDING, inside one forward pass. > > Moose raised the architecture that dissolves the wall this project kept hitting: @@ -22722,6 +22695,33 @@ - `def route(runtime, router, text, tokenize)` -- Would this model choose to use the capability on this prompt? - `def install_routed(weights, cfg, operator, router, layer, gain, temperature)` -- Install a circuit whose GATE is the router, not a constant. +### holographic_router.py + +> Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. +> +> WHY THIS EXISTS (measured, backlog N9/N28): +> The catalog's find_capability scores by shared content words. That is deterministic and needs no model, +> but it has no notion of MEANING: "squish a big array down for storage" shares no token with +> `holographic_coldstore`, and "airspeed velocity of an unladen swallow" confidently matched a physics +> module on the single word "velocity". Measured on the 12-ask suite: token overlap ~2/12 top-1, median +> rank 13 of 503; the nomic embedding router hit 7/12 top-1, median rank 1. +> +> WHAT SHIPS (and what does NOT): +> A 96 KB index -- 503 module vectors at 64d q8, plus the ABTT correction (mu, pc) baked in -- extracted +> from the build cache by tools/semantic/export_index.py. NO model ships. So this router can only score a +> query it already has a VECTOR for: +> * queries embedded at build time and cached (the exam's asks, an app's fixed vocabulary) +> * a caller that supplies its own query vector (an app that ran the encoder itself) +> A brand-new free-text query with no vector and no model present CANNOT be embedded here -- and this +> router says so and returns None, so the caller falls back to the token router rather than guessing. That +> honesty is the whole point: silence beats a confident wrong route. +> +> DETERMINISM: q8 dequant + fixed ABTT transform + argsort with a name tie-break. No RNG, no model. + +**Public API:** + +- `class EmbeddingRouter` -- Loads the shipped q8 index (any dim; 128d is the measured champion) and routes a query VECTOR (or a cached query text) to modules. + ### holographic_sampler.py > holographic_sampler.py -- the SAMPLER: a placeable read-probe (modeling-app backlog, the capstone item). @@ -24352,6 +24352,39 @@ ### holographic_session.py +> SESSION -- never compute the same conversation prefix twice. +> +> Moose runs a 0.8B on a CPU laptop and it is slow. The single largest waste in a +> conversation is not the model's arithmetic -- it is that every turn RE-PREFILLS +> the entire history. MEASURED on a realistic six-turn exchange: 489 tokens +> processed, of which only 137 were new. SEVENTY-TWO PERCENT OF THE WORK WAS +> REPEATED, and the fraction grows with every turn. +> +> leCore already had the pieces and never joined them: the runtime exposes +> prefill/step over an InferenceState with copy(), galvacache memoises pure +> functions at a measured 75% hit rate with bit-identical output, and sessions +> exist. What was missing is the RADIX TREE -- the structure that answers "what is +> the longest prefix of this prompt that I have already computed?" +> +> WHY A TREE AND NOT A DICTIONARY: turn 4 of a conversation shares its first three +> turns with turn 3, and a dictionary keyed on the whole prompt misses that +> completely. vLLM and SGLang call this RadixAttention; the idea is the same +> whatever the model: index by prefix, resume from the deepest match, compute only +> the tail. +> +> THE GUARANTEE THIS KEEPS, because a cache that changes answers is worse than no +> cache: resuming from a cached state reproduces a full recompute TO FLOAT +> ROUNDING -- measured 7.1e-15, machine epsilon. Not bit-identical, and the +> difference is real rather than pedantic: resuming STEPS the tail while a fresh +> call PREFILLS it, and the two associate their sums differently. The selftest +> asserts the measured bound against the live runtime rather than assuming it. + +**Public API:** + +- `class PrefixCache` -- A radix tree over token sequences, holding inference states. + +### holographic_session.py + > SESSION STORE -- contexts that outlive the process. > > A Galvatron's context is not a transcript, it is STATE: the GDN recurrent @@ -24384,39 +24417,6 @@ - `class SessionStore` -- Named, persistent, independent contexts on disk. - `def runtime_fingerprint(runtime)` -- A cheap, deterministic id for the checkpoint behind a runtime, so a -### holographic_session.py - -> SESSION -- never compute the same conversation prefix twice. -> -> Moose runs a 0.8B on a CPU laptop and it is slow. The single largest waste in a -> conversation is not the model's arithmetic -- it is that every turn RE-PREFILLS -> the entire history. MEASURED on a realistic six-turn exchange: 489 tokens -> processed, of which only 137 were new. SEVENTY-TWO PERCENT OF THE WORK WAS -> REPEATED, and the fraction grows with every turn. -> -> leCore already had the pieces and never joined them: the runtime exposes -> prefill/step over an InferenceState with copy(), galvacache memoises pure -> functions at a measured 75% hit rate with bit-identical output, and sessions -> exist. What was missing is the RADIX TREE -- the structure that answers "what is -> the longest prefix of this prompt that I have already computed?" -> -> WHY A TREE AND NOT A DICTIONARY: turn 4 of a conversation shares its first three -> turns with turn 3, and a dictionary keyed on the whole prompt misses that -> completely. vLLM and SGLang call this RadixAttention; the idea is the same -> whatever the model: index by prefix, resume from the deepest match, compute only -> the tail. -> -> THE GUARANTEE THIS KEEPS, because a cache that changes answers is worse than no -> cache: resuming from a cached state reproduces a full recompute TO FLOAT -> ROUNDING -- measured 7.1e-15, machine epsilon. Not bit-identical, and the -> difference is real rather than pedantic: resuming STEPS the tail while a fresh -> call PREFILLS it, and the two associate their sums differently. The selftest -> asserts the measured bound against the live runtime rather than assuming it. - -**Public API:** - -- `class PrefixCache` -- A radix tree over token sequences, holding inference states. - ### holographic_shader.py > holographic_shader.py -- N filter passes in ONE evaluation. Two things a GPU structurally cannot do. @@ -27945,6 +27945,48 @@ ### holographic_transform.py +> holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). +> +> The engine has scattered transform bits (scenegraph.translation/rotation/compose_transforms; cosserat's +> quaternion helpers; splatexport's rotation<->quaternion), but not the full kit a gizmo and a property panel need +> in one place. This gathers the standard, well-known math: +> +> * decompose(M) -> (translate, rotation-quaternion, scale) -- what a gizmo reads off a matrix to show handles, +> and what a property panel shows as the T/R/S fields; +> * compose_trs(translate, quat, scale) -> 4x4 -- the inverse (build a matrix from panel values); +> * a quaternion kit -- from/to matrix, from/to axis-angle, from/to euler, multiply, SLERP, rotate a vector +> (quaternions are what rotation UI and animation want: no gimbal lock, and slerp gives smooth interpolation); +> * look_at(eye, target, up) -> 4x4 view matrix -- for a camera or an object aimed at a point. +> +> Conventions, stated ONCE and held (the backlog's coordinate-convention note): matrices are 4x4 and act on COLUMN +> vectors, p' = M @ [x, y, z, 1]; compose(A, B) = A @ B means "apply B, then A"; quaternions are (w, x, y, z), unit +> length; euler angles are (rx, ry, rz) applied X then Y then Z, i.e. R = Rz @ Ry @ Rx; look_at returns an OpenGL +> view matrix (the camera looks down -z, y is up) to match the engine's Camera. Nothing here is holographic -- it is +> plain linear algebra a modeling app needs -- so it is kept as a small, readable utility, not dressed up as a bind. +> Deterministic; NumPy + stdlib only. + +**Public API:** + +- `def translation(t)` -- A 4x4 translation matrix from a 3-vector. +- `def scaling(s)` -- A 4x4 scale matrix. `s` is a scalar (uniform) or a 3-vector (per-axis). +- `def rotation_axis_angle(axis, angle)` -- A 4x4 rotation of `angle` radians about `axis` (Rodrigues' formula). +- `def compose(*mats)` -- Matrix product M0 @ M1 @ ... -- with the column-vector convention this applies the RIGHTMOST first. +- `def decompose(M)` -- Split a 4x4 affine transform into (translate (3,), rotation quaternion (4,), scale (3,)). Assumes no shear +- `def compose_trs(translate, quat, scale)` -- Build a 4x4 from translate (3,), a rotation quaternion (4,), and scale (3,) -- the inverse of decompose. +- `def quat_normalize(q)` +- `def quat_mul(a, b)` -- The Hamilton product a*b: the rotation "apply b, then a". +- `def quat_from_axis_angle(axis, angle)` -- A quaternion for a rotation of `angle` radians about `axis`. +- `def quat_to_axis_angle(q)` -- Recover (axis, angle) from a quaternion. +- `def quat_to_matrix(q)` -- The 3x3 rotation matrix for a quaternion. +- `def quat_from_matrix(R)` -- The quaternion for a 3x3 rotation matrix (Shepperd's method: branch on the largest diagonal term for +- `def quat_from_euler(rx, ry, rz)` -- A quaternion from euler angles applied X then Y then Z (R = Rz @ Ry @ Rx). +- `def quat_to_euler(q)` -- Recover euler angles (rx, ry, rz) from a quaternion, inverting R = Rz @ Ry @ Rx. Handles gimbal lock +- `def quat_slerp(a, b, t)` -- Spherical linear interpolation between two rotations -- constant angular speed, the smooth in-between an +- `def quat_rotate(q, v)` -- Rotate a 3-vector by a quaternion. +- `def look_at(eye, target, up)` -- An OpenGL view matrix for a camera at `eye` looking at `target` (the engine's convention: the camera looks + +### holographic_transform.py + > TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding. > > Everything before this applied leCore's levers uniformly: grow a memory channel @@ -27987,48 +28029,6 @@ - `def plan(weights, cfg, target_tokens, kv_rank, grow_gain)` -- Decide what to do to each layer, from the analysis rather than by rule. - `def apply_plan(weights, cfg, the_plan, progress)` -- Carry out the growth actions. KV compression is a RUNTIME setting and is -### holographic_transform.py - -> holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). -> -> The engine has scattered transform bits (scenegraph.translation/rotation/compose_transforms; cosserat's -> quaternion helpers; splatexport's rotation<->quaternion), but not the full kit a gizmo and a property panel need -> in one place. This gathers the standard, well-known math: -> -> * decompose(M) -> (translate, rotation-quaternion, scale) -- what a gizmo reads off a matrix to show handles, -> and what a property panel shows as the T/R/S fields; -> * compose_trs(translate, quat, scale) -> 4x4 -- the inverse (build a matrix from panel values); -> * a quaternion kit -- from/to matrix, from/to axis-angle, from/to euler, multiply, SLERP, rotate a vector -> (quaternions are what rotation UI and animation want: no gimbal lock, and slerp gives smooth interpolation); -> * look_at(eye, target, up) -> 4x4 view matrix -- for a camera or an object aimed at a point. -> -> Conventions, stated ONCE and held (the backlog's coordinate-convention note): matrices are 4x4 and act on COLUMN -> vectors, p' = M @ [x, y, z, 1]; compose(A, B) = A @ B means "apply B, then A"; quaternions are (w, x, y, z), unit -> length; euler angles are (rx, ry, rz) applied X then Y then Z, i.e. R = Rz @ Ry @ Rx; look_at returns an OpenGL -> view matrix (the camera looks down -z, y is up) to match the engine's Camera. Nothing here is holographic -- it is -> plain linear algebra a modeling app needs -- so it is kept as a small, readable utility, not dressed up as a bind. -> Deterministic; NumPy + stdlib only. - -**Public API:** - -- `def translation(t)` -- A 4x4 translation matrix from a 3-vector. -- `def scaling(s)` -- A 4x4 scale matrix. `s` is a scalar (uniform) or a 3-vector (per-axis). -- `def rotation_axis_angle(axis, angle)` -- A 4x4 rotation of `angle` radians about `axis` (Rodrigues' formula). -- `def compose(*mats)` -- Matrix product M0 @ M1 @ ... -- with the column-vector convention this applies the RIGHTMOST first. -- `def decompose(M)` -- Split a 4x4 affine transform into (translate (3,), rotation quaternion (4,), scale (3,)). Assumes no shear -- `def compose_trs(translate, quat, scale)` -- Build a 4x4 from translate (3,), a rotation quaternion (4,), and scale (3,) -- the inverse of decompose. -- `def quat_normalize(q)` -- `def quat_mul(a, b)` -- The Hamilton product a*b: the rotation "apply b, then a". -- `def quat_from_axis_angle(axis, angle)` -- A quaternion for a rotation of `angle` radians about `axis`. -- `def quat_to_axis_angle(q)` -- Recover (axis, angle) from a quaternion. -- `def quat_to_matrix(q)` -- The 3x3 rotation matrix for a quaternion. -- `def quat_from_matrix(R)` -- The quaternion for a 3x3 rotation matrix (Shepperd's method: branch on the largest diagonal term for -- `def quat_from_euler(rx, ry, rz)` -- A quaternion from euler angles applied X then Y then Z (R = Rz @ Ry @ Rx). -- `def quat_to_euler(q)` -- Recover euler angles (rx, ry, rz) from a quaternion, inverting R = Rz @ Ry @ Rx. Handles gimbal lock -- `def quat_slerp(a, b, t)` -- Spherical linear interpolation between two rotations -- constant angular speed, the smooth in-between an -- `def quat_rotate(q, v)` -- Rotate a 3-vector by a quaternion. -- `def look_at(eye, target, up)` -- An OpenGL view matrix for a camera at `eye` looking at `target` (the engine's convention: the camera looks - ### holographic_transform_space.py > holographic_transform_space.py -- the TRANSFORM + SPACE model behind a gizmo. A gizmo is a UI; the backend it From 17f6951100c5aea0699dc45133efef7c906283c2 Mon Sep 17 00:00:00 2001 From: atimics Date: Wed, 12 Aug 2026 14:20:35 -0700 Subject: [PATCH 08/16] fix: separate qwen installation and evaluation corpora --- experiments/qwen35_acceptance/README.md | 8 +++- experiments/qwen35_acceptance/generate.py | 45 +++++++++++++++++------ experiments/qwen35_acceptance/run.py | 43 +++++++++++++++++----- tests/test_qwen_acceptance.py | 31 ++++++++++++++-- 4 files changed, 101 insertions(+), 26 deletions(-) diff --git a/experiments/qwen35_acceptance/README.md b/experiments/qwen35_acceptance/README.md index b10a68a..9f8bda8 100644 --- a/experiments/qwen35_acceptance/README.md +++ b/experiments/qwen35_acceptance/README.md @@ -1,7 +1,8 @@ # Qwen3.5 acceptance experiment This directory turns the open Qwen integration questions into a frozen ilxyr -project. It does not claim that the full run has passed. It generates the +project for one owner-authorized, bounded attempt. It does not claim that the +full run has passed. It generates the hypothesis, methodology contributions, experiment contract, two explicit model forecasts, funding record, and ordered ilxyr commands for one real checkpoint. @@ -19,11 +20,14 @@ Generate a project from the leCore checkout: ```bash python experiments/qwen35_acceptance/generate.py \ /absolute/path/to/Qwen3.5-0.8B \ + /absolute/path/to/installation-corpus.txt \ /absolute/path/to/evaluation-corpus.txt \ /absolute/path/to/ilxyr-project ``` -`project.json` records the exact model/corpus digests and the ordered commands +The two corpora must have different contents. + +`project.json` records the exact model and both corpus digests and the ordered commands for `~/develop/ilxyr/target/debug/ilxyr`. Review the forecasts and frozen thresholds before contributing them to an ilxyr workspace. ilxyr executes the runner without a shell and records its strict `metrics`/`source` envelope. diff --git a/experiments/qwen35_acceptance/generate.py b/experiments/qwen35_acceptance/generate.py index c4c28a5..632b260 100644 --- a/experiments/qwen35_acceptance/generate.py +++ b/experiments/qwen35_acceptance/generate.py @@ -69,7 +69,10 @@ def write_json(path, payload): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("model_dir", type=Path) - ap.add_argument("corpus", type=Path) + ap.add_argument("installation_corpus", type=Path, + help="text used only to ground the installation") + ap.add_argument("evaluation_corpus", type=Path, + help="separate held-out text used only for evaluation") ap.add_argument("out_dir", type=Path, help="directory that will receive the ilxyr project JSON") ap.add_argument("--installed-dir", type=Path, @@ -83,22 +86,28 @@ def main(argv=None): args = ap.parse_args(argv) model_dir = args.model_dir.expanduser().resolve() - corpus = args.corpus.expanduser().resolve() + installation_corpus = args.installation_corpus.expanduser().resolve() + evaluation_corpus = args.evaluation_corpus.expanduser().resolve() out_dir = args.out_dir.expanduser().resolve() installed_dir = ((args.installed_dir.expanduser().resolve()) if args.installed_dir else out_dir / "installed-checkpoint") python = args.python.expanduser().resolve() - if not model_dir.is_dir() or not corpus.is_file() or not python.is_file(): - ap.error("model_dir, corpus, and --python must exist") + if (not model_dir.is_dir() or not installation_corpus.is_file() + or not evaluation_corpus.is_file() or not python.is_file()): + ap.error("model_dir, both corpora, and --python must exist") if int(args.min_tokens) < 1000: ap.error("--min-tokens must be at least 1000") model_digest, model_files = model_manifest(model_dir) - corpus_digest = sha256_file(corpus) + installation_digest = sha256_file(installation_corpus) + evaluation_digest = sha256_file(evaluation_corpus) + if installation_digest == evaluation_digest: + ap.error("installation and evaluation corpora must have distinct contents") source_commit = subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip() - stem = "lecore.qwen35.install.%s.%s.v1" % ( - model_digest[:12], source_commit[:12]) + stem = "lecore.qwen35.install.%s.%s.%s.%s.v1" % ( + model_digest[:12], installation_digest[:12], evaluation_digest[:12], + source_commit[:12]) ids = { "hypothesis": stem + ".hypothesis", "foundation": stem + ".foundation", @@ -135,7 +144,7 @@ def main(argv=None): design = contribution( ids["design"], "experiment_design", "experiment-designer", "One-shot Qwen3.5 installation acceptance run", - "Execute once against the content-bound public checkpoint and corpus. Do not tune thresholds or replace the corpus after admission. Resolve accepted only when source cleanliness, tokenizer parity, reference-logit parity, the paired statistical gate, disk reload, official text generation, and official vision smoke all pass. A cleanly executed no-go is preserved as rejected evidence.", + "Execute once against the content-bound public checkpoint, installation corpus, and separate held-out evaluation corpus. Do not tune thresholds or replace either corpus after admission. Resolve accepted only when source cleanliness, tokenizer parity, reference-logit parity, the paired statistical gate, disk reload, official text generation, and official vision smoke all pass. A cleanly executed no-go is preserved as rejected evidence.", [ids["hypothesis"], ids["foundation"], ids["engineering"]], ["Accepted and rejected are exhaustive for a valid metrics envelope.", "Runtime or dependency failure resolves separately as execution_failure."], @@ -155,7 +164,10 @@ def main(argv=None): "experiment_design": ids["design"], }, "baseline": "baseline://Qwen/Qwen3.5-0.8B/%s" % model_digest, - "datasets": ["dataset://lecore/qwen-acceptance/%s" % corpus_digest], + "datasets": [ + "dataset://lecore/qwen-installation/%s" % installation_digest, + "dataset://lecore/qwen-evaluation/%s" % evaluation_digest, + ], "models": ["weight://Qwen/Qwen3.5-0.8B/%s" % model_digest], "metrics": METRIC_SPECS, "seeds": [0], @@ -175,7 +187,8 @@ def main(argv=None): "executor": "local-command", "program": str(python), "args": [str(HERE / "run.py"), str(model_dir), str(installed_dir), - str(corpus), "--min-tokens", str(int(args.min_tokens))], + str(installation_corpus), str(evaluation_corpus), + "--min-tokens", str(int(args.min_tokens))], "timeout_seconds": int(args.timeout_seconds), "max_cost_credits": int(args.compute_credits), "network": "open", @@ -187,7 +200,7 @@ def main(argv=None): "evidence_authority": { "level": "corpus_proxy", "scope": {"seeds": [0], - "eval_set": "dataset://lecore/qwen-acceptance/%s" % corpus_digest, + "eval_set": "dataset://lecore/qwen-evaluation/%s" % evaluation_digest, "coverage": 1.0}, # ilxyr artifact refs name objects already present in its local # content-addressed store; raw file SHA-256 values are not object @@ -251,7 +264,15 @@ def main(argv=None): write_json(out_dir / "project.json", { "schema": "lecore.ilxyr-project.v1", "experiment_id": ids["experiment"], "source_commit": source_commit, "model_digest": model_digest, - "model_files": model_files, "corpus_sha256": corpus_digest, + "model_files": model_files, + "corpora": { + "installation": {"path": str(installation_corpus), + "sha256": installation_digest, + "bytes": installation_corpus.stat().st_size}, + "evaluation": {"path": str(evaluation_corpus), + "sha256": evaluation_digest, + "bytes": evaluation_corpus.stat().st_size}, + }, "installed_dir": str(installed_dir), "commands": commands, }) print(json.dumps({"project_dir": str(out_dir), diff --git a/experiments/qwen35_acceptance/run.py b/experiments/qwen35_acceptance/run.py index 8b3ee71..71b60fa 100644 --- a/experiments/qwen35_acceptance/run.py +++ b/experiments/qwen35_acceptance/run.py @@ -122,7 +122,7 @@ def reference_parity(model_dir, token_ids, tolerance): return float(tokenizer_pass), rel, float(rel <= float(tolerance)) -def run_installer(model_dir, installed_dir, corpus, transcript): +def run_installer(model_dir, installed_dir, installation_corpus, transcript): command = [ sys.executable, str(REPO / "assimilation" / "install.py"), @@ -130,7 +130,7 @@ def run_installer(model_dir, installed_dir, corpus, transcript): str(model_dir), str(installed_dir), "--doc", - str(corpus), + str(installation_corpus), "--device", "cpu", ] @@ -182,7 +182,7 @@ def official_output_smokes(installed_dir): return float(text_pass), float(vision_pass), peak_gpu -def source_snapshot(): +def source_snapshot(installation_corpus, evaluation_corpus): repository = subprocess.check_output( ["git", "remote", "get-url", "origin"], cwd=REPO, text=True).strip() commit = subprocess.check_output( @@ -200,6 +200,18 @@ def source_snapshot(): return float(not dirty), { "repository": repository, "commit": commit, + "inputs": { + "installation_corpus": { + "path": str(installation_corpus), + "sha256": sha256_file(installation_corpus), + "bytes": installation_corpus.stat().st_size, + }, + "evaluation_corpus": { + "path": str(evaluation_corpus), + "sha256": sha256_file(evaluation_corpus), + "bytes": evaluation_corpus.stat().st_size, + }, + }, "artifacts": [{"path": str(path.relative_to(REPO)), "sha256": sha256_file(path)} for path in paths], } @@ -209,7 +221,10 @@ def main(argv=None): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("model_dir", type=Path) ap.add_argument("installed_dir", type=Path) - ap.add_argument("corpus", type=Path) + ap.add_argument("installation_corpus", type=Path, + help="text used only to ground the experimental installation") + ap.add_argument("evaluation_corpus", type=Path, + help="separate held-out text used only for paired evaluation") ap.add_argument("--min-tokens", type=int, default=4096) ap.add_argument("--chunk-size", type=int, default=128) ap.add_argument("--max-regression", type=float, default=0.01) @@ -218,7 +233,16 @@ def main(argv=None): model_dir = args.model_dir.resolve() installed_dir = args.installed_dir.resolve() - corpus = args.corpus.resolve() + installation_corpus = args.installation_corpus.resolve() + evaluation_corpus = args.evaluation_corpus.resolve() + if not model_dir.is_dir(): + raise SystemExit("model_dir does not exist: %s" % model_dir) + if not installation_corpus.is_file() or not evaluation_corpus.is_file(): + raise SystemExit("both installation and evaluation corpora must exist") + installation_digest = sha256_file(installation_corpus) + evaluation_digest = sha256_file(evaluation_corpus) + if installation_digest == evaluation_digest: + raise SystemExit("installation and evaluation corpora must have distinct contents") if installed_dir.exists() and any(installed_dir.iterdir()): raise SystemExit("installed_dir must be absent or empty: %s" % installed_dir) if int(args.min_tokens) < 1000: @@ -228,15 +252,15 @@ def main(argv=None): from holographic.io_and_interop.holographic_gdnruntime import load_runtime from holographic.io_and_interop.holographic_measure import better_than - text = corpus.read_text(encoding="utf-8", errors="replace") + text = evaluation_corpus.read_text(encoding="utf-8", errors="replace") token_ids = BPE.from_dir(model_dir).encode(text) needed = int(args.min_tokens) + 1 if len(token_ids) < needed: - raise SystemExit("corpus produced %d tokens; acceptance requires at least %d" + raise SystemExit("evaluation corpus produced %d tokens; acceptance requires at least %d" % (len(token_ids), needed)) token_ids = token_ids[:needed] - source_clean, source = source_snapshot() + source_clean, source = source_snapshot(installation_corpus, evaluation_corpus) tok_pass, ref_error, ref_pass = reference_parity( model_dir, token_ids, args.logit_tolerance) @@ -246,7 +270,8 @@ def main(argv=None): gc.collect() artifact_dir = installed_dir.parent / (installed_dir.name + ".acceptance") - run_installer(model_dir, installed_dir, corpus, artifact_dir / "install.log") + run_installer(model_dir, installed_dir, installation_corpus, + artifact_dir / "install.log") child_peak = peak_rss_mb("RUSAGE_CHILDREN") installed, _ = load_runtime(installed_dir) diff --git a/tests/test_qwen_acceptance.py b/tests/test_qwen_acceptance.py index 896561a..2d4f59c 100644 --- a/tests/test_qwen_acceptance.py +++ b/tests/test_qwen_acceptance.py @@ -115,12 +115,17 @@ def test_streamed_measure_matches_whole_forward(tmp_path): def test_generator_emits_complete_ilxyr_contract(tmp_path): output, _ = build_fixture(tmp_path) - corpus = tmp_path / "corpus.txt" - corpus.write_text("Evidence must be paired, replayable, and recorded. " * 400) + installation_corpus = tmp_path / "installation.txt" + evaluation_corpus = tmp_path / "evaluation.txt" + installation_corpus.write_text( + "Installation grounding is separate from evaluation. " * 400) + evaluation_corpus.write_text( + "Evidence must be paired, replayable, and recorded. " * 400) project = tmp_path / "project" command = [sys.executable, str(ROOT / "experiments" / "qwen35_acceptance" / "generate.py"), - str(output), str(corpus), str(project), "--min-tokens", "1000"] + str(output), str(installation_corpus), str(evaluation_corpus), + str(project), "--min-tokens", "1000"] completed = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True, check=False) assert completed.returncode == 0, completed.stderr @@ -132,6 +137,12 @@ def test_generator_emits_complete_ilxyr_contract(tmp_path): assert [metric["name"] for metric in experiment["metrics"]] == list(contract.METRIC_NAMES) assert experiment["execution"]["program"] == str(Path(sys.executable).resolve()) assert "--research-spectral" not in experiment["execution"]["args"] + assert experiment["execution"]["args"][3:5] == [ + str(installation_corpus.resolve()), str(evaluation_corpus.resolve())] + assert len(experiment["datasets"]) == 2 + assert manifest["corpora"]["installation"]["sha256"] != \ + manifest["corpora"]["evaluation"]["sha256"] + assert "corpus_sha256" not in manifest assert len(manifest["commands"]) == 12 for name in ("hypothesis.json", "foundation.json", "engineering-review.json", "experiment-design.json", "forecast-empirical.json", @@ -139,6 +150,20 @@ def test_generator_emits_complete_ilxyr_contract(tmp_path): assert (project / name).is_file() +def test_generator_rejects_reused_installation_and_evaluation_corpus(tmp_path): + output, _ = build_fixture(tmp_path) + corpus = tmp_path / "corpus.txt" + corpus.write_text("The exact same bytes may not serve both experiment roles. " * 200) + command = [sys.executable, + str(ROOT / "experiments" / "qwen35_acceptance" / "generate.py"), + str(output), str(corpus), str(corpus), str(tmp_path / "project"), + "--min-tokens", "1000"] + completed = subprocess.run(command, cwd=tmp_path, capture_output=True, + text=True, check=False) + assert completed.returncode == 2 + assert "distinct contents" in completed.stderr + + def test_risky_entrypoints_are_explicitly_opt_in(tmp_path, capsys): installer = subprocess.run( [sys.executable, str(ROOT / "assimilation" / "install.py")], From a04ab5692be38f06120aba4b0bc5e2a284eb2c79 Mon Sep 17 00:00:00 2001 From: atimics Date: Wed, 12 Aug 2026 14:31:25 -0700 Subject: [PATCH 09/16] fix: enforce qwen pre-install controls --- experiments/qwen35_acceptance/generate.py | 9 ++++++--- experiments/qwen35_acceptance/run.py | 9 +++++++++ tests/test_qwen_acceptance.py | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/experiments/qwen35_acceptance/generate.py b/experiments/qwen35_acceptance/generate.py index 632b260..0527eee 100644 --- a/experiments/qwen35_acceptance/generate.py +++ b/experiments/qwen35_acceptance/generate.py @@ -31,9 +31,12 @@ def sha256_file(path): def model_manifest(model_dir): - names = ["config.json", "tokenizer.json", "vocab.json", "merges.txt"] - paths = [model_dir / name for name in names if (model_dir / name).is_file()] - paths += sorted(model_dir.glob("*.safetensors")) + # Bind every top-level file in the materialized snapshot. The processor, + # chat template, and index are part of the official vision/text smoke just + # as surely as config.json and the weights are. Hidden local cache state + # is deliberately excluded. + paths = sorted(path for path in model_dir.iterdir() + if path.is_file() and not path.name.startswith(".")) if not any(path.suffix == ".safetensors" for path in paths): raise ValueError("no safetensors checkpoint in %s" % model_dir) records = [{"path": path.name, "sha256": sha256_file(path), diff --git a/experiments/qwen35_acceptance/run.py b/experiments/qwen35_acceptance/run.py index 71b60fa..48c0923 100644 --- a/experiments/qwen35_acceptance/run.py +++ b/experiments/qwen35_acceptance/run.py @@ -263,6 +263,15 @@ def main(argv=None): source_clean, source = source_snapshot(installation_corpus, evaluation_corpus) tok_pass, ref_error, ref_pass = reference_parity( model_dir, token_ids, args.logit_tolerance) + if not source_clean: + raise RuntimeError("refusing installation from a dirty source checkout") + if not tok_pass: + raise RuntimeError("refusing installation after tokenizer parity failed") + if not ref_pass: + raise RuntimeError( + "refusing installation after reference-logit parity failed " + "(relative error %.9g > %.9g)" % + (ref_error, float(args.logit_tolerance))) original, _ = load_runtime(model_dir) before = streamed_measure(original, token_ids, chunk_size=args.chunk_size) diff --git a/tests/test_qwen_acceptance.py b/tests/test_qwen_acceptance.py index 2d4f59c..39a3f4f 100644 --- a/tests/test_qwen_acceptance.py +++ b/tests/test_qwen_acceptance.py @@ -143,6 +143,9 @@ def test_generator_emits_complete_ilxyr_contract(tmp_path): assert manifest["corpora"]["installation"]["sha256"] != \ manifest["corpora"]["evaluation"]["sha256"] assert "corpus_sha256" not in manifest + assert {item["path"] for item in manifest["model_files"]} >= { + "config.json", "tokenizer.json", "model.safetensors", + } assert len(manifest["commands"]) == 12 for name in ("hypothesis.json", "foundation.json", "engineering-review.json", "experiment-design.json", "forecast-empirical.json", @@ -164,6 +167,22 @@ def test_generator_rejects_reused_installation_and_evaluation_corpus(tmp_path): assert "distinct contents" in completed.stderr +def test_model_manifest_binds_processor_and_template_files(tmp_path): + output, _ = build_fixture(tmp_path) + (output / "preprocessor_config.json").write_text('{"size": 32}\n') + (output / "chat_template.jinja").write_text("{{ messages }}\n") + (output / ".cache-marker").write_text("local cache state\n") + generator = load_script( + "qwen_acceptance_generator", + ROOT / "experiments" / "qwen35_acceptance" / "generate.py") + + _digest, files = generator.model_manifest(output) + names = {item["path"] for item in files} + assert "preprocessor_config.json" in names + assert "chat_template.jinja" in names + assert ".cache-marker" not in names + + def test_risky_entrypoints_are_explicitly_opt_in(tmp_path, capsys): installer = subprocess.run( [sys.executable, str(ROOT / "assimilation" / "install.py")], From 0bda4f135fb577e4e2c1f0ef29ac5b1756b7ccc2 Mon Sep 17 00:00:00 2001 From: atimics Date: Wed, 12 Aug 2026 14:50:47 -0700 Subject: [PATCH 10/16] fix: preserve qwen experiment venv interpreter --- experiments/qwen35_acceptance/generate.py | 6 +++++- tests/test_qwen_acceptance.py | 26 ++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/experiments/qwen35_acceptance/generate.py b/experiments/qwen35_acceptance/generate.py index 0527eee..a2101c7 100644 --- a/experiments/qwen35_acceptance/generate.py +++ b/experiments/qwen35_acceptance/generate.py @@ -94,7 +94,11 @@ def main(argv=None): out_dir = args.out_dir.expanduser().resolve() installed_dir = ((args.installed_dir.expanduser().resolve()) if args.installed_dir else out_dir / "installed-checkpoint") - python = args.python.expanduser().resolve() + # Preserve the selected executable path instead of dereferencing its final + # symlink. A venv's ``python`` normally points at the system interpreter; + # resolving it records /usr/bin/python and silently drops the venv's site + # packages when ilxyr executes the absolute program. + python = args.python.expanduser().absolute() if (not model_dir.is_dir() or not installation_corpus.is_file() or not evaluation_corpus.is_file() or not python.is_file()): ap.error("model_dir, both corpora, and --python must exist") diff --git a/tests/test_qwen_acceptance.py b/tests/test_qwen_acceptance.py index 39a3f4f..7c402bf 100644 --- a/tests/test_qwen_acceptance.py +++ b/tests/test_qwen_acceptance.py @@ -135,7 +135,7 @@ def test_generator_emits_complete_ilxyr_contract(tmp_path): experiment = json.loads((project / "experiment.json").read_text()) manifest = json.loads((project / "project.json").read_text()) assert [metric["name"] for metric in experiment["metrics"]] == list(contract.METRIC_NAMES) - assert experiment["execution"]["program"] == str(Path(sys.executable).resolve()) + assert experiment["execution"]["program"] == str(Path(sys.executable).absolute()) assert "--research-spectral" not in experiment["execution"]["args"] assert experiment["execution"]["args"][3:5] == [ str(installation_corpus.resolve()), str(evaluation_corpus.resolve())] @@ -153,6 +153,30 @@ def test_generator_emits_complete_ilxyr_contract(tmp_path): assert (project / name).is_file() +def test_generator_preserves_virtual_environment_python_path(tmp_path): + output, _ = build_fixture(tmp_path) + installation_corpus = tmp_path / "installation.txt" + evaluation_corpus = tmp_path / "evaluation.txt" + installation_corpus.write_text("Installation material. " * 400) + evaluation_corpus.write_text("Held-out evaluation material. " * 400) + venv_python = tmp_path / "venv" / "bin" / "python" + venv_python.parent.mkdir(parents=True) + venv_python.symlink_to(Path(sys.executable).resolve()) + project = tmp_path / "project" + + completed = subprocess.run( + [str(venv_python), + str(ROOT / "experiments" / "qwen35_acceptance" / "generate.py"), + str(output), str(installation_corpus), str(evaluation_corpus), + str(project), "--python", str(venv_python), "--min-tokens", "1000"], + cwd=tmp_path, capture_output=True, text=True, check=False) + assert completed.returncode == 0, completed.stderr + + experiment = json.loads((project / "experiment.json").read_text()) + assert experiment["execution"]["program"] == str(venv_python.absolute()) + assert experiment["execution"]["program"] != str(venv_python.resolve()) + + def test_generator_rejects_reused_installation_and_evaluation_corpus(tmp_path): output, _ = build_fixture(tmp_path) corpus = tmp_path / "corpus.txt" From 3a421d0582171db87919be7d99d103a90ee54a3f Mon Sep 17 00:00:00 2001 From: docs-bot Date: Wed, 12 Aug 2026 21:51:20 +0000 Subject: [PATCH 11/16] docs: refresh generated docs (REFERENCE, CAPABILITIES, API_QUICKREF, PIPELINE_MAP, FACULTY_MAP, DOC_MAP) [skip ci] --- REFERENCE.md | 278 +++++++++++++++++++++++++-------------------------- 1 file changed, 139 insertions(+), 139 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index e06473d..164a4cd 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -76,8 +76,8 @@ | module | what it is | lines | |---|---|---| | [`holographic_session.py`](#holographic-session) | holographic_session.py -- ONE render session that ties the disconnected rendering threads together. | 227 | -| [`holographic_session.py`](#holographic-session) | SESSION -- never compute the same conversation prefix twice. | 253 | | [`holographic_session.py`](#holographic-session) | SESSION STORE -- contexts that outlive the process. | 261 | +| [`holographic_session.py`](#holographic-session) | SESSION -- never compute the same conversation prefix twice. | 253 | ### `splat*` family (6) @@ -423,8 +423,8 @@ | [`holographic_materialio.py`](#holographic-materialio) | PBR material interchange (G2-IO): the standard-format bridge for materials, and a VSA-native carrier. | 307 | | [`holographic_matlib.py`](#holographic-matlib) | holographic_matlib.py -- a comprehensive RENDER material library: plain diffuse -> a fractal planet. | 801 | | [`holographic_meaning_predict.py`](#holographic-meaning-predict) | Generation with structure: predict a next-MEANING vector and settle it, rather | 198 | -| [`holographic_measure.py`](#holographic-measure) | The variance harness: every headline number gets a mean, a spread, and a confidence | 187 | | [`holographic_measure.py`](#holographic-measure) | MEASURE -- perplexity with error bars, and decisions that respect them. | 239 | +| [`holographic_measure.py`](#holographic-measure) | The variance harness: every headline number gets a mean, a spread, and a confidence | 187 | | [`holographic_memory.py`](#holographic-memory) | MEMORY -- the Galvatron's own store, built on leCore's holographic database. | 241 | | [`holographic_memoryhome.py`](#holographic-memoryhome) | holographic_memoryhome.py -- the MEMORY home (consolidation backlog H6): keep the hot working set where the CP | 165 | | [`holographic_memsearch.py`](#holographic-memsearch) | MEMSEARCH -- searchable memory that lives in the weights and runs in the pass. | 212 | @@ -577,8 +577,8 @@ | [`holographic_rolling.py`](#holographic-rolling) | holographic_rolling.py -- the CAUSAL rolling / streaming statistics kit: trailing mean, std, min, max, | 378 | | [`holographic_roomacoustic.py`](#holographic-roomacoustic) | holographic_roomacoustic.py -- A6: GEOMETRIC ROOM ACOUSTICS. How a room echoes -- reflections and reverb. | 161 | | [`holographic_route.py`](#holographic-route) | Representation routing (ARCH-7): route each operation to the representation that supports it -- the policy lay | 183 | -| [`holographic_router.py`](#holographic-router) | ROUTER -- the model DECIDING, inside one forward pass. | 229 | | [`holographic_router.py`](#holographic-router) | Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. | 173 | +| [`holographic_router.py`](#holographic-router) | ROUTER -- the model DECIDING, inside one forward pass. | 229 | | [`holographic_sampler.py`](#holographic-sampler) | holographic_sampler.py -- the SAMPLER: a placeable read-probe (modeling-app backlog, the capstone item). | 223 | | [`holographic_sampling.py`](#holographic-sampling) | Blue-noise / Poisson-disk point sampling -- the EXCLUSION principle, done right. | 134 | | [`holographic_samplinghome.py`](#holographic-samplinghome) | holographic_samplinghome.py -- the SAMPLING home (consolidation backlog R4): one place for the Monte-Carlo | 127 | @@ -683,8 +683,8 @@ | [`holographic_toolclient.py`](#holographic-toolclient) | holographic_toolclient.py -- call another node the same way leCore is called. | 98 | | [`holographic_topology.py`](#holographic-topology) | Principled topology by persistent homology (EXP-7). | 288 | | [`holographic_transfer.py`](#holographic-transfer) | holographic_transfer.py -- KERNEL SCATTER / GATHER: the ONE bundle/readout under every particle<->grid transfe | 224 | -| [`holographic_transform.py`](#holographic-transform) | holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). | 297 | | [`holographic_transform.py`](#holographic-transform) | TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding. | 189 | +| [`holographic_transform.py`](#holographic-transform) | holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). | 297 | | [`holographic_transform_space.py`](#holographic-transform-space) | holographic_transform_space.py -- the TRANSFORM + SPACE model behind a gizmo. A gizmo is a UI; the backend it | 176 | | [`holographic_transformbank.py`](#holographic-transformbank) | holographic_transformbank.py -- a prebuilt map of hypervector transforms, and what it can and cannot hold. | 358 | | [`holographic_transformhome.py`](#holographic-transformhome) | holographic_transformhome.py -- the TRANSFORM home (consolidation backlog H5): one facade over "move / rotate | 223 | @@ -14848,39 +14848,6 @@ ### holographic_measure.py -> The variance harness: every headline number gets a mean, a spread, and a confidence -> interval across seeds -- so a lucky-seed point estimate can't pass as a real result. -> -> WHY this exists -> --------------- -> This whole engine is built on RANDOM vectors. Atoms are random, the RP-tree's -> hyperplanes are random, the reservoir is random, train/test splits are shuffled. A -> single-seed score is therefore a sample from a distribution, and reporting it alone hides -> how wide that distribution is. For an engine whose entire pitch is "measured, not -> promised," reporting a number without its noise is the sharpest blind spot -- so this -> points that same discipline at the numbers themselves. -> -> measure(run_once, seeds) runs a scored experiment once per seed and returns the mean, the -> sample standard deviation, and a 95% percentile-bootstrap confidence interval (no -> distributional assumptions). assert_robust(stats, floor) passes only if the LOWER CI -> bound clears the floor -- which is what stops a single fortunate seed from passing a -> test the typical seed would fail. report() formats "mean +/- std (95% CI [lo, hi], n)". -> -> USE REAL DATA. The point of the harness is to characterise the real distribution of a -> real claim; running it on a toy makes the spread meaningless. The measurements wired -> through it here all run on real corpora (Gutenberg Alice, UDHR, Reuters, Brown). - -**Public API:** - -- `def time_call(fn, repeats, warmup)` -- Time `fn()` honestly: warm up first, then take the MEDIAN of `repeats` timings in microseconds. -- `def measure(run_once, seeds, n_boot, boot_seed)` -- Run a scored experiment across seeds; return mean, std, and a 95% bootstrap CI. -- `def fdr_gate(rows, alpha)` -- P9 -- false-discovery control across a whole ABLATION TABLE, from the measurement home. -- `def assert_robust(stats, floor)` -- Pass only if the LOWER CI bound clears the floor -- not just the mean. This is -- `def is_fragile(stats, margin_floor)` -- A claim is FRAGILE if its spread is large relative to how far its mean sits above -- `def report(name, stats, floor)` -- Format a stats dict as 'name: mean +/- std (95% CI [lo, hi], n)', with a - -### holographic_measure.py - > MEASURE -- perplexity with error bars, and decisions that respect them. > > Moose asked what assimilation is actually doing. The answer, from his own run: @@ -14913,6 +14880,39 @@ - `def better_than(a, b, alpha, seed, resamples)` -- Is model A better than model B, or is the difference undecidable? - `def tokens_needed(reference, effect_pct, alpha)` -- How many tokens would be needed to RESOLVE an effect of this size. +### holographic_measure.py + +> The variance harness: every headline number gets a mean, a spread, and a confidence +> interval across seeds -- so a lucky-seed point estimate can't pass as a real result. +> +> WHY this exists +> --------------- +> This whole engine is built on RANDOM vectors. Atoms are random, the RP-tree's +> hyperplanes are random, the reservoir is random, train/test splits are shuffled. A +> single-seed score is therefore a sample from a distribution, and reporting it alone hides +> how wide that distribution is. For an engine whose entire pitch is "measured, not +> promised," reporting a number without its noise is the sharpest blind spot -- so this +> points that same discipline at the numbers themselves. +> +> measure(run_once, seeds) runs a scored experiment once per seed and returns the mean, the +> sample standard deviation, and a 95% percentile-bootstrap confidence interval (no +> distributional assumptions). assert_robust(stats, floor) passes only if the LOWER CI +> bound clears the floor -- which is what stops a single fortunate seed from passing a +> test the typical seed would fail. report() formats "mean +/- std (95% CI [lo, hi], n)". +> +> USE REAL DATA. The point of the harness is to characterise the real distribution of a +> real claim; running it on a toy makes the spread meaningless. The measurements wired +> through it here all run on real corpora (Gutenberg Alice, UDHR, Reuters, Brown). + +**Public API:** + +- `def time_call(fn, repeats, warmup)` -- Time `fn()` honestly: warm up first, then take the MEDIAN of `repeats` timings in microseconds. +- `def measure(run_once, seeds, n_boot, boot_seed)` -- Run a scored experiment across seeds; return mean, std, and a 95% bootstrap CI. +- `def fdr_gate(rows, alpha)` -- P9 -- false-discovery control across a whole ABLATION TABLE, from the measurement home. +- `def assert_robust(stats, floor)` -- Pass only if the LOWER CI bound clears the floor -- not just the mean. This is +- `def is_fragile(stats, margin_floor)` -- A claim is FRAGILE if its spread is large relative to how far its mean sits above +- `def report(name, stats, floor)` -- Format a stats dict as 'name: mean +/- std (95% CI [lo, hi], n)', with a + ### holographic_memory.py > MEMORY -- the Galvatron's own store, built on leCore's holographic database. @@ -22656,6 +22656,33 @@ ### holographic_router.py +> Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. +> +> WHY THIS EXISTS (measured, backlog N9/N28): +> The catalog's find_capability scores by shared content words. That is deterministic and needs no model, +> but it has no notion of MEANING: "squish a big array down for storage" shares no token with +> `holographic_coldstore`, and "airspeed velocity of an unladen swallow" confidently matched a physics +> module on the single word "velocity". Measured on the 12-ask suite: token overlap ~2/12 top-1, median +> rank 13 of 503; the nomic embedding router hit 7/12 top-1, median rank 1. +> +> WHAT SHIPS (and what does NOT): +> A 96 KB index -- 503 module vectors at 64d q8, plus the ABTT correction (mu, pc) baked in -- extracted +> from the build cache by tools/semantic/export_index.py. NO model ships. So this router can only score a +> query it already has a VECTOR for: +> * queries embedded at build time and cached (the exam's asks, an app's fixed vocabulary) +> * a caller that supplies its own query vector (an app that ran the encoder itself) +> A brand-new free-text query with no vector and no model present CANNOT be embedded here -- and this +> router says so and returns None, so the caller falls back to the token router rather than guessing. That +> honesty is the whole point: silence beats a confident wrong route. +> +> DETERMINISM: q8 dequant + fixed ABTT transform + argsort with a name tie-break. No RNG, no model. + +**Public API:** + +- `class EmbeddingRouter` -- Loads the shipped q8 index (any dim; 128d is the measured champion) and routes a query VECTOR (or a cached query text) to modules. + +### holographic_router.py + > ROUTER -- the model DECIDING, inside one forward pass. > > Moose raised the architecture that dissolves the wall this project kept hitting: @@ -22695,33 +22722,6 @@ - `def route(runtime, router, text, tokenize)` -- Would this model choose to use the capability on this prompt? - `def install_routed(weights, cfg, operator, router, layer, gain, temperature)` -- Install a circuit whose GATE is the router, not a constant. -### holographic_router.py - -> Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. -> -> WHY THIS EXISTS (measured, backlog N9/N28): -> The catalog's find_capability scores by shared content words. That is deterministic and needs no model, -> but it has no notion of MEANING: "squish a big array down for storage" shares no token with -> `holographic_coldstore`, and "airspeed velocity of an unladen swallow" confidently matched a physics -> module on the single word "velocity". Measured on the 12-ask suite: token overlap ~2/12 top-1, median -> rank 13 of 503; the nomic embedding router hit 7/12 top-1, median rank 1. -> -> WHAT SHIPS (and what does NOT): -> A 96 KB index -- 503 module vectors at 64d q8, plus the ABTT correction (mu, pc) baked in -- extracted -> from the build cache by tools/semantic/export_index.py. NO model ships. So this router can only score a -> query it already has a VECTOR for: -> * queries embedded at build time and cached (the exam's asks, an app's fixed vocabulary) -> * a caller that supplies its own query vector (an app that ran the encoder itself) -> A brand-new free-text query with no vector and no model present CANNOT be embedded here -- and this -> router says so and returns None, so the caller falls back to the token router rather than guessing. That -> honesty is the whole point: silence beats a confident wrong route. -> -> DETERMINISM: q8 dequant + fixed ABTT transform + argsort with a name tie-break. No RNG, no model. - -**Public API:** - -- `class EmbeddingRouter` -- Loads the shipped q8 index (any dim; 128d is the measured champion) and routes a query VECTOR (or a cached query text) to modules. - ### holographic_sampler.py > holographic_sampler.py -- the SAMPLER: a placeable read-probe (modeling-app backlog, the capstone item). @@ -24352,39 +24352,6 @@ ### holographic_session.py -> SESSION -- never compute the same conversation prefix twice. -> -> Moose runs a 0.8B on a CPU laptop and it is slow. The single largest waste in a -> conversation is not the model's arithmetic -- it is that every turn RE-PREFILLS -> the entire history. MEASURED on a realistic six-turn exchange: 489 tokens -> processed, of which only 137 were new. SEVENTY-TWO PERCENT OF THE WORK WAS -> REPEATED, and the fraction grows with every turn. -> -> leCore already had the pieces and never joined them: the runtime exposes -> prefill/step over an InferenceState with copy(), galvacache memoises pure -> functions at a measured 75% hit rate with bit-identical output, and sessions -> exist. What was missing is the RADIX TREE -- the structure that answers "what is -> the longest prefix of this prompt that I have already computed?" -> -> WHY A TREE AND NOT A DICTIONARY: turn 4 of a conversation shares its first three -> turns with turn 3, and a dictionary keyed on the whole prompt misses that -> completely. vLLM and SGLang call this RadixAttention; the idea is the same -> whatever the model: index by prefix, resume from the deepest match, compute only -> the tail. -> -> THE GUARANTEE THIS KEEPS, because a cache that changes answers is worse than no -> cache: resuming from a cached state reproduces a full recompute TO FLOAT -> ROUNDING -- measured 7.1e-15, machine epsilon. Not bit-identical, and the -> difference is real rather than pedantic: resuming STEPS the tail while a fresh -> call PREFILLS it, and the two associate their sums differently. The selftest -> asserts the measured bound against the live runtime rather than assuming it. - -**Public API:** - -- `class PrefixCache` -- A radix tree over token sequences, holding inference states. - -### holographic_session.py - > SESSION STORE -- contexts that outlive the process. > > A Galvatron's context is not a transcript, it is STATE: the GDN recurrent @@ -24417,6 +24384,39 @@ - `class SessionStore` -- Named, persistent, independent contexts on disk. - `def runtime_fingerprint(runtime)` -- A cheap, deterministic id for the checkpoint behind a runtime, so a +### holographic_session.py + +> SESSION -- never compute the same conversation prefix twice. +> +> Moose runs a 0.8B on a CPU laptop and it is slow. The single largest waste in a +> conversation is not the model's arithmetic -- it is that every turn RE-PREFILLS +> the entire history. MEASURED on a realistic six-turn exchange: 489 tokens +> processed, of which only 137 were new. SEVENTY-TWO PERCENT OF THE WORK WAS +> REPEATED, and the fraction grows with every turn. +> +> leCore already had the pieces and never joined them: the runtime exposes +> prefill/step over an InferenceState with copy(), galvacache memoises pure +> functions at a measured 75% hit rate with bit-identical output, and sessions +> exist. What was missing is the RADIX TREE -- the structure that answers "what is +> the longest prefix of this prompt that I have already computed?" +> +> WHY A TREE AND NOT A DICTIONARY: turn 4 of a conversation shares its first three +> turns with turn 3, and a dictionary keyed on the whole prompt misses that +> completely. vLLM and SGLang call this RadixAttention; the idea is the same +> whatever the model: index by prefix, resume from the deepest match, compute only +> the tail. +> +> THE GUARANTEE THIS KEEPS, because a cache that changes answers is worse than no +> cache: resuming from a cached state reproduces a full recompute TO FLOAT +> ROUNDING -- measured 7.1e-15, machine epsilon. Not bit-identical, and the +> difference is real rather than pedantic: resuming STEPS the tail while a fresh +> call PREFILLS it, and the two associate their sums differently. The selftest +> asserts the measured bound against the live runtime rather than assuming it. + +**Public API:** + +- `class PrefixCache` -- A radix tree over token sequences, holding inference states. + ### holographic_shader.py > holographic_shader.py -- N filter passes in ONE evaluation. Two things a GPU structurally cannot do. @@ -27945,48 +27945,6 @@ ### holographic_transform.py -> holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). -> -> The engine has scattered transform bits (scenegraph.translation/rotation/compose_transforms; cosserat's -> quaternion helpers; splatexport's rotation<->quaternion), but not the full kit a gizmo and a property panel need -> in one place. This gathers the standard, well-known math: -> -> * decompose(M) -> (translate, rotation-quaternion, scale) -- what a gizmo reads off a matrix to show handles, -> and what a property panel shows as the T/R/S fields; -> * compose_trs(translate, quat, scale) -> 4x4 -- the inverse (build a matrix from panel values); -> * a quaternion kit -- from/to matrix, from/to axis-angle, from/to euler, multiply, SLERP, rotate a vector -> (quaternions are what rotation UI and animation want: no gimbal lock, and slerp gives smooth interpolation); -> * look_at(eye, target, up) -> 4x4 view matrix -- for a camera or an object aimed at a point. -> -> Conventions, stated ONCE and held (the backlog's coordinate-convention note): matrices are 4x4 and act on COLUMN -> vectors, p' = M @ [x, y, z, 1]; compose(A, B) = A @ B means "apply B, then A"; quaternions are (w, x, y, z), unit -> length; euler angles are (rx, ry, rz) applied X then Y then Z, i.e. R = Rz @ Ry @ Rx; look_at returns an OpenGL -> view matrix (the camera looks down -z, y is up) to match the engine's Camera. Nothing here is holographic -- it is -> plain linear algebra a modeling app needs -- so it is kept as a small, readable utility, not dressed up as a bind. -> Deterministic; NumPy + stdlib only. - -**Public API:** - -- `def translation(t)` -- A 4x4 translation matrix from a 3-vector. -- `def scaling(s)` -- A 4x4 scale matrix. `s` is a scalar (uniform) or a 3-vector (per-axis). -- `def rotation_axis_angle(axis, angle)` -- A 4x4 rotation of `angle` radians about `axis` (Rodrigues' formula). -- `def compose(*mats)` -- Matrix product M0 @ M1 @ ... -- with the column-vector convention this applies the RIGHTMOST first. -- `def decompose(M)` -- Split a 4x4 affine transform into (translate (3,), rotation quaternion (4,), scale (3,)). Assumes no shear -- `def compose_trs(translate, quat, scale)` -- Build a 4x4 from translate (3,), a rotation quaternion (4,), and scale (3,) -- the inverse of decompose. -- `def quat_normalize(q)` -- `def quat_mul(a, b)` -- The Hamilton product a*b: the rotation "apply b, then a". -- `def quat_from_axis_angle(axis, angle)` -- A quaternion for a rotation of `angle` radians about `axis`. -- `def quat_to_axis_angle(q)` -- Recover (axis, angle) from a quaternion. -- `def quat_to_matrix(q)` -- The 3x3 rotation matrix for a quaternion. -- `def quat_from_matrix(R)` -- The quaternion for a 3x3 rotation matrix (Shepperd's method: branch on the largest diagonal term for -- `def quat_from_euler(rx, ry, rz)` -- A quaternion from euler angles applied X then Y then Z (R = Rz @ Ry @ Rx). -- `def quat_to_euler(q)` -- Recover euler angles (rx, ry, rz) from a quaternion, inverting R = Rz @ Ry @ Rx. Handles gimbal lock -- `def quat_slerp(a, b, t)` -- Spherical linear interpolation between two rotations -- constant angular speed, the smooth in-between an -- `def quat_rotate(q, v)` -- Rotate a 3-vector by a quaternion. -- `def look_at(eye, target, up)` -- An OpenGL view matrix for a camera at `eye` looking at `target` (the engine's convention: the camera looks - -### holographic_transform.py - > TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding. > > Everything before this applied leCore's levers uniformly: grow a memory channel @@ -28029,6 +27987,48 @@ - `def plan(weights, cfg, target_tokens, kv_rank, grow_gain)` -- Decide what to do to each layer, from the analysis rather than by rule. - `def apply_plan(weights, cfg, the_plan, progress)` -- Carry out the growth actions. KV compression is a RUNTIME setting and is +### holographic_transform.py + +> holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). +> +> The engine has scattered transform bits (scenegraph.translation/rotation/compose_transforms; cosserat's +> quaternion helpers; splatexport's rotation<->quaternion), but not the full kit a gizmo and a property panel need +> in one place. This gathers the standard, well-known math: +> +> * decompose(M) -> (translate, rotation-quaternion, scale) -- what a gizmo reads off a matrix to show handles, +> and what a property panel shows as the T/R/S fields; +> * compose_trs(translate, quat, scale) -> 4x4 -- the inverse (build a matrix from panel values); +> * a quaternion kit -- from/to matrix, from/to axis-angle, from/to euler, multiply, SLERP, rotate a vector +> (quaternions are what rotation UI and animation want: no gimbal lock, and slerp gives smooth interpolation); +> * look_at(eye, target, up) -> 4x4 view matrix -- for a camera or an object aimed at a point. +> +> Conventions, stated ONCE and held (the backlog's coordinate-convention note): matrices are 4x4 and act on COLUMN +> vectors, p' = M @ [x, y, z, 1]; compose(A, B) = A @ B means "apply B, then A"; quaternions are (w, x, y, z), unit +> length; euler angles are (rx, ry, rz) applied X then Y then Z, i.e. R = Rz @ Ry @ Rx; look_at returns an OpenGL +> view matrix (the camera looks down -z, y is up) to match the engine's Camera. Nothing here is holographic -- it is +> plain linear algebra a modeling app needs -- so it is kept as a small, readable utility, not dressed up as a bind. +> Deterministic; NumPy + stdlib only. + +**Public API:** + +- `def translation(t)` -- A 4x4 translation matrix from a 3-vector. +- `def scaling(s)` -- A 4x4 scale matrix. `s` is a scalar (uniform) or a 3-vector (per-axis). +- `def rotation_axis_angle(axis, angle)` -- A 4x4 rotation of `angle` radians about `axis` (Rodrigues' formula). +- `def compose(*mats)` -- Matrix product M0 @ M1 @ ... -- with the column-vector convention this applies the RIGHTMOST first. +- `def decompose(M)` -- Split a 4x4 affine transform into (translate (3,), rotation quaternion (4,), scale (3,)). Assumes no shear +- `def compose_trs(translate, quat, scale)` -- Build a 4x4 from translate (3,), a rotation quaternion (4,), and scale (3,) -- the inverse of decompose. +- `def quat_normalize(q)` +- `def quat_mul(a, b)` -- The Hamilton product a*b: the rotation "apply b, then a". +- `def quat_from_axis_angle(axis, angle)` -- A quaternion for a rotation of `angle` radians about `axis`. +- `def quat_to_axis_angle(q)` -- Recover (axis, angle) from a quaternion. +- `def quat_to_matrix(q)` -- The 3x3 rotation matrix for a quaternion. +- `def quat_from_matrix(R)` -- The quaternion for a 3x3 rotation matrix (Shepperd's method: branch on the largest diagonal term for +- `def quat_from_euler(rx, ry, rz)` -- A quaternion from euler angles applied X then Y then Z (R = Rz @ Ry @ Rx). +- `def quat_to_euler(q)` -- Recover euler angles (rx, ry, rz) from a quaternion, inverting R = Rz @ Ry @ Rx. Handles gimbal lock +- `def quat_slerp(a, b, t)` -- Spherical linear interpolation between two rotations -- constant angular speed, the smooth in-between an +- `def quat_rotate(q, v)` -- Rotate a 3-vector by a quaternion. +- `def look_at(eye, target, up)` -- An OpenGL view matrix for a camera at `eye` looking at `target` (the engine's convention: the camera looks + ### holographic_transform_space.py > holographic_transform_space.py -- the TRANSFORM + SPACE model behind a gizmo. A gizmo is a UI; the backend it From 41450108b4477572b45af89d66d40d04f53a159f Mon Sep 17 00:00:00 2001 From: atimics Date: Wed, 12 Aug 2026 17:02:00 -0700 Subject: [PATCH 12/16] ci: scope qwen experiment artifacts --- tests/test_select_tests.py | 17 +++++++++++++++++ tools/select_tests.py | 15 +++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/tests/test_select_tests.py b/tests/test_select_tests.py index 11e20a9..746f200 100644 --- a/tests/test_select_tests.py +++ b/tests/test_select_tests.py @@ -76,6 +76,23 @@ def test_unknown_archive_elsewhere_still_forces_full(): assert affected_tests(["features/mystery_dataset.zip"], root=ROOT) == "ALL" +def test_qwen_result_artifacts_select_the_qwen_contract_only(): + artifact = ( + "experiments/qwen35_acceptance/results/" + "v2-example/result/metrics.json" + ) + assert affected_tests([artifact], root=ROOT) == ["tests/test_qwen_acceptance.py"] + assert affected_tests( + ["experiments/qwen35_acceptance/launch-manifest.json"], root=ROOT + ) == ["tests/test_qwen_acceptance.py"] + + +def test_unknown_experiment_artifact_still_forces_full(): + assert affected_tests( + ["experiments/unknown/results/v1/metrics.json"], root=ROOT + ) == "ALL" + + def test_a_committed_data_artifact_scopes_to_its_readers_not_the_world(): """The CI-cost fix, pinned. Non-.py changes used to return ALL unconditionally, so ROUTINE BOT PUSHES -- docs.yml commits capabilities.json, semantic-coverage.yml commits the routing index and seed -- took the diff --git a/tools/select_tests.py b/tools/select_tests.py index e339d6c..6eccddb 100644 --- a/tools/select_tests.py +++ b/tools/select_tests.py @@ -54,6 +54,13 @@ _BUILD_ARTIFACT_NAMES = {"holographic_vsa_complete.zip"} _BUILD_DIR_PREFIXES = ("dist/", "build/", "build_pkg/") +# Immutable experiment records are inputs to their contract test, not executable +# project data. Keep these mappings narrow: an unknown artifact anywhere else must +# still take the conservative ALL path. +_SCOPED_DATA_PREFIXES = { + "experiments/qwen35_acceptance/": "tests/test_qwen_acceptance.py", +} + def _is_build_artifact(path): """True for the repo's own build outputs (the packaging zip, dist/, build/, *.egg-info/). Regenerated files, never @@ -237,6 +244,14 @@ def affected_tests(changed_files, root="."): else: return "ALL" # a .py we can't place in the map -> don't risk it else: + scoped_test = next((test for prefix, test in _SCOPED_DATA_PREFIXES.items() + if f.startswith(prefix)), None) + if scoped_test is not None: + test_module = _path_to_module(scoped_test) + if test_module not in modules: + return "ALL" # mapping drifted -> fail safe + data_tests.add(test_module) + continue data_hits = _data_file_dependents(f, root, modules, direct, _data_cache) if data_hits is None: return "ALL" # genuinely unplaceable -> run the whole suite From 6cb7706e0f723b266b0450620f30683921084d10 Mon Sep 17 00:00:00 2001 From: atimics Date: Wed, 12 Aug 2026 17:03:14 -0700 Subject: [PATCH 13/16] tools: add qwen evidence manifest builder --- .../qwen35_acceptance/build_publication.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 experiments/qwen35_acceptance/build_publication.py diff --git a/experiments/qwen35_acceptance/build_publication.py b/experiments/qwen35_acceptance/build_publication.py new file mode 100644 index 0000000..0a61078 --- /dev/null +++ b/experiments/qwen35_acceptance/build_publication.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Build a deterministic publication manifest for one Qwen result directory.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import mimetypes +from pathlib import Path +from typing import Any + + +EXCLUDED_NAMES = {"publication-manifest.json", "publication-receipt.json"} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def read_json(path: Path) -> Any: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def evidence_ref(result_dir: Path) -> str: + events_path = result_dir / "ledger" / "events.jsonl" + with events_path.open("r", encoding="utf-8") as handle: + events = [json.loads(line) for line in handle if line.strip()] + recorded = [ + event["artifact_ref"] + for event in events + if event.get("event_type") == "EvidenceRecorded" + ] + if len(recorded) != 1: + raise ValueError(f"expected exactly one EvidenceRecorded event, got {len(recorded)}") + return recorded[0] + + +def build_manifest(result_dir: Path) -> dict[str, Any]: + project = read_json(result_dir / "project" / "project.json") + status = read_json(result_dir / "result" / "ilxyr-status.json") + verification = read_json(result_dir / "result" / "ilxyr-verify.json") + latest_evidence = status["latest_evidence"] + + files = [] + for path in sorted(item for item in result_dir.rglob("*") if item.is_file()): + if path.name in EXCLUDED_NAMES: + continue + relative = path.relative_to(result_dir).as_posix() + media_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream" + files.append( + { + "path": relative, + "bytes": path.stat().st_size, + "sha256": sha256(path), + "media_type": media_type, + } + ) + + return { + "schema": "lecore.qwen35-publication-manifest.v1", + "experiment_id": project["experiment_id"], + "resolved_outcome": latest_evidence["resolved_outcome"], + "run_ref": latest_evidence["run_ref"], + "evidence_ref": evidence_ref(result_dir), + "ledger_verification": verification, + "files": files, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("result_dir", type=Path) + args = parser.parse_args() + result_dir = args.result_dir.expanduser().resolve() + manifest = build_manifest(result_dir) + output = result_dir / "publication-manifest.json" + output.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(f"{sha256(output)} {output}") + + +if __name__ == "__main__": + main() From 0580567388e9b4dfa61a95fae5445670baf60348 Mon Sep 17 00:00:00 2001 From: docs-bot Date: Thu, 13 Aug 2026 00:03:42 +0000 Subject: [PATCH 14/16] docs: refresh generated docs (REFERENCE, CAPABILITIES, API_QUICKREF, PIPELINE_MAP, FACULTY_MAP, DOC_MAP) [skip ci] --- REFERENCE.md | 278 +++++++++++++++++++++++++-------------------------- 1 file changed, 139 insertions(+), 139 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index 164a4cd..e06473d 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -76,8 +76,8 @@ | module | what it is | lines | |---|---|---| | [`holographic_session.py`](#holographic-session) | holographic_session.py -- ONE render session that ties the disconnected rendering threads together. | 227 | -| [`holographic_session.py`](#holographic-session) | SESSION STORE -- contexts that outlive the process. | 261 | | [`holographic_session.py`](#holographic-session) | SESSION -- never compute the same conversation prefix twice. | 253 | +| [`holographic_session.py`](#holographic-session) | SESSION STORE -- contexts that outlive the process. | 261 | ### `splat*` family (6) @@ -423,8 +423,8 @@ | [`holographic_materialio.py`](#holographic-materialio) | PBR material interchange (G2-IO): the standard-format bridge for materials, and a VSA-native carrier. | 307 | | [`holographic_matlib.py`](#holographic-matlib) | holographic_matlib.py -- a comprehensive RENDER material library: plain diffuse -> a fractal planet. | 801 | | [`holographic_meaning_predict.py`](#holographic-meaning-predict) | Generation with structure: predict a next-MEANING vector and settle it, rather | 198 | -| [`holographic_measure.py`](#holographic-measure) | MEASURE -- perplexity with error bars, and decisions that respect them. | 239 | | [`holographic_measure.py`](#holographic-measure) | The variance harness: every headline number gets a mean, a spread, and a confidence | 187 | +| [`holographic_measure.py`](#holographic-measure) | MEASURE -- perplexity with error bars, and decisions that respect them. | 239 | | [`holographic_memory.py`](#holographic-memory) | MEMORY -- the Galvatron's own store, built on leCore's holographic database. | 241 | | [`holographic_memoryhome.py`](#holographic-memoryhome) | holographic_memoryhome.py -- the MEMORY home (consolidation backlog H6): keep the hot working set where the CP | 165 | | [`holographic_memsearch.py`](#holographic-memsearch) | MEMSEARCH -- searchable memory that lives in the weights and runs in the pass. | 212 | @@ -577,8 +577,8 @@ | [`holographic_rolling.py`](#holographic-rolling) | holographic_rolling.py -- the CAUSAL rolling / streaming statistics kit: trailing mean, std, min, max, | 378 | | [`holographic_roomacoustic.py`](#holographic-roomacoustic) | holographic_roomacoustic.py -- A6: GEOMETRIC ROOM ACOUSTICS. How a room echoes -- reflections and reverb. | 161 | | [`holographic_route.py`](#holographic-route) | Representation routing (ARCH-7): route each operation to the representation that supports it -- the policy lay | 183 | -| [`holographic_router.py`](#holographic-router) | Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. | 173 | | [`holographic_router.py`](#holographic-router) | ROUTER -- the model DECIDING, inside one forward pass. | 229 | +| [`holographic_router.py`](#holographic-router) | Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. | 173 | | [`holographic_sampler.py`](#holographic-sampler) | holographic_sampler.py -- the SAMPLER: a placeable read-probe (modeling-app backlog, the capstone item). | 223 | | [`holographic_sampling.py`](#holographic-sampling) | Blue-noise / Poisson-disk point sampling -- the EXCLUSION principle, done right. | 134 | | [`holographic_samplinghome.py`](#holographic-samplinghome) | holographic_samplinghome.py -- the SAMPLING home (consolidation backlog R4): one place for the Monte-Carlo | 127 | @@ -683,8 +683,8 @@ | [`holographic_toolclient.py`](#holographic-toolclient) | holographic_toolclient.py -- call another node the same way leCore is called. | 98 | | [`holographic_topology.py`](#holographic-topology) | Principled topology by persistent homology (EXP-7). | 288 | | [`holographic_transfer.py`](#holographic-transfer) | holographic_transfer.py -- KERNEL SCATTER / GATHER: the ONE bundle/readout under every particle<->grid transfe | 224 | -| [`holographic_transform.py`](#holographic-transform) | TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding. | 189 | | [`holographic_transform.py`](#holographic-transform) | holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). | 297 | +| [`holographic_transform.py`](#holographic-transform) | TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding. | 189 | | [`holographic_transform_space.py`](#holographic-transform-space) | holographic_transform_space.py -- the TRANSFORM + SPACE model behind a gizmo. A gizmo is a UI; the backend it | 176 | | [`holographic_transformbank.py`](#holographic-transformbank) | holographic_transformbank.py -- a prebuilt map of hypervector transforms, and what it can and cannot hold. | 358 | | [`holographic_transformhome.py`](#holographic-transformhome) | holographic_transformhome.py -- the TRANSFORM home (consolidation backlog H5): one facade over "move / rotate | 223 | @@ -14848,6 +14848,39 @@ ### holographic_measure.py +> The variance harness: every headline number gets a mean, a spread, and a confidence +> interval across seeds -- so a lucky-seed point estimate can't pass as a real result. +> +> WHY this exists +> --------------- +> This whole engine is built on RANDOM vectors. Atoms are random, the RP-tree's +> hyperplanes are random, the reservoir is random, train/test splits are shuffled. A +> single-seed score is therefore a sample from a distribution, and reporting it alone hides +> how wide that distribution is. For an engine whose entire pitch is "measured, not +> promised," reporting a number without its noise is the sharpest blind spot -- so this +> points that same discipline at the numbers themselves. +> +> measure(run_once, seeds) runs a scored experiment once per seed and returns the mean, the +> sample standard deviation, and a 95% percentile-bootstrap confidence interval (no +> distributional assumptions). assert_robust(stats, floor) passes only if the LOWER CI +> bound clears the floor -- which is what stops a single fortunate seed from passing a +> test the typical seed would fail. report() formats "mean +/- std (95% CI [lo, hi], n)". +> +> USE REAL DATA. The point of the harness is to characterise the real distribution of a +> real claim; running it on a toy makes the spread meaningless. The measurements wired +> through it here all run on real corpora (Gutenberg Alice, UDHR, Reuters, Brown). + +**Public API:** + +- `def time_call(fn, repeats, warmup)` -- Time `fn()` honestly: warm up first, then take the MEDIAN of `repeats` timings in microseconds. +- `def measure(run_once, seeds, n_boot, boot_seed)` -- Run a scored experiment across seeds; return mean, std, and a 95% bootstrap CI. +- `def fdr_gate(rows, alpha)` -- P9 -- false-discovery control across a whole ABLATION TABLE, from the measurement home. +- `def assert_robust(stats, floor)` -- Pass only if the LOWER CI bound clears the floor -- not just the mean. This is +- `def is_fragile(stats, margin_floor)` -- A claim is FRAGILE if its spread is large relative to how far its mean sits above +- `def report(name, stats, floor)` -- Format a stats dict as 'name: mean +/- std (95% CI [lo, hi], n)', with a + +### holographic_measure.py + > MEASURE -- perplexity with error bars, and decisions that respect them. > > Moose asked what assimilation is actually doing. The answer, from his own run: @@ -14880,39 +14913,6 @@ - `def better_than(a, b, alpha, seed, resamples)` -- Is model A better than model B, or is the difference undecidable? - `def tokens_needed(reference, effect_pct, alpha)` -- How many tokens would be needed to RESOLVE an effect of this size. -### holographic_measure.py - -> The variance harness: every headline number gets a mean, a spread, and a confidence -> interval across seeds -- so a lucky-seed point estimate can't pass as a real result. -> -> WHY this exists -> --------------- -> This whole engine is built on RANDOM vectors. Atoms are random, the RP-tree's -> hyperplanes are random, the reservoir is random, train/test splits are shuffled. A -> single-seed score is therefore a sample from a distribution, and reporting it alone hides -> how wide that distribution is. For an engine whose entire pitch is "measured, not -> promised," reporting a number without its noise is the sharpest blind spot -- so this -> points that same discipline at the numbers themselves. -> -> measure(run_once, seeds) runs a scored experiment once per seed and returns the mean, the -> sample standard deviation, and a 95% percentile-bootstrap confidence interval (no -> distributional assumptions). assert_robust(stats, floor) passes only if the LOWER CI -> bound clears the floor -- which is what stops a single fortunate seed from passing a -> test the typical seed would fail. report() formats "mean +/- std (95% CI [lo, hi], n)". -> -> USE REAL DATA. The point of the harness is to characterise the real distribution of a -> real claim; running it on a toy makes the spread meaningless. The measurements wired -> through it here all run on real corpora (Gutenberg Alice, UDHR, Reuters, Brown). - -**Public API:** - -- `def time_call(fn, repeats, warmup)` -- Time `fn()` honestly: warm up first, then take the MEDIAN of `repeats` timings in microseconds. -- `def measure(run_once, seeds, n_boot, boot_seed)` -- Run a scored experiment across seeds; return mean, std, and a 95% bootstrap CI. -- `def fdr_gate(rows, alpha)` -- P9 -- false-discovery control across a whole ABLATION TABLE, from the measurement home. -- `def assert_robust(stats, floor)` -- Pass only if the LOWER CI bound clears the floor -- not just the mean. This is -- `def is_fragile(stats, margin_floor)` -- A claim is FRAGILE if its spread is large relative to how far its mean sits above -- `def report(name, stats, floor)` -- Format a stats dict as 'name: mean +/- std (95% CI [lo, hi], n)', with a - ### holographic_memory.py > MEMORY -- the Galvatron's own store, built on leCore's holographic database. @@ -22656,33 +22656,6 @@ ### holographic_router.py -> Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. -> -> WHY THIS EXISTS (measured, backlog N9/N28): -> The catalog's find_capability scores by shared content words. That is deterministic and needs no model, -> but it has no notion of MEANING: "squish a big array down for storage" shares no token with -> `holographic_coldstore`, and "airspeed velocity of an unladen swallow" confidently matched a physics -> module on the single word "velocity". Measured on the 12-ask suite: token overlap ~2/12 top-1, median -> rank 13 of 503; the nomic embedding router hit 7/12 top-1, median rank 1. -> -> WHAT SHIPS (and what does NOT): -> A 96 KB index -- 503 module vectors at 64d q8, plus the ABTT correction (mu, pc) baked in -- extracted -> from the build cache by tools/semantic/export_index.py. NO model ships. So this router can only score a -> query it already has a VECTOR for: -> * queries embedded at build time and cached (the exam's asks, an app's fixed vocabulary) -> * a caller that supplies its own query vector (an app that ran the encoder itself) -> A brand-new free-text query with no vector and no model present CANNOT be embedded here -- and this -> router says so and returns None, so the caller falls back to the token router rather than guessing. That -> honesty is the whole point: silence beats a confident wrong route. -> -> DETERMINISM: q8 dequant + fixed ABTT transform + argsort with a name tie-break. No RNG, no model. - -**Public API:** - -- `class EmbeddingRouter` -- Loads the shipped q8 index (any dim; 128d is the measured champion) and routes a query VECTOR (or a cached query text) to modules. - -### holographic_router.py - > ROUTER -- the model DECIDING, inside one forward pass. > > Moose raised the architecture that dissolves the wall this project kept hitting: @@ -22722,6 +22695,33 @@ - `def route(runtime, router, text, tokenize)` -- Would this model choose to use the capability on this prompt? - `def install_routed(weights, cfg, operator, router, layer, gain, temperature)` -- Install a circuit whose GATE is the router, not a constant. +### holographic_router.py + +> Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. +> +> WHY THIS EXISTS (measured, backlog N9/N28): +> The catalog's find_capability scores by shared content words. That is deterministic and needs no model, +> but it has no notion of MEANING: "squish a big array down for storage" shares no token with +> `holographic_coldstore`, and "airspeed velocity of an unladen swallow" confidently matched a physics +> module on the single word "velocity". Measured on the 12-ask suite: token overlap ~2/12 top-1, median +> rank 13 of 503; the nomic embedding router hit 7/12 top-1, median rank 1. +> +> WHAT SHIPS (and what does NOT): +> A 96 KB index -- 503 module vectors at 64d q8, plus the ABTT correction (mu, pc) baked in -- extracted +> from the build cache by tools/semantic/export_index.py. NO model ships. So this router can only score a +> query it already has a VECTOR for: +> * queries embedded at build time and cached (the exam's asks, an app's fixed vocabulary) +> * a caller that supplies its own query vector (an app that ran the encoder itself) +> A brand-new free-text query with no vector and no model present CANNOT be embedded here -- and this +> router says so and returns None, so the caller falls back to the token router rather than guessing. That +> honesty is the whole point: silence beats a confident wrong route. +> +> DETERMINISM: q8 dequant + fixed ABTT transform + argsort with a name tie-break. No RNG, no model. + +**Public API:** + +- `class EmbeddingRouter` -- Loads the shipped q8 index (any dim; 128d is the measured champion) and routes a query VECTOR (or a cached query text) to modules. + ### holographic_sampler.py > holographic_sampler.py -- the SAMPLER: a placeable read-probe (modeling-app backlog, the capstone item). @@ -24352,6 +24352,39 @@ ### holographic_session.py +> SESSION -- never compute the same conversation prefix twice. +> +> Moose runs a 0.8B on a CPU laptop and it is slow. The single largest waste in a +> conversation is not the model's arithmetic -- it is that every turn RE-PREFILLS +> the entire history. MEASURED on a realistic six-turn exchange: 489 tokens +> processed, of which only 137 were new. SEVENTY-TWO PERCENT OF THE WORK WAS +> REPEATED, and the fraction grows with every turn. +> +> leCore already had the pieces and never joined them: the runtime exposes +> prefill/step over an InferenceState with copy(), galvacache memoises pure +> functions at a measured 75% hit rate with bit-identical output, and sessions +> exist. What was missing is the RADIX TREE -- the structure that answers "what is +> the longest prefix of this prompt that I have already computed?" +> +> WHY A TREE AND NOT A DICTIONARY: turn 4 of a conversation shares its first three +> turns with turn 3, and a dictionary keyed on the whole prompt misses that +> completely. vLLM and SGLang call this RadixAttention; the idea is the same +> whatever the model: index by prefix, resume from the deepest match, compute only +> the tail. +> +> THE GUARANTEE THIS KEEPS, because a cache that changes answers is worse than no +> cache: resuming from a cached state reproduces a full recompute TO FLOAT +> ROUNDING -- measured 7.1e-15, machine epsilon. Not bit-identical, and the +> difference is real rather than pedantic: resuming STEPS the tail while a fresh +> call PREFILLS it, and the two associate their sums differently. The selftest +> asserts the measured bound against the live runtime rather than assuming it. + +**Public API:** + +- `class PrefixCache` -- A radix tree over token sequences, holding inference states. + +### holographic_session.py + > SESSION STORE -- contexts that outlive the process. > > A Galvatron's context is not a transcript, it is STATE: the GDN recurrent @@ -24384,39 +24417,6 @@ - `class SessionStore` -- Named, persistent, independent contexts on disk. - `def runtime_fingerprint(runtime)` -- A cheap, deterministic id for the checkpoint behind a runtime, so a -### holographic_session.py - -> SESSION -- never compute the same conversation prefix twice. -> -> Moose runs a 0.8B on a CPU laptop and it is slow. The single largest waste in a -> conversation is not the model's arithmetic -- it is that every turn RE-PREFILLS -> the entire history. MEASURED on a realistic six-turn exchange: 489 tokens -> processed, of which only 137 were new. SEVENTY-TWO PERCENT OF THE WORK WAS -> REPEATED, and the fraction grows with every turn. -> -> leCore already had the pieces and never joined them: the runtime exposes -> prefill/step over an InferenceState with copy(), galvacache memoises pure -> functions at a measured 75% hit rate with bit-identical output, and sessions -> exist. What was missing is the RADIX TREE -- the structure that answers "what is -> the longest prefix of this prompt that I have already computed?" -> -> WHY A TREE AND NOT A DICTIONARY: turn 4 of a conversation shares its first three -> turns with turn 3, and a dictionary keyed on the whole prompt misses that -> completely. vLLM and SGLang call this RadixAttention; the idea is the same -> whatever the model: index by prefix, resume from the deepest match, compute only -> the tail. -> -> THE GUARANTEE THIS KEEPS, because a cache that changes answers is worse than no -> cache: resuming from a cached state reproduces a full recompute TO FLOAT -> ROUNDING -- measured 7.1e-15, machine epsilon. Not bit-identical, and the -> difference is real rather than pedantic: resuming STEPS the tail while a fresh -> call PREFILLS it, and the two associate their sums differently. The selftest -> asserts the measured bound against the live runtime rather than assuming it. - -**Public API:** - -- `class PrefixCache` -- A radix tree over token sequences, holding inference states. - ### holographic_shader.py > holographic_shader.py -- N filter passes in ONE evaluation. Two things a GPU structurally cannot do. @@ -27945,6 +27945,48 @@ ### holographic_transform.py +> holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). +> +> The engine has scattered transform bits (scenegraph.translation/rotation/compose_transforms; cosserat's +> quaternion helpers; splatexport's rotation<->quaternion), but not the full kit a gizmo and a property panel need +> in one place. This gathers the standard, well-known math: +> +> * decompose(M) -> (translate, rotation-quaternion, scale) -- what a gizmo reads off a matrix to show handles, +> and what a property panel shows as the T/R/S fields; +> * compose_trs(translate, quat, scale) -> 4x4 -- the inverse (build a matrix from panel values); +> * a quaternion kit -- from/to matrix, from/to axis-angle, from/to euler, multiply, SLERP, rotate a vector +> (quaternions are what rotation UI and animation want: no gimbal lock, and slerp gives smooth interpolation); +> * look_at(eye, target, up) -> 4x4 view matrix -- for a camera or an object aimed at a point. +> +> Conventions, stated ONCE and held (the backlog's coordinate-convention note): matrices are 4x4 and act on COLUMN +> vectors, p' = M @ [x, y, z, 1]; compose(A, B) = A @ B means "apply B, then A"; quaternions are (w, x, y, z), unit +> length; euler angles are (rx, ry, rz) applied X then Y then Z, i.e. R = Rz @ Ry @ Rx; look_at returns an OpenGL +> view matrix (the camera looks down -z, y is up) to match the engine's Camera. Nothing here is holographic -- it is +> plain linear algebra a modeling app needs -- so it is kept as a small, readable utility, not dressed up as a bind. +> Deterministic; NumPy + stdlib only. + +**Public API:** + +- `def translation(t)` -- A 4x4 translation matrix from a 3-vector. +- `def scaling(s)` -- A 4x4 scale matrix. `s` is a scalar (uniform) or a 3-vector (per-axis). +- `def rotation_axis_angle(axis, angle)` -- A 4x4 rotation of `angle` radians about `axis` (Rodrigues' formula). +- `def compose(*mats)` -- Matrix product M0 @ M1 @ ... -- with the column-vector convention this applies the RIGHTMOST first. +- `def decompose(M)` -- Split a 4x4 affine transform into (translate (3,), rotation quaternion (4,), scale (3,)). Assumes no shear +- `def compose_trs(translate, quat, scale)` -- Build a 4x4 from translate (3,), a rotation quaternion (4,), and scale (3,) -- the inverse of decompose. +- `def quat_normalize(q)` +- `def quat_mul(a, b)` -- The Hamilton product a*b: the rotation "apply b, then a". +- `def quat_from_axis_angle(axis, angle)` -- A quaternion for a rotation of `angle` radians about `axis`. +- `def quat_to_axis_angle(q)` -- Recover (axis, angle) from a quaternion. +- `def quat_to_matrix(q)` -- The 3x3 rotation matrix for a quaternion. +- `def quat_from_matrix(R)` -- The quaternion for a 3x3 rotation matrix (Shepperd's method: branch on the largest diagonal term for +- `def quat_from_euler(rx, ry, rz)` -- A quaternion from euler angles applied X then Y then Z (R = Rz @ Ry @ Rx). +- `def quat_to_euler(q)` -- Recover euler angles (rx, ry, rz) from a quaternion, inverting R = Rz @ Ry @ Rx. Handles gimbal lock +- `def quat_slerp(a, b, t)` -- Spherical linear interpolation between two rotations -- constant angular speed, the smooth in-between an +- `def quat_rotate(q, v)` -- Rotate a 3-vector by a quaternion. +- `def look_at(eye, target, up)` -- An OpenGL view matrix for a camera at `eye` looking at `target` (the engine's convention: the camera looks + +### holographic_transform.py + > TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding. > > Everything before this applied leCore's levers uniformly: grow a memory channel @@ -27987,48 +28029,6 @@ - `def plan(weights, cfg, target_tokens, kv_rank, grow_gain)` -- Decide what to do to each layer, from the analysis rather than by rule. - `def apply_plan(weights, cfg, the_plan, progress)` -- Carry out the growth actions. KV compression is a RUNTIME setting and is -### holographic_transform.py - -> holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). -> -> The engine has scattered transform bits (scenegraph.translation/rotation/compose_transforms; cosserat's -> quaternion helpers; splatexport's rotation<->quaternion), but not the full kit a gizmo and a property panel need -> in one place. This gathers the standard, well-known math: -> -> * decompose(M) -> (translate, rotation-quaternion, scale) -- what a gizmo reads off a matrix to show handles, -> and what a property panel shows as the T/R/S fields; -> * compose_trs(translate, quat, scale) -> 4x4 -- the inverse (build a matrix from panel values); -> * a quaternion kit -- from/to matrix, from/to axis-angle, from/to euler, multiply, SLERP, rotate a vector -> (quaternions are what rotation UI and animation want: no gimbal lock, and slerp gives smooth interpolation); -> * look_at(eye, target, up) -> 4x4 view matrix -- for a camera or an object aimed at a point. -> -> Conventions, stated ONCE and held (the backlog's coordinate-convention note): matrices are 4x4 and act on COLUMN -> vectors, p' = M @ [x, y, z, 1]; compose(A, B) = A @ B means "apply B, then A"; quaternions are (w, x, y, z), unit -> length; euler angles are (rx, ry, rz) applied X then Y then Z, i.e. R = Rz @ Ry @ Rx; look_at returns an OpenGL -> view matrix (the camera looks down -z, y is up) to match the engine's Camera. Nothing here is holographic -- it is -> plain linear algebra a modeling app needs -- so it is kept as a small, readable utility, not dressed up as a bind. -> Deterministic; NumPy + stdlib only. - -**Public API:** - -- `def translation(t)` -- A 4x4 translation matrix from a 3-vector. -- `def scaling(s)` -- A 4x4 scale matrix. `s` is a scalar (uniform) or a 3-vector (per-axis). -- `def rotation_axis_angle(axis, angle)` -- A 4x4 rotation of `angle` radians about `axis` (Rodrigues' formula). -- `def compose(*mats)` -- Matrix product M0 @ M1 @ ... -- with the column-vector convention this applies the RIGHTMOST first. -- `def decompose(M)` -- Split a 4x4 affine transform into (translate (3,), rotation quaternion (4,), scale (3,)). Assumes no shear -- `def compose_trs(translate, quat, scale)` -- Build a 4x4 from translate (3,), a rotation quaternion (4,), and scale (3,) -- the inverse of decompose. -- `def quat_normalize(q)` -- `def quat_mul(a, b)` -- The Hamilton product a*b: the rotation "apply b, then a". -- `def quat_from_axis_angle(axis, angle)` -- A quaternion for a rotation of `angle` radians about `axis`. -- `def quat_to_axis_angle(q)` -- Recover (axis, angle) from a quaternion. -- `def quat_to_matrix(q)` -- The 3x3 rotation matrix for a quaternion. -- `def quat_from_matrix(R)` -- The quaternion for a 3x3 rotation matrix (Shepperd's method: branch on the largest diagonal term for -- `def quat_from_euler(rx, ry, rz)` -- A quaternion from euler angles applied X then Y then Z (R = Rz @ Ry @ Rx). -- `def quat_to_euler(q)` -- Recover euler angles (rx, ry, rz) from a quaternion, inverting R = Rz @ Ry @ Rx. Handles gimbal lock -- `def quat_slerp(a, b, t)` -- Spherical linear interpolation between two rotations -- constant angular speed, the smooth in-between an -- `def quat_rotate(q, v)` -- Rotate a 3-vector by a quaternion. -- `def look_at(eye, target, up)` -- An OpenGL view matrix for a camera at `eye` looking at `target` (the engine's convention: the camera looks - ### holographic_transform_space.py > holographic_transform_space.py -- the TRANSFORM + SPACE model behind a gizmo. A gizmo is a UI; the backend it From 2f32151c6d44c80a7e2ff98c0696650cd1d9c6f1 Mon Sep 17 00:00:00 2001 From: atimics Date: Wed, 12 Aug 2026 17:08:37 -0700 Subject: [PATCH 15/16] experiments: version formal qwen attempts --- experiments/qwen35_acceptance/generate.py | 12 +++++++++--- tests/test_qwen_acceptance.py | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/experiments/qwen35_acceptance/generate.py b/experiments/qwen35_acceptance/generate.py index a2101c7..011bcab 100644 --- a/experiments/qwen35_acceptance/generate.py +++ b/experiments/qwen35_acceptance/generate.py @@ -84,6 +84,8 @@ def main(argv=None): ap.add_argument("--min-tokens", type=int, default=4096) ap.add_argument("--timeout-seconds", type=int, default=21600) ap.add_argument("--compute-credits", type=int, default=100) + ap.add_argument("--experiment-version", type=int, default=1, + help="monotonic formal-attempt identity (default: 1)") ap.add_argument("--ilxyr-cli", type=Path, default=Path.home() / "develop" / "ilxyr" / "target" / "debug" / "ilxyr") args = ap.parse_args(argv) @@ -104,6 +106,8 @@ def main(argv=None): ap.error("model_dir, both corpora, and --python must exist") if int(args.min_tokens) < 1000: ap.error("--min-tokens must be at least 1000") + if int(args.experiment_version) < 1: + ap.error("--experiment-version must be at least 1") model_digest, model_files = model_manifest(model_dir) installation_digest = sha256_file(installation_corpus) @@ -112,9 +116,9 @@ def main(argv=None): ap.error("installation and evaluation corpora must have distinct contents") source_commit = subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip() - stem = "lecore.qwen35.install.%s.%s.%s.%s.v1" % ( + stem = "lecore.qwen35.install.%s.%s.%s.%s.v%d" % ( model_digest[:12], installation_digest[:12], evaluation_digest[:12], - source_commit[:12]) + source_commit[:12], int(args.experiment_version)) ids = { "hypothesis": stem + ".hypothesis", "foundation": stem + ".foundation", @@ -216,7 +220,8 @@ def main(argv=None): # frozen in their handles and the runner attests checker files. "provenance": {"artifact_hashes": [], "model_lineage": "model://Qwen/Qwen3.5-0.8B/%s" % model_digest, - "checker": "checker://lecore/qwen35-acceptance/v1"}, + "checker": "checker://lecore/qwen35-acceptance/v%d" + % int(args.experiment_version)}, }, "expected_outputs": (["metrics.%s" % name for name in METRIC_NAMES] + ["resolved_outcome", "forecast_settlements"]), @@ -270,6 +275,7 @@ def main(argv=None): [cli, "verify", ""]]) write_json(out_dir / "project.json", { "schema": "lecore.ilxyr-project.v1", "experiment_id": ids["experiment"], + "experiment_version": int(args.experiment_version), "source_commit": source_commit, "model_digest": model_digest, "model_files": model_files, "corpora": { diff --git a/tests/test_qwen_acceptance.py b/tests/test_qwen_acceptance.py index 7c402bf..4ec8614 100644 --- a/tests/test_qwen_acceptance.py +++ b/tests/test_qwen_acceptance.py @@ -177,6 +177,29 @@ def test_generator_preserves_virtual_environment_python_path(tmp_path): assert experiment["execution"]["program"] != str(venv_python.resolve()) +def test_generator_can_freeze_a_second_formal_attempt(tmp_path): + output, _ = build_fixture(tmp_path) + installation_corpus = tmp_path / "installation.txt" + evaluation_corpus = tmp_path / "evaluation.txt" + installation_corpus.write_text("Installation material. " * 400) + evaluation_corpus.write_text("Held-out evaluation material. " * 400) + project = tmp_path / "project-v2" + + completed = subprocess.run( + [sys.executable, + str(ROOT / "experiments" / "qwen35_acceptance" / "generate.py"), + str(output), str(installation_corpus), str(evaluation_corpus), + str(project), "--min-tokens", "1000", "--experiment-version", "2"], + cwd=tmp_path, capture_output=True, text=True, check=False) + assert completed.returncode == 0, completed.stderr + + experiment = json.loads((project / "experiment.json").read_text()) + manifest = json.loads((project / "project.json").read_text()) + assert experiment["id"].endswith(".v2.acceptance") + assert experiment["evidence_authority"]["provenance"]["checker"].endswith("/v2") + assert manifest["experiment_version"] == 2 + + def test_generator_rejects_reused_installation_and_evaluation_corpus(tmp_path): output, _ = build_fixture(tmp_path) corpus = tmp_path / "corpus.txt" From 5ad686ef19d1629c75a16c2318a2560ab38d0d5d Mon Sep 17 00:00:00 2001 From: docs-bot Date: Thu, 13 Aug 2026 00:09:14 +0000 Subject: [PATCH 16/16] docs: refresh generated docs (REFERENCE, CAPABILITIES, API_QUICKREF, PIPELINE_MAP, FACULTY_MAP, DOC_MAP) [skip ci] --- REFERENCE.md | 278 +++++++++++++++++++++++++-------------------------- 1 file changed, 139 insertions(+), 139 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index e06473d..164a4cd 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -76,8 +76,8 @@ | module | what it is | lines | |---|---|---| | [`holographic_session.py`](#holographic-session) | holographic_session.py -- ONE render session that ties the disconnected rendering threads together. | 227 | -| [`holographic_session.py`](#holographic-session) | SESSION -- never compute the same conversation prefix twice. | 253 | | [`holographic_session.py`](#holographic-session) | SESSION STORE -- contexts that outlive the process. | 261 | +| [`holographic_session.py`](#holographic-session) | SESSION -- never compute the same conversation prefix twice. | 253 | ### `splat*` family (6) @@ -423,8 +423,8 @@ | [`holographic_materialio.py`](#holographic-materialio) | PBR material interchange (G2-IO): the standard-format bridge for materials, and a VSA-native carrier. | 307 | | [`holographic_matlib.py`](#holographic-matlib) | holographic_matlib.py -- a comprehensive RENDER material library: plain diffuse -> a fractal planet. | 801 | | [`holographic_meaning_predict.py`](#holographic-meaning-predict) | Generation with structure: predict a next-MEANING vector and settle it, rather | 198 | -| [`holographic_measure.py`](#holographic-measure) | The variance harness: every headline number gets a mean, a spread, and a confidence | 187 | | [`holographic_measure.py`](#holographic-measure) | MEASURE -- perplexity with error bars, and decisions that respect them. | 239 | +| [`holographic_measure.py`](#holographic-measure) | The variance harness: every headline number gets a mean, a spread, and a confidence | 187 | | [`holographic_memory.py`](#holographic-memory) | MEMORY -- the Galvatron's own store, built on leCore's holographic database. | 241 | | [`holographic_memoryhome.py`](#holographic-memoryhome) | holographic_memoryhome.py -- the MEMORY home (consolidation backlog H6): keep the hot working set where the CP | 165 | | [`holographic_memsearch.py`](#holographic-memsearch) | MEMSEARCH -- searchable memory that lives in the weights and runs in the pass. | 212 | @@ -577,8 +577,8 @@ | [`holographic_rolling.py`](#holographic-rolling) | holographic_rolling.py -- the CAUSAL rolling / streaming statistics kit: trailing mean, std, min, max, | 378 | | [`holographic_roomacoustic.py`](#holographic-roomacoustic) | holographic_roomacoustic.py -- A6: GEOMETRIC ROOM ACOUSTICS. How a room echoes -- reflections and reverb. | 161 | | [`holographic_route.py`](#holographic-route) | Representation routing (ARCH-7): route each operation to the representation that supports it -- the policy lay | 183 | -| [`holographic_router.py`](#holographic-router) | ROUTER -- the model DECIDING, inside one forward pass. | 229 | | [`holographic_router.py`](#holographic-router) | Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. | 173 | +| [`holographic_router.py`](#holographic-router) | ROUTER -- the model DECIDING, inside one forward pass. | 229 | | [`holographic_sampler.py`](#holographic-sampler) | holographic_sampler.py -- the SAMPLER: a placeable read-probe (modeling-app backlog, the capstone item). | 223 | | [`holographic_sampling.py`](#holographic-sampling) | Blue-noise / Poisson-disk point sampling -- the EXCLUSION principle, done right. | 134 | | [`holographic_samplinghome.py`](#holographic-samplinghome) | holographic_samplinghome.py -- the SAMPLING home (consolidation backlog R4): one place for the Monte-Carlo | 127 | @@ -683,8 +683,8 @@ | [`holographic_toolclient.py`](#holographic-toolclient) | holographic_toolclient.py -- call another node the same way leCore is called. | 98 | | [`holographic_topology.py`](#holographic-topology) | Principled topology by persistent homology (EXP-7). | 288 | | [`holographic_transfer.py`](#holographic-transfer) | holographic_transfer.py -- KERNEL SCATTER / GATHER: the ONE bundle/readout under every particle<->grid transfe | 224 | -| [`holographic_transform.py`](#holographic-transform) | holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). | 297 | | [`holographic_transform.py`](#holographic-transform) | TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding. | 189 | +| [`holographic_transform.py`](#holographic-transform) | holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). | 297 | | [`holographic_transform_space.py`](#holographic-transform-space) | holographic_transform_space.py -- the TRANSFORM + SPACE model behind a gizmo. A gizmo is a UI; the backend it | 176 | | [`holographic_transformbank.py`](#holographic-transformbank) | holographic_transformbank.py -- a prebuilt map of hypervector transforms, and what it can and cannot hold. | 358 | | [`holographic_transformhome.py`](#holographic-transformhome) | holographic_transformhome.py -- the TRANSFORM home (consolidation backlog H5): one facade over "move / rotate | 223 | @@ -14848,39 +14848,6 @@ ### holographic_measure.py -> The variance harness: every headline number gets a mean, a spread, and a confidence -> interval across seeds -- so a lucky-seed point estimate can't pass as a real result. -> -> WHY this exists -> --------------- -> This whole engine is built on RANDOM vectors. Atoms are random, the RP-tree's -> hyperplanes are random, the reservoir is random, train/test splits are shuffled. A -> single-seed score is therefore a sample from a distribution, and reporting it alone hides -> how wide that distribution is. For an engine whose entire pitch is "measured, not -> promised," reporting a number without its noise is the sharpest blind spot -- so this -> points that same discipline at the numbers themselves. -> -> measure(run_once, seeds) runs a scored experiment once per seed and returns the mean, the -> sample standard deviation, and a 95% percentile-bootstrap confidence interval (no -> distributional assumptions). assert_robust(stats, floor) passes only if the LOWER CI -> bound clears the floor -- which is what stops a single fortunate seed from passing a -> test the typical seed would fail. report() formats "mean +/- std (95% CI [lo, hi], n)". -> -> USE REAL DATA. The point of the harness is to characterise the real distribution of a -> real claim; running it on a toy makes the spread meaningless. The measurements wired -> through it here all run on real corpora (Gutenberg Alice, UDHR, Reuters, Brown). - -**Public API:** - -- `def time_call(fn, repeats, warmup)` -- Time `fn()` honestly: warm up first, then take the MEDIAN of `repeats` timings in microseconds. -- `def measure(run_once, seeds, n_boot, boot_seed)` -- Run a scored experiment across seeds; return mean, std, and a 95% bootstrap CI. -- `def fdr_gate(rows, alpha)` -- P9 -- false-discovery control across a whole ABLATION TABLE, from the measurement home. -- `def assert_robust(stats, floor)` -- Pass only if the LOWER CI bound clears the floor -- not just the mean. This is -- `def is_fragile(stats, margin_floor)` -- A claim is FRAGILE if its spread is large relative to how far its mean sits above -- `def report(name, stats, floor)` -- Format a stats dict as 'name: mean +/- std (95% CI [lo, hi], n)', with a - -### holographic_measure.py - > MEASURE -- perplexity with error bars, and decisions that respect them. > > Moose asked what assimilation is actually doing. The answer, from his own run: @@ -14913,6 +14880,39 @@ - `def better_than(a, b, alpha, seed, resamples)` -- Is model A better than model B, or is the difference undecidable? - `def tokens_needed(reference, effect_pct, alpha)` -- How many tokens would be needed to RESOLVE an effect of this size. +### holographic_measure.py + +> The variance harness: every headline number gets a mean, a spread, and a confidence +> interval across seeds -- so a lucky-seed point estimate can't pass as a real result. +> +> WHY this exists +> --------------- +> This whole engine is built on RANDOM vectors. Atoms are random, the RP-tree's +> hyperplanes are random, the reservoir is random, train/test splits are shuffled. A +> single-seed score is therefore a sample from a distribution, and reporting it alone hides +> how wide that distribution is. For an engine whose entire pitch is "measured, not +> promised," reporting a number without its noise is the sharpest blind spot -- so this +> points that same discipline at the numbers themselves. +> +> measure(run_once, seeds) runs a scored experiment once per seed and returns the mean, the +> sample standard deviation, and a 95% percentile-bootstrap confidence interval (no +> distributional assumptions). assert_robust(stats, floor) passes only if the LOWER CI +> bound clears the floor -- which is what stops a single fortunate seed from passing a +> test the typical seed would fail. report() formats "mean +/- std (95% CI [lo, hi], n)". +> +> USE REAL DATA. The point of the harness is to characterise the real distribution of a +> real claim; running it on a toy makes the spread meaningless. The measurements wired +> through it here all run on real corpora (Gutenberg Alice, UDHR, Reuters, Brown). + +**Public API:** + +- `def time_call(fn, repeats, warmup)` -- Time `fn()` honestly: warm up first, then take the MEDIAN of `repeats` timings in microseconds. +- `def measure(run_once, seeds, n_boot, boot_seed)` -- Run a scored experiment across seeds; return mean, std, and a 95% bootstrap CI. +- `def fdr_gate(rows, alpha)` -- P9 -- false-discovery control across a whole ABLATION TABLE, from the measurement home. +- `def assert_robust(stats, floor)` -- Pass only if the LOWER CI bound clears the floor -- not just the mean. This is +- `def is_fragile(stats, margin_floor)` -- A claim is FRAGILE if its spread is large relative to how far its mean sits above +- `def report(name, stats, floor)` -- Format a stats dict as 'name: mean +/- std (95% CI [lo, hi], n)', with a + ### holographic_memory.py > MEMORY -- the Galvatron's own store, built on leCore's holographic database. @@ -22656,6 +22656,33 @@ ### holographic_router.py +> Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. +> +> WHY THIS EXISTS (measured, backlog N9/N28): +> The catalog's find_capability scores by shared content words. That is deterministic and needs no model, +> but it has no notion of MEANING: "squish a big array down for storage" shares no token with +> `holographic_coldstore`, and "airspeed velocity of an unladen swallow" confidently matched a physics +> module on the single word "velocity". Measured on the 12-ask suite: token overlap ~2/12 top-1, median +> rank 13 of 503; the nomic embedding router hit 7/12 top-1, median rank 1. +> +> WHAT SHIPS (and what does NOT): +> A 96 KB index -- 503 module vectors at 64d q8, plus the ABTT correction (mu, pc) baked in -- extracted +> from the build cache by tools/semantic/export_index.py. NO model ships. So this router can only score a +> query it already has a VECTOR for: +> * queries embedded at build time and cached (the exam's asks, an app's fixed vocabulary) +> * a caller that supplies its own query vector (an app that ran the encoder itself) +> A brand-new free-text query with no vector and no model present CANNOT be embedded here -- and this +> router says so and returns None, so the caller falls back to the token router rather than guessing. That +> honesty is the whole point: silence beats a confident wrong route. +> +> DETERMINISM: q8 dequant + fixed ABTT transform + argsort with a name tie-break. No RNG, no model. + +**Public API:** + +- `class EmbeddingRouter` -- Loads the shipped q8 index (any dim; 128d is the measured champion) and routes a query VECTOR (or a cached query text) to modules. + +### holographic_router.py + > ROUTER -- the model DECIDING, inside one forward pass. > > Moose raised the architecture that dissolves the wall this project kept hitting: @@ -22695,33 +22722,6 @@ - `def route(runtime, router, text, tokenize)` -- Would this model choose to use the capability on this prompt? - `def install_routed(weights, cfg, operator, router, layer, gain, temperature)` -- Install a circuit whose GATE is the router, not a constant. -### holographic_router.py - -> Embedding router -- route a request to the right module by COSINE in nomic's space, not token overlap. -> -> WHY THIS EXISTS (measured, backlog N9/N28): -> The catalog's find_capability scores by shared content words. That is deterministic and needs no model, -> but it has no notion of MEANING: "squish a big array down for storage" shares no token with -> `holographic_coldstore`, and "airspeed velocity of an unladen swallow" confidently matched a physics -> module on the single word "velocity". Measured on the 12-ask suite: token overlap ~2/12 top-1, median -> rank 13 of 503; the nomic embedding router hit 7/12 top-1, median rank 1. -> -> WHAT SHIPS (and what does NOT): -> A 96 KB index -- 503 module vectors at 64d q8, plus the ABTT correction (mu, pc) baked in -- extracted -> from the build cache by tools/semantic/export_index.py. NO model ships. So this router can only score a -> query it already has a VECTOR for: -> * queries embedded at build time and cached (the exam's asks, an app's fixed vocabulary) -> * a caller that supplies its own query vector (an app that ran the encoder itself) -> A brand-new free-text query with no vector and no model present CANNOT be embedded here -- and this -> router says so and returns None, so the caller falls back to the token router rather than guessing. That -> honesty is the whole point: silence beats a confident wrong route. -> -> DETERMINISM: q8 dequant + fixed ABTT transform + argsort with a name tie-break. No RNG, no model. - -**Public API:** - -- `class EmbeddingRouter` -- Loads the shipped q8 index (any dim; 128d is the measured champion) and routes a query VECTOR (or a cached query text) to modules. - ### holographic_sampler.py > holographic_sampler.py -- the SAMPLER: a placeable read-probe (modeling-app backlog, the capstone item). @@ -24352,39 +24352,6 @@ ### holographic_session.py -> SESSION -- never compute the same conversation prefix twice. -> -> Moose runs a 0.8B on a CPU laptop and it is slow. The single largest waste in a -> conversation is not the model's arithmetic -- it is that every turn RE-PREFILLS -> the entire history. MEASURED on a realistic six-turn exchange: 489 tokens -> processed, of which only 137 were new. SEVENTY-TWO PERCENT OF THE WORK WAS -> REPEATED, and the fraction grows with every turn. -> -> leCore already had the pieces and never joined them: the runtime exposes -> prefill/step over an InferenceState with copy(), galvacache memoises pure -> functions at a measured 75% hit rate with bit-identical output, and sessions -> exist. What was missing is the RADIX TREE -- the structure that answers "what is -> the longest prefix of this prompt that I have already computed?" -> -> WHY A TREE AND NOT A DICTIONARY: turn 4 of a conversation shares its first three -> turns with turn 3, and a dictionary keyed on the whole prompt misses that -> completely. vLLM and SGLang call this RadixAttention; the idea is the same -> whatever the model: index by prefix, resume from the deepest match, compute only -> the tail. -> -> THE GUARANTEE THIS KEEPS, because a cache that changes answers is worse than no -> cache: resuming from a cached state reproduces a full recompute TO FLOAT -> ROUNDING -- measured 7.1e-15, machine epsilon. Not bit-identical, and the -> difference is real rather than pedantic: resuming STEPS the tail while a fresh -> call PREFILLS it, and the two associate their sums differently. The selftest -> asserts the measured bound against the live runtime rather than assuming it. - -**Public API:** - -- `class PrefixCache` -- A radix tree over token sequences, holding inference states. - -### holographic_session.py - > SESSION STORE -- contexts that outlive the process. > > A Galvatron's context is not a transcript, it is STATE: the GDN recurrent @@ -24417,6 +24384,39 @@ - `class SessionStore` -- Named, persistent, independent contexts on disk. - `def runtime_fingerprint(runtime)` -- A cheap, deterministic id for the checkpoint behind a runtime, so a +### holographic_session.py + +> SESSION -- never compute the same conversation prefix twice. +> +> Moose runs a 0.8B on a CPU laptop and it is slow. The single largest waste in a +> conversation is not the model's arithmetic -- it is that every turn RE-PREFILLS +> the entire history. MEASURED on a realistic six-turn exchange: 489 tokens +> processed, of which only 137 were new. SEVENTY-TWO PERCENT OF THE WORK WAS +> REPEATED, and the fraction grows with every turn. +> +> leCore already had the pieces and never joined them: the runtime exposes +> prefill/step over an InferenceState with copy(), galvacache memoises pure +> functions at a measured 75% hit rate with bit-identical output, and sessions +> exist. What was missing is the RADIX TREE -- the structure that answers "what is +> the longest prefix of this prompt that I have already computed?" +> +> WHY A TREE AND NOT A DICTIONARY: turn 4 of a conversation shares its first three +> turns with turn 3, and a dictionary keyed on the whole prompt misses that +> completely. vLLM and SGLang call this RadixAttention; the idea is the same +> whatever the model: index by prefix, resume from the deepest match, compute only +> the tail. +> +> THE GUARANTEE THIS KEEPS, because a cache that changes answers is worse than no +> cache: resuming from a cached state reproduces a full recompute TO FLOAT +> ROUNDING -- measured 7.1e-15, machine epsilon. Not bit-identical, and the +> difference is real rather than pedantic: resuming STEPS the tail while a fresh +> call PREFILLS it, and the two associate their sums differently. The selftest +> asserts the measured bound against the live runtime rather than assuming it. + +**Public API:** + +- `class PrefixCache` -- A radix tree over token sequences, holding inference states. + ### holographic_shader.py > holographic_shader.py -- N filter passes in ONE evaluation. Two things a GPU structurally cannot do. @@ -27945,48 +27945,6 @@ ### holographic_transform.py -> holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). -> -> The engine has scattered transform bits (scenegraph.translation/rotation/compose_transforms; cosserat's -> quaternion helpers; splatexport's rotation<->quaternion), but not the full kit a gizmo and a property panel need -> in one place. This gathers the standard, well-known math: -> -> * decompose(M) -> (translate, rotation-quaternion, scale) -- what a gizmo reads off a matrix to show handles, -> and what a property panel shows as the T/R/S fields; -> * compose_trs(translate, quat, scale) -> 4x4 -- the inverse (build a matrix from panel values); -> * a quaternion kit -- from/to matrix, from/to axis-angle, from/to euler, multiply, SLERP, rotate a vector -> (quaternions are what rotation UI and animation want: no gimbal lock, and slerp gives smooth interpolation); -> * look_at(eye, target, up) -> 4x4 view matrix -- for a camera or an object aimed at a point. -> -> Conventions, stated ONCE and held (the backlog's coordinate-convention note): matrices are 4x4 and act on COLUMN -> vectors, p' = M @ [x, y, z, 1]; compose(A, B) = A @ B means "apply B, then A"; quaternions are (w, x, y, z), unit -> length; euler angles are (rx, ry, rz) applied X then Y then Z, i.e. R = Rz @ Ry @ Rx; look_at returns an OpenGL -> view matrix (the camera looks down -z, y is up) to match the engine's Camera. Nothing here is holographic -- it is -> plain linear algebra a modeling app needs -- so it is kept as a small, readable utility, not dressed up as a bind. -> Deterministic; NumPy + stdlib only. - -**Public API:** - -- `def translation(t)` -- A 4x4 translation matrix from a 3-vector. -- `def scaling(s)` -- A 4x4 scale matrix. `s` is a scalar (uniform) or a 3-vector (per-axis). -- `def rotation_axis_angle(axis, angle)` -- A 4x4 rotation of `angle` radians about `axis` (Rodrigues' formula). -- `def compose(*mats)` -- Matrix product M0 @ M1 @ ... -- with the column-vector convention this applies the RIGHTMOST first. -- `def decompose(M)` -- Split a 4x4 affine transform into (translate (3,), rotation quaternion (4,), scale (3,)). Assumes no shear -- `def compose_trs(translate, quat, scale)` -- Build a 4x4 from translate (3,), a rotation quaternion (4,), and scale (3,) -- the inverse of decompose. -- `def quat_normalize(q)` -- `def quat_mul(a, b)` -- The Hamilton product a*b: the rotation "apply b, then a". -- `def quat_from_axis_angle(axis, angle)` -- A quaternion for a rotation of `angle` radians about `axis`. -- `def quat_to_axis_angle(q)` -- Recover (axis, angle) from a quaternion. -- `def quat_to_matrix(q)` -- The 3x3 rotation matrix for a quaternion. -- `def quat_from_matrix(R)` -- The quaternion for a 3x3 rotation matrix (Shepperd's method: branch on the largest diagonal term for -- `def quat_from_euler(rx, ry, rz)` -- A quaternion from euler angles applied X then Y then Z (R = Rz @ Ry @ Rx). -- `def quat_to_euler(q)` -- Recover euler angles (rx, ry, rz) from a quaternion, inverting R = Rz @ Ry @ Rx. Handles gimbal lock -- `def quat_slerp(a, b, t)` -- Spherical linear interpolation between two rotations -- constant angular speed, the smooth in-between an -- `def quat_rotate(q, v)` -- Rotate a 3-vector by a quaternion. -- `def look_at(eye, target, up)` -- An OpenGL view matrix for a camera at `eye` looking at `target` (the engine's convention: the camera looks - -### holographic_transform.py - > TRANSFORM -- rebuild a model where the MEASUREMENT says it needs rebuilding. > > Everything before this applied leCore's levers uniformly: grow a memory channel @@ -28029,6 +27987,48 @@ - `def plan(weights, cfg, target_tokens, kv_rank, grow_gain)` -- Decide what to do to each layer, from the analysis rather than by rule. - `def apply_plan(weights, cfg, the_plan, progress)` -- Carry out the growth actions. KV compression is a RUNTIME setting and is +### holographic_transform.py + +> holographic_transform.py -- TRANSFORM UTILITIES for a modeling app (modeling-app backlog, item G). +> +> The engine has scattered transform bits (scenegraph.translation/rotation/compose_transforms; cosserat's +> quaternion helpers; splatexport's rotation<->quaternion), but not the full kit a gizmo and a property panel need +> in one place. This gathers the standard, well-known math: +> +> * decompose(M) -> (translate, rotation-quaternion, scale) -- what a gizmo reads off a matrix to show handles, +> and what a property panel shows as the T/R/S fields; +> * compose_trs(translate, quat, scale) -> 4x4 -- the inverse (build a matrix from panel values); +> * a quaternion kit -- from/to matrix, from/to axis-angle, from/to euler, multiply, SLERP, rotate a vector +> (quaternions are what rotation UI and animation want: no gimbal lock, and slerp gives smooth interpolation); +> * look_at(eye, target, up) -> 4x4 view matrix -- for a camera or an object aimed at a point. +> +> Conventions, stated ONCE and held (the backlog's coordinate-convention note): matrices are 4x4 and act on COLUMN +> vectors, p' = M @ [x, y, z, 1]; compose(A, B) = A @ B means "apply B, then A"; quaternions are (w, x, y, z), unit +> length; euler angles are (rx, ry, rz) applied X then Y then Z, i.e. R = Rz @ Ry @ Rx; look_at returns an OpenGL +> view matrix (the camera looks down -z, y is up) to match the engine's Camera. Nothing here is holographic -- it is +> plain linear algebra a modeling app needs -- so it is kept as a small, readable utility, not dressed up as a bind. +> Deterministic; NumPy + stdlib only. + +**Public API:** + +- `def translation(t)` -- A 4x4 translation matrix from a 3-vector. +- `def scaling(s)` -- A 4x4 scale matrix. `s` is a scalar (uniform) or a 3-vector (per-axis). +- `def rotation_axis_angle(axis, angle)` -- A 4x4 rotation of `angle` radians about `axis` (Rodrigues' formula). +- `def compose(*mats)` -- Matrix product M0 @ M1 @ ... -- with the column-vector convention this applies the RIGHTMOST first. +- `def decompose(M)` -- Split a 4x4 affine transform into (translate (3,), rotation quaternion (4,), scale (3,)). Assumes no shear +- `def compose_trs(translate, quat, scale)` -- Build a 4x4 from translate (3,), a rotation quaternion (4,), and scale (3,) -- the inverse of decompose. +- `def quat_normalize(q)` +- `def quat_mul(a, b)` -- The Hamilton product a*b: the rotation "apply b, then a". +- `def quat_from_axis_angle(axis, angle)` -- A quaternion for a rotation of `angle` radians about `axis`. +- `def quat_to_axis_angle(q)` -- Recover (axis, angle) from a quaternion. +- `def quat_to_matrix(q)` -- The 3x3 rotation matrix for a quaternion. +- `def quat_from_matrix(R)` -- The quaternion for a 3x3 rotation matrix (Shepperd's method: branch on the largest diagonal term for +- `def quat_from_euler(rx, ry, rz)` -- A quaternion from euler angles applied X then Y then Z (R = Rz @ Ry @ Rx). +- `def quat_to_euler(q)` -- Recover euler angles (rx, ry, rz) from a quaternion, inverting R = Rz @ Ry @ Rx. Handles gimbal lock +- `def quat_slerp(a, b, t)` -- Spherical linear interpolation between two rotations -- constant angular speed, the smooth in-between an +- `def quat_rotate(q, v)` -- Rotate a 3-vector by a quaternion. +- `def look_at(eye, target, up)` -- An OpenGL view matrix for a camera at `eye` looking at `target` (the engine's convention: the camera looks + ### holographic_transform_space.py > holographic_transform_space.py -- the TRANSFORM + SPACE model behind a gizmo. A gizmo is a UI; the backend it