diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19a6525..eae4907 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,21 @@ concurrency: env: HF_HUB_DISABLE_TELEMETRY: "1" + # Authenticates every Hub call in this workflow. `huggingface_hub` and + # `transformers` both read HF_TOKEN from the environment, so nothing in the code + # needs to know about it. + # + # Why it is here: unauthenticated CI gets rate-limited hard, and three builds + # across two sessions died on 429s. The failure is not cosmetic — when + # `model_info` cannot resolve a revision to a commit sha, `export_static_assets.py` + # REFUSES to publish rather than emit weight URLs pinned to a moving `main` + # (issue #5). That refusal is correct, so the fix belongs here rather than in a + # looser guard. + # + # The secret is optional by design: forks and PRs from forks get an empty string, + # which `huggingface_hub` treats as anonymous — exactly today's behaviour, with + # today's retries. Nothing breaks without it; it is only more likely to succeed. + HF_TOKEN: ${{ secrets.HF_TOKEN }} jobs: backend: @@ -162,8 +177,19 @@ jobs: run: | sh ../../scripts/npm-ci-retry.sh npx playwright install --with-deps chromium - - name: e2e (both projects — chromium against the real backend, static against the built Pages bundle) + - name: e2e (all three projects — chromium against the real backend, static against the built Pages bundle, webgpu against a real adapter when the runner has one) working-directory: code/frontend + # KNOWN GAP, stated so a green tick is not read as more than it is: the + # `webgpu` project needs a WebGPU adapter, and GitHub-hosted runners have no + # GPU (the software adapter they can offer advertises no shader-f16). It + # therefore SKIPS here, printing why, instead of passing vacuously on the WASM + # rung — so THIS JOB DOES NOT VERIFY THE WEBGPU PATH. It is verified by running + # `npx playwright test --project webgpu` on a machine with a real GPU. + # What this job does always verify about that defect: the non-degeneracy + # invariant and the dtype ladder (tests/unit/logitsSanity.test.ts, in the + # frontend job) and a real session passing the same load-time gate on the WASM + # rung (tests/e2e/static.spec.ts). Do not "fix" the skip by deleting the + # project — the absence of this coverage is what let webgpu/q4f16 ship. run: npm run test:e2e - name: Upload traces on failure if: failure() diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index c3244f4..81d6a93 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -20,6 +20,14 @@ concurrency: env: HF_HUB_DISABLE_TELEMETRY: "1" + # Read by `huggingface_hub` and `transformers` directly; no code change needed. + # This job is the one that most needs it: it runs the REAL backend as a build tool + # and resolves every curated model's revision to a commit sha. On a 429 that + # resolution fails, and `export_static_assets.py` refuses to publish rather than + # ship weight URLs pointing at a moving `main` (issue #5) — so an unauthenticated + # rate limit does not degrade the deploy, it blocks it. Optional: absent, the + # value is empty and the Hub is called anonymously, exactly as before. + HF_TOKEN: ${{ secrets.HF_TOKEN }} jobs: build-and-deploy: diff --git a/.specify/feature.json b/.specify/feature.json index 47c7985..3e4e114 100644 --- a/.specify/feature.json +++ b/.specify/feature.json @@ -1 +1 @@ -{"feature_directory":"specs/006-lexicon-lab-tiny"} +{"feature_directory":"specs/007-vacancy-transform-field"} diff --git a/code/backend/src/llm_geometry/api/routes_arch.py b/code/backend/src/llm_geometry/api/routes_arch.py index f9a1794..a529def 100644 --- a/code/backend/src/llm_geometry/api/routes_arch.py +++ b/code/backend/src/llm_geometry/api/routes_arch.py @@ -20,7 +20,9 @@ from pydantic import BaseModel from ..arch import build_graph, check_model_size, generate, trace_forward, weight_window +from ..arch.vacancy_score import vacancy_score from ..config import ARCH_DEFAULT_MAX_CONTEXT, ARCH_WEIGHTS_MAX_CELLS +from ..errors import InvalidParamError from .encoding import jsonable_6sig router = APIRouter(prefix="/arch", tags=["arch"]) @@ -57,6 +59,27 @@ class GenerateBody(BaseModel): seed: int | None = None +class VacancyScoreBody(BaseModel): + """`POST /api/arch/vacancy-score` request body (feature 007, contract §8). + + `passages` defaults to the six evenly spaced excerpts of the shipped corpus the + measurement of §8.3a used, so an empty request reproduces the reference configuration. + `passage` is the singular sugar the panel uses when a reader edits one excerpt. + + The transform's other knobs are deliberately absent: `consistent` and `reveal_after` + are fixed at the theorem's condition, because the number this endpoint produces is + only interpretable beside the tiny arm's exact zero, and that zero holds only there. + """ + + model_id: str + passage: str | None = None + passages: list[str] | None = None + p: float = 1.0 + seed: int = 0 + match_prosody: bool = True + keep: list[str] = [] + + @router.get("/graph") def arch_graph(model_id: str) -> dict[str, Any]: """The cached traced architecture graph (nodes/edges/meta/schema_version).""" @@ -107,3 +130,39 @@ def arch_generate(body: GenerateBody) -> dict[str, Any]: seed=body.seed, ) ) + + +@router.post("/vacancy-score") +def arch_vacancy_score(body: VacancyScoreBody) -> dict[str, Any]: + """What a word's FORM is worth to a model that has one for it (contract §8). + + Three variants of each passage — English, a real-word swap, and nonce — are scored by + three real forward passes each, and the mean NLL over the tokens of the words that + survive all three is reported per variant, together with the two differences that + decompose the damage: `nll(swap) − nll(english)` is the cost of **wrong content**, + `nll(nonce) − nll(swap)` the cost of **unknown form**. `nll(nonce) − nll(english)` + is returned but flagged `headline: false` — it is their sum and conflates them. + + This is the full stack, at float32, so it reports everything. The static build runs a + quantized export and may report only what the measurement of §8.3a bounded for that + dtype; it refuses the rest by name rather than inventing an error bar. + """ + _gate(body.model_id) + if body.passage is not None and body.passages is not None: + raise InvalidParamError( + "send either `passage` (one) or `passages` (several), not both — they would " + "silently disagree about what was scored" + ) + passages = body.passages + if body.passage is not None: + passages = [body.passage] + return _sig6( + vacancy_score( + body.model_id, + passages, + p=body.p, + seed=body.seed, + match_prosody=body.match_prosody, + keep=frozenset(body.keep), + ) + ) diff --git a/code/backend/src/llm_geometry/api/routes_lex.py b/code/backend/src/llm_geometry/api/routes_lex.py index 536c221..654f92c 100644 --- a/code/backend/src/llm_geometry/api/routes_lex.py +++ b/code/backend/src/llm_geometry/api/routes_lex.py @@ -28,6 +28,7 @@ from __future__ import annotations import base64 +import dataclasses import hashlib import threading from functools import lru_cache @@ -551,6 +552,225 @@ async def coverage(request: Request) -> dict[str, Any]: ) +# -- POST /api/lex/vacancy ----------------------------------------------------------------- + +#: Characters of vacated text returned inline by default, and the ceiling a caller may ask +#: for. See :func:`vacancy` for why this endpoint returns an excerpt plus a digest rather +#: than the whole corpus. Both numbers are mirrored in +#: `code/frontend/src/lib/staticClient/lex.ts`; the API-parity fixture pins them together. +VACANCY_PREVIEW_CHARS = 2000 +VACANCY_PREVIEW_MAX = 20000 + + +def _vacancy_params(payload: dict[str, Any]) -> Any: + """Coerce contract §7.1's knobs out of a raw JSON object. + + Every value goes through the same coercers the rest of this module uses, and the + range checks live in ``VacancyParams.__post_init__`` — which raises + ``InvalidParamError`` itself, so a bad `p` is a 422 in the shared envelope rather than + a 500 out of the dataclass. + + Passed by keyword on purpose: the transform's parameter set is still growing (the + swap-mint control of §8.3), and a keyword call keeps a new optional field from + landing in the wrong slot here. + """ + from ..lex.vacancy import VacancyParams + + keep = payload.get("keep", ()) + if keep is None: + keep = () + if isinstance(keep, str): + # Same trap `vacancy_domain` guards: a bare string iterates character by + # character, so `keep: "little"` would quietly protect six single letters. + raise InvalidParamError( + f"keep must be a list of words, not a string (got {keep!r}); a string would " + "be read letter by letter" + ) + if not isinstance(keep, (list, tuple, set, frozenset)): + raise InvalidParamError(f"keep must be a list of words, got {keep!r}") + for word in keep: + if not isinstance(word, str): + raise InvalidParamError(f"keep must contain strings, got {word!r}") + return VacancyParams( + p=_as_float(payload.get("p", 0.0), "p"), + seed=_as_int(payload.get("seed", 0), "seed"), + consistent=_as_bool(payload.get("consistent", True), "consistent"), + match_prosody=_as_bool(payload.get("match_prosody", True), "match_prosody"), + reveal_after=_as_int(payload.get("reveal_after", 0), "reveal_after"), + keep=frozenset(str(w) for w in keep), + ) + + +def _vacancy_key(params: Any) -> dict[str, Any] | None: + """A canonical cache-key fragment for the transform's parameters, or ``None``. + + Read off the dataclass's OWN fields rather than an enumerated list, so a knob added to + the transform is in the cache key the day it is added rather than the day someone + remembers to add it here — the failure mode of the enumerated version being a cache + hit that serves a run made under a setting that did not exist. Leading-underscore + fields are derived state, never inputs. + """ + if params is None: + return None + out: dict[str, Any] = {} + for spec in dataclasses.fields(params): + if spec.name.startswith("_"): + continue + value = getattr(params, spec.name) + out[spec.name] = sorted(value) if isinstance(value, (set, frozenset)) else value + return out + + +def _vacate(text: str, params: Any) -> tuple[Any, str]: + """Build the `p`-independent map over this corpus and rewrite the text with it. + + The domain is :func:`vacancy_domain` — the corpus's own types UNION the full Dolch + list — never the active budget, so the map is a function of `(corpus, seed, + match_prosody)` alone and switching budgets in the UI cannot re-mint the corpus + underneath a panel whose whole claim is that nonces are stable (§5.2). + + Deliberately not cached. The whole thing is ~70 ms on the shipped corpus, and a cache + keyed on the parameters that exist today would silently serve the wrong map the day + the transform grows another one. + """ + from ..lex.vacancy import build_vacancy_map, vacancy_domain, vacate_text + + vmap = build_vacancy_map(vacancy_domain(tokenize(text)), params) + return vmap, vacate_text(text, vmap, params) + + +def _is_mapped(params: Any) -> bool: + """Contract §7.2: the mapped vocabulary is defined only in the theorem's condition.""" + return bool(params.consistent) and params.reveal_after == 0 + + +def _vacancy_vocab( + *, + params: Any, + vmap: Any, + original_text: str, + vacated_text: str, + source: str, + budget: str, + size: int | None, +) -> tuple[LexVocab, str]: + """The vocabulary a vacated corpus is read with, and which of §7.2's two rules gave it. + + **Mapped** (`consistent`, `reveal_after = 0`): the budget is resolved against the + ENGLISH corpus and then pushed through the same `transformWord`, preserving order. The + map is injective, so every word keeps the id its pre-image had and the token id stream + is unchanged — which is the whole invariance theorem of §7.3. + + **Rebuilt** (every other condition): a source type no longer has a single image, so + the budget is rebuilt from the vacated corpus by the tab's normal rule. Coverage then + collapses, and the collapse IS the measurement (FR-715). + """ + from ..lex.vacancy import map_vocab_words + + if _is_mapped(params): + english = _resolve_budget(source, budget, size, original_text) + return ( + LexVocab( + words=tuple(map_vocab_words(english.words, vmap, params)), + source=english.source, + budget_name=english.budget_name, + ), + "mapped", + ) + return _resolve_budget(source, budget, size, vacated_text), "rebuilt" + + +@router.post("/vacancy") +async def vacancy(request: Request) -> dict[str, Any]: + """The vacancy transform applied to a corpus, with the statistics of contract §10. + + Same corpus-source rule as `/api/lex/coverage` — `text`, `hf_dataset`, or (with + neither) the shipped corpus — and the same budget triple, because the interesting + question about a vacated corpus is always "under which vocabulary?". + + **Why an excerpt and a digest rather than the whole vacated text.** The shipped corpus + is ~86 kB of body text and the panel re-runs this on every tick of the `p` slider, so + returning it whole would put megabytes on the wire across one sweep to show a reader a + screenful. Nothing needs it whole: the panel shows an excerpt (the source's own figure + is its first 400 characters), and a caller that wants to *train* on the vacated corpus + sends the same parameters to `/api/lex/train`, which vacates server-side rather than + round-tripping the text. What the excerpt cannot do by itself is prove which text it + came from, so `vacated_sha256` covers all of it in 64 bytes — that digest is also the + single value the static build's in-browser transform is checked against, which is the + parity this feature rests on. `preview_chars` raises the excerpt up to + ``VACANCY_PREVIEW_MAX``. + + Every number here is measured on the corpus in the request. The source document's own + prosody figures are its numbers on a corpus we do not have and are transcribed + nowhere (§10). + """ + from ..lex.vacancy import vacancy_stats + + payload = await _json_body(request) + original = await _text_source(payload) + params = _vacancy_params(payload) + + preview_chars = _as_int(payload.get("preview_chars", VACANCY_PREVIEW_CHARS), "preview_chars") + if not 0 <= preview_chars <= VACANCY_PREVIEW_MAX: + raise InvalidParamError( + f"preview_chars must be in 0..{VACANCY_PREVIEW_MAX}, got {preview_chars}. " + "The whole vacated corpus is never returned; `vacated_sha256` identifies it, " + "and /api/lex/train vacates server-side so the text never needs a round trip." + ) + + source = str(payload.get("source", DEFAULT_BUDGET_SOURCE)) + budget = str(payload.get("budget", DEFAULT_BUDGET)) + raw_size = payload.get("size") + size = _as_int(raw_size, "size") if raw_size is not None else None + + vmap, vacated = _vacate(original, params) + vocab, rule = _vacancy_vocab( + params=params, + vmap=vmap, + original_text=original, + vacated_text=vacated, + source=source, + budget=budget, + size=size, + ) + stats = vacancy_stats(original, vacated, vmap, params) + + return _jsonable( + { + "p": params.p, + "seed": params.seed, + "consistent": params.consistent, + "match_prosody": params.match_prosody, + "reveal_after": params.reveal_after, + "keep": sorted(params.keep), + # Which of §7.2's two rules produced `words`. A client must not have to infer + # it from the parameters: "mapped" is the only condition under which the ids + # are the English ids, and that is the difference between an invariance result + # and a coverage collapse. + "vocabulary_rule": rule, + "words": list(vocab.words), + "budget": _budget_payload(vocab, vacated), + "corpus": _corpus_stats(vacated), + # §10's field names verbatim, camelCase inside a snake_case envelope on + # purpose: they are a cross-language contract, not this API's naming. + "vacancy_stats": stats, + # Also inside `vacancy_stats`; surfaced here because injectivity is the + # guarantee the mapped vocabulary rests on, and a caller checking it should + # not have to reach into a statistics block to do so. + "bijective": stats["bijective"], + "remint_rounds": stats["remintRounds"], + "preview": vacated[:preview_chars], + "original_preview": original[:preview_chars], + "preview_chars": preview_chars, + "truncated": len(vacated) > preview_chars, + "vacated_chars": len(vacated), + "vacated_sha256": hashlib.sha256(vacated.encode("utf-8")).hexdigest(), + "original_chars": len(original), + "original_sha256": hashlib.sha256(original.encode("utf-8")).hexdigest(), + } + ) + + # -- POST /api/lex/train ------------------------------------------------------------------- @@ -577,10 +797,37 @@ async def train(request: Request, response: Response) -> dict[str, Any]: With `base` set, the *existing model's* vocabulary is used and travels with the result — feature 004's issue #6 was exactly the opposite mistake, and repeating it would silently re-tokenize a fine-tune against a vocabulary the weights never saw. + + **`vacancy` (feature 007, optional, additive).** An object of contract §7.1's knobs. + Absent, everything below behaves exactly as it did before it existed. Present, the + resolved corpus is vacated first and the model trains on the vacated text under the + vocabulary §7.2 assigns it — mapped when the theorem's conditions hold, rebuilt from + the vacated corpus otherwise. The transform happens here rather than in the client so + the corpus never round-trips: `/api/lex/vacancy` deliberately returns an excerpt. + + Under the mapped condition this is a *pure relabelling*: same token id stream, same + losses, bit for bit (§7.3). That is not a caveat, it is the result — and it is why + `p = 0` and a `p = 0.5` mapped run land on the *same* cache entry only when their + (text, vocabulary) pairs really coincide, which at `p = 0` they do, the transform + being the identity there (`u ∈ [0, 1)`). """ payload = await _json_body(request) text = await _text_source(payload) + vacancy_payload = payload.get("vacancy") + vac_params = None + vmap = None + original_text = text + if vacancy_payload is not None: + if not isinstance(vacancy_payload, dict): + raise InvalidParamError( + "vacancy must be an object of the transform's parameters " + "(p, seed, consistent, match_prosody, reveal_after, keep), " + f"got {vacancy_payload!r}" + ) + vac_params = _vacancy_params(vacancy_payload) + vmap, text = _vacate(original_text, vac_params) + steps = _as_int(payload.get("steps", DEFAULT_STEPS), "steps") if not 1 <= steps <= MAX_STEPS: raise InvalidParamError(f"steps must be in 1..{MAX_STEPS}, got {steps}") @@ -623,9 +870,19 @@ async def train(request: Request, response: Response) -> dict[str, Any]: source = str(payload.get("source", DEFAULT_BUDGET_SOURCE)) budget = str(payload.get("budget", DEFAULT_BUDGET)) size = payload.get("size") - vocab = _resolve_budget( - source, budget, _as_int(size, "size") if size is not None else None, text - ) + size_int = _as_int(size, "size") if size is not None else None + if vac_params is None: + vocab = _resolve_budget(source, budget, size_int, text) + else: + vocab, _rule = _vacancy_vocab( + params=vac_params, + vmap=vmap, + original_text=original_text, + vacated_text=text, + source=source, + budget=budget, + size=size_int, + ) config = _model_config_from(payload, vocab_rows=vocab.rows) key, _ = _train_cache_key( @@ -634,6 +891,11 @@ async def train(request: Request, response: Response) -> dict[str, Any]: "config": config, "vocab": list(vocab.words), "vocab_source": vocab.source, + # Redundant with (corpus, vocabulary) today — every knob that can change a + # training run changes one of those two — and in the key anyway, so that a + # knob added to the transform later cannot silently collide with a run made + # before it existed. + "vacancy": _vacancy_key(vac_params), "base": base, "steps": steps, "lr": lr, diff --git a/code/backend/src/llm_geometry/arch/vacancy_score.py b/code/backend/src/llm_geometry/arch/vacancy_score.py new file mode 100644 index 0000000..51dc68c --- /dev/null +++ b/code/backend/src/llm_geometry/arch/vacancy_score.py @@ -0,0 +1,665 @@ +"""The pretrained arm of the vacancy instrument (contract §8). + +What this measures, in one sentence: **does a model that knows English still predict the +closed-class scaffolding when the content words have been vacated?** The scaffolding is +character-identical across the variants, so the same words — very often the same token +ids, at the same word positions — are scored in each, and the difference is attributable +to what happened around them. + +Three variants of one passage are scored (§8.3): + +* ``english`` — the passage as written; +* ``swap`` — every vacated stem replaced by a REAL, frequency-rank-matched English + word. Equally nonsensical, ordinarily tokenized, every form known; +* ``nonce`` — every vacated stem replaced by a phonotactically legal invention. + +and the two differences that are worth anything are the *labelled* ones: + +* ``nll(swap) − nll(english)`` — the cost of **wrong content**; +* ``nll(nonce) − nll(swap)`` — the cost of **unknown form**. + +``nll(nonce) − nll(english)`` is never a headline: it is the sum of the two and conflates +them. The second difference is also only an UPPER BOUND on what a word's *location* was +worth, because nonce forms fragment into more subword tokens than real words do and that +residual is not separable without a tokenizer-level control (§8.3, ``UNKNOWN_FORM_NOTE``). + +ALIGNMENT (§8.2, FR-718). Tokens are attributed to words by **UTF-8 byte spans** derived +from the tokenizer's byte-level pieces, never by "characters": + +* transformers.js exposes no offsets at all, so HF's ``return_offsets_mapping`` is not a + mechanism the two stacks can share; +* per-token decoding emits U+FFFD and destroys the text on any split multi-byte character, + in both stacks; +* Python indexes code points where JavaScript indexes UTF-16 units, so the two disagree on + the same string (31 vs 32 for one probe text) — bytes are the only safe contract unit; +* HF's own offsets OVERLAP on multi-byte characters, so per-token quantities summed over a + word would be double-counted; the byte spans are a true partition. + +Every step is verified rather than trusted: the concatenated token bytes must equal +``utf8(text)`` exactly, and a mismatch RAISES (never mis-attributes). The input is +NFC-normalized once up front because Qwen's tokenizer carries an NFC normalizer while +gpt2's and SmolLM2's do not — without that, a decomposed character silently shifts every +span after it, and the check above would fire on a passage that is perfectly fine. +""" + +from __future__ import annotations + +import math +import unicodedata +from dataclasses import dataclass +from functools import lru_cache +from typing import Any + +import torch + +from ..errors import ComputeError, InvalidParamError +from ..lex.vacancy import ( + VacancyParams, + build_vacancy_map, + type_counts, + vacancy_domain, + vacate_text, +) +from ..lex.vocab import WORD_RE, tokenize +from ..models.loader import LoadedModel, load_model +from .tracing import _TRACE_LOCK + +#: The three variants, in the order the UI reads them. ``english`` is the reference; +#: ``swap`` sits between the other two by construction, which is what makes the +#: decomposition of §8.3 a decomposition rather than two unrelated numbers. +VARIANTS: tuple[str, ...] = ("english", "swap", "nonce") + +#: `mint` values for the two vacated variants (§8.3). ``english`` is not minted at all. +VARIANT_MINT: dict[str, str] = {"swap": "swap", "nonce": "nonce"} + +#: Ceiling on the passages accepted in one request. Each one costs three real forward +#: passes; the measurement of §8.3a used six. +MAX_PASSAGES = 12 + +#: Words per default passage, and how many of them — the shape of the run the numbers in +#: §8.3a come from, so the panel's default reproduces the measured configuration. +DEFAULT_PASSAGE_WORDS = 250 +DEFAULT_PASSAGE_COUNT = 6 + +#: Fraction of the shipped corpus's 250-word blocks that are front matter (title page + +#: the alphabetical index of first lines). Measured: the index ends in block 11 of 63. +FRONT_MATTER_FRACTION = 0.2 + +#: Verbatim in the response, and rendered by the panel. The residual is stated, not hidden. +UNKNOWN_FORM_NOTE = ( + "Nonce forms fragment into more subword tokens than real words do, so this difference " + "is the cost of an unknown form TOGETHER WITH the cost of a stranger, longer context. " + "The two are not separable without a tokenizer-level control, so treat this as an " + "UPPER BOUND on what a word's location was worth — never as pure location." +) + +#: The entropy confound (§8.4), stated wherever a delta is. +CONFOUND_NOTE = ( + "A vacated passage genuinely has higher entropy, so every prediction inside it gets " + "worse — the scaffolding included. A positive difference is therefore expected, not a " + "surprise: its MAGNITUDE is the result, and it is only interpretable against the tiny " + "arm's exact zero." +) + +#: The tiny arm's side of the 2×2 (§7.3, §8.3a). Not a measurement made here — it is the +#: Lexicon Lab's, restated so the panel can put the two numbers side by side, which is the +#: entire reason this panel exists. +TINY_ARM = { + "delta_nats": 0.0, + "exact": True, + "label": "the same measurement on a model with no locations", + "note": ( + "For the from-scratch word-level GeoTransformer of the Lexicon Lab, the vacancy " + "transform is a pure relabelling of the vocabulary: with consistent=true and " + "revealAfter=0 the token id stream is element-for-element identical, so the " + "training loss is bit-identical and a word's FORM is worth exactly 0. That is not " + "a rounding — it is an identity, and it is asserted in that tab, not assumed." + ), +} + + +# --- byte-level alignment (§8.2) ---------------------------------------------------------- + + +@lru_cache(maxsize=1) +def byte_decoder() -> dict[str, int]: + """GPT-2's unicode→byte table: the inverse of ``bytes_to_unicode``. + + Byte-level BPE renders every byte as a printable character so the vocabulary is + text; this undoes that, which is what turns a piece into a byte count and therefore + into a span. All four curated models use a byte-level BPE, and a piece carrying a + character outside this table is a tokenizer this code has not been verified against — + so it raises rather than guessing a width. + """ + bs = ( + list(range(ord("!"), ord("~") + 1)) + + list(range(0xA1, 0xAC + 1)) + + list(range(0xAE, 0xFF + 1)) + ) + cs = list(bs) + n = 0 + for b in range(256): + if b not in bs: + bs.append(b) + cs.append(256 + n) + n += 1 + return {chr(c): b for c, b in zip(cs, bs)} + + +def token_byte_spans(pieces: list[str], text: str) -> list[tuple[int, int]]: + """``[start, end)`` UTF-8 byte range owned by each token, verified by reconstruction. + + The spans tile ``utf8(text)`` exactly once — contiguous, covering, non-overlapping — + so a per-token quantity may be summed over a word without double-counting. A token + that is a bare continuation byte gets a degenerate empty span, which is the honest + answer for it; it still resolves to the right word because its start byte lies strictly + inside the multi-byte character it continues. + + Raises ``ComputeError`` if the reconstruction is not byte-identical to the input + (FR-718). There is no fallback: a wrong span silently attributes a token to the wrong + word, which would corrupt the very number this module exists to report. + """ + table = byte_decoder() + spans: list[tuple[int, int]] = [] + rebuilt = bytearray() + cursor = 0 + for i, piece in enumerate(pieces): + try: + raw = bytes(table[c] for c in piece) + except KeyError as exc: + raise ComputeError( + f"token {i} ({piece!r}) contains a character outside the byte-level BPE " + "table, so its byte width cannot be determined", + {"index": i, "piece": piece}, + ) from exc + rebuilt += raw + spans.append((cursor, cursor + len(raw))) + cursor += len(raw) + + expected = text.encode("utf-8") + if bytes(rebuilt) != expected: + raise ComputeError( + "token→text alignment failed: the concatenated byte-level pieces do not " + f"reproduce the passage ({len(rebuilt)} bytes rebuilt vs {len(expected)} " + "expected). Refusing to attribute tokens to words rather than mis-attribute " + "them.", + {"rebuilt_bytes": len(rebuilt), "text_bytes": len(expected)}, + ) + return spans + + +@dataclass(frozen=True) +class WordSpan: + """One ``WORD_RE`` match of a passage, in UTF-8 byte coordinates.""" + + index: int + word: str + start: int + end: int + + +def word_spans(text: str) -> list[WordSpan]: + """Every word of `text`, in byte coordinates, using the TOKENIZER'S OWN regex. + + The same ``WORD_RE`` the vacancy transform rewrites with, so a word here is exactly a + word there — otherwise the two would disagree about ``good-bye`` and the preserved set + would not be the set that was actually preserved. + """ + out: list[WordSpan] = [] + for i, m in enumerate(WORD_RE.finditer(text)): + start = len(text[: m.start()].encode("utf-8")) + out.append(WordSpan(i, m.group(0), start, start + len(m.group(0).encode("utf-8")))) + return out + + +def preserved_token_indices( + spans: list[tuple[int, int]], + words: list[WordSpan], + preserved: frozenset[int], +) -> list[int]: + """Indices of the tokens belonging to a PRESERVED word. + + A token belongs to a word when their byte ranges **overlap** — not "starts inside". + Byte-level BPE folds a word's leading space into the word's own token, so the token + that *is* the function word starts one byte before the word does; the start rule would + drop nearly every one of them. + + A token overlapping words with different preserved-ness cannot be attributed, and that + RAISES. It cannot happen with the byte-level pretokenizers of the curated models + (each token lies within one word plus its leading whitespace), so if it ever does, the + assumption behind this whole attribution has changed and the number must not be + reported (§8.2). + """ + out: list[int] = [] + for i, (a, b) in enumerate(spans): + hits = [w for w in words if a < w.end and b > w.start] + if not hits: + continue # punctuation, whitespace, line breaks — scored, attributed to no word + flags = {w.index in preserved for w in hits} + if len(flags) > 1: + raise ComputeError( + f"token {i} spans both a preserved and a vacated word " + f"({', '.join(repr(w.word) for w in hits)}); refusing to attribute it", + {"index": i, "words": [w.word for w in hits]}, + ) + if flags == {True}: + out.append(i) + return out + + +# --- the three variants (§8.3) ------------------------------------------------------------ + + +def _params(p: float, seed: int, match_prosody: bool, keep: frozenset[str], mint: str) -> Any: + """`VacancyParams` for one variant, with the swap control wired in when it exists. + + `mint` is the newest knob of §7.1 and lands with the transform itself. Rather than + silently scoring the nonce variant twice if it is not there yet — which would report a + 'cost of wrong content' of ~0 and look like a finding — this raises and names the + control that is missing. + """ + if mint not in VARIANT_MINT.values(): + raise InvalidParamError(f"mint must be one of {sorted(set(VARIANT_MINT.values()))}") + fields = {f.name for f in VacancyParams.__dataclass_fields__.values()} + if "mint" not in fields: + raise ComputeError( + "this build's vacancy transform has no `mint` parameter, so the swap control " + "of contract §8.3 does not exist and the decomposition cannot be computed", + {"available": sorted(fields)}, + ) + return VacancyParams( + p=float(p), + seed=int(seed), + consistent=True, + match_prosody=bool(match_prosody), + reveal_after=0, + keep=frozenset(keep), + mint=mint, + ) + + +def variant_texts( + passage: str, + *, + p: float, + seed: int, + match_prosody: bool, + keep: frozenset[str], +) -> dict[str, str]: + """The passage and its two vacated twins, keyed by variant name. + + The map is built over ``vacancy_domain(passage types)`` — the passage's own types plus + the full Dolch list — exactly as everywhere else, so the nonce a stem gets here is the + nonce it gets in the Lexicon Lab for the same seed. + + `consistent=True` and `reveal_after=0` are not options here: they are the condition the + invariance theorem is stated for, and the whole point of this panel is to put the tiny + arm's exact zero (which holds only there) beside the pretrained number. + """ + tokens = tokenize(passage) + domain = vacancy_domain(tokens) + counts = type_counts(tokens) + texts = {"english": passage} + for variant, mint in VARIANT_MINT.items(): + params = _params(p, seed, match_prosody, keep, mint) + # `counts` is the passage's own type frequencies: the swap control ranks the + # replacement pool by them (§8.3), and the nonce strategy ignores them entirely. + vmap = build_vacancy_map(domain, params, counts) + texts[variant] = vacate_text(passage, vmap, params) + return texts + + +def preserved_word_indices(texts: dict[str, str]) -> tuple[list[WordSpan], frozenset[int]]: + """Word spans of the English passage, and the indices PRESERVED in every variant. + + "Preserved" is character identity across all three variants, which is stronger than + "closed class": it also covers eligible stems the `u(stem) < p` decision happened to + spare, and it is exactly the property §8.1 relies on ("character-identical in both + passages"). Restricting all three NLLs to the SAME word set is what makes them + comparable at all. + + The transform guarantees each rewritten word is itself a single complete ``WORD_RE`` + match, so the variants have the same word count in the same order; a mismatch means + that guarantee broke and it raises rather than aligning by luck. + """ + per_variant = {name: word_spans(text) for name, text in texts.items()} + counts = {name: len(spans) for name, spans in per_variant.items()} + if len(set(counts.values())) != 1: + raise ComputeError( + "the variants do not have the same number of words, so preserved words " + f"cannot be aligned: {counts}", + {"word_counts": counts}, + ) + english = per_variant["english"] + preserved = frozenset( + w.index for w in english if all(per_variant[name][w.index].word == w.word for name in texts) + ) + return english, preserved + + +# --- scoring ------------------------------------------------------------------------------ + + +@dataclass(frozen=True) +class ScoredText: + """One real forward pass over one variant of one passage.""" + + text: str + n_tokens: int # tokens in the passage (position 0 has no prediction) + nll: list[float] # nats, index i = cost of predicting token i given tokens < i + preserved: list[int] # indices into `nll` (all > 0) of preserved-word tokens + + @property + def scored(self) -> list[float]: + return self.nll[1:] + + +def _max_positions(lm: LoadedModel) -> int | None: + config = getattr(lm.model, "config", None) + for attr in ("max_position_embeddings", "n_positions", "n_ctx"): + value = getattr(config, attr, None) + if isinstance(value, int) and value > 0: + return value + return None + + +def score_text( + lm: LoadedModel, text: str, words: list[WordSpan], preserved: frozenset[int] +) -> ScoredText: + """Per-token NLL from ONE teacher-forced forward pass, plus the preserved indices. + + No special tokens and no chat template: this scores the passage as written, so that + the three variants differ by the transform and by nothing else. Position 0 has no + prediction and is excluded everywhere — it is not a zero, it is absent. + """ + tokenizer = lm.tokenizer + enc = tokenizer(text, return_tensors="pt", add_special_tokens=False) + ids = enc["input_ids"] + n = int(ids.shape[1]) + if n < 2: + raise InvalidParamError( + f"a passage must tokenize to at least 2 tokens to be scored, got {n}" + ) + limit = _max_positions(lm) + if limit is not None and n > limit: + raise InvalidParamError( + f"the passage tokenizes to {n} tokens but {lm.model_id} has a context of " + f"{limit}. Shorten the passage (or split it across several) — this " + "measurement is one forward pass over the whole thing, never a truncation, " + "because a truncated variant would be scoring different text.", + {"n_tokens": n, "max_context": limit}, + ) + + pieces = tokenizer.convert_ids_to_tokens(ids[0].tolist()) + spans = token_byte_spans(list(pieces), text) + preserved_idx = [i for i in preserved_token_indices(spans, words, preserved) if i > 0] + + with _TRACE_LOCK, torch.no_grad(): + out = lm.model(input_ids=ids, attention_mask=enc["attention_mask"]) + logprobs = torch.log_softmax(out.logits[0].float(), dim=-1) + targets = ids[0, 1:] + per_token = -logprobs[:-1].gather(1, targets.unsqueeze(1)).squeeze(1) + nll = [math.nan] + [float(v) for v in per_token] + return ScoredText(text=text, n_tokens=n, nll=nll, preserved=preserved_idx) + + +def _mean(values: list[float]) -> float: + if not values: + raise ComputeError("no tokens to average — the passage has no scored positions") + return sum(values) / len(values) + + +def _stats(scored: ScoredText) -> dict[str, Any]: + """The fields of §8.1 for one scored variant. + + ``nTokens`` is the number of SCORED positions (every token but the first), because it + is the count ``nllAll`` averages over — which is what makes + ``bitsPerChar = nllAll · nTokens / (ln 2 · nChars)`` the passage's total surprisal + per character rather than an off-by-one approximation of it. + """ + all_nll = scored.scored + n_chars = len(scored.text) + nll_all = _mean(all_nll) + return { + "nllPreserved": _mean([scored.nll[i] for i in scored.preserved]), + "nllAll": nll_all, + "bitsPerChar": nll_all * len(all_nll) / (math.log(2.0) * n_chars) if n_chars else 0.0, + "nTokens": len(all_nll), + "nPreservedTokens": len(scored.preserved), + "nChars": n_chars, + } + + +def _pooled(scores: list[ScoredText]) -> dict[str, Any]: + """§8.1's fields over several passages, pooled at the TOKEN level. + + Token-weighted, never a mean of means: a passage with twice the tokens carries twice + the weight, which is the only pooling for which the pooled `nllPreserved` is the mean + surprisal of a preserved token and the measured quantization bound applies. + """ + all_nll = [v for s in scores for v in s.scored] + preserved = [s.nll[i] for s in scores for i in s.preserved] + n_chars = sum(len(s.text) for s in scores) + nll_all = _mean(all_nll) + return { + "nllPreserved": _mean(preserved), + "nllAll": nll_all, + "bitsPerChar": nll_all * len(all_nll) / (math.log(2.0) * n_chars) if n_chars else 0.0, + "nTokens": len(all_nll), + "nPreservedTokens": len(preserved), + "nChars": n_chars, + } + + +def _paired_difference(a: list[ScoredText], b: list[ScoredText], label: str) -> dict[str, Any]: + """``mean(nll_b − nll_a)`` over preserved tokens, PAIRED, with its standard error. + + The pairing is exact and is not an assumption: preserved words are character-identical + across variants, and the curated models' pretokenizers never merge across a word + boundary, so each preserved word yields the same pieces in every variant and the + preserved token lists correspond one-for-one. That is checked here; if it ever fails, + the difference is refused rather than computed over mismatched tokens. + + Pairing also removes the between-token variance — the same function word is compared + with itself in the other condition — which is what makes a standard error on an effect + of ~0.1 nats worth printing at all. + """ + diffs: list[float] = [] + for sa, sb in zip(a, b): + if len(sa.preserved) != len(sb.preserved): + raise ComputeError( + f"{label}: the variants have {len(sa.preserved)} and {len(sb.preserved)} " + "preserved tokens, so they cannot be paired", + {"a": len(sa.preserved), "b": len(sb.preserved)}, + ) + for ia, ib in zip(sa.preserved, sb.preserved): + diffs.append(sb.nll[ib] - sa.nll[ia]) + n = len(diffs) + mean = _mean(diffs) + if n > 1: + var = sum((d - mean) ** 2 for d in diffs) / (n - 1) + se = math.sqrt(var / n) + else: + se = math.nan + return {"nats": mean, "se": se, "nPairs": n} + + +# --- default passages --------------------------------------------------------------------- + + +def default_passages( + count: int = DEFAULT_PASSAGE_COUNT, words: int = DEFAULT_PASSAGE_WORDS +) -> list[str]: + """Evenly spaced excerpts of the shipped corpus — the measured configuration (§8.3a). + + Contiguous whole lines, so verse structure survives, and the front matter is skipped: + *The Real Mother Goose* opens with a title page and an alphabetical index of first + lines, and a "passage" cut from that is a column of titles rather than English. It + would raise every condition's NLL together and dilute the contrast with text that is + not the thing being measured. MEASURED on the shipped corpus: the index runs to block + 11 of 63, so :data:`FRONT_MATTER_FRACTION` of the blocks are dropped. The constant is + corpus-specific on purpose — this function only ever reads the shipped corpus, and a + detector for "is this an index?" that worked on one book and silently mis-fired on the + next would be worse than a measured number with its measurement written down. + + Deterministic: the same corpus gives the same passages, which is what lets the panel's + default reproduce the run the reference numbers came from, and what the cross-stack + digest fixture pins. + """ + from ..lex.corpus import load_corpus_text + + text = unicodedata.normalize("NFC", load_corpus_text()) + blocks: list[str] = [] + current: list[str] = [] + n = 0 + for line in text.split("\n"): + current.append(line) + n += len(WORD_RE.findall(line)) + if n >= words: + blocks.append("\n".join(current).strip("\n")) + current, n = [], 0 + if not blocks: + raise ComputeError("the shipped corpus produced no passage of the requested size") + start = max(1, round(len(blocks) * FRONT_MATTER_FRACTION)) + step = max(1, (len(blocks) - start) // max(1, count)) + out = [blocks[min(start + i * step, len(blocks) - 1)] for i in range(count)] + return out + + +# --- the endpoint's computation ----------------------------------------------------------- + + +def vacancy_score( + model_id: str, + passages: list[str] | None = None, + *, + p: float = 1.0, + seed: int = 0, + match_prosody: bool = True, + keep: frozenset[str] = frozenset(), +) -> dict[str, Any]: + """Score every variant of every passage and return the payload of §8.1 and §8.3. + + One real forward pass per (passage, variant) — 3n passes in total — on the real + pretrained weights at float32. Nothing here is cached across requests: the passages + are user text and the whole computation is a few seconds of CPU on the curated + models. + """ + if passages is None: + passages = default_passages() + if not isinstance(passages, list) or not passages: + raise InvalidParamError("passages must be a non-empty list of strings") + if len(passages) > MAX_PASSAGES: + raise InvalidParamError( + f"at most {MAX_PASSAGES} passages per request, got {len(passages)}; each one " + "costs three real forward passes" + ) + for passage in passages: + if not isinstance(passage, str) or not passage.strip(): + raise InvalidParamError("every passage must be a non-empty string") + if not 0.0 <= float(p) <= 1.0: + raise InvalidParamError(f"p must lie in [0, 1], got {p!r}") + + # NFC once, up front, and everything downstream indexes THIS string (§8.2). Qwen's + # tokenizer normalizes internally and the others do not; without this the byte-span + # check would fire on a passage that is perfectly well formed. + normalized = [unicodedata.normalize("NFC", passage) for passage in passages] + + lm = load_model(model_id) + scored: dict[str, list[ScoredText]] = {name: [] for name in VARIANTS} + per_passage: list[dict[str, Any]] = [] + for index, passage in enumerate(normalized): + texts = variant_texts(passage, p=p, seed=seed, match_prosody=match_prosody, keep=keep) + words, preserved = preserved_word_indices(texts) + if not preserved: + raise ComputeError( + f"passage {index} has no word that survives the transform, so there is no " + "scaffolding to score. Lower p, or use a passage with closed-class words.", + {"passage": index}, + ) + row: dict[str, Any] = { + "index": index, + "nWords": len(words), + "nPreservedWords": len(preserved), + "variants": {}, + } + for name in VARIANTS: + # The preserved WORD indices are the English passage's; each variant's own + # word spans are recomputed because a nonce is a different number of bytes. + variant_words = word_spans(texts[name]) + result = score_text(lm, texts[name], variant_words, preserved) + scored[name].append(result) + row["variants"][name] = _stats(result) + per_passage.append(row) + + differences = [ + { + "id": "wrong_content", + "label": "the cost of wrong content", + "expr": "nll(swap) − nll(english)", + "headline": True, + **_paired_difference(scored["english"], scored["swap"], "swap − english"), + }, + { + "id": "unknown_form", + "label": "the cost of unknown form", + "expr": "nll(nonce) − nll(swap)", + "headline": True, + "upperBound": True, + "note": UNKNOWN_FORM_NOTE, + **_paired_difference(scored["swap"], scored["nonce"], "nonce − swap"), + }, + { + # Reported, never headlined (§8.3): it is the SUM of the two above, so + # showing it as "what location was worth" would credit the cost of nonsense + # to the cost of an unknown word. It is here because it is the one contrast + # the quantized static build has a measured bound for. + "id": "total", + "label": "both costs together", + "expr": "nll(nonce) − nll(english)", + "headline": False, + "note": ( + "The sum of the two differences above. It conflates wrong content with " + "unknown form and is never the headline." + ), + **_paired_difference(scored["english"], scored["nonce"], "nonce − english"), + }, + ] + + return { + "model_id": lm.model_id, + "revision": lm.revision, + # What actually ran, so a reader never has to guess which error bounds apply. + "stack": "backend", + "dtype": "float32", + "p": float(p), + "seed": int(seed), + "match_prosody": bool(match_prosody), + "keep": sorted(keep), + "alignment": { + "mechanism": "byte-level pieces → UTF-8 byte spans", + "unit": "utf8_bytes", + "verified": True, + "note": ( + "Token→word attribution is verified at run time by reconstructing the " + "passage from the token byte spans; a mismatch raises rather than " + "mis-attributing." + ), + }, + "variants": [ + { + "id": name, + "pooled": _pooled(scored[name]), + "preview": scored[name][0].text[:400], + } + for name in VARIANTS + ], + # The English passages exactly as scored (NFC-normalized), so the panel can show + # the reader the text the number came from and let them edit it. Without this the + # default set would be a black box the UI could only describe. + "passages_used": normalized, + "differences": differences, + "passages": per_passage, + "tiny_arm": TINY_ARM, + "confound": CONFOUND_NOTE, + } diff --git a/code/backend/src/llm_geometry/lex/vacancy.py b/code/backend/src/llm_geometry/lex/vacancy.py new file mode 100644 index 0000000..fbde8aa --- /dev/null +++ b/code/backend/src/llm_geometry/lex/vacancy.py @@ -0,0 +1,1359 @@ +"""The vacancy transform — field without location, at a controlled rate `p`. + +Carroll's trick, stated operationally: the closed-class scaffolding is left character-for- +character intact — function words, inflectional morphology, punctuation, syntax, line +structure — while open-class stems are replaced by phonotactically legal nonce forms that +carry the same syllable count and stress. A reader parses *the slithy toves did gyre and +gimble* because every grammatical signal survives and only lexical content is vacant. Such +a token has a FIELD (its neighbourhood is fully specified by context) and no LOCATION (no +prior embedding). + +Normative source: `specs/007-vacancy-transform-field/architecture.md`. That document — not +the Python or the TypeScript — is what both stacks implement, so **every departure from the +original `tiny-seuss/synth/jabberwockify.py` here is one the contract lists in its §9**: + +1. Words are found with the tokenizer's own :data:`~llm_geometry.lex.vocab.WORD_RE`, not the + source's `[A-Za-z][A-Za-z']*`. Otherwise the transform and the trainer disagree about + `good-bye` and the relabelling theorem (contract §7.3) is simply false. +2. ``u = (top64 >> 11) / 2**53``, not ``top64 / 2**64``. A 64-bit integer over 2**64 is not + exactly representable as a float64, so Python and JavaScript can land on different + doubles for the same digest and disagree about a word at the boundary. +3. `random.Random` is replaced by a sha256 counter stream: MT19937 seeded from a string is + not reproducible in TypeScript. +4. The map is built **once over the whole type set in canonical order**, never lazily while + rewriting. The source's `used` set and give-up counter make a word's nonce depend on `p` + and on document order, which breaks the stability property the source claims for itself. +5. The domain is avoided **implicitly** — there is no caller-supplied `avoid` parameter — so + a minted form can never merge with a real English type and the map is a pure function of + `(domain, seed, match_prosody)`. The source accepts an `avoid` argument and never passes + one; making it optional reproduces the same failure a level up, where the map depends on + what the caller remembered (measured: seed 0 gives `remintRounds` 0 with the domain passed + and 1 without, with different nonces either way — both valid, which is the problem). +6. The give-up path is a deterministic salt relaxation, not `syllable + str(len(used))`. +7. Seams (`wee` + `er`) are repaired from a hash of `(stem, suffix)`, not a shared RNG. +8. Injectivity is **verified** over assembled surface forms, at every `p`, and re-minted on + collision — the theorem depends on it, so it is checked on every build and reported as + ``bijective`` / ``remintRounds``. +9. `split_suffix` carries the audited copy's ``SPLIT_EXCEPTIONS`` (`brother`, not + `broth`+`er`). +10. No claim of exact prosody: :data:`STRESS_TABLE` is 61 hand entries over the Dolch list, + described by its own author as "seeded by rule; wants roughly an hour of human + checking", so every prosody number ships with the three-way `stressFrom*` split beside it. +11. ``corpusTypesVacated`` counts types actually vacated, measured from the output text, not + ``len(map)``. Every count in §10 names its scope — corpus or domain — because an + unprefixed "types" is ambiguous between the two and cost the stacks two round trips. + +The properties that make a `p`-sweep interpretable are structural here rather than hoped +for: `u` depends only on `(seed, stem)`, so vacated sets are **nested** in `p`; the map is +built independently of `p`, so a stem's nonce is **stable** across the whole sweep. + +**Two minting strategies** (``VacancyParams.mint``, contract §8.3). ``"nonce"`` invents the +replacement; ``"swap"`` draws a REAL English word from the domain's own open-class types by +frequency rank, which is the control that separates *wrong content* from *unknown form* for +the pretrained arm. They differ in exactly one property, and §5.2a proves the difference is +forced rather than a shortcoming of this implementation: a swap map's images ARE domain +types, so a vacated word can land on one that has not moved, and no `p`-stable such map is +injective at intermediate `p` unless it is the identity. Swap is therefore a bijection of +the domain at full vacancy — where the pretrained arm measures, and where the invariance +theorem holds for it exactly as for nonce — and :func:`map_vocab_words` refuses the rest +rather than manufacturing a vocabulary with two words on one row. + +Two defects that the first implementation of this contract exposed are now fixed in the +contract itself, and each is covered by a named test here: + +* **The transform commutes with lowercasing** (§5.7): everything — the seam test, the seam + hash, the assembly — happens on the lowercased word, and `match_case` is applied to the + WHOLE assembled surface with the original whole word as the case source. Slicing the suffix + case-preserved made `gums` -> `flels` but `GUMS` -> `FLESS`, giving one type two surfaces. +* **Injectivity is checked over surface forms and holds at every `p`** (§5.2 conditions A and + B). A bare-nonce check at `p = 1` missed `hang` -> `wak`, whose surface `wak` + `ed` is the + real English word `waked`; the two collided at `p = 0.25` and `p = 0.5`, where `waked` + itself was not vacated. +""" + +from __future__ import annotations + +import hashlib +import math +import re +from dataclasses import dataclass, field +from typing import Iterable, Mapping, Sequence + +from ..errors import ComputeError, InvalidParamError +from .dolch import dolch_budget +from .vocab import WORD_RE + +# --- the closed class ------------------------------------------------------------------- + +#: Contract §2.1, ported verbatim from the source and whitespace-split. The source carries a +#: warning we keep: an earlier version unioned this with the short Dolch service words, which +#: silently protected content verbs (`run`, `eat`, `see`, `get`, `let`, `put`) and understated +#: the vacancy rate. The closed class is THIS CURATED LIST ONLY. +FUNCTION_WORDS: frozenset[str] = frozenset("""a an the this that these those my your his her its our +their some any all both each every no none i me you he she it we they him them +us who whom whose which what where when why how is am are was were be been being +do does did done have has had having will would shall should can could may might +must not and or but so if then than as of to in on at by for with from into onto +up down out off over under again once here there very too also only just even +still yet ever never always about after before while because though although +unless until since during between among against through above below near far +one two three four five six seven eight nine ten""".lower().split()) + +# --- suffix splitting ------------------------------------------------------------------- + +#: Contract §3, order significant — the first match wins. +SUFFIXES: tuple[str, ...] = ("ing", "edly", "est", "ies", "'s", "n't", "ed", "es", "er", "ly", "s") + +#: Words that are never split. From the AUDITED copy of the source (contract §3, departure 9); +#: without it `brother -> broth+er` and `morning -> morn+ing`, which the source itself flags as +#: a known artifact. This is a spelling heuristic, not a morphological analyser, and it stays +#: wrong outside the list (`ladder -> ladd+er`) — acceptable, because the nonce still carries a +#: consistent identity and an inflected-looking surface, but it must be stated in the UI rather +#: than quietly tolerated. +SPLIT_EXCEPTIONS: frozenset[str] = frozenset( + { + "brother", + "father", + "mother", + "sister", + "never", + "over", + "under", + "morning", + "giving", + "thing", + } +) + +#: The stem must be ASCII letters only. `str.isalpha()` is Unicode-aware and would accept +#: letters JavaScript's `^[A-Za-z]+$` rejects; nothing in the shipped corpus exercises the +#: difference, a pasted corpus would (contract §2.2). +_STEM_RE = re.compile(r"[A-Za-z]+") + +#: A vacated word must itself be a single complete `WORD_RE` match — checked, not assumed. +_WHOLE_WORD_RE = re.compile(WORD_RE.pattern + r"\Z") + +# --- the phonotactic tables ------------------------------------------------------------- +# Contract §5.4, ported verbatim, ORDER SIGNIFICANT: the index into each list is what the +# byte stream selects, so reordering silently changes every nonce in both stacks. + +ONSETS: tuple[str, ...] = ( + "b", "br", "bl", "d", "dr", "f", "fl", "fr", "g", "gl", "gr", "h", + "j", "k", "kl", "kr", "l", "m", "n", "p", "pl", "pr", "r", "s", "sk", + "sl", "sm", "sn", "sp", "st", "str", "sw", "t", "tr", "th", "thr", + "v", "w", "wr", "y", "z", "sh", "shr", "ch", "gn", "sc", "sq", +) # fmt: skip + +NUCLEI: tuple[str, ...] = ( + "a", "e", "i", "o", "u", "ai", "ee", "ea", "oo", "ou", "oa", "ie", + "y", "au", "ur", "ir", "or", "ar", "er", +) # fmt: skip + +CODAS: tuple[str, ...] = ( + "", "b", "d", "f", "g", "k", "l", "m", "n", "p", "r", "s", "t", "v", + "z", "sh", "ch", "th", "ck", "ff", "ll", "mp", "nd", "ng", "nk", "nt", + "sk", "sp", "st", "ft", "lt", "lk", "rd", "rk", "rm", "rn", "rt", "ble", + "dle", "gle", "kle", "tle", "mble", "ndle", "ffle", "zzle", +) # fmt: skip + +UNSTRESSED_TAILS: tuple[str, ...] = ( + "y", "le", "er", "ow", "en", "el", "ish", "ous", "id", + "ic", "um", "ent", "ing", +) # fmt: skip + +#: Prefixes for an unstressed FIRST syllable. +UNSTRESSED_ONSETS: tuple[str, ...] = ("a", "be", "re", "de", "un", "en") + +#: The reduced coda set for an unstressed syllable. The duplicated empty string doubles its +#: weight; keep it (contract §5.4). +#: +#: **This table is UNREACHABLE and that is correct.** §5.5 step 2 has exactly three branches +#: and none of them draws from here, exactly as in the source, where `_syl(stressed=False)` +#: is never called. An earlier draft of the contract implied a fourth branch; it was +#: self-contradictory and has been removed. It is retained here for fidelity to the source's +#: tables and because **adding a use would shift the list indices the byte stream selects and +#: change every multi-syllable nonce in both stacks**. Do not "fix" it. +REDUCED_CODAS: tuple[str, ...] = ("", "", "l", "n", "r", "s") + +# --- prosody ---------------------------------------------------------------------------- + +#: Contract §6.1, ported verbatim from `tiny-seuss/synth/lexicon.py`: 61 polysyllables of the +#: Dolch list, "seeded by rule and then overridden by a hand table", and listed by the source +#: itself under *not yet exercised*. So we do not claim exact prosody and no UI string may: +#: the shipped corpus is *The Real Mother Goose*, most of whose types are not in this table +#: and therefore fall through to :func:`rule_syllables`. Every prosody statistic must be shown +#: with ``stressFromTable``, which is the honesty of every other prosody number. +#: `Santa Claus` is retained verbatim even though a word tokenizer can never match it. +STRESS_TABLE: dict[str, str] = { + "away": "01", "funny": "10", "little": "100", "yellow": "10", + "into": "10", "over": "10", "pretty": "10", "under": "10", + "after": "10", "again": "01", "any": "10", "every": "100", + "giving": "10", "once": "1", "open": "10", + "always": "100", "around": "01", "because": "01", "before": "01", + "seven": "10", "eight": "1", "myself": "01", "never": "10", + "only": "10", "today": "01", "together": "0100", "better": "10", + "carry": "10", "many": "10", "upon": "01", "very": "100", + "apple": "10", "baby": "10", "birthday": "100", "brother": "10", + "chicken": "10", "children": "100", "Christmas": "10", + "farmer": "10", "flower": "10", "garden": "10", "good-bye": "01", + "horse": "1", "kitty": "10", "letter": "10", "money": "10", + "morning": "10", "mother": "10", "paper": "100", "party": "10", + "picture": "10", "rabbit": "10", "robin": "10", "squirrel": "1", + "table": "10", "water": "10", "window": "10", "Santa Claus": "101", + "father": "10", "sister": "10", "summer": "10", +} # fmt: skip + +#: Contract §6.4. `anapest` is the Seuss engine (and Byron's *The Destruction of +#: Sennacherib*, which is where he got it). +FEET: dict[str, str] = {"anapest": "001", "iamb": "01", "trochee": "10", "dactyl": "100"} + +_VOWEL_GROUP_RE = re.compile(r"[aeiouy]+") +_NON_LOWER_RE = re.compile(r"[^a-z]") + +# --- minting ---------------------------------------------------------------------------- + +#: Collapse a run of three or more identical consonants to two (contract §5.5 step 3). +_RUN_RE = re.compile(r"([bcdfghjklmnpqrstvwxz])\1{2,}") + +#: Contract §5.5 step 5: deterministic, order-independent relaxations, unlike the source's +#: give-up counter. Reaching :data:`MINT_MAX_SALT` raises — it has never happened and if it +#: does we want to know. +#: +#: **These thresholds are on the ATTEMPT COUNTER `a`, not on the absolute salt.** A mint call +#: carries a base salt `S`; the byte stream is keyed on `S + a` for `a = 0, 1, 2, …`, and a +#: re-mint restarts `a` at 0. So a re-mint is held to exactly the same quality bar as an +#: original mint — which is why the seed-7 re-mint of `hang` comes back monosyllabic +#: (`smeeg`) rather than falling out of a loop with every check already relaxed. +MINT_RELAX_SYLLABLES_SALT = 400 +MINT_RELAX_LENGTH_SALT = 800 +MINT_MAX_SALT = 1200 + +#: Contract §5.2/§7.3: injectivity is verified, not assumed, and re-minted on collision. +MAX_REMINT_ROUNDS = 8 + +# --- the swap control (contract §8.3) ---------------------------------------------------- + +#: Half-width of the frequency-rank window a swap replacement is drawn from: attempt `a` +#: proposes ``pool[(r + delta) % len(pool)]`` for ``delta`` in ``[-w, -1] | [1, w]``. +SWAP_WINDOW = 32 + +#: The window doubles every this many attempts, up to the whole pool — the same deterministic +#: relaxation §5.5 uses, and necessary for the same reason: "anything already used" depletes a +#: fixed window for the stems late in canonical order. +SWAP_WIDEN_EVERY = 64 + +#: Attempt at which the prosody filter is dropped, mirroring :data:`MINT_RELAX_SYLLABLES_SALT`. +SWAP_RELAX_PROSODY = 1024 + +#: Reaching this many attempts raises rather than looping — the pool is finite, so unlike +#: minting this bound is reachable in principle and we want to know if it ever is. +SWAP_MAX_ATTEMPTS = 4096 + +#: The two minting strategies of contract §7.1 / §8.3. +MINT_STRATEGIES: tuple[str, ...] = ("nonce", "swap") + +#: Contract §5.8: a re-mint jumps to a fresh region of the byte stream deterministically. +REMINT_SALT_STRIDE = 1000 + +#: Characters a seam repair may substitute (contract §5.7). +_SEAM_CHARS = "lnrtk" + + +# --- word segmentation ------------------------------------------------------------------ + + +def _reject_bare_str(value: object, func: str, param: str) -> None: + """Raise if a text was passed where an iterable of TYPES was expected. + + ``Iterable[str]`` happily accepts a ``str`` and iterates it character by character, so + ``vacancy_domain(corpus_text)`` silently yields a domain of single letters. That failure + surfaces much later and somewhere else — as `stem 'real' is outside the vacancy map's + domain` — so it is caught here, at the actual mistake. + + ``TypeError`` rather than :class:`InvalidParamError`: this is a wrong argument TYPE, not a + value out of range, and it can never reach an HTTP boundary as a user's fault. + """ + if isinstance(value, str): + raise TypeError( + f"{func}({param}=...) takes an iterable of word types, not a text; " + f"pass tokenize(text) or set(tokenize(text)), not the text itself" + ) + + +def stem_and_suffix(word: str) -> tuple[str, str]: + """Split an inflectional suffix off `word` (contract §3). + + The suffixes are tried in :data:`SUFFIXES` order and the first match wins. A suffix `s` + matches iff ``lower(word).endswith(s)`` and ``len(word) - len(s) >= 3``. The split slices + the ORIGINAL word, so case is preserved. Words in :data:`SPLIT_EXCEPTIONS` are never + split. + + Preserving the suffix is what keeps the syntax parseable: the stem is vacated and the + suffix re-attached, so the nonce still looks inflected. + """ + lower = word.lower() + if lower in SPLIT_EXCEPTIONS: + return word, "" + for suffix in SUFFIXES: + if lower.endswith(suffix) and len(word) - len(suffix) >= 3: + cut = len(word) - len(suffix) + return word[:cut], word[cut:] + return word, "" + + +def is_eligible(stem: str, keep: frozenset[str] = FUNCTION_WORDS) -> bool: + """Is this stem open-class, i.e. may it be vacated at all? (Contract §2.2.) + + `keep` is the EFFECTIVE closed class — :data:`FUNCTION_WORDS` unioned with any caller + extras. Pass :attr:`VacancyParams.keep_set`, never :attr:`VacancyParams.keep`, or the + function words lose their protection. + + Test 2 is what makes hyphenated and apostrophised words behave as they do, and both + stacks must agree on it exactly: `good-bye` matches no suffix, so its stem contains a + hyphen and it is **never** vacated; `dog's` splits to `dog`, which passes, giving + ``'s``. + """ + return stem.lower() not in keep and _STEM_RE.fullmatch(stem) is not None and len(stem) > 2 + + +def match_case(src: str, new: str) -> str: + """Carry `src`'s capitalisation onto `new`, so `Jack` becomes `Flim`, not `flim`. + + Applied to the WHOLE assembled surface form with the ORIGINAL WHOLE WORD as `src` + (contract §5.7), never to the nonce alone with a case-preserved suffix appended after — + that is what broke `GUMS`. + """ + if src.isupper() and len(src) > 1: + return new.upper() + if src[:1].isupper(): + return new[:1].upper() + new[1:] + return new + + +# --- the vacancy decision --------------------------------------------------------------- + + +def vacancy_u(stem: str, seed: int) -> float: + """The stem's position in [0, 1) — vacate iff ``u(stem) < p`` (contract §4). + + ``u`` is a function of ``(seed, stem)`` alone: not of `p`, not of traversal order, not of + which other words exist. That is what makes the vacated sets NESTED as `p` grows, which is + the first of the two properties a `p`-sweep needs to be interpretable. + + The ``>> 11`` is mandatory and is a departure from the source (which used + ``top64 / 2**64``). A 64-bit integer divided by 2**64 is not exactly representable as a + float64, so Python and JavaScript can round to different doubles for the same digest and + disagree about a word at the boundary. Shifting to 53 bits makes the numerator exactly + representable, so ``u`` is *the same double* in both languages. + """ + digest = hashlib.sha256(f"{seed}:{stem.lower()}".encode("utf-8")).digest() + return (int.from_bytes(digest[:8], "big") >> 11) / 2**53 + + +# --- prosody ---------------------------------------------------------------------------- + + +def rule_syllables(word: str) -> int: + """The spelling fallback for syllable counting (contract §6.2). + + The source has a further ``if w.endswith("le") ...: pass`` branch; it is dead code and is + not ported, so this is byte-identical to the source's behaviour. + """ + w = _NON_LOWER_RE.sub("", word.lower().strip("'-")) + if not w: + return 1 + n = len(_VOWEL_GROUP_RE.findall(w)) + if w.endswith("e") and n > 1 and not w.endswith(("le", "ee", "ye")): + n -= 1 + return max(1, n) + + +def stress_source(word: str, minted_stress: Mapping[str, str] | None = None) -> str: + """Where :func:`stress` got its answer: ``"minted"``, ``"table"``, or ``"rule"``. + + ``"table"`` is the fraction the UI must publish next to every prosody number — it is the + part that a human has (nominally) checked. ``"minted"`` is exact by construction but is a + form we invented, and ``"rule"`` is a spelling guess. + """ + lower = word.lower() + if minted_stress and lower in minted_stress: + return "minted" + if word in STRESS_TABLE or lower in STRESS_TABLE: + return "table" + return "rule" + + +def stress(word: str, minted_stress: Mapping[str, str] | None = None) -> str: + """The stress pattern of `word` as a string of ``0``/``1`` (contract §6.3). + + Lookup order, exactly: the minted patterns (so prosody scoring on a vacated corpus is + exact *for the forms we minted*), then :data:`STRESS_TABLE` case-sensitively (for + `Christmas`), then case-insensitively, then the spelling rule. + + `minted_stress` is a parameter rather than the source's module-level global: a global + would make the answer depend on which corpora had been transformed earlier in the process. + """ + lower = word.lower() + if minted_stress and lower in minted_stress: + return minted_stress[lower] + if word in STRESS_TABLE: + return STRESS_TABLE[word] + if lower in STRESS_TABLE: + return STRESS_TABLE[lower] + n = rule_syllables(lower) + return "1" if n == 1 else "1" + "0" * (n - 1) + + +def syllables(word: str, minted_stress: Mapping[str, str] | None = None) -> int: + """``len(stress(word))`` — the syllable count implied by the stress pattern.""" + return len(stress(word, minted_stress)) + + +def scan(line: str, minted_stress: Mapping[str, str] | None = None) -> str: + """The line's concatenated stress string, e.g. ``0100100100``. + + Words are found with the tokenizer's regex but NOT lowercased, so the case-sensitive + :data:`STRESS_TABLE` lookup can still fire. + """ + return "".join(stress(t, minted_stress) for t in WORD_RE.findall(line)) + + +def meter_score( + line: str, foot: str = "anapest", minted_stress: Mapping[str, str] | None = None +) -> float: + """Fraction of syllable positions in `line` that match the repeating `foot`. + + ``0.0`` for a line with no syllables (contract §6.4). + """ + if foot not in FEET: + raise InvalidParamError( + f"unknown foot {foot!r}; expected one of {sorted(FEET)}", + {"foot": foot}, + ) + s = scan(line, minted_stress) + if not s: + return 0.0 + pattern = FEET[foot] + target = (pattern * (len(s) // len(pattern) + 1))[: len(s)] + return sum(a == b for a, b in zip(s, target)) / len(s) + + +# --- the deterministic byte stream ------------------------------------------------------ + + +class _ByteStream: + """A sha256 counter stream (contract §5.3), trivially identical in both languages. + + ``random.Random`` cannot be ported: MT19937 seeded from a string is not reproducible in + TypeScript without reimplementing the generator *and* `Random.choice`'s masking. + """ + + __slots__ = ("_prefix", "_counter", "_block", "_offset") + + def __init__(self, seed: int, stem: str, salt: int, tag: str = "mint") -> None: + #: `tag` separates the minting stream from the swap-draw stream of §8.3. It is part + #: of the hashed prefix, so the two can never alias however the salts line up. + self._prefix = f"{seed}:{tag}:{stem}:{salt}:" + self._counter = 0 + self._block = hashlib.sha256(f"{self._prefix}0".encode("utf-8")).digest() + self._offset = 0 + + def u32(self) -> int: + """The next big-endian unsigned 32-bit word, refilling from the next counter.""" + if self._offset + 4 > len(self._block): + self._counter += 1 + self._block = hashlib.sha256(f"{self._prefix}{self._counter}".encode("utf-8")).digest() + self._offset = 0 + value = int.from_bytes(self._block[self._offset : self._offset + 4], "big") + self._offset += 4 + return value + + def choice(self, options: Sequence[str]) -> str: + """``options[u32() % len(options)]``. + + Every list here is shorter than 256, so the modulo bias is aesthetic rather than + statistical — but both stacks must bias IDENTICALLY, which this does. + """ + return options[self.u32() % len(options)] + + +def _mint( + key: str, + seed: int, + match_prosody: bool, + forbidden: frozenset[str] | set[str], + start_salt: int = 0, + stem: str | None = None, +) -> tuple[str, str, int]: + """Mint one nonce for `key`, returning ``(nonce, intended stress pattern, salt)``. + + Contract §5.5. Depends only on ``(seed, key, match_prosody, forbidden, start_salt)`` — + and `forbidden` depends only on the canonically-ordered prefix of stems before this one — + so a stem's nonce is the same at every `p`. That is the stability property (§5.6). + + **`key` feeds the byte stream and the uniqueness check ONLY; the stress pattern comes + from `stem`.** They differ under ``consistent=False``, where the key is + ``f"{stem}#{idx}"`` (§5.8). Letting the key reach the prosody lookup gives + ``stress("little#0") == "10"`` instead of ``stress("little") == "100"``, so `Little` + mints as `Wrerken` rather than `Wrerkenle` — §7.1 says the nonce carries *the stem's* + syllable count and stress, and a mint key is not a word. `stem` defaults to `key`, which + is the consistent case where they are the same string. + + `start_salt` is the base salt `S`; the byte stream is keyed on ``S + a`` for the attempt + counter ``a = 0, 1, 2, …``, and the quality relaxations are thresholds on ``a``. The + ACCEPTING salt (``S + a``) is returned because §5.8 needs it: a re-mint restarts at + ``REMINT_SALT_STRIDE * round + previousSalt + 1``, with its own ``a`` back at 0. + + Three branches in step 2, in that order, exactly as §5.5 spells them out. There is no + fourth: :data:`REDUCED_CODAS` is unreachable here, as it is in the source, where + ``_syl(stressed=False)`` is never called. **Do not "fix" this** — a fourth branch would + shift every list index the byte stream selects and change every multi-syllable nonce. + """ + pattern = stress(key if stem is None else stem) if match_prosody else "1" + n_syl = len(pattern) + for attempt in range(MINT_MAX_SALT): # `a` of §5.5; the stream is keyed on `S + a` + salt = start_salt + attempt + rnd = _ByteStream(seed, key, salt) + parts: list[str] = [] + for i, mark in enumerate(pattern): + if mark == "1": + parts.append(rnd.choice(ONSETS) + rnd.choice(NUCLEI) + rnd.choice(CODAS)) + elif i == 0: + parts.append(rnd.choice(UNSTRESSED_ONSETS)) + else: + parts.append(rnd.choice(UNSTRESSED_TAILS)) + w = _RUN_RE.sub(r"\1\1", "".join(parts)) + long_enough = len(w) >= 3 or attempt >= MINT_RELAX_LENGTH_SALT + right_length = syllables(w) == n_syl or attempt >= MINT_RELAX_SYLLABLES_SALT + if long_enough and right_length and w not in forbidden: + return w, pattern, salt + raise ComputeError( + f"could not mint a nonce for {key!r} in {MINT_MAX_SALT} attempts", + {"key": key, "stem": stem, "seed": seed, "pattern": pattern, "start_salt": start_salt}, + ) + + +def type_counts(tokens: Iterable[str]) -> dict[str, int]: + """Occurrences per lowercased type — the frequency source the swap control ranks by. + + Takes the TOKEN STREAM (``tokenize(text)``), not the type set: a set has no frequencies, + and ``mint="swap"`` needs them. Passing the deduplicated domain here would silently rank + every type equally and give a map that is alphabetical rather than frequency-matched, so + :func:`build_vacancy_map` requires this to be passed explicitly and raises without it + rather than inventing a fallback. + """ + _reject_bare_str(tokens, "type_counts", "tokens") + counts: dict[str, int] = {} + for t in tokens: + key = t.lower() + counts[key] = counts.get(key, 0) + 1 + return counts + + +def swap_pool(domain: Iterable[str], counts: Mapping[str, int], keep: frozenset[str]) -> list[str]: + """The replacement pool of contract §8.3: the domain's open-class TYPES by frequency rank. + + Ordered by ``(count descending, type ascending)`` — the tie rule + :func:`~llm_geometry.lex.vocab.frequency_budget` already uses, so "frequency rank" means + one thing in this codebase. A type absent from the corpus (the 22 Dolch-only words) has + count 0 and ranks last, alphabetically among its equals. + + **Types, not stems.** The stem set is exactly the set of keys the map assigns, so drawing + from it would consume the pool exactly and leave an A-collision with nowhere to move. On + the shipped corpus the pool is 1944 types against 1680 stems, which is the slack the + re-draw rounds spend. + """ + return sorted( + {t.lower() for t in domain if is_eligible(stem_and_suffix(t)[0], keep)}, + key=lambda t: (-counts.get(t, 0), t), + ) + + +def swap_rank( + stem: str, family: Iterable[str], pool: Sequence[str], counts: Mapping[str, int] +) -> int: + """Where `stem` sits in the frequency-ranked pool (contract §8.3). + + Not ``pool.index(stem)``: 375 of the shipped corpus's 1680 eligible stems are not domain + types at all — `hang` and `gum` reach the map only as the stems of `hanged` and `gums` — + so a lookup would raise on a fifth of them. The rank is instead the position the stem's + own key would take in the pool's order, with the stem's frequency taken over its whole + INFLECTIONAL FAMILY: `hang` is as frequent as `hanged` + `hanging` make it, which is the + frequency a reader of the corpus actually meets. + + Defined as the number of pool entries whose key sorts strictly before the stem's, so it + is a plain count and cannot be read two ways; the binary search is only how it is + computed. Ties fall to the stem's own alphabetical position, exactly as the pool order + does. + """ + freq = sum(counts.get(t, 0) for t in family) + key = (-freq, stem) + lo, hi = 0, len(pool) + while lo < hi: + mid = (lo + hi) // 2 + other = pool[mid] + if (-counts.get(other, 0), other) < key: + lo = mid + 1 + else: + hi = mid + return lo + + +def _draw_swap( + stem: str, + seed: int, + match_prosody: bool, + pool: Sequence[str], + rank: Mapping[str, int], + used: frozenset[str] | set[str], + suffixes: Sequence[str], + claimed: frozenset[str] | set[str], + barred: frozenset[str] | set[str], + start_salt: int = 0, +) -> tuple[str, int, list[str]]: + """Draw one real-word replacement for `stem` (contract §8.3). + + Returns ``(word, salt, surface forms)`` — the forms the stem now owns, one per suffix it + occurs with, so the caller can claim them. + + Deterministic in ``(seed, stem, pool, used, claimed, start_salt)`` and independent of `p`, + exactly as :func:`_mint` is, so §5.6's stability property survives the swap control. + + Attempt `a` widens the window every :data:`SWAP_WIDEN_EVERY` attempts and drops the + prosody filter at :data:`SWAP_RELAX_PROSODY`; both relaxations are functions of `a` alone, + never of how many stems happen to have been assigned first. The byte stream carries the + ``swap`` tag, so it can never alias the ``mint`` stream at the same salt. + + **Conditions A and B₁ are enforced here, at draw time, not only checked afterwards.** A + candidate is rejected if any of the surfaces it would produce is already claimed by an + earlier stem, is an ineligible domain type, or equals the very type it replaces. Checking + only afterwards costs 29 collision rounds on the shipped corpus at seed 0 and does not + converge inside the 8 §5.2 allows: the pool holds inflected types, so a bare stem drawing + `years` and a suffixed one drawing `year` land on the same surface, and re-drawing one of + them at random walks into the next such pair. Enforcing at draw time makes the map + correct by construction and the verification loop below a check rather than a search. + """ + n = len(pool) + if n == 0: + raise ComputeError( + "the swap pool is empty — no domain type has an eligible stem, so there is no " + "real word to draw (contract §8.3)", + {"stem": stem, "seed": seed}, + ) + r = rank[stem] + pattern = stress(stem) if match_prosody else None + family = {stem + suffix for suffix in suffixes} + for attempt in range(SWAP_MAX_ATTEMPTS): + salt = start_salt + attempt + # The doubling is capped at 20 before the min, not because 20 is meaningful but + # because JavaScript's `<<` takes its shift count modulo 32: an uncapped + # `attempt // 64` reaches 63 at the give-up bound and the two stacks would compute + # different widths from the same attempt. 32 << 20 already exceeds any pool. + width = min(SWAP_WINDOW << min(attempt // SWAP_WIDEN_EVERY, 20), n) + offset = _ByteStream(seed, stem, salt, tag="swap").u32() % (2 * width) + delta = offset - width if offset < width else offset - width + 1 + candidate = pool[(r + delta) % n] + if candidate == stem or candidate in used or candidate in family: + continue + if pattern is not None and attempt < SWAP_RELAX_PROSODY and stress(candidate) != pattern: + continue + forms = [surface_form(candidate, stem, suffix, seed) for suffix in suffixes] + if any(f in claimed or f in barred or f in family for f in forms): + continue + if len(set(forms)) != len(forms): + continue + return candidate, salt, forms + raise ComputeError( + f"could not draw a swap replacement for {stem!r} in {SWAP_MAX_ATTEMPTS} attempts — " + "the pool is finite, so report this rather than raising the bound", + {"stem": stem, "seed": seed, "pool": n, "used": len(used), "start_salt": start_salt}, + ) + + +def _seam_fix(nonce: str, suffix: str, stem: str, seed: int) -> str: + """Repair a seam like ``wee`` + ``er`` -> ``weeer`` (contract §5.7). + + Deterministic in `(stem, suffix)`, so it is order-independent — the source used a shared + RNG here, which is not. The substitution is applied once, not looped: the hash is fixed, + so a loop would never terminate on a repeat. + + `stem` and `suffix` must already be LOWERCASED; see :func:`surface_form`. + """ + if not suffix or not nonce or nonce[-1] != suffix[0]: + return nonce + digest = hashlib.sha256(f"{seed}:seam:{stem}:{suffix}".encode("utf-8")).digest() + return nonce[:-1] + _SEAM_CHARS[int.from_bytes(digest[:4], "big") % len(_SEAM_CHARS)] + + +def surface_form(nonce: str, stem: str, suffix: str, seed: int) -> str: + """The assembled, LOWERCASED output of §5.7 — seam repaired, suffix re-attached. + + This is the unit the injectivity conditions of §5.2 are stated over, and the only place + the transform assembles anything, so the case-commuting invariant + + ``lower(transform_word(w)) == transform_word(lower(w))`` + + holds by construction: nothing downstream of here branches on case. + """ + key, tail = stem.lower(), suffix.lower() + return _seam_fix(nonce, tail, key, seed) + tail + + +# --- parameters ------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class VacancyParams: + """The knobs of contract §7.1. + + Three of them — `p`, `seed`, `match_prosody` — are INVISIBLE to a word-level model + trained from scratch, because such a model never sees the letters: with + ``consistent=True`` and ``reveal_after=0`` the transform is a pure relabelling of the + vocabulary and training is bit-identical. Only the knobs that break type identity + (``consistent=False``, ``reveal_after>0``) can move a loss. That is the honest tiny-arm + result and it is worth stating plainly rather than dressing a null up as a curve. + """ + + #: Fraction of eligible TYPES vacated. Compared as given; the UI emits two decimal + #: places and both stacks parse it as a float64. + p: float = 0.0 + #: Selects both `u` and the nonce assignment. + seed: int = 0 + #: One nonce per source type, corpus-wide. ``False`` is the source's "inconsistent + #: assignment" control: same vacancy rate, no learnable identity, and DELIBERATELY no + #: stability property. + consistent: bool = True + #: The nonce carries the stem's syllable count and stress. + match_prosody: bool = True + #: The first N occurrences of a vacated stem keep their English form, seeding a partial + #: location. ``0`` is the pure case. + reveal_after: int = 0 + #: Extra words added to the closed class. The effective set is + #: ``FUNCTION_WORDS | lower(keep)`` — see :attr:`keep_set`. + keep: frozenset[str] = frozenset() + #: How a replacement is produced (contract §8.3). ``"nonce"`` invents a phonotactically + #: legal form; ``"swap"`` draws a REAL English word from the domain's own open-class types + #: by frequency rank, so the passage stays equally nonsensical while every form remains a + #: known word with ordinary tokenization. That is the control that separates "wrong + #: content" from "unknown form" in the pretrained arm — and §5.2a proves it can only be + #: injective at full vacancy, which is where the pretrained arm measures. + mint: str = "nonce" + + _keep_set: frozenset[str] = field(default=frozenset(), repr=False, compare=False) + + def __post_init__(self) -> None: + if isinstance(self.p, bool) or not isinstance(self.p, (int, float)): + raise InvalidParamError(f"p must be a number, got {self.p!r}", {"p": self.p}) + if not math.isfinite(self.p) or not 0.0 <= self.p <= 1.0: + raise InvalidParamError(f"p must lie in [0, 1], got {self.p!r}", {"p": self.p}) + if isinstance(self.seed, bool) or not isinstance(self.seed, int): + raise InvalidParamError(f"seed must be an int, got {self.seed!r}", {"seed": self.seed}) + if isinstance(self.reveal_after, bool) or not isinstance(self.reveal_after, int): + raise InvalidParamError( + f"reveal_after must be an int, got {self.reveal_after!r}", + {"reveal_after": self.reveal_after}, + ) + if self.reveal_after < 0: + raise InvalidParamError( + f"reveal_after must be >= 0, got {self.reveal_after}", + {"reveal_after": self.reveal_after}, + ) + if self.mint not in MINT_STRATEGIES: + raise InvalidParamError( + f"mint must be one of {list(MINT_STRATEGIES)}, got {self.mint!r}", + {"mint": self.mint}, + ) + if self.mint == "swap" and not self.consistent: + raise InvalidParamError( + "mint='swap' requires consistent=True: the inconsistent control needs a fresh " + "type per occurrence and the corpus has 1680 open-class stems against 8202 " + "vacated tokens, so there is no supply of real words to draw (contract §8.3)", + {"mint": self.mint, "consistent": self.consistent}, + ) + # Same trap as `vacancy_domain`: `keep="little"` would quietly protect six letters. + _reject_bare_str(self.keep, "VacancyParams", "keep") + for w in self.keep: + if not isinstance(w, str): + raise InvalidParamError( + f"keep must contain strings, got {w!r}", {"keep": sorted(map(str, self.keep))} + ) + object.__setattr__(self, "keep", frozenset(self.keep)) + object.__setattr__(self, "_keep_set", FUNCTION_WORDS | {w.lower() for w in self.keep}) + + @property + def keep_set(self) -> frozenset[str]: + """The EFFECTIVE closed class: :data:`FUNCTION_WORDS` plus the caller's extras.""" + return self._keep_set + + +# --- the map ---------------------------------------------------------------------------- + + +class _RewriteState: + """Per-rewrite bookkeeping for the order-DEPENDENT conditions only. + + ``consistent=True, reveal_after=0`` — the condition the invariance theorem is stated for — + touches nothing here except the counter, and the counter never changes an output. + + `used` is seeded from the map's STORED :attr:`VacancyMap.forbidden` (§5.8) — the domain + plus every nonce ever handed out, superseded ones included — not from + ``mapping.values()``, which drops the superseded ones and is the reconstruction the two + stacks disagreed on. + """ + + __slots__ = ("counts", "used") + + def __init__(self, used: set[str]) -> None: + self.counts: dict[str, int] = {} + self.used = used + + +@dataclass(frozen=True) +class VacancyMap: + """A `p`-independent stem -> nonce assignment, plus the evidence that it is injective. + + Built once over the whole type set in canonical order (contract §5.2), so the map at any + `p` is just the restriction of this one map to ``{stem : u(stem) < p}``. Nesting and + stability are therefore structural facts rather than properties to be hoped for. + + The two mappings are plain dicts for cheap lookup and must not be mutated by callers; the + only writer is the inconsistent-assignment control, which registers the patterns of the + forms it mints so that prosody scoring stays exact. + """ + + #: ``lower(stem) -> nonce``. + mapping: dict[str, str] + #: ``nonce -> intended stress pattern``, for :func:`stress`'s first lookup. + minted_stress: dict[str, str] + seed: int + match_prosody: bool + #: The lowercased domain this map was built over — :func:`vacancy_domain`, i.e. the + #: corpus's types plus the full Dolch list. It is also the set a nonce may never equal, + #: so a minted form can never silently merge with a real English word. Kept because §10's + #: `domainTypes*` counts are over it and it is NOT recoverable from the corpus text: + #: budget-only words have images but never appear. + domain: frozenset[str] + #: ``|image|`` of the domain, at full vacancy. + image_size: int + #: ``image_size == type_count``. The relabelling theorem depends on it. + bijective: bool + #: How many collision-driven re-mint rounds were needed. Expected to be 0. + remint_rounds: int + #: Every replacement ever handed out, PLUS the whole domain — including nonces a re-mint + #: round superseded (contract §5.8). STORED, never reconstructed as + #: ``domain | mapping.values()``: that reconstruction silently drops the superseded ones + #: (`wak` at seed 7), and the per-occurrence path of the ``consistent=False`` control + #: draws against this set, so reusing a form that was rejected for cause can recreate the + #: very collision the re-mint resolved. + forbidden: frozenset[str] + #: Is the map injective at EVERY `p`, or only at full vacancy? ``True`` for + #: ``mint="nonce"``, where condition B keeps every image out of the domain; ``False`` for + #: ``mint="swap"``, whose images ARE domain types — §5.2a proves no `p`-stable swap can do + #: better, and :func:`map_vocab_words` refuses the cases where it matters. + injective_at_every_p: bool + + @property + def type_count(self) -> int: + """``|domain|`` — what `image_size` must equal for the map to be injective.""" + return len(self.domain) + + def nonce_for(self, stem: str) -> str | None: + """The nonce assigned to `stem`, or ``None`` if the stem is outside the domain.""" + return self.mapping.get(stem.lower()) + + def apply_word(self, word: str, params: VacancyParams) -> str: + """Transform a single word in isolation. + + Convenience for one-off queries and for the order-INDEPENDENT conditions. Rewriting a + text goes through :func:`vacate_text`, which threads the occurrence counters that + ``reveal_after`` and ``consistent=False`` need. + """ + return self._transform(word, params, _RewriteState(set(self.forbidden))) + + def _transform(self, word: str, params: VacancyParams, state: _RewriteState) -> str: + stem, suffix = stem_and_suffix(word) + if not is_eligible(stem, params.keep_set): + return word + key = stem.lower() + if vacancy_u(key, params.seed) >= params.p: + return word + seen = state.counts.get(key, 0) + 1 + state.counts[key] = seen + if seen <= params.reveal_after: + return word + + if params.consistent: + nonce = self.mapping.get(key) + if nonce is None: + raise ComputeError( + f"stem {key!r} is outside the vacancy map's domain — the map must be " + "built over the union of the corpus types and the budget's words", + {"stem": key}, + ) + else: + # The inconsistent-assignment control: the nonce is derived from + # (stem, occurrence index) in document order, so every occurrence is a fresh + # type. This condition has NO stability property; destroying the field while + # holding the vacancy rate fixed is its entire purpose. + # §5.8 pins the key: `f"{stem}#{idx}"` with `idx` the 0-based occurrence index + # of the STEM in document order. `#` is not a legal `WORD_RE` character, so the + # key can never collide with a real stem — and the key must NOT reach the + # prosody lookup, or the pattern becomes `stress("little#0")` rather than + # `stress("little")` and the nonce loses the stem's syllable count. + # + # **Condition B applies here too** (§5.8): the nonce may equal neither a domain + # type nor THE STEM IT REPLACES, and `{key}` is not redundant with `self.domain` + # — a stem need not be a type. Measured: at seed 7, `p = 1`, `tak` minted `tak`, + # so `Taking -> Taking` and one token silently failed to vacate + # (`corpus_types_vacated` 1921 against the consistent path's 1922). §7.1 denies + # this control a *stability* property, which is about a nonce being reused + # across occurrences; it does not license a word surviving the transform, and a + # control whose vacancy rate is not the stated rate is not a control. Adding the + # stem to `forbidden` puts it through §5.5's ordinary re-mint loop, so the + # replacement is held to the same quality bar as any other nonce. + # `state.used` starts as the map's STORED `forbidden` (domain + every nonce ever + # handed out, superseded ones included — §5.8), so this is exactly the set the + # TypeScript control draws against. `{key}` is not redundant with it: a stem need + # not be a type. + nonce, pattern, _salt = _mint( + f"{key}#{seen - 1}", + params.seed, + params.match_prosody, + state.used | {key}, + stem=key, + ) + state.used.add(nonce) + self.minted_stress.setdefault(nonce, pattern) + + out = match_case(word, surface_form(nonce, key, suffix, params.seed)) + if _WHOLE_WORD_RE.fullmatch(out) is None: + raise ComputeError( + f"vacating {word!r} produced {out!r}, which is not a single complete word " + "token — the token stream would no longer align with the original", + {"word": word, "output": out}, + ) + return out + + +def _image_of(word: str, mapping: Mapping[str, str], keep: frozenset[str], seed: int) -> str: + """A lowercased type at full vacancy, used only by the injectivity check.""" + stem, suffix = stem_and_suffix(word) + if not is_eligible(stem, keep): + return word + nonce = mapping.get(stem.lower()) + if nonce is None: + return word + return surface_form(nonce, stem, suffix, seed) + + +def vacancy_domain(types: Iterable[str]) -> list[str]: + """The map's domain: the corpus's types UNION the **full** Dolch list (contract §5.2). + + Always the full list, **never the active budget**. A budget word absent from the corpus + still needs an image, or the mapped vocabulary of §7.2 has a hole in it — but if the + domain tracked the *active* budget, switching budgets in the UI would rebuild the map and + re-mint the corpus underneath a panel whose whole claim is that nonces are stable. Using + the full list makes the map a function of `(corpus, seed, match_prosody)` alone. + + Since the domain is also the set a nonce may not equal (§5.2, no caller-supplied + `avoid`), a SMALLER domain genuinely mints differently: measured, building over + ``corpus ∪ dolch_budget(name)`` for any name below `full` moves exactly one stem, because + `floor` is a full-list Dolch word that never appears in the corpus and so is forbidden in + the full domain and free in the smaller ones. Unioning the full list here is what makes + that unreachable, so there is only ever one map + (`test_the_map_does_not_move_when_the_active_budget_changes`). + + Takes an iterable of TYPES, not a text: see :func:`_reject_bare_str`. + """ + _reject_bare_str(types, "vacancy_domain", "types") + return sorted({t.lower() for t in types} | {w.lower() for w in dolch_budget("full")}) + + +def build_vacancy_map( + types: Iterable[str], + params: VacancyParams, + counts: Mapping[str, int] | None = None, +) -> VacancyMap: + """Assign every eligible stem a nonce, once, in canonical order (contract §5.2). + + `types` is the domain — pass :func:`vacancy_domain(corpus_types)`, which pins it to the + corpus's types plus the full Dolch list. + + **The domain is avoided implicitly; there is no caller-supplied `avoid` parameter.** With + one, the map depends on what the caller remembered to pass: measured, at seed 0 the same + corpus and seed give ``remint_rounds`` 0 with the domain passed and 1 without, and + different nonces either way. Both maps are valid — that is the problem, because one call + site passing it and another not (the panel and the golden fixture, say) is a silent + divergence with no failing test. Condition B below already forbids a surface form equal to + any domain type, so avoiding the domain at mint time is not extra policy, only the cheaper + route to the same fixed point. The map is now a pure function of + ``(domain, seed, match_prosody)``. + + `p` is deliberately unused: the map is built over ALL eligible stems and restricted to + ``{u < p}`` at rewrite time. That is what makes nesting and stability structural. + + **Injectivity is verified over assembled SURFACE FORMS, and the condition is + `p`-independent** (§5.2). Both must hold over the domain: + + * **A.** the surface forms are pairwise distinct + * **B.** no surface form equals any lowercased domain type, eligible or not + + A bare-nonce check is not enough — the collision arrives through the suffix — and a check + performed at `p = 1` only is not enough either, because at full vacancy every eligible + type has moved and nothing is left for a minted form to collide with. B is deliberately + conservative: it forbids a minted form from equalling a word that would always have been + vacated alongside it, and that costs a re-mint but buys a condition independent of `p`, + which is what the theorem needs. Measured cost on the shipped corpus: one re-mint at + seed 7, where `hang` first minted `wak` and `hanged` surfaced as the real word `waked`. + + On violation only the LOSING stem is re-minted — the one later in ASCII-ascending order + among those involved — at salt ``1000 * round + previousSalt + 1``, so a re-mint never + cascades (§5.8). + + **Under ``mint="swap"`` the replacement is a real English word and B cannot apply**, since + the replacement is a domain type by construction. §5.2a works out what B was standing in + for and what swap can therefore satisfy: A unchanged, and **B₁** — no surface form equals + an INELIGIBLE domain type, and none equals its own source type. That makes the map a + bijection of the domain at full vacancy, which is where the pretrained arm measures, and + §5.2a proves no `p`-stable swap can do better. :attr:`VacancyMap.injective_at_every_p` + records which regime the map is in and :func:`map_vocab_words` refuses the rest. + + `counts` is the corpus's per-type occurrence count (:func:`type_counts`), REQUIRED by + ``mint="swap"`` and unused by ``mint="nonce"`` — the nonce map stays a pure function of + ``(domain, seed, match_prosody)``, asserted in the tests. Swap raises without it rather + than falling back to an alphabetical rank, which would be a frequency match in name only. + """ + _reject_bare_str(types, "build_vacancy_map", "types") + keep = params.keep_set + type_set = {t.lower() for t in types} + swapping = params.mint == "swap" + if swapping and counts is None: + raise InvalidParamError( + "mint='swap' needs the corpus's type counts to rank the replacement pool by " + "frequency (contract §8.3); pass counts=type_counts(tokenize(text))", + {"mint": params.mint}, + ) + + pairs: list[tuple[str, str]] = [] + stem_set: set[str] = set() + families: dict[str, set[str]] = {} + suffixes: dict[str, list[str]] = {} + eligible_types: set[str] = set() + for t in sorted(type_set): # ASCII order, so `suffixes[stem]` is canonical + stem, suffix = stem_and_suffix(t) + if not is_eligible(stem, keep): + continue + pairs.append((stem.lower(), suffix.lower())) + stem_set.add(stem.lower()) + families.setdefault(stem.lower(), set()).add(t) + suffixes.setdefault(stem.lower(), []).append(suffix.lower()) + eligible_types.add(t) + + # Condition B's scope: EVERY domain type under `nonce`, and only the types that can never + # be vacated under `swap` — §5.2a's B₁, which is what full-vacancy injectivity needs and + # all a map drawing from the domain can possibly satisfy. + barred = type_set - eligible_types if swapping else type_set + + mapping: dict[str, str] = {} + minted_stress: dict[str, str] = {} + salts: dict[str, int] = {} + # `forbidden = used ∪ domain` — the domain, always, with nothing left to the caller. It + # accumulates and is never pruned, so a superseded nonce stays out of circulation (§5.8), + # and it is STORED on the map rather than reconstructed from `mapping.values()`. + forbidden: set[str] = set(type_set) + pool: list[str] = [] + rank: dict[str, int] = {} + if swapping: + assert counts is not None # narrowed by the guard above + pool = swap_pool(type_set, counts, keep) + rank = {s: swap_rank(s, families[s], pool, counts) for s in sorted(stem_set)} + used: set[str] = set() + claimed_forms: set[str] = set() + for stem in sorted(stem_set): + word, salt, forms = _draw_swap( + stem, + params.seed, + params.match_prosody, + pool, + rank, + used, + suffixes[stem], + claimed_forms, + barred, + ) + used.add(word) + claimed_forms.update(forms) + forbidden.add(word) + mapping[stem] = word + salts[stem] = salt + else: + for stem in sorted(stem_set): + nonce, pattern, salt = _mint(stem, params.seed, params.match_prosody, forbidden) + forbidden.add(nonce) + mapping[stem] = nonce + minted_stress[nonce] = pattern + salts[stem] = salt + + rounds = 0 + while True: + claimed: dict[str, str] = {} # surface -> the stem that owns it + losers: set[str] = set() + for stem, suffix in pairs: + form = surface_form(mapping[stem], stem, suffix, params.seed) + if form in barred or form == stem + suffix: # condition B (B₁ under swap) + losers.add(stem) + if form in claimed: # condition A + losers.add(max(stem, claimed[form])) + else: + claimed[form] = stem + if not losers: + break + if rounds >= MAX_REMINT_ROUNDS: + raise ComputeError( + f"vacancy map still collides after {MAX_REMINT_ROUNDS} re-mint rounds", + {"stems": sorted(losers)[:20], "rounds": rounds, "mint": params.mint}, + ) + rounds += 1 + for stem in sorted(losers): + others = {n for s, n in mapping.items() if s != stem} + start_salt = REMINT_SALT_STRIDE * rounds + salts[stem] + 1 + if swapping: + # A superseded REPLACEMENT returns to the pool — unlike a superseded nonce, + # which stays forbidden forever. The pool is finite (1944 real words against + # 1680 stems on the shipped corpus), so retiring words permanently would + # starve later rounds; and a real word cannot "recreate the collision it was + # rejected for" the way a nonce can, because the collision was with a + # different stem's surface, which has itself moved. + elsewhere = { + surface_form(mapping[s], s, suffix, params.seed) + for s, suffix in pairs + if s != stem + } + word, salt, _forms = _draw_swap( + stem, + params.seed, + params.match_prosody, + pool, + rank, + others, + suffixes[stem], + elsewhere, + barred, + start_salt=start_salt, + ) + forbidden.add(word) + mapping[stem] = word + salts[stem] = salt + continue + nonce, pattern, salt = _mint( + stem, + params.seed, + params.match_prosody, + forbidden, + start_salt=start_salt, + ) + minted_stress.pop(mapping[stem], None) + forbidden.add(nonce) + mapping[stem] = nonce + minted_stress[nonce] = pattern + salts[stem] = salt + + seen = {_image_of(t, mapping, keep, params.seed) for t in type_set} + return VacancyMap( + mapping=mapping, + minted_stress=minted_stress, + seed=params.seed, + match_prosody=params.match_prosody, + domain=frozenset(type_set), + image_size=len(seen), + bijective=len(seen) == len(type_set), + remint_rounds=rounds, + forbidden=frozenset(forbidden), + injective_at_every_p=not swapping, + ) + + +# --- rewriting -------------------------------------------------------------------------- + + +def vacate_text(text: str, vmap: VacancyMap, params: VacancyParams) -> str: + """Rewrite `text` in place, vacating every eligible stem with ``u(stem) < p``. + + Words are found with **exactly the tokenizer's regex** and everything else — whitespace, + punctuation, digits, line breaks — passes through unchanged, byte for byte. Every output + is itself a single complete `WORD_RE` match (checked in :meth:`VacancyMap._transform`), + so ``tokenize(vacate(text))`` has the same length and ordering as ``tokenize(text)``, and + because line breaks are untouched the ````-per-line rule produces the same number of + ```` in the same places. + """ + state = _RewriteState(set(vmap.forbidden)) + return WORD_RE.sub(lambda m: vmap._transform(m.group(0), params, state), text) + + +def map_vocab_words(words: Sequence[str], vmap: VacancyMap, params: VacancyParams) -> list[str]: + """Push a budget's word list through the same transform, PRESERVING ORDER (§7.2). + + Since the map is injective, ``itos_p = SPECIALS ++ map_vocab_words(words, ...)`` assigns + every word the id its pre-image had, which is why the token id stream is unchanged and + training is bit-identical. + + This rule is only valid in the condition it is stated for. Under ``consistent=False`` or + ``reveal_after > 0`` a source type no longer has a single image, so the budget must be + REBUILT from the vacated corpus instead — the collapse in coverage is the measurement. + Calling this there would quietly manufacture a vocabulary that matches no corpus. + """ + _reject_bare_str(words, "map_vocab_words", "words") + if not params.consistent or params.reveal_after: + raise InvalidParamError( + "the mapped vocabulary is only defined for consistent=True, reveal_after=0; " + "every other condition rebuilds the budget from the vacated corpus", + {"consistent": params.consistent, "reveal_after": params.reveal_after}, + ) + if not vmap.injective_at_every_p and 0.0 < params.p < 1.0: + # §5.2a: swap's images ARE domain types, so at intermediate `p` a vacated type can + # land on an un-vacated one and two budget words would share a row. That is not a + # defect to be re-drawn away — the theorem there shows no `p`-stable swap avoids it — + # so the mapped vocabulary is refused, exactly as it is for the two controls above. + raise InvalidParamError( + f"mint='swap' has no mapped vocabulary at p={params.p}: its replacements are " + "domain types, so a vacated type can collide with an un-vacated one and the map " + "is injective only at full vacancy (contract §5.2a). Use p=0 or p=1, or rebuild " + "the budget from the vacated corpus", + {"mint": params.mint, "p": params.p}, + ) + state = _RewriteState(set(vmap.forbidden)) + return [vmap._transform(w, params, state) for w in words] + + +# --- statistics ------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _Prosody: + """One side of the prosody statistics, token-weighted. + + ``from_table + from_minted + from_rule == 1`` by construction: :func:`stress_source` + partitions the tokens. + """ + + mean_syllables: float + mean_anapest: float + from_table: float + from_minted: float + from_rule: float + + +def _prosody(text: str, minted_stress: Mapping[str, str] | None) -> _Prosody: + words = WORD_RE.findall(text) + if not words: + return _Prosody(0.0, 0.0, 0.0, 0.0, 0.0) + total_syllables = sum(syllables(w, minted_stress) for w in words) + sources = [stress_source(w, minted_stress) for w in words] + lines = [ln for ln in text.splitlines() if WORD_RE.search(ln)] + anapest = ( + sum(meter_score(ln, "anapest", minted_stress) for ln in lines) / len(lines) + if lines + else 0.0 + ) + return _Prosody( + mean_syllables=total_syllables / len(words), + mean_anapest=anapest, + from_table=sources.count("table") / len(words), + from_minted=sources.count("minted") / len(words), + from_rule=sources.count("rule") / len(words), + ) + + +def vacancy_stats( + original: str, vacated: str, vmap: VacancyMap, params: VacancyParams +) -> dict[str, float | int | bool]: + """The statistics contract (§10), with exactly these field names. + + **Every count names its scope; an unprefixed ``types*`` is forbidden.** "Types" is + ambiguous between the corpus (2 211 types of *Mother Goose*) and the domain of §5.2 + (2 233 = corpus plus the full Dolch list), and the ambiguity cost two round trips between + the two stacks — which agreed on ``tokensVacated`` to the token and disagreed only on + what they were counting. + + * ``domainTypes*`` — over :attr:`VacancyMap.domain`. This governs the map and the + vocabulary, so it is the diagnostic number, and it is a property of ``(map, p)``: the + 22 domain-only Dolch words (`funny`, `squirrel`, `today`, …) have images but never + appear in the text, so they cannot be measured from it. + * ``corpusTypes*`` — over the corpus's own type set. **This is what the panel shows a + reader**: counting words the reader cannot see inflates the vacancy rate being shown. + ``corpusTypesVacated`` and ``tokensVacated`` are MEASURED from the two texts, so they + respect ``reveal_after`` and the inconsistent-assignment control — the source reports + ``len(self.map)``, which is the size of the assignment and not what was vacated. + * ``stemsTotal`` is ``|map|``; ``stemsVacated`` counts stems with ``u(stem) < p``. + + At ``p = 1`` every eligible stem vacates, because ``u ∈ [0, 1)`` by construction, so + ``stemsVacated == stemsTotal`` and ``*TypesVacated == *TypesEligible``. The first of + those identities is what exposed the scope confusion. + + The prosody numbers ship with a THREE-WAY split of where each token's stress came from, + token-weighted and summing to 1 on each side (§10). A single "table coverage" number was + ambiguous the moment minted forms existed — read literally it counts only the hand table, + read as "stress we actually know" it also counts forms we minted, and the two readings + differ by a factor of thirty. So: + + * ``stressFromTable*`` — the 61-entry hand table of §6.1, the honesty number for English + words; + * ``stressFromMinted*`` — forms we minted and registered a pattern for. Known by + construction but ASSERTED rather than verified: §5.5 accepts a candidate on syllable + COUNT, so the count is checked and the pattern is not; + * ``stressFromRule*`` — the spelling heuristic of §6.2, i.e. a guess. + + The source's own numbers (mean anapest 0.351 -> 0.345, mean syllables 1.224 -> 1.211) are + its numbers on a corpus we do not have. They are not transcribed anywhere. + """ + before_words = WORD_RE.findall(original) + after_words = WORD_RE.findall(vacated) + if len(before_words) != len(after_words): + raise ComputeError( + f"vacating changed the token count ({len(before_words)} -> {len(after_words)}); " + "the token streams no longer align", + {"tokens_before": len(before_words), "tokens_after": len(after_words)}, + ) + + keep = params.keep_set + types = {w.lower() for w in before_words} + eligible = {t for t in types if is_eligible(stem_and_suffix(t)[0], keep)} + changed = [b.lower() for b, a in zip(before_words, after_words) if b.lower() != a.lower()] + + domain_eligible = {t for t in vmap.domain if is_eligible(stem_and_suffix(t)[0], keep)} + domain_vacated = { + t for t in domain_eligible if vacancy_u(stem_and_suffix(t)[0], params.seed) < params.p + } + stems_vacated = sum(1 for s in vmap.mapping if vacancy_u(s, params.seed) < params.p) + + # The original text is English, so it is scored WITHOUT the minted patterns; `avoid` + # guarantees no English type is also a nonce, so `stressFromMintedBefore` is 0 by + # construction rather than by omission. + before = _prosody(original, None) + after = _prosody(vacated, vmap.minted_stress) + + return { + "domainTypesTotal": len(vmap.domain), + "domainTypesEligible": len(domain_eligible), + "domainTypesVacated": len(domain_vacated), + "corpusTypesTotal": len(types), + "corpusTypesEligible": len(eligible), + "corpusTypesVacated": len(set(changed)), + "stemsTotal": len(vmap.mapping), + "stemsVacated": stems_vacated, + "tokensTotal": len(before_words), + "tokensVacated": len(changed), + "meanSyllablesBefore": before.mean_syllables, + "meanSyllablesAfter": after.mean_syllables, + "meanAnapestBefore": before.mean_anapest, + "meanAnapestAfter": after.mean_anapest, + "stressFromTableBefore": before.from_table, + "stressFromTableAfter": after.from_table, + "stressFromMintedBefore": before.from_minted, + "stressFromMintedAfter": after.from_minted, + "stressFromRuleBefore": before.from_rule, + "stressFromRuleAfter": after.from_rule, + "bijective": vmap.bijective, + "imageSize": vmap.image_size, + "remintRounds": vmap.remint_rounds, + } diff --git a/code/backend/tests/contract/test_api_arch.py b/code/backend/tests/contract/test_api_arch.py index 1d4432d..df82130 100644 --- a/code/backend/tests/contract/test_api_arch.py +++ b/code/backend/tests/contract/test_api_arch.py @@ -308,3 +308,70 @@ def test_generate_oversized_model_is_422_envelope(): ) assert resp.status_code == 422 assert resp.json()["error"]["type"] == "ModelTooLargeError" + + +# --- POST /api/arch/vacancy-score (feature 007, contract §8) ------------------------- + + +VACANCY_PASSAGE = ( + "Little Jack Horner sat in a corner,\n" + "Eating a Christmas pie;\n" + "He put in his thumb, and pulled out a plum,\n" + "And said, What a good boy am I!\n" + "Hey diddle diddle, the cat and the fiddle,\n" + "The cow jumped over the moon;\n" + "The little dog laughed to see such sport,\n" + "And the dish ran away with the spoon.\n" +) + + +def _vacancy(payload): + return client.post("/api/arch/vacancy-score", json={"model_id": MODEL, **payload}) + + +def test_vacancy_score_shape_and_the_labelled_decomposition(): + resp = _vacancy({"passage": VACANCY_PASSAGE, "p": 1.0, "seed": 0}) + assert resp.status_code == 200 + body = resp.json() + + assert [v["id"] for v in body["variants"]] == ["english", "swap", "nonce"] + for v in body["variants"]: + stats = v["pooled"] + assert stats["nllPreserved"] == sig6(stats["nllPreserved"]) + assert stats["nPreservedTokens"] > 0 + # Every variant scores the SAME scaffolding — the property the deltas rest on. + assert len({v["pooled"]["nPreservedTokens"] for v in body["variants"]}) == 1 + + by_id = {d["id"]: d for d in body["differences"]} + assert by_id["wrong_content"]["expr"] == "nll(swap) − nll(english)" + assert by_id["unknown_form"]["expr"] == "nll(nonce) − nll(swap)" + # The conflated difference is present but explicitly NOT a headline (§8.3). + assert by_id["total"]["expr"] == "nll(nonce) − nll(english)" + assert by_id["total"]["headline"] is False + assert by_id["unknown_form"]["upperBound"] is True + + # The full stack reports everything: per-passage rows, and the tiny arm's exact 0. + assert len(body["passages"]) == 1 + assert body["tiny_arm"]["delta_nats"] == 0.0 + assert body["dtype"] == "float32" + assert body["alignment"]["unit"] == "utf8_bytes" + + +def test_vacancy_score_rejects_both_passage_and_passages(): + resp = _vacancy({"passage": VACANCY_PASSAGE, "passages": [VACANCY_PASSAGE]}) + assert resp.status_code == 400 + assert resp.json()["error"]["type"] == "InvalidParamError" + + +def test_vacancy_score_bad_p_is_400_envelope(): + resp = _vacancy({"passage": VACANCY_PASSAGE, "p": 2.0}) + assert resp.status_code == 400 + assert resp.json()["error"]["type"] == "InvalidParamError" + + +def test_vacancy_score_oversized_model_is_422_envelope(): + resp = client.post( + "/api/arch/vacancy-score", json={"model_id": TOO_BIG, "passage": VACANCY_PASSAGE} + ) + assert resp.status_code == 422 + assert resp.json()["error"]["type"] == "ModelTooLargeError" diff --git a/code/backend/tests/contract/test_api_lex.py b/code/backend/tests/contract/test_api_lex.py index c4b23fc..940ed24 100644 --- a/code/backend/tests/contract/test_api_lex.py +++ b/code/backend/tests/contract/test_api_lex.py @@ -12,9 +12,11 @@ from __future__ import annotations +import hashlib import json import math import time +from pathlib import Path from typing import Any import numpy as np @@ -36,6 +38,11 @@ client = TestClient(app) +#: `/code/backend/tests/contract/test_api_lex.py` -> ``. The vacancy parity +#: fixture lives beside the frontend tests that consume it, and this file asserts against +#: the same copy so the two stacks cannot be pinned to two different documents. +REPO_ROOT = Path(__file__).resolve().parents[4] + #: A real but deliberately cheap model: a full run of this is a fraction of a second. TINY_TRAIN: dict[str, Any] = { "source": "dolch", @@ -673,6 +680,264 @@ def test_model_export_rejects_an_unknown_token() -> None: ) +# -- POST /api/lex/vacancy (feature 007) ----------------------------------------------- + + +def _vacancy(**body: Any) -> dict[str, Any]: + resp = client.post("/api/lex/vacancy", json=body) + assert resp.status_code == 200, resp.text + payload = resp.json() + _assert_rounded6(payload) + return payload + + +def test_vacancy_reports_every_statistic_the_contract_names() -> None: + """§10's field list, verbatim and complete — an unprefixed `types*` is forbidden.""" + payload = _vacancy(p=1.0, seed=0) + stats = payload["vacancy_stats"] + assert set(stats) == { + "domainTypesTotal", + "domainTypesEligible", + "domainTypesVacated", + "corpusTypesTotal", + "corpusTypesEligible", + "corpusTypesVacated", + "stemsTotal", + "stemsVacated", + "tokensTotal", + "tokensVacated", + "meanSyllablesBefore", + "meanSyllablesAfter", + "meanAnapestBefore", + "meanAnapestAfter", + "stressFromTableBefore", + "stressFromTableAfter", + "stressFromMintedBefore", + "stressFromMintedAfter", + "stressFromRuleBefore", + "stressFromRuleAfter", + "bijective", + "imageSize", + "remintRounds", + } + # The measured numbers of §10 on the shipped corpus, at full vacancy. + assert stats["domainTypesTotal"] == 2233 + assert stats["domainTypesEligible"] == 1944 + assert stats["corpusTypesTotal"] == 2211 + assert stats["corpusTypesEligible"] == 1922 + assert stats["corpusTypesVacated"] == 1922 + assert stats["tokensVacated"] == 8202 + assert stats["stemsTotal"] == 1680 + # Identities, not observations: `u ∈ [0, 1)`, so p = 1 vacates everything eligible. + assert stats["stemsVacated"] == stats["stemsTotal"] + assert stats["domainTypesVacated"] == stats["domainTypesEligible"] + assert stats["corpusTypesVacated"] == stats["corpusTypesEligible"] + # The three-way stress split is token-weighted and sums to 1 on each side. + for side in ("Before", "After"): + total = sum(stats[f"stressFrom{k}{side}"] for k in ("Table", "Minted", "Rule")) + assert abs(total - 1.0) < 1e-6, (side, total) + # Injectivity is REPORTED, not assumed, and is surfaced beside the statistics block. + assert payload["bijective"] is True + assert stats["bijective"] is True + assert payload["remint_rounds"] == stats["remintRounds"] == 0 + + +def test_vacancy_reports_the_measured_remint_at_seed_7() -> None: + """Seed 7 needs exactly one re-mint: `hang` first minted `wak`, and `hanged` would + have surfaced as the real English word `waked` (§5.2 condition B).""" + payload = _vacancy(p=1.0, seed=7) + assert payload["remint_rounds"] == 1 + assert payload["bijective"] is True + # The transform is unharmed by it: the same 1 922 types and 8 202 tokens move. + assert payload["vacancy_stats"]["corpusTypesVacated"] == 1922 + assert payload["vacancy_stats"]["tokensVacated"] == 8202 + + +def test_vacancy_returns_an_excerpt_and_a_digest_rather_than_the_corpus() -> None: + payload = _vacancy(p=1.0, seed=0) + assert payload["preview_chars"] == 2000 + assert len(payload["preview"]) == 2000 + assert payload["truncated"] is True + # ~86 kB of corpus behind a 2 kB excerpt, pinned by 64 hex characters. + assert payload["vacated_chars"] > 20 * payload["preview_chars"] + assert len(payload["vacated_sha256"]) == 64 + assert payload["vacated_sha256"] != payload["original_sha256"] + assert ( + payload["original_sha256"] == hashlib.sha256(load_corpus_text().encode("utf-8")).hexdigest() + ) + + short = _vacancy(p=1.0, seed=0, preview_chars=10) + assert short["preview"] == payload["preview"][:10] + assert short["vacated_sha256"] == payload["vacated_sha256"] + + +def test_vacancy_is_the_identity_at_p_zero() -> None: + payload = _vacancy(p=0.0, seed=0) + assert payload["vacated_sha256"] == payload["original_sha256"] + assert payload["preview"] == payload["original_preview"] + for field in ("corpusTypesVacated", "domainTypesVacated", "tokensVacated", "stemsVacated"): + assert payload["vacancy_stats"][field] == 0, field + + +def test_vacancy_maps_the_vocabulary_and_leaves_coverage_untouched() -> None: + """SC-703 in the units the panel shows. + + Under `consistent=true, reveal_after=0` the transform is a pure relabelling, so the + budget's measured coverage of the VACATED corpus is bit-identical to its coverage of + the English one — the same tokens in budget, `` in exactly the same places. + """ + english = client.post("/api/lex/coverage", json={"source": "dolch", "budget": "primer"}).json() + for p in (0.0, 0.25, 0.5, 0.75, 1.0): + for seed in (0, 7): + payload = _vacancy(p=p, seed=seed, source="dolch", budget="primer") + assert payload["vocabulary_rule"] == "mapped" + assert payload["budget"]["coverage"] == english["coverage"], (p, seed) + assert payload["budget"]["rows"] == english["rows"] + assert len(payload["words"]) == len(english["words"]) + assert payload["corpus"]["n_tokens"] == english["corpus"]["n_tokens"] + assert payload["corpus"]["n_lines"] == english["corpus"]["n_lines"] + if p > 0: + assert payload["words"] != english["words"] + + +def test_vacancy_controls_rebuild_the_budget_and_collapse_coverage() -> None: + """SC-705: the conditions that break type identity break the invariance, measurably.""" + mapped = _vacancy(p=0.5, seed=0, source="dolch", budget="primer") + inconsistent = _vacancy(p=0.5, seed=0, consistent=False, source="dolch", budget="primer") + revealed = _vacancy(p=0.5, seed=0, reveal_after=2, source="dolch", budget="primer") + assert mapped["vocabulary_rule"] == "mapped" + assert inconsistent["vocabulary_rule"] == "rebuilt" + assert revealed["vocabulary_rule"] == "rebuilt" + for control in (inconsistent, revealed): + assert control["budget"]["coverage"]["unk_rate"] > mapped["budget"]["coverage"]["unk_rate"] + # §10: `corpusTypesVacated` is measured from the two TEXTS, so a type whose every + # occurrence falls inside the reveal window does not count — the reading that matches + # what the number claims to a reader. + assert ( + revealed["vacancy_stats"]["corpusTypesVacated"] + < mapped["vacancy_stats"]["corpusTypesVacated"] + ) + + +def test_vacancy_nests_and_stays_stable_as_p_rises() -> None: + """SC-701 / SC-702 through the API: a word rewritten at a low `p` is byte-identical at + every higher one, and the vacated set only grows.""" + seen: list[tuple[float, int, list[str]]] = [] + for p in (0.0, 0.25, 0.5, 0.75, 1.0): + payload = _vacancy(p=p, seed=0, source="dolch", budget="primer") + seen.append((p, payload["vacancy_stats"]["stemsVacated"], payload["words"])) + for (_, lower_n, lower_words), (_, higher_n, higher_words) in zip(seen, seen[1:]): + assert higher_n >= lower_n + for english, low, high in zip(seen[0][2], lower_words, higher_words): + if low != english: + assert high == low, (english, low, high) + + +def test_vacancy_transforms_a_users_own_text() -> None: + text = "The little brown squirrel ate the pretty acorn.\nThe squirrel ran away.\n" + payload = _vacancy(text=text, p=1.0, seed=0, preview_chars=200) + assert payload["original_preview"] == text + assert payload["preview"] != text + # §1: only WORD_RE matches move; punctuation and line breaks pass through byte for byte. + assert payload["preview"].count("\n") == text.count("\n") + assert payload["preview"].count(".") == 2 + assert payload["vacancy_stats"]["tokensTotal"] == 12 + assert payload["vacancy_stats"]["tokensVacated"] == 9 # the three `the`s are preserved + + +def test_vacancy_rejects_parameters_outside_the_contract() -> None: + for body in ( + {"p": 1.5}, + {"p": -0.1}, + {"p": "half"}, + {"seed": "zero"}, + {"reveal_after": -1}, + {"preview_chars": 20001}, + {"preview_chars": -1}, + {"keep": "little"}, # a bare string would protect six single letters + {"keep": [3]}, + {"source": "dolch", "size": 50}, + {"budget": "not-a-budget"}, + {"text": "!!! ???"}, + ): + _assert_error_envelope(client.post("/api/lex/vacancy", json=body), 400, "InvalidParamError") + + +def test_vacancy_matches_the_static_client_fixture() -> None: + """**The parity assertion this feature rests on.** + + `code/frontend/tests/fixtures/vacancy-api-golden.json` is a transcript of this route, + written by `scripts/export_vacancy_api_golden.py`. `staticVacancy.test.ts` asserts the + browser's in-page implementation reproduces it field for field; this asserts the live + route still does. One document, two stacks, and no way for either to drift alone. + """ + fixture = json.loads( + ( + REPO_ROOT / "code" / "frontend" / "tests" / "fixtures" / "vacancy-api-golden.json" + ).read_text(encoding="utf-8") + ) + assert fixture["format"] == "vacancy-api-golden-v1" + assert fixture["endpoint"] == "/api/lex/vacancy" + assert fixture["defaults"]["preview_chars"] == 2000 + assert fixture["defaults"]["preview_max"] == 20000 + assert len(fixture["cases"]) >= 6 + for case in fixture["cases"]: + resp = client.post("/api/lex/vacancy", json=case["request"]) + assert resp.status_code == 200, (case["label"], resp.text) + assert resp.json() == case["response"], ( + f"{case['label']}: the route no longer returns what the parity fixture " + "records. Regenerate it with `python scripts/export_vacancy_api_golden.py` " + "ONLY after confirming the change was intended — the browser asserts against " + "the same file." + ) + + +# -- POST /api/lex/train with vacancy (feature 007) ------------------------------------- + + +def test_train_on_a_vacated_corpus_is_bit_identical_under_the_mapped_vocabulary() -> None: + """SC-703's corollary, run for real: `p` is invisible to a word-level model. + + The mapped vocabulary gives every word the id its pre-image had, so the token id + stream is unchanged and the losses are IDENTICAL — not close. That is the tiny arm's + headline result, and it is asserted here rather than described. + """ + english = _train(seed=11) + vacated = _train(seed=11, vacancy={"p": 0.5, "seed": 0}) + for field in ("first_loss", "final_loss", "val_loss", "n_tokens", "vocab_rows"): + assert vacated[field] == english[field], field + # Same numbers, different words: the relabelling is real, the model is blind to it. + assert vacated["model_token"] != english["model_token"] + assert vacated["sample"] != english["sample"] + + +def test_train_without_vacancy_is_untouched_by_the_new_parameter() -> None: + """Additive means additive: the same request keeps hitting the same cache entry.""" + first = _train(seed=12) + again = _train(seed=12) + assert again["model_token"] == first["model_token"] + # `vacancy: null` is "no vacancy", not "vacancy with defaults". + explicit = _train(seed=12, vacancy=None) + assert explicit["model_token"] == first["model_token"] + # …and `p = 0` really is the identity, so it lands on that entry too. + identity = _train(seed=12, vacancy={"p": 0.0, "seed": 0}) + assert identity["model_token"] == first["model_token"] + + +def test_train_rejects_a_vacancy_block_that_is_not_an_object() -> None: + for bad in (0.5, "p=0.5", [0.5]): + _assert_error_envelope( + client.post("/api/lex/train", json={**TINY_TRAIN, "vacancy": bad}), + 400, + "InvalidParamError", + ) + _assert_error_envelope( + client.post("/api/lex/train", json={**TINY_TRAIN, "vacancy": {"p": 2}}), + 400, + "InvalidParamError", + ) + + # -- the frozen 002 contract is untouched ---------------------------------------------- diff --git a/code/backend/tests/integration/test_arch_vacancy_score.py b/code/backend/tests/integration/test_arch_vacancy_score.py new file mode 100644 index 0000000..76e8116 --- /dev/null +++ b/code/backend/tests/integration/test_arch_vacancy_score.py @@ -0,0 +1,157 @@ +"""The pretrained arm, end to end on a REAL model (contract §8, FR-717/719/719a). + +Runtime cost: `gpt2` (124M, float32, CPU) over ONE ~250-word passage is three forward +passes of ~350 tokens — about 1 s once the weights are cached, ~10 s on a cold cache +including the download. The measurement of §8.3a pools six passages; one is enough to +assert every structural property, and the six-passage run is what the panel does. +""" + +from __future__ import annotations + +import math + +import pytest + +from llm_geometry.arch.vacancy_score import ( + VARIANTS, + default_passages, + vacancy_score, +) +from llm_geometry.errors import InvalidParamError + +MODEL = "gpt2" # the smallest curated model, and one of the two §8.3a measured + + +@pytest.fixture(scope="module") +def scored() -> dict: + return vacancy_score(MODEL, default_passages(count=1), p=1.0, seed=0) + + +def test_reports_every_field_of_section_8_1(scored: dict) -> None: + assert [v["id"] for v in scored["variants"]] == list(VARIANTS) + for variant in scored["variants"]: + stats = variant["pooled"] + assert set(stats) == { + "nllPreserved", + "nllAll", + "bitsPerChar", + "nTokens", + "nPreservedTokens", + "nChars", + } + assert stats["nTokens"] > 0 + assert stats["nPreservedTokens"] > 0 + assert stats["nllPreserved"] > 0 + # bitsPerChar is the passage's total surprisal per character, so it must agree + # with nllAll·nTokens/(ln2·nChars) — the definition, not an independent number. + assert stats["bitsPerChar"] == pytest.approx( + stats["nllAll"] * stats["nTokens"] / (math.log(2) * stats["nChars"]), rel=1e-4 + ) + assert scored["dtype"] == "float32" + assert scored["stack"] == "backend" + assert scored["alignment"]["verified"] is True + assert scored["alignment"]["unit"] == "utf8_bytes" + + +def test_the_same_scaffolding_is_scored_in_every_variant(scored: dict) -> None: + """The preserved token sets correspond one-for-one, which is what makes the + differences paired and the comparison meaningful at all.""" + counts = {v["id"]: v["pooled"]["nPreservedTokens"] for v in scored["variants"]} + assert len(set(counts.values())) == 1, counts + # …while the variants as a whole tokenize differently: nonce forms fragment, which + # is the residual the "unknown form" difference is an upper bound because of. + tokens = {v["id"]: v["pooled"]["nTokens"] for v in scored["variants"]} + assert tokens["nonce"] > tokens["swap"], tokens + + +def test_the_decomposition_is_labelled_and_adds_up(scored: dict) -> None: + by_id = {d["id"]: d for d in scored["differences"]} + assert set(by_id) == {"wrong_content", "unknown_form", "total"} + + wrong = by_id["wrong_content"] + form = by_id["unknown_form"] + total = by_id["total"] + + assert wrong["expr"] == "nll(swap) − nll(english)" + assert form["expr"] == "nll(nonce) − nll(swap)" + assert total["expr"] == "nll(nonce) − nll(english)" + # FR-719a: the conflated difference is never a headline. + assert wrong["headline"] and form["headline"] + assert total["headline"] is False + assert form["upperBound"] is True and "UPPER BOUND" in form["note"] + + # Paired means over the same tokens are exactly additive — a real invariant, not a + # tolerance: if it ever fails, the three differences were not computed over one + # aligned token set and none of them means what its label says. + assert total["nats"] == pytest.approx(wrong["nats"] + form["nats"], abs=1e-9) + assert wrong["nPairs"] == form["nPairs"] == total["nPairs"] + for d in (wrong, form, total): + assert d["se"] > 0 + + +def test_both_costs_are_real_and_wrong_content_dominates(scored: dict) -> None: + """SC-707/707b: the measured result, asserted as a sign and an ordering. + + Not asserted as a fixed value — it is a real model on real text and would be a + brittle golden — but the ORDER is the finding: most of the damage is saying the + wrong thing, and only a minority of it is the form being unknown. + """ + by_id = {d["id"]: d for d in scored["differences"]} + wrong = by_id["wrong_content"]["nats"] + form = by_id["unknown_form"]["nats"] + total = by_id["total"]["nats"] + assert wrong > 0, "swapping in real but wrong words must cost something" + assert form > 0, "nonce forms must cost more than known wrong words" + assert wrong > form, (wrong, form) + assert 0.4 < total < 2.0, total + + +def test_the_tiny_arms_exact_zero_travels_with_the_number(scored: dict) -> None: + """FR-719: the pretrained delta is only interpretable beside the exact 0.""" + assert scored["tiny_arm"]["delta_nats"] == 0.0 + assert scored["tiny_arm"]["exact"] is True + assert "exactly 0" in scored["tiny_arm"]["note"] + assert "higher entropy" in scored["confound"] + + +def test_per_passage_rows_are_reported_by_the_full_stack(scored: dict) -> None: + rows = scored["passages"] + assert len(rows) == 1 + assert set(rows[0]["variants"]) == set(VARIANTS) + assert rows[0]["nPreservedWords"] > 0 + # The English text is returned as scored, so the panel shows the reader the passage + # the number came from rather than describing it. + used = scored["passages_used"] + assert len(used) == 1 and used[0].strip() + assert used[0].startswith(scored["variants"][0]["preview"][:40]) + + +def test_pooling_is_token_weighted_across_passages() -> None: + """Two passages pooled must equal the token-weighted combination of their rows.""" + result = vacancy_score(MODEL, default_passages(count=2), p=1.0, seed=0) + for variant in result["variants"]: + rows = [p["variants"][variant["id"]] for p in result["passages"]] + expected = sum(r["nllAll"] * r["nTokens"] for r in rows) / sum(r["nTokens"] for r in rows) + assert variant["pooled"]["nllAll"] == pytest.approx(expected, rel=1e-6) + + +def test_p_zero_is_the_identity_and_costs_nothing() -> None: + """u ∈ [0,1), so p = 0 vacates nothing and every variant is the same text.""" + result = vacancy_score(MODEL, default_passages(count=1), p=0.0, seed=0) + previews = {v["preview"] for v in result["variants"]} + assert len(previews) == 1 + for d in result["differences"]: + assert d["nats"] == pytest.approx(0.0, abs=1e-9) + + +def test_bad_parameters_raise_typed_errors() -> None: + with pytest.raises(InvalidParamError): + vacancy_score(MODEL, [], p=1.0) + with pytest.raises(InvalidParamError): + vacancy_score(MODEL, [" "], p=1.0) + with pytest.raises(InvalidParamError): + vacancy_score(MODEL, default_passages(count=1), p=1.5) + with pytest.raises(InvalidParamError, match="context"): + # gpt2 holds 1024 positions; a passage past it is refused rather than truncated, + # because a truncated variant is not the same text as the one it is compared to. + vacancy_score(MODEL, ["the cow jumped over the moon. " * 400], p=1.0) diff --git a/code/backend/tests/unit/test_arch_vacancy_align.py b/code/backend/tests/unit/test_arch_vacancy_align.py new file mode 100644 index 0000000..73e569b --- /dev/null +++ b/code/backend/tests/unit/test_arch_vacancy_align.py @@ -0,0 +1,149 @@ +"""Token→word alignment for the pretrained arm (contract §8.2, FR-718). + +Real tokenizers, real text, no mocks — but no model weights, so this file is fast: it +downloads tokenizer files only. The alignment is the part of the measurement that can be +silently wrong (a mis-attributed token still produces a plausible number), so it is +tested harder than the arithmetic around it. +""" + +from __future__ import annotations + +import unicodedata + +import pytest +from transformers import AutoTokenizer + +from llm_geometry.arch.vacancy_score import ( + byte_decoder, + default_passages, + preserved_token_indices, + preserved_word_indices, + token_byte_spans, + variant_texts, + word_spans, +) +from llm_geometry.errors import ComputeError + +# Every curated model is a byte-level BPE; gpt2 and SmolLM2 have no normalizer and Qwen +# has an NFC one, which is the case the up-front normalization exists for. +TOKENIZERS = ["gpt2", "HuggingFaceTB/SmolLM2-135M-Instruct", "Qwen/Qwen2.5-0.5B-Instruct"] + +TEXTS = [ + "The cow jumped over the moon, and the little dog laughed.", + # Multi-byte characters that BPE splits ACROSS tokens — the case per-token decoding + # corrupts and the case HF's own offsets overlap on. + "café naïve — “owl” ≈ ç√ 東京 end", + " leading and\ttabbed\nnewlines ", + "don't good-bye o'clock", +] + + +@pytest.mark.parametrize("model_id", TOKENIZERS) +@pytest.mark.parametrize("text", TEXTS) +def test_byte_spans_reconstruct_the_text_exactly(model_id: str, text: str) -> None: + tok = AutoTokenizer.from_pretrained(model_id) + normalized = unicodedata.normalize("NFC", text) + ids = tok(normalized, add_special_tokens=False)["input_ids"] + pieces = tok.convert_ids_to_tokens(ids) + spans = token_byte_spans(list(pieces), normalized) + + raw = normalized.encode("utf-8") + assert len(spans) == len(ids) + # A true partition: contiguous, covering, in order. That is what lets a per-token + # quantity be summed over a word without double-counting. + assert spans[0][0] == 0 + assert spans[-1][1] == len(raw) + for (a, b), (c, _d) in zip(spans, spans[1:]): + assert a <= b == c + assert b"".join(raw[a:b] for a, b in spans) == raw + + +def test_byte_spans_raise_when_the_pieces_do_not_rebuild_the_text() -> None: + """A mismatch must RAISE, never mis-attribute (FR-718).""" + tok = AutoTokenizer.from_pretrained("gpt2") + text = "The cow jumped over the moon." + pieces = list(tok.convert_ids_to_tokens(tok(text, add_special_tokens=False)["input_ids"])) + with pytest.raises(ComputeError, match="alignment failed"): + token_byte_spans(pieces[:-1], text) + + +def test_decomposed_input_is_caught_rather_than_silently_shifted() -> None: + """NFD input rebuilds to different bytes — the check fires instead of drifting.""" + tok = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B-Instruct") + decomposed = unicodedata.normalize("NFD", "café swirl") + assert decomposed != unicodedata.normalize("NFC", decomposed) + ids = tok(decomposed, add_special_tokens=False)["input_ids"] + pieces = list(tok.convert_ids_to_tokens(ids)) + with pytest.raises(ComputeError): + token_byte_spans(pieces, decomposed) + # …and NFC first (what `vacancy_score` does) makes it exact. + nfc = unicodedata.normalize("NFC", decomposed) + nfc_ids = tok(nfc, add_special_tokens=False)["input_ids"] + spans = token_byte_spans(list(tok.convert_ids_to_tokens(nfc_ids)), nfc) + assert spans[-1][1] == len(nfc.encode("utf-8")) + + +def test_byte_decoder_is_the_inverse_of_bytes_to_unicode() -> None: + table = byte_decoder() + assert len(table) == 256 + assert sorted(table.values()) == list(range(256)) + # 0x20 is NOT printable in this scheme, so it is rendered as U+0120 ("Ġ") — which is + # exactly why a piece cannot be read as text and needs this table to be measured. + assert table["Ġ"] == 0x20 + assert " " not in table + assert table["Ċ"] == 0x0A + + +def test_leading_space_tokens_are_attributed_to_their_word() -> None: + """The overlap rule, not "starts inside": byte-level BPE folds the space in.""" + tok = AutoTokenizer.from_pretrained("gpt2") + text = "the cow and the moon" + ids = tok(text, add_special_tokens=False)["input_ids"] + spans = token_byte_spans(list(tok.convert_ids_to_tokens(ids)), text) + words = word_spans(text) + assert [w.word for w in words] == ["the", "cow", "and", "the", "moon"] + # "the", "and", "the" preserved; "cow", "moon" vacated. + preserved = frozenset({0, 2, 3}) + got = preserved_token_indices(spans, words, preserved) + assert got, "no token was attributed to a preserved word" + decoded = "".join(tok.decode([ids[i]]) for i in got) + assert decoded.replace(" ", "") == "theandthe" + + +def test_a_token_spanning_a_preserved_and_a_vacated_word_raises() -> None: + """Ambiguous attribution is refused, not resolved by a heuristic.""" + words = word_spans("the cow") + # One synthetic token covering both words: the failure mode the assertion guards. + with pytest.raises(ComputeError, match="spans both"): + preserved_token_indices([(0, 7)], words, frozenset({0})) + + +def test_variants_preserve_word_count_and_the_scaffolding() -> None: + passage = default_passages(count=1)[0] + texts = variant_texts(passage, p=1.0, seed=0, match_prosody=True, keep=frozenset()) + assert set(texts) == {"english", "swap", "nonce"} + words, preserved = preserved_word_indices(texts) + assert preserved, "full vacancy left no scaffolding at all" + # Preserved means character-identical in EVERY variant — the property §8.1 rests on. + for name in ("swap", "nonce"): + variant = word_spans(texts[name]) + assert len(variant) == len(words) + for i in preserved: + assert variant[i].word == words[i].word + # …and the vacated words really did move, in both variants and differently. + vacated = [i for i in range(len(words)) if i not in preserved] + assert vacated + swap_words = word_spans(texts["swap"]) + nonce_words = word_spans(texts["nonce"]) + assert any(swap_words[i].word != words[i].word for i in vacated) + assert any(nonce_words[i].word != swap_words[i].word for i in vacated) + + +def test_default_passages_are_deterministic_and_sized() -> None: + a = default_passages() + b = default_passages() + assert a == b + assert len(a) == 6 + assert len({t for t in a}) == 6, "the default set repeats a passage" + for text in a: + assert len(word_spans(text)) >= 200 diff --git a/code/backend/tests/unit/test_lex_vacancy.py b/code/backend/tests/unit/test_lex_vacancy.py new file mode 100644 index 0000000..910c490 --- /dev/null +++ b/code/backend/tests/unit/test_lex_vacancy.py @@ -0,0 +1,1260 @@ +"""The vacancy transform, checked against the properties a `p`-sweep depends on. + +Everything here runs on the REAL committed corpus (`The Real Mother Goose`) and the real +Dolch budgets. No fixtures, no mocks: the transform's whole claim is about what happens to a +real English text, and a synthetic string would not exercise the cases that matter +(`good-bye`, `don't`, `dog's`, capitalised line openers, hyphenated compounds). + +The four properties, in the order the contract states them +(`specs/007-vacancy-transform-field/architecture.md`): + +* NESTING (SC-701) — `u` depends only on `(seed, stem)`, so the vacated set at `p` is a + subset of the vacated set at any larger `p`. +* STABILITY (SC-702) — the map is built once, in canonical order, independently of `p` and of + document order, so a stem's nonce is the same everywhere it appears. +* INVARIANCE (SC-703) — with `consistent=True, reveal_after=0` and a mapped vocabulary the + token id stream is unchanged, so a word-level model is *exactly* invariant to `p`. +* INJECTIVITY (SC-704) — verified on the real type set, not assumed; the theorem depends on + it. +""" + +from __future__ import annotations + +import random + +import pytest + +from llm_geometry.errors import ComputeError, InvalidParamError +from llm_geometry.lex.corpus import load_corpus_text +from llm_geometry.lex.dolch import DOLCH_ORDER, dolch_budget +from llm_geometry.lex.vacancy import ( + FUNCTION_WORDS, + REMINT_SALT_STRIDE, + SPLIT_EXCEPTIONS, + STRESS_TABLE, + SUFFIXES, + VacancyMap, + VacancyParams, + _mint, # private: the salt semantics of §5.5/§5.8 are a contract detail, so they are pinned + build_vacancy_map, + is_eligible, + map_vocab_words, + meter_score, + stem_and_suffix, + stress, + stress_source, + syllables, + type_counts, + vacancy_domain, + vacancy_stats, + vacancy_u, + vacate_text, +) +from llm_geometry.lex.vocab import WORD_RE, LexVocab, frequency_budget, tokenize + +P_GRID = (0.0, 0.25, 0.5, 0.75, 1.0) +SEEDS = (0, 7) + + +@pytest.fixture(scope="module") +def corpus() -> str: + return load_corpus_text() + + +@pytest.fixture(scope="module") +def corpus_types(corpus: str) -> list[str]: + return sorted(set(tokenize(corpus))) + + +@pytest.fixture(scope="module") +def budget() -> list[str]: + """The full Dolch list — the budget whose words must also be in the map's domain.""" + return dolch_budget("full") + + +@pytest.fixture(scope="module") +def domain(corpus_types: list[str]) -> list[str]: + """Contract §5.2: the corpus's types UNION the FULL Dolch list, never the active budget.""" + return vacancy_domain(corpus_types) + + +@pytest.fixture(scope="module") +def maps(domain: list[str]) -> dict[int, VacancyMap]: + """One map per seed, built once — they are `p`-independent by construction.""" + return {seed: build_vacancy_map(domain, VacancyParams(p=1.0, seed=seed)) for seed in SEEDS} + + +@pytest.fixture(scope="module") +def vacated(corpus: str, maps: dict[int, VacancyMap]) -> dict[tuple[int, float], str]: + """The vacated corpus at every (seed, p) on the grid.""" + out = {} + for seed in SEEDS: + for p in P_GRID: + params = VacancyParams(p=p, seed=seed) + out[(seed, p)] = vacate_text(corpus, maps[seed], params) + return out + + +def _changed_types(original: str, vacated_text: str) -> set[str]: + """The source types whose surface actually changed, measured from the two texts.""" + before = WORD_RE.findall(original) + after = WORD_RE.findall(vacated_text) + assert len(before) == len(after) + return {b.lower() for b, a in zip(before, after) if b != a} + + +def _images(original: str, vacated_text: str) -> dict[str, set[str]]: + """source TYPE -> the set of lower-cased surfaces it was rewritten to. + + Keyed on the lower-cased word, which is what the tokenizer sees. That is the strong + question, and the transform must survive it — see + :func:`test_transform_commutes_with_lowercasing_over_the_whole_corpus`. + """ + images: dict[str, set[str]] = {} + for b, a in zip(WORD_RE.findall(original), WORD_RE.findall(vacated_text)): + images.setdefault(b.lower(), set()).add(a.lower()) + return images + + +# --- the verbatim tables ---------------------------------------------------------------- + + +def test_closed_class_is_the_curated_list_only(): + """The source's warning, kept: an earlier version unioned this with the short Dolch + service words, which silently protected content verbs and understated the vacancy rate.""" + assert len(FUNCTION_WORDS) == 137 + for w in ("run", "eat", "see", "get", "let", "put"): + assert w not in FUNCTION_WORDS, f"{w} is a content verb, not closed class" + for w in ("the", "and", "never", "ten"): + assert w in FUNCTION_WORDS + assert all(w == w.lower() for w in FUNCTION_WORDS) + + +def test_suffix_order_is_load_bearing(): + """`ies` must be tried before `es` and `s`, `edly` before `ed`.""" + assert SUFFIXES.index("ies") < SUFFIXES.index("es") < SUFFIXES.index("s") + assert SUFFIXES.index("edly") < SUFFIXES.index("ed") + assert stem_and_suffix("berries") == ("berr", "ies") + + +def test_split_exceptions_come_from_the_audited_copy(): + """Without them `brother -> broth+er` and `morning -> morn+ing`, which the source itself + flags as a known artifact.""" + assert "brother" in SPLIT_EXCEPTIONS and "morning" in SPLIT_EXCEPTIONS + assert stem_and_suffix("brother") == ("brother", "") + assert stem_and_suffix("morning") == ("morning", "") + # ... and it is a spelling heuristic, still wrong outside the list. Documented, not fixed. + assert stem_and_suffix("ladder") == ("ladd", "er") + + +# --- eligibility, contract §2.2 --------------------------------------------------------- + + +def test_good_bye_is_never_vacated(maps): + """No suffix matches, so the stem contains a hyphen and fails the ASCII-letters test.""" + assert stem_and_suffix("good-bye") == ("good-bye", "") + assert not is_eligible("good-bye") + for seed in SEEDS: + assert maps[seed].apply_word("good-bye", VacancyParams(p=1.0, seed=seed)) == "good-bye" + assert "good-bye" not in maps[seed].mapping + + +def test_dont_is_never_vacated(maps): + """The MECHANISM, not just the outcome (§2.2). The `n't` suffix does NOT split `don't`, + because §3's length rule needs `len(word) - len(suffix) >= 3` and `5 - 3 = 2`. The stem is + therefore the whole `don't`, which contains an apostrophe and fails eligibility test 2 — + the ASCII-letters test — rather than test 3, the length test.""" + assert stem_and_suffix("don't") == ("don't", "") + assert not is_eligible("don't") + assert len("don't") - len("n't") == 2 # < 3, which is why no split happens + assert len("don't") > 2 # so it is test 2 that rejects it, not test 3 + for seed in SEEDS: + assert maps[seed].apply_word("don't", VacancyParams(p=1.0, seed=seed)) == "don't" + + +def test_dogs_apostrophe_s_keeps_its_suffix(maps): + """`dog's` splits to `dog`, which passes; the output is `'s`.""" + assert stem_and_suffix("dog's") == ("dog", "'s") + assert is_eligible("dog") + for seed in SEEDS: + params = VacancyParams(p=1.0, seed=seed) + out = maps[seed].apply_word("dog's", params) + assert out.endswith("'s") + assert out[:-2].lower() == maps[seed].mapping["dog"] or out[:-2].lower() != "dog" + assert out != "dog's" + + +def test_short_and_closed_class_stems_are_ineligible(): + assert not is_eligible("cat"[:2]) # len 2 + assert not is_eligible("the") + assert is_eligible("cat") + # `keep` extends the closed class. + assert not is_eligible("cat", FUNCTION_WORDS | {"cat"}) + + +def test_unicode_letters_are_not_ascii_letters(): + """`str.isalpha()` would accept these; JavaScript's `^[A-Za-z]+$` does not.""" + assert not is_eligible("café") + assert not is_eligible("naïve") + + +# --- the vacancy decision, contract §4 -------------------------------------------------- + + +def test_u_is_a_53_bit_double_and_depends_only_on_seed_and_stem(): + for stem in ("candle", "mother", "jack", "z"): + u = vacancy_u(stem, 0) + assert 0.0 <= u < 1.0 + # The `>> 11` exists so the numerator is exactly representable; if it were not, this + # product would not be an integer and the two stacks could disagree at the boundary. + assert (u * 2**53).is_integer() + assert vacancy_u(stem, 0) == u + assert vacancy_u(stem.upper(), 0) == u + assert vacancy_u(stem, 1) != u + + +def test_nesting_across_p_grid_and_two_seeds(corpus, vacated): + """SC-701. The vacated sets grow monotonically with `p`, measured from the output.""" + for seed in SEEDS: + sets = [_changed_types(corpus, vacated[(seed, p)]) for p in P_GRID] + for smaller, larger in zip(sets, sets[1:]): + assert smaller <= larger + assert sets[0] == set() + assert len(sets[-1]) > len(sets[0]) + # And it is genuinely graded, not a step function. + assert len(sets[1]) < len(sets[2]) < len(sets[3]) < len(sets[4]) + + +def test_vacancy_rate_tracks_p(corpus, corpus_types, vacated): + """`p` is the fraction of ELIGIBLE TYPES vacated, so the measured rate should sit close + to `p` — this is a distributional check on `u`, not a re-implementation of it.""" + eligible = {t for t in corpus_types if is_eligible(stem_and_suffix(t)[0])} + for seed in SEEDS: + for p in (0.25, 0.5, 0.75): + rate = len(_changed_types(corpus, vacated[(seed, p)])) / len(eligible) + assert abs(rate - p) < 0.05, f"seed={seed} p={p} rate={rate}" + + +# --- stability, contract §5.6 ----------------------------------------------------------- + + +def test_a_stems_nonce_is_identical_at_every_p(corpus, maps, vacated): + """SC-702. Nothing about the assignment may depend on `p`.""" + for seed in SEEDS: + per_p = [_images(corpus, vacated[(seed, p)]) for p in P_GRID] + for word in per_p[-1]: + surfaces = {img for images in per_p for img in images.get(word, set()) if img != word} + assert len(surfaces) <= 1, f"{word} took {surfaces} across the p-sweep" + # ... and the assignment itself, which is what SC-702 is stated over. + for stem, nonce in maps[seed].mapping.items(): + for p in P_GRID: + if vacancy_u(stem, seed) < p: + assert maps[seed].nonce_for(stem) == nonce + + +# --- the case-commuting invariant, contract §5.7 ---------------------------------------- + + +def test_transform_commutes_with_lowercasing_over_the_whole_corpus(corpus, maps): + """The normative invariant of §5.7: `lower(transform(w)) == transform(lower(w))`. + + The tokenizer lowercases, so a step that branches on case — a seam test against a + case-preserved suffix, say — gives one TYPE two surface forms and §7.3 is false. This is + the test that would have caught `gums -> flels` while `GUMS -> FLESS`. + """ + params = VacancyParams(p=1.0, seed=0) + vmap = maps[0] + for word in WORD_RE.findall(corpus): + assert vmap.apply_word(word, params).lower() == vmap.apply_word(word.lower(), params) + # The specific pair that broke it, now equal. + assert vmap.apply_word("GUMS", params) == "FLELS" + assert vmap.apply_word("gums", params) == "flels" + + +def test_transform_commutes_over_every_type_in_three_casings(corpus_types, maps): + """Same invariant, but exercising casings the corpus does not happen to contain.""" + for seed in SEEDS: + params = VacancyParams(p=1.0, seed=seed) + vmap = maps[seed] + for word in corpus_types: + lowered = vmap.apply_word(word, params) + assert lowered == lowered.lower() + for variant in (word.upper(), word.capitalize(), word.lower()): + assert vmap.apply_word(variant, params).lower() == lowered + # ... and the case marking itself survives, so the corpus still reads as English. + assert vmap.apply_word(word.upper(), params).isupper() or len(word) == 1 + assert vmap.apply_word(word.capitalize(), params)[0].isupper() + + +def test_map_is_unchanged_by_shuffling_the_input_types(domain, corpus_types, maps): + """The build order is canonical (sorted), never document order.""" + shuffled = list(domain) + random.Random(1234).shuffle(shuffled) + rebuilt = build_vacancy_map(shuffled, VacancyParams(p=1.0, seed=0)) + assert rebuilt.mapping == maps[0].mapping + assert rebuilt.minted_stress == maps[0].minted_stress + + +def test_map_does_not_depend_on_p(domain, corpus_types, maps): + for p in (0.0, 0.35, 1.0): + built = build_vacancy_map(domain, VacancyParams(p=p, seed=0)) + assert built.mapping == maps[0].mapping + + +def test_seeds_give_different_assignments(maps): + assert maps[0].mapping != maps[7].mapping + shared = set(maps[0].mapping) & set(maps[7].mapping) + differing = sum(1 for k in shared if maps[0].mapping[k] != maps[7].mapping[k]) + assert differing > 0.99 * len(shared) + + +# --- injectivity, contract §7.3 --------------------------------------------------------- + + +def test_map_is_injective_on_the_real_corpus(maps, domain): + """SC-704. Two distinct source types can collide through the stem+suffix construction; + the build verifies the assembled SURFACE FORMS and re-mints until they do not.""" + for seed in SEEDS: + vmap = maps[seed] + assert vmap.bijective is True + assert vmap.image_size == vmap.type_count == len(domain) + assert len(set(vmap.mapping.values())) == len(vmap.mapping) + # Condition B costs exactly one re-mint on this corpus, at seed 7. Measured, not assumed: + # if a change to the minter makes it zero or two, this number is the first thing to look at. + assert maps[0].remint_rounds == 0 + assert maps[7].remint_rounds == 1 + + +def test_injectivity_holds_at_every_p_not_just_the_endpoints(corpus, vacated): + """Trap 1 of §7.3: checking at `p = 1` only is insufficient, because at full vacancy every + eligible type has moved and a minted form has nothing left to collide with.""" + for seed in SEEDS: + for p in P_GRID: + pairs = list(zip(WORD_RE.findall(corpus), WORD_RE.findall(vacated[(seed, p)]))) + image_of: dict[str, str] = {} + for src, out in pairs: + image_of.setdefault(src.lower(), out.lower()) + assert len(set(image_of.values())) == len( + image_of + ), f"seed={seed} p={p}: two source types share a surface form" + + +def test_a_remint_is_held_to_the_same_quality_bar_as_an_original_mint(maps): + """§5.5's thresholds are on the ATTEMPT COUNTER `a`, not on the absolute salt. + + A mint call carries a base salt `S`; the stream is keyed on `S + a` and a re-mint restarts + `a` at 0. Read the thresholds absolutely instead and a re-mint at `S = 1001` would begin + with the length and syllable checks already relaxed — and a second round, at `S = 2001`, + could not run at all, contradicting "raise after 8 rounds". + + The observable: `hang` is monosyllabic, and its RE-MINTED nonce still is. + """ + assert syllables("hang") == 1 + assert syllables(maps[7].mapping["hang"]) == 1 + assert maps[7].minted_stress[maps[7].mapping["hang"]] == stress("hang") + assert len(maps[7].mapping["hang"]) >= 3 + # Directly: a mint from a base salt past every threshold still enforces every check. + for base in (0, 401, 801, 1001, REMINT_SALT_STRIDE + 500): + nonce, pattern, salt = _mint("hang", 7, True, frozenset(), start_salt=base) + assert syllables(nonce) == len(pattern) == 1 + assert len(nonce) >= 3 + assert base <= salt < base + 1200 + # ... and the stream is keyed on `S + a`: forbidding the first candidate at `S` gives + # exactly what a fresh call at `S + 1` produces. + first, _pattern, first_salt = _mint("hang", 7, True, frozenset(), start_salt=100) + assert first_salt == 100 + assert _mint("hang", 7, True, frozenset({first}), start_salt=100)[0] == ( + _mint("hang", 7, True, frozenset(), start_salt=101)[0] + ) + + +def test_hanged_no_longer_surfaces_as_the_english_word_waked(corpus, corpus_types, maps, vacated): + """Regression for trap 2 of §7.3, the case that motivated condition B. + + At seed 7 the stem `hang` first minted `wak`, so `hanged` assembled to `waked` — a real + word of this corpus, and one that is NOT vacated at p = 0.25 or p = 0.5, where it therefore + merged with the vacated `hanged`. A bare-nonce `avoid` check passed (`wak` is not a corpus + type) and a p = 1 image check passed (there, `waked` had moved too). + """ + assert "waked" in corpus_types and "hanged" in corpus_types + vmap = maps[7] + assert vmap.mapping["hang"] != "wak" + for p in (0.25, 0.5): + params = VacancyParams(p=p, seed=7) + assert vmap.apply_word("hanged", params) != "waked" + text = vacated[(7, p)] + pairs = list(zip(WORD_RE.findall(corpus), WORD_RE.findall(text))) + survivors = {b.lower() for b, a in pairs if b == a} + moved = {a.lower() for b, a in pairs if b != a} + assert not (moved & survivors) + + +def test_no_nonce_is_a_real_corpus_type(maps, corpus_types): + """FR-706. The source accepts an `avoid` parameter and never passes one, which lets a + minted form silently merge with an English type.""" + real = set(corpus_types) + for seed in SEEDS: + assert not (set(maps[seed].mapping.values()) & real) + + +def test_domain_is_the_corpus_plus_the_full_dolch_list_never_the_active_budget( + corpus_types, domain, budget +): + """§5.2. If the domain tracked the ACTIVE budget, switching budgets in the UI would + rebuild the map and re-mint the corpus underneath a panel demonstrating that nonces are + stable.""" + assert set(domain) == {t.lower() for t in corpus_types} | {w.lower() for w in budget} + assert set(vacancy_domain([])) == {w.lower() for w in dolch_budget("full")} + assert domain == sorted(domain) # canonical order, ASCII ascending + + +def test_map_is_a_pure_function_of_domain_seed_and_prosody(corpus, corpus_types, domain, maps): + """§5.2: there is no caller-supplied `avoid`, so no call path can build a different map. + + The parameter existed, defaulted to empty, and both stacks passed the type set — so they + agreed and no parity test could catch it. But the map was then a function of what the + caller remembered: at seed 0 the same corpus gives `remint_rounds` 0 with the domain + passed and 1 without, with different nonces either way. Both maps valid; that is the + problem. Here the same map is built through four different call paths. + """ + reference = maps[0].mapping + paths = [ + build_vacancy_map(domain, VacancyParams(p=1.0, seed=0)), + build_vacancy_map(vacancy_domain(corpus_types), VacancyParams(p=0.0, seed=0)), + build_vacancy_map(vacancy_domain(tokenize(corpus)), VacancyParams(p=0.5, seed=0)), + build_vacancy_map(vacancy_domain(set(corpus_types)), VacancyParams(p=1.0, seed=0)), + ] + for built in paths: + assert built.mapping == reference + assert built.minted_stress == maps[0].minted_stress + assert built.domain == maps[0].domain + assert built.remint_rounds == maps[0].remint_rounds + # ... and the domain is what a nonce is forbidden to equal, with nothing left to a caller. + assert not set(reference.values()) & maps[0].domain + + +def test_iterables_of_types_reject_a_bare_text(corpus, domain, budget, maps): + """`Iterable[str]` accepts a `str` and iterates it character by character, so passing the + corpus text built a domain of single letters and failed much later, somewhere else.""" + for call in ( + lambda: vacancy_domain(corpus), + lambda: build_vacancy_map(corpus, VacancyParams(p=1.0, seed=0)), + lambda: map_vocab_words(corpus, maps[0], VacancyParams(p=1.0, seed=0)), + lambda: VacancyParams(p=1.0, seed=0, keep="little"), + ): + with pytest.raises(TypeError, match="not a text"): + call() + # The correct forms still work. + assert vacancy_domain(tokenize(corpus)) == domain + assert len(map_vocab_words(budget, maps[0], VacancyParams(p=1.0, seed=0))) == len(budget) + assert VacancyParams(keep=frozenset({"little"})).keep_set == FUNCTION_WORDS | {"little"} + + +def test_the_map_does_not_move_when_the_active_budget_changes(corpus_types, maps): + """The property the panel depends on, and the reason the domain is the FULL Dolch list. + + Once `avoid` became implicit (§5.2), the domain IS the forbidden set — so a *smaller* + domain forbids fewer words and genuinely mints differently. Measured: building over + `corpus ∪ dolch_budget(name)` for any name below `full` moves exactly one stem, `jam`, + because `floor` is a full-list Dolch word that never appears in the corpus and so is + forbidden in the full domain and free in the smaller ones. + + That is not a defect, it is the argument: if the domain tracked the ACTIVE budget, a + reader switching budgets would watch the corpus re-mint under a panel demonstrating that + nonces are stable. `vacancy_domain` unions the full list regardless, so the smaller + domains are unreachable through the sanctioned API and there is only ever one map. + """ + for name in DOLCH_ORDER: + assert vacancy_domain(corpus_types) == vacancy_domain(corpus_types + dolch_budget(name)) + for seed in SEEDS: + built = build_vacancy_map( + vacancy_domain(corpus_types + dolch_budget(name)), + VacancyParams(p=1.0, seed=seed), + ) + assert built.mapping == maps[seed].mapping, f"{name}/{seed}: the map moved" + + # The counterexample that makes the rule load-bearing, pinned so it cannot drift silently. + assert "floor" in dolch_budget("full") + assert "floor" not in corpus_types + smaller = sorted({t.lower() for t in corpus_types} | set(dolch_budget("pre_primer"))) + off_contract = build_vacancy_map(smaller, VacancyParams(p=1.0, seed=0)) + moved = [s for s, n in off_contract.mapping.items() if maps[0].mapping[s] != n] + assert moved == ["jam"] + assert off_contract.mapping["jam"] == "floor" != maps[0].mapping["jam"] + + +def test_no_surface_form_is_ever_an_english_word_of_the_domain(corpus, domain, maps, vacated): + """Condition B of §5.2, stated over the whole domain and therefore `p`-independent. + + Deliberately conservative: it forbids a minted form from equalling a word that would + always have been vacated alongside it. That costs a re-mint and buys a condition that + holds simultaneously at every `p`, which is what the theorem needs. + """ + real = set(domain) + for seed in SEEDS: + params = VacancyParams(p=1.0, seed=seed) + for t in domain: + out = maps[seed].apply_word(t, params) + if out != t: + assert out not in real, f"seed={seed}: {t} surfaced as the English word {out}" + for p in P_GRID: + pairs = zip(WORD_RE.findall(corpus), WORD_RE.findall(vacated[(seed, p)])) + assert not {a.lower() for b, a in pairs if b != a} & real + + +# --- the rewrite, contract §1 ----------------------------------------------------------- + + +def test_every_output_is_one_complete_word_token(corpus, vacated): + for seed in SEEDS: + for p in P_GRID: + for word in WORD_RE.findall(vacated[(seed, p)]): + m = WORD_RE.fullmatch(word) + assert m is not None and m.group(0) == word + + +def test_token_count_and_order_are_preserved(corpus, vacated): + original = tokenize(corpus) + for seed in SEEDS: + for p in P_GRID: + assert len(tokenize(vacated[(seed, p)])) == len(original) + + +def test_line_structure_is_preserved(corpus, vacated): + """Line breaks are untouched, so the ``-per-line rule fires in the same places.""" + lines = corpus.splitlines() + token_lines = [i for i, ln in enumerate(lines) if WORD_RE.search(ln)] + for seed in SEEDS: + for p in P_GRID: + out_lines = vacated[(seed, p)].splitlines() + assert len(out_lines) == len(lines) + assert [i for i, ln in enumerate(out_lines) if WORD_RE.search(ln)] == token_lines + + +def test_non_word_characters_pass_through_byte_for_byte(corpus, vacated): + """Whitespace, punctuation and digits are not the transform's business.""" + stripped = WORD_RE.sub("", corpus) + for seed in SEEDS: + for p in P_GRID: + assert WORD_RE.sub("", vacated[(seed, p)]) == stripped + + +def test_p_zero_is_the_identity(corpus, vacated): + for seed in SEEDS: + assert vacated[(seed, 0.0)] == corpus + + +def test_p_one_vacates_every_eligible_type(corpus, corpus_types, vacated): + eligible = {t for t in corpus_types if is_eligible(stem_and_suffix(t)[0])} + for seed in SEEDS: + assert _changed_types(corpus, vacated[(seed, 1.0)]) == eligible + + +def test_capitalisation_is_carried_onto_the_nonce(maps): + params = VacancyParams(p=1.0, seed=0) + assert maps[0].apply_word("Jack", params)[0].isupper() + assert maps[0].apply_word("JACK", params).isupper() + assert maps[0].apply_word("jack", params).islower() + + +# --- the invariance theorem, contract §7.3 ---------------------------------------------- + + +def test_mapped_vocabulary_leaves_the_token_id_stream_unchanged(corpus, budget, maps, vacated): + """SC-703, on the real corpus: the transform is a pure relabelling of the vocabulary, so + a word-level model sees the identical id stream and trains bit-identically.""" + base = LexVocab(tuple(budget), source="dolch", budget_name="full") + reference = base.encode(tokenize(corpus)) + for seed in SEEDS: + for p in P_GRID: + params = VacancyParams(p=p, seed=seed) + words = map_vocab_words(budget, maps[seed], params) + assert len(set(words)) == len(words) + mapped = LexVocab(tuple(words), source="dolch", budget_name="full") + assert mapped.rows == base.rows + assert mapped.encode(tokenize(vacated[(seed, p)])) == reference + + +def test_map_vocab_words_preserves_order(budget, maps): + params = VacancyParams(p=1.0, seed=0) + words = map_vocab_words(budget, maps[0], params) + assert len(words) == len(budget) + for src, out in zip(budget, words): + if is_eligible(stem_and_suffix(src)[0]): + assert out != src + else: + assert out == src + + +def test_map_vocab_words_refuses_the_conditions_it_is_undefined_for(budget, maps): + """Under those conditions the budget must be REBUILT from the vacated corpus; the + coverage collapse is the measurement, and manufacturing a mapped vocabulary instead + would silently hide it.""" + for params in ( + VacancyParams(p=1.0, seed=0, consistent=False), + VacancyParams(p=1.0, seed=0, reveal_after=2), + ): + with pytest.raises(InvalidParamError): + map_vocab_words(budget, maps[0], params) + + +# --- the control conditions, contract §6/§7.1 ------------------------------------------- + + +def test_the_control_conditions_all_differ_from_each_other(corpus, maps, vacated): + """An invariance is only worth showing against something that breaks it.""" + baseline = vacated[(0, 1.0)] + inconsistent = vacate_text(corpus, maps[0], VacancyParams(p=1.0, seed=0, consistent=False)) + revealed = vacate_text(corpus, maps[0], VacancyParams(p=1.0, seed=0, reveal_after=2)) + flat = vacate_text(corpus, maps[0], VacancyParams(p=1.0, seed=0, match_prosody=False)) + kept = vacate_text(corpus, maps[0], VacancyParams(p=1.0, seed=0, keep=frozenset({"little"}))) + variants = { + "baseline": baseline, + "inconsistent": inconsistent, + "revealed": revealed, + "kept": kept, + } + for name, text in variants.items(): + assert text != corpus, name + assert len(tokenize(text)) == len(tokenize(corpus)), name + assert len(set(variants.values())) == len(variants) + + # `match_prosody=False` needs its own map: the flag changes what is minted, not how it + # is applied, so re-using a prosody-matched map cannot show a difference. + assert flat == baseline + flat_map = build_vacancy_map( + vacancy_domain(set(tokenize(corpus))), + VacancyParams(p=1.0, seed=0, match_prosody=False), + ) + flat = vacate_text(corpus, flat_map, VacancyParams(p=1.0, seed=0, match_prosody=False)) + assert flat != baseline + assert all(syllables(n) == 1 for n in flat_map.mapping.values()) + + +def test_inconsistent_assignment_destroys_type_identity(corpus, maps): + """Same vacancy rate, no learnable identity — that is the point of the control.""" + params = VacancyParams(p=1.0, seed=0, consistent=False) + text = vacate_text(corpus, maps[0], params) + assert len(_changed_types(corpus, text)) == len( + _changed_types(corpus, vacate_text(corpus, maps[0], VacancyParams(p=1.0, seed=0))) + ) + images = _images(corpus, text) + multiplied = {t for t, surfaces in images.items() if len(surfaces) > 1} + assert len(multiplied) > 100 + assert len(set(tokenize(text))) > len(set(tokenize(corpus))) + + +def test_the_inconsistent_mint_key_never_reaches_the_prosody_lookup(corpus, maps): + """§5.8. The key `f"{stem}#{idx}"` feeds the byte stream and the uniqueness check ONLY. + + Let it reach the prosody lookup and the pattern becomes `stress("little#0") == "10"` + instead of `stress("little") == "100"`, so `Little` mints as a disyllable. §7.1 says the + nonce carries THE STEM'S syllable count and stress; a mint key is not a word, and the + spelling rule has no business being asked about one. Caught by the golden fixture, not by + either stack's tests — hence this one. + """ + assert stress("little") == "100" != stress("little#0") == "10" + # At the minter: the key drives the byte stream, `stem` drives the pattern. + correct = _mint("little#0", 0, True, frozenset(), stem="little") + as_if_key_were_the_stem = _mint("little#0", 0, True, frozenset()) + assert correct[1] == "100" and syllables(correct[0]) == 3 + assert as_if_key_were_the_stem[1] == "10" + assert correct[0] != as_if_key_were_the_stem[0] + + # End to end: every minted form still carries its stem's syllable count, occurrence by + # occurrence. Restricted to un-suffixed words so the seam repair of §5.7 cannot muddy + # the comparison — it is covered by its own test. + params = VacancyParams(p=1.0, seed=0, consistent=False) + text = vacate_text(corpus, maps[0], params) + checked = 0 + for src, out in zip(WORD_RE.findall(corpus), WORD_RE.findall(text)): + stem, suffix = stem_and_suffix(src) + if src == out or suffix: + continue + assert maps[0].minted_stress[out.lower()] == stress(stem.lower()) + assert syllables(out.lower()) == syllables(stem.lower()) + checked += 1 + assert checked > 4000 + + +def test_condition_b_applies_to_the_per_occurrence_path_too(corpus, maps): + """§5.8, and the `tak` case that exposed it. + + Condition B — no minted form may equal a domain type — was enforced when building the + map and NOT on the `consistent=False` minting path. The gap is observable: at seed 7, + `p = 1`, the stem `tak` (of `taking`) minted the nonce `tak`, so `Taking -> Taking` and + one token silently failed to vacate. `corpus_types_vacated` read 1921 against the + consistent path's 1922 and `tokens_vacated` 8201 against 8202. + + §7.1 denies this control a STABILITY property — that is about a nonce being reused + across occurrences — and it does not license a word surviving the transform. A control + whose vacancy rate is not the stated rate is not a control, so a per-occurrence nonce + must equal neither a domain type nor the stem it replaces, under the same re-mint loop. + """ + # `tak` is a stem, not a type, which is exactly why the domain did not already forbid + # it: the domain is the corpus's TYPES (`taking`, `takes`, …) plus the Dolch list. + assert stem_and_suffix("taking") == ("tak", "ing") + assert "tak" not in maps[7].domain and "taking" in maps[7].domain + + for seed in (0, 7): + params = VacancyParams(p=1.0, seed=seed, consistent=False) + # A fresh map per condition: `consistent=False` writes to `minted_stress`. + vmap = build_vacancy_map(vacancy_domain(set(tokenize(corpus))), params) + text = vacate_text(corpus, vmap, params) + stats = vacancy_stats(corpus, text, vmap, params) + # At `p = 1` every eligible type vacates (§10) — in this control exactly as in the + # mapped condition, which is the whole claim. + assert stats["corpusTypesVacated"] == stats["corpusTypesEligible"] == 1922, seed + assert stats["tokensVacated"] == 8202, seed + + # No token survives the transform, and none survives as itself least of all. + for src, out in zip(WORD_RE.findall(corpus), WORD_RE.findall(text)): + stem = stem_and_suffix(src)[0] + if is_eligible(stem): + assert out.lower() != src.lower(), (seed, src) + + # And the mechanism, directly: forbidding the stem is not implied by forbidding the + # domain, so `_mint` must be handed it. The losing draw is the FOURTH occurrence of + # `tak` in document order — `tak#3` at seed 7 mints `tak` on its first attempt. + assert _mint("tak#3", 7, True, frozenset(), stem="tak")[0] == "tak" + assert _mint("tak#3", 7, True, frozenset({"tak"}), stem="tak")[0] != "tak" + + +def test_reveal_after_keeps_the_first_n_occurrences(corpus, maps): + params = VacancyParams(p=1.0, seed=0, reveal_after=3) + text = vacate_text(corpus, maps[0], params) + before, after = WORD_RE.findall(corpus), WORD_RE.findall(text) + seen: dict[str, int] = {} + for b, a in zip(before, after): + stem = stem_and_suffix(b)[0] + if not is_eligible(stem): + continue + key = stem.lower() + seen[key] = seen.get(key, 0) + 1 + assert (a == b) == (seen[key] <= 3) + + +# --- prosody, contract §6 --------------------------------------------------------------- + + +def test_syllable_rule_matches_the_contract(): + assert syllables("cat") == 1 + assert syllables("candle") == 2 # trailing `le` keeps its syllable + assert syllables("make") == 1 # silent `e` dropped + assert syllables("tree") == 1 + assert syllables("") == 1 + assert syllables("'-") == 1 + + +def test_stress_table_lookup_is_case_sensitive_before_it_is_case_insensitive(): + assert stress("Christmas") == STRESS_TABLE["Christmas"] + assert stress_source("Christmas") == "table" + # Only the capitalised key is in the table, so the lower-cased form falls to the rule. + assert stress_source("christmas") == "rule" + assert stress_source("little") == "table" + assert stress_source("candlestick") == "rule" + + +def test_minted_stress_wins_over_everything(maps): + vmap = maps[0] + nonce, pattern = next(iter(vmap.minted_stress.items())) + assert stress(nonce, vmap.minted_stress) == pattern + assert stress_source(nonce, vmap.minted_stress) == "minted" + + +def test_prosody_is_matched_when_asked(maps, corpus_types): + """`match_prosody` means the nonce carries the stem's syllable count and stress.""" + vmap = maps[0] + checked = 0 + for stem, nonce in vmap.mapping.items(): + assert vmap.minted_stress[nonce] == stress(stem) + assert syllables(nonce) == len(stress(stem)) + checked += 1 + assert checked > 1000 + + +def test_meter_score_is_a_fraction_and_rejects_unknown_feet(): + assert meter_score("") == 0.0 + assert 0.0 <= meter_score("Hickory dickory dock") <= 1.0 + assert meter_score("cat", "trochee") == 1.0 + with pytest.raises(InvalidParamError): + meter_score("cat", "spondee") + + +# --- statistics, contract §10 ----------------------------------------------------------- + + +def test_stats_have_exactly_the_contract_field_names(corpus, maps, vacated): + stats = vacancy_stats(corpus, vacated[(0, 0.5)], maps[0], VacancyParams(p=0.5, seed=0)) + assert set(stats) == { + "domainTypesTotal", + "domainTypesEligible", + "domainTypesVacated", + "corpusTypesTotal", + "corpusTypesEligible", + "corpusTypesVacated", + "stemsTotal", + "stemsVacated", + "tokensTotal", + "tokensVacated", + "meanSyllablesBefore", + "meanSyllablesAfter", + "meanAnapestBefore", + "meanAnapestAfter", + "stressFromTableBefore", + "stressFromTableAfter", + "stressFromMintedBefore", + "stressFromMintedAfter", + "stressFromRuleBefore", + "stressFromRuleAfter", + "bijective", + "imageSize", + "remintRounds", + } + assert not [k for k in stats if k.startswith("types")], "an unprefixed types* is forbidden" + + +def test_the_three_way_stress_split_sums_to_one_on_each_side(corpus, maps, vacated): + """§10: token-weighted fractions, unambiguous where a single coverage number was not.""" + for seed in SEEDS: + for p in P_GRID: + params = VacancyParams(p=p, seed=seed) + s = vacancy_stats(corpus, vacated[(seed, p)], maps[seed], params) + for side in ("Before", "After"): + total = ( + s[f"stressFromTable{side}"] + + s[f"stressFromMinted{side}"] + + s[f"stressFromRule{side}"] + ) + assert total == pytest.approx(1.0) + # No English word of the corpus is also a minted form — `avoid` guarantees it. + assert s["stressFromMintedBefore"] == 0.0 + # The split moves the right way: vacating replaces guessed stress with declared stress. + at_zero = vacancy_stats(corpus, vacated[(0, 0.0)], maps[0], VacancyParams(p=0.0, seed=0)) + at_one = vacancy_stats(corpus, vacated[(0, 1.0)], maps[0], VacancyParams(p=1.0, seed=0)) + assert at_zero["stressFromMintedAfter"] == 0.0 + assert at_one["stressFromMintedAfter"] > 0.4 + assert at_one["stressFromRuleAfter"] < at_zero["stressFromRuleAfter"] + + +def test_stats_are_measured_not_asserted(corpus, corpus_types, maps, vacated): + eligible = {t for t in corpus_types if is_eligible(stem_and_suffix(t)[0])} + for seed in SEEDS: + previous = -1 + for p in P_GRID: + params = VacancyParams(p=p, seed=seed) + stats = vacancy_stats(corpus, vacated[(seed, p)], maps[seed], params) + assert stats["corpusTypesTotal"] == len(set(tokenize(corpus))) + assert stats["corpusTypesEligible"] == len(eligible) + assert stats["domainTypesTotal"] == len(maps[seed].domain) + assert stats["stemsTotal"] == len(maps[seed].mapping) + assert stats["tokensTotal"] == len(tokenize(corpus)) + assert stats["corpusTypesVacated"] <= stats["corpusTypesEligible"] + assert stats["corpusTypesVacated"] > previous # strictly graded in p + previous = stats["corpusTypesVacated"] + assert stats["bijective"] is True + assert stats["remintRounds"] == maps[seed].remint_rounds + assert 0.0 <= stats["stressFromTableAfter"] <= 1.0 + assert stats["corpusTypesVacated"] == len(eligible) + assert stats["tokensVacated"] > 0 + + +def test_corpus_types_vacated_is_measured_from_the_texts_not_from_map_membership(corpus, maps): + """§10. A type counts as vacated iff at least one of its occurrences ACTUALLY changed. + + Under `reveal_after` the two readings diverge sharply: a type whose every occurrence falls + inside the reveal window is still in the map and still has `u(stem) < p`, so asking the map + over-reports roughly 2x. The panel would then tell a reader that 1337 types are vacant in a + text where 665 of them are printed in plain English. + """ + params = VacancyParams(p=0.7, seed=0, reveal_after=2) + text = vacate_text(corpus, maps[0], params) + stats = vacancy_stats(corpus, text, maps[0], params) + + measured = len(_changed_types(corpus, text)) + by_membership = len( + { + t + for t in set(tokenize(corpus)) + if is_eligible(stem_and_suffix(t)[0]) + and vacancy_u(stem_and_suffix(t)[0], params.seed) < params.p + } + ) + assert stats["corpusTypesVacated"] == measured == 665 + assert by_membership == 1337 # the map-membership reading, over-reporting 2.01x + # Every type the two readings disagree about really is still English in the output. + unchanged = {w for w, surfaces in _images(corpus, text).items() if surfaces == {w}} + assert len(unchanged & {t.lower() for t in tokenize(corpus)}) >= by_membership - measured + assert not (unchanged & _changed_types(corpus, text)) + # tokensVacated is measured the same way, and the reveal window is why it drops. + without_reveal = VacancyParams(p=0.7, seed=0) + assert ( + stats["tokensVacated"] + < vacancy_stats( + corpus, vacate_text(corpus, maps[0], without_reveal), maps[0], without_reveal + )["tokensVacated"] + ) + + +def test_the_two_counting_scopes_and_their_identities(corpus, corpus_types, maps, vacated): + """§10: the scope is in the name, and the identities are what exposed the confusion. + + The domain has 22 more eligible types than the corpus — Dolch words like `funny`, + `squirrel` and `today` that are in the budget but never appear in the text. Counting them + in what the panel shows a reader would inflate the vacancy rate they are being shown; + leaving them out of the diagnostic would misstate what the map covers. Hence both. + """ + eligible = {t for t in corpus_types if is_eligible(stem_and_suffix(t)[0])} + for seed in SEEDS: + for p in P_GRID: + params = VacancyParams(p=p, seed=seed) + s = vacancy_stats(corpus, vacated[(seed, p)], maps[seed], params) + assert s["domainTypesTotal"] == 2233 + assert s["domainTypesEligible"] == 1944 + assert s["corpusTypesTotal"] == 2211 + assert s["corpusTypesEligible"] == 1922 + assert s["domainTypesEligible"] == s["corpusTypesEligible"] + 22 + assert s["stemsTotal"] == 1680 <= s["domainTypesEligible"] + assert s["domainTypesVacated"] >= s["corpusTypesVacated"] + if p == 1.0: + # `u` lands in [0, 1), so at p = 1 every eligible stem vacates. + assert s["stemsVacated"] == s["stemsTotal"] + assert s["domainTypesVacated"] == s["domainTypesEligible"] + assert s["corpusTypesVacated"] == s["corpusTypesEligible"] == len(eligible) + + +def test_both_scopes_reproduce_the_numbers_the_typescript_stack_measured(corpus, maps, vacated): + """Cross-stack parity, pinned as data. These are the sequences the TS side reported; if + either stack's counting drifts, this is where it shows up.""" + expected = { + 0: {"corpus": [0, 461, 954, 1430, 1922], "domain": [0, 469, 966, 1448, 1944]}, + 7: {"corpus": [0, 434, 975, 1440, 1922], "domain": [0, 440, 985, 1455, 1944]}, + } + for seed in SEEDS: + stats = [ + vacancy_stats(corpus, vacated[(seed, p)], maps[seed], VacancyParams(p=p, seed=seed)) + for p in P_GRID + ] + assert [s["corpusTypesVacated"] for s in stats] == expected[seed]["corpus"] + assert [s["domainTypesVacated"] for s in stats] == expected[seed]["domain"] + assert [s["tokensVacated"] for s in stats][-1] == 8202 + + +def test_prosody_survives_full_vacancy(corpus, maps, vacated): + """The claim is that meter is untouched, so this pins the size of the drift rather than + asserting equality — and it quotes OUR corpus, never the source's numbers.""" + stats = vacancy_stats(corpus, vacated[(0, 1.0)], maps[0], VacancyParams(p=1.0, seed=0)) + assert abs(stats["meanSyllablesAfter"] - stats["meanSyllablesBefore"]) < 0.02 + assert abs(stats["meanAnapestAfter"] - stats["meanAnapestBefore"]) < 0.02 + # And the honesty number: most of the corpus is not in the 61-entry hand table. + assert 0.0 < stats["stressFromTableBefore"] < 0.15 + assert stats["stressFromRuleBefore"] > 0.85 + + +def test_stats_reject_texts_that_do_not_align(corpus, maps): + with pytest.raises(ComputeError): + vacancy_stats(corpus, corpus + " extra", maps[0], VacancyParams(p=1.0, seed=0)) + + +# --- parameter validation --------------------------------------------------------------- + + +@pytest.mark.parametrize( + "kwargs", + [ + {"p": -0.01}, + {"p": 1.5}, + {"p": float("nan")}, + {"p": "0.5"}, + {"seed": 1.5}, + {"reveal_after": -1}, + {"keep": frozenset({3})}, + ], +) +def test_bad_parameters_raise_invalid_param_error(kwargs): + with pytest.raises(InvalidParamError): + VacancyParams(**kwargs) + + +def test_defaults_are_the_contracts_defaults(): + params = VacancyParams() + assert (params.p, params.seed, params.consistent) == (0.0, 0, True) + assert (params.match_prosody, params.reveal_after, params.keep) == (True, 0, frozenset()) + assert params.keep_set == FUNCTION_WORDS + assert VacancyParams(keep=frozenset({"Little"})).keep_set == FUNCTION_WORDS | {"little"} + + +def test_a_stem_outside_the_maps_domain_is_a_compute_error(maps): + """The map's domain must include the budget's words as well as the corpus's types.""" + with pytest.raises(ComputeError): + maps[0].apply_word("zzzqqqx", VacancyParams(p=1.0, seed=0)) + + +# --- `forbidden`, contract §5.8 --------------------------------------------------------- + + +def test_forbidden_is_stored_and_keeps_superseded_remint_nonces(maps, domain): + """§5.8: `forbidden` is STORED, not rebuilt as `domain | mapping.values()`. + + THE CASE THAT DISTINGUISHES THEM, and the only one the shipped corpus produces: at + seed 7 the stem `hang` first minted `wak`, whose surface `wak` + `ed` is the real English + word `waked`; condition B rejected it and the re-mint returned `smeeg`. `wak` is now no + stem's nonce, so a reconstruction from `mapping.values()` drops it — but it must stay + forbidden, because it was rejected for cause and the `consistent=False` control draws + against this very set. + """ + seed7 = maps[7] + assert seed7.mapping["hang"] == "smeeg" + assert seed7.remint_rounds == 1 + assert "wak" in seed7.forbidden + assert "wak" not in set(seed7.mapping.values()) # exactly what a rebuild would lose + assert "waked" in seed7.domain # ... and why it was superseded + + # The rest of the field, so "stored" cannot decay into "stored but wrong". + for seed, vmap in maps.items(): + assert vmap.domain <= vmap.forbidden, seed + assert set(vmap.mapping.values()) <= vmap.forbidden, seed + assert maps[0].forbidden == maps[0].domain | set(maps[0].mapping.values()) # 0 re-mints + assert maps[7].forbidden == maps[7].domain | set(maps[7].mapping.values()) | {"wak"} + + +def test_the_inconsistent_control_draws_against_the_stored_forbidden_set(corpus, maps): + """The per-occurrence path must never hand out a superseded nonce (§5.8).""" + params = VacancyParams(p=1.0, seed=7, consistent=False) + text = vacate_text(corpus, maps[7], params) + assert "wak" not in {w.lower() for w in WORD_RE.findall(text)} + + +# --- the swap control, contract §8.3 / §5.2a -------------------------------------------- + + +@pytest.fixture(scope="module") +def counts(corpus: str) -> dict[str, int]: + """The corpus's per-type occurrence counts — the frequency source swap ranks by.""" + return type_counts(tokenize(corpus)) + + +@pytest.fixture(scope="module") +def swap_maps(domain: list[str], counts: dict[str, int]) -> dict[int, VacancyMap]: + return { + seed: build_vacancy_map(domain, VacancyParams(seed=seed, mint="swap"), counts) + for seed in SEEDS + } + + +def test_swap_replaces_stems_with_real_corpus_words(swap_maps, corpus_types, budget): + """The whole point of the control: every replacement is a word English already had.""" + real = {t.lower() for t in corpus_types} | {w.lower() for w in budget} + for seed, vmap in swap_maps.items(): + assert vmap.mapping, seed + assert set(vmap.mapping.values()) <= real, seed + # ... and no stem keeps its own form, which would be a word that failed to vacate. + assert all(stem != word for stem, word in vmap.mapping.items()), seed + + +def test_swap_needs_the_frequency_counts_and_says_so(domain): + """No silent fallback to an alphabetical rank, which would be frequency in name only.""" + with pytest.raises(InvalidParamError): + build_vacancy_map(domain, VacancyParams(mint="swap")) + + +def test_swap_refuses_the_inconsistent_control(domain): + """1680 open-class stems against 8202 vacated tokens — there is no supply (§8.3).""" + with pytest.raises(InvalidParamError): + VacancyParams(mint="swap", consistent=False) + + +def test_an_unknown_mint_strategy_is_rejected(): + with pytest.raises(InvalidParamError): + VacancyParams(mint="real-words") + + +def test_counts_do_not_reach_the_nonce_map(domain, counts, maps): + """The nonce map stays a pure function of `(domain, seed, match_prosody)` (§5.2).""" + for seed in SEEDS: + with_counts = build_vacancy_map(domain, VacancyParams(seed=seed), counts) + assert with_counts.mapping == maps[seed].mapping + assert with_counts.remint_rounds == maps[seed].remint_rounds + + +def test_swap_is_stable_in_seed_and_stem(domain, counts, swap_maps, corpus): + """SC-702 for swap: the map is built once, independently of `p` (§5.6).""" + for seed in SEEDS: + again = build_vacancy_map(domain, VacancyParams(p=1.0, seed=seed, mint="swap"), counts) + assert again.mapping == swap_maps[seed].mapping + # ... and a stem's surface is byte-identical at every `p` at which it is vacated. + forms: dict[str, set[str]] = {} + for p in P_GRID: + params = VacancyParams(p=p, seed=0, mint="swap") + for stem in ("little", "moon", "crown"): + if vacancy_u(stem, 0) < p: + forms.setdefault(stem, set()).add(swap_maps[0].apply_word(stem, params)) + assert forms and all(len(v) == 1 for v in forms.values()) + + +def test_swap_is_nested_in_p(corpus, swap_maps): + """SC-701 for swap: `u(stem) < p` is untouched, so the vacated sets are still nested.""" + previous: set[str] = set() + for p in P_GRID: + text = vacate_text(corpus, swap_maps[0], VacancyParams(p=p, seed=0, mint="swap")) + changed = _changed_types(corpus, text) + assert previous <= changed, p + previous = changed + + +def test_swap_is_a_bijection_of_the_domain_at_full_vacancy(swap_maps): + """A + B₁ of §5.2a, which is what the invariance theorem needs at `p = 1`.""" + for seed, vmap in swap_maps.items(): + assert vmap.bijective, seed + assert vmap.image_size == vmap.type_count, seed + assert vmap.remint_rounds == 0, seed + assert vmap.injective_at_every_p is False, seed + + +def test_swap_satisfies_the_invariance_theorem_where_it_is_defined(corpus, budget, swap_maps): + """SC-703 for swap, at the `p` where §5.2a proves a swap map CAN be injective. + + At `p in {0, 1}` the id stream is element-for-element identical to the untransformed + stream, exactly as it is for `mint="nonce"` — the tiny model is exactly as blind to a + real-word swap as to an invented form, which is the check that the control is right. + """ + base = LexVocab(tuple(budget), source="dolch", budget_name="full") + reference = base.encode(tokenize(corpus)) + for seed in SEEDS: + for p in (0.0, 1.0): + params = VacancyParams(p=p, seed=seed, mint="swap") + text = vacate_text(corpus, swap_maps[seed], params) + words = map_vocab_words(budget, swap_maps[seed], params) + assert len(set(words)) == len(words) + mapped = LexVocab(tuple(words), source="dolch", budget_name="full") + assert mapped.rows == base.rows + assert mapped.encode(tokenize(text)) == reference + + +def test_swap_refuses_the_mapped_vocabulary_at_intermediate_p(budget, swap_maps): + """§5.2a: no `p`-stable swap into the domain is injective at `0 < p < 1`, so the mapped + vocabulary does not exist there and is refused rather than silently duplicated.""" + for p in (0.25, 0.5, 0.75): + with pytest.raises(InvalidParamError): + map_vocab_words(budget, swap_maps[0], VacancyParams(p=p, seed=0, mint="swap")) + + +def test_why_swap_cannot_be_injective_at_intermediate_p(corpus, swap_maps): + """The theorem of §5.2a, measured rather than merely proved. + + A vacated type lands on a real English word; at intermediate `p` that word's own + occurrences may not have moved, so two source types share one surface. This pins the + collision count so the refusal above can never be mistaken for over-caution — if a future + change makes swap injective at `p = 0.5`, this test fails and the contract is wrong. + """ + params = VacancyParams(p=0.5, seed=0, mint="swap") + vmap = swap_maps[0] + images: dict[str, str] = {} + collisions = 0 + for t in sorted(vmap.domain): + image = vmap.apply_word(t, params).lower() + if image in images: + collisions += 1 + images[image] = t + assert collisions > 0 + # ... whereas at full vacancy there are none, which is what B₁ buys. + full = VacancyParams(p=1.0, seed=0, mint="swap") + assert len({vmap.apply_word(t, full).lower() for t in vmap.domain}) == len(vmap.domain) + + +def test_swap_honours_match_prosody(domain, counts, swap_maps): + """`matchProsody` is a real filter under swap, and — unlike minting — it is a filter over + a FINITE pool, so it cannot always be honoured. + + Measured at seed 0: 1586 of 1680 stems get a stress-matched real word, i.e. 94.4 %. The + other 94 exhaust the 1024 attempts of :data:`SWAP_RELAX_PROSODY` because their pattern is + rare in the corpus and the few words carrying it are already used — the relaxation of + §5.5, applied to a pool that can genuinely run out. Minting has no such limit, which is + exactly the difference between inventing a form and borrowing one, so the bound here is + the measurement rather than a claim of exactness. + """ + matched = swap_maps[0].mapping + hits = sum(1 for stem, word in matched.items() if stress(word) == stress(stem)) + assert hits / len(matched) > 0.9 + flat = build_vacancy_map( + domain, VacancyParams(seed=0, mint="swap", match_prosody=False), counts + ) + assert flat.mapping != matched + flat_hits = sum(1 for stem, word in flat.mapping.items() if stress(word) == stress(stem)) + assert flat_hits < hits + + +def test_swap_replacements_are_not_registered_as_minted_stress(swap_maps): + """They are real English words: their stress comes from the table or the rule, so + `stressFromMinted` must stay 0 on both sides of a swap (§8.3).""" + for seed, vmap in swap_maps.items(): + assert vmap.minted_stress == {}, seed + + +def test_swap_statistics_report_the_same_vacancy_rate_as_nonce(corpus, maps, swap_maps): + """The control holds the vacancy rate fixed and changes only what replaces the word.""" + for seed in SEEDS: + nonce = vacancy_stats( + corpus, + vacate_text(corpus, maps[seed], VacancyParams(p=1.0, seed=seed)), + maps[seed], + VacancyParams(p=1.0, seed=seed), + ) + params = VacancyParams(p=1.0, seed=seed, mint="swap") + swap = vacancy_stats( + corpus, vacate_text(corpus, swap_maps[seed], params), swap_maps[seed], params + ) + for field in ("corpusTypesVacated", "tokensVacated", "stemsVacated", "stemsTotal"): + assert swap[field] == nonce[field], (seed, field) + + +def test_sc703_over_the_full_grid_for_both_mint_strategies(corpus, domain, counts): + """SC-703 as the spec states it, for BOTH minting strategies. + + The grid the spec names: all five Dolch budgets plus a frequency budget, + ``p in {0, 0.25, 0.5, 0.75, 1}``, ``seed in {0, 7}``, both ``match_prosody`` settings — + 120 cases. + + ``mint="nonce"`` passes all 120: condition B keeps every image out of the domain, so the + map is injective at every `p` and the id stream is element-for-element unchanged. + + ``mint="swap"`` passes 48 — every case at `p in {0, 1}` — and REFUSES the other 72. That + is not a weaker test of the same claim; it is the claim §5.2a proves. A swap map's images + are domain types, so at intermediate `p` a vacated type can land on one that has not + moved, and no `p`-stable map avoids it short of the identity. Where a swap map can be + injective at all it is exactly as invisible to the model as an invented form, which is + what makes the control trustworthy. The counts are asserted, so neither number can drift + without this failing. + """ + budgets = {name: list(dolch_budget(name)) for name in DOLCH_ORDER} + budgets["frequency-top300"] = frequency_budget(corpus, 300) + assert len(budgets) == 6 + + passed = {"nonce": 0, "swap": 0} + refused = {"nonce": 0, "swap": 0} + for mint in ("nonce", "swap"): + for seed in SEEDS: + for match_prosody in (True, False): + vmap = build_vacancy_map( + domain, + VacancyParams(seed=seed, mint=mint, match_prosody=match_prosody), + counts, + ) + for p in P_GRID: + params = VacancyParams(p=p, seed=seed, mint=mint, match_prosody=match_prosody) + text = vacate_text(corpus, vmap, params) + for words in budgets.values(): + base = LexVocab(tuple(words), source="dolch", budget_name="full") + try: + mapped_words = map_vocab_words(words, vmap, params) + except InvalidParamError: + refused[mint] += 1 + continue + assert len(set(mapped_words)) == len(mapped_words), (mint, seed, p) + mapped = LexVocab(tuple(mapped_words), source="dolch", budget_name="full") + assert mapped.rows == base.rows + assert mapped.encode(tokenize(text)) == base.encode(tokenize(corpus)) + passed[mint] += 1 + + assert (passed["nonce"], refused["nonce"]) == (120, 0) + assert (passed["swap"], refused["swap"]) == (48, 72) diff --git a/code/frontend/playwright.config.ts b/code/frontend/playwright.config.ts index 12c9731..b40ceff 100644 --- a/code/frontend/playwright.config.ts +++ b/code/frontend/playwright.config.ts @@ -5,6 +5,11 @@ import { defineConfig, devices } from "@playwright/test"; // static — tests/e2e/static.spec.ts against the BUILT static site (`npm run // preview:static` on :4173, Pages base path, NO Python backend), proving // the GitHub Pages build stands alone. +// webgpu — tests/e2e/webgpu.spec.ts against the same static site, but launched with +// --enable-unsafe-webgpu so Chromium exposes the machine's REAL adapter. +// Without that flag headless Chromium has no adapter at all, which is how +// a broken WebGPU dtype (q4f16) shipped: the suite only ever ran WASM. +// Skips loudly where there is no adapter (e.g. Linux CI runners). // Playwright's webServer list is global (not per-project), so the servers to boot are // chosen from the --project filter on the command line: `--project static` must not // require a backend venv, and `--project chromium` must not pay for a static build. @@ -19,6 +24,21 @@ argv.forEach((a, i) => { const wants = (name: string): boolean => projectFilters.length === 0 || projectFilters.includes(name); +// Flags that make headless Chromium hand back the machine's REAL GPU adapter instead +// of SwiftShader (measured on macOS arm64 with the harness in the q4f16 investigation): +// --enable-unsafe-webgpu alone → google/swiftshader, no shader-f16 +// --enable-unsafe-webgpu --use-angle=metal → apple/metal-3, shader-f16 ✔ +// The ANGLE backend name is platform-specific, so it is chosen per platform; where no +// hardware adapter turns up, tests/e2e/webgpu.spec.ts skips loudly rather than +// quietly measuring software. +const isMac: boolean = + (globalThis as { process?: { platform?: string } }).process?.platform === "darwin"; +const WEBGPU_ARGS = [ + "--enable-unsafe-webgpu", + "--ignore-gpu-blocklist", + ...(isMac ? ["--use-angle=metal"] : ["--enable-features=Vulkan"]), +]; + export default defineConfig({ testDir: "tests/e2e", timeout: 240_000, @@ -36,7 +56,7 @@ export default defineConfig({ { name: "chromium", use: { ...devices["Desktop Chrome"], baseURL: "http://localhost:5173" }, - testIgnore: /static\.spec\.ts/, + testIgnore: /(static|webgpu)\.spec\.ts/, }, { name: "static", @@ -45,6 +65,19 @@ export default defineConfig({ use: { ...devices["Desktop Chrome"], baseURL: "http://localhost:4173" }, testMatch: /static\.spec\.ts/, }, + { + name: "webgpu", + use: { + ...devices["Desktop Chrome"], + baseURL: "http://localhost:4173", + // See WEBGPU_ARGS. The suite-wide --enable-unsafe-swiftshader is deliberately + // NOT inherited here: with it, `requestAdapter()` hands back + // google/swiftshader, and software WebGPU is exactly what this project must + // not measure. + launchOptions: { args: WEBGPU_ARGS }, + }, + testMatch: /webgpu\.spec\.ts/, + }, ], webServer: [ ...(wants("chromium") @@ -65,7 +98,7 @@ export default defineConfig({ }, ] : []), - ...(wants("static") + ...(wants("static") || wants("webgpu") ? [ { // Builds the static bundle first (same flags as the Pages deploy), then diff --git a/code/frontend/src/lib/StaticRuntimeBadge.svelte b/code/frontend/src/lib/StaticRuntimeBadge.svelte index ac6fa46..d9af5bf 100644 --- a/code/frontend/src/lib/StaticRuntimeBadge.svelte +++ b/code/frontend/src/lib/StaticRuntimeBadge.svelte @@ -5,9 +5,14 @@ import { staticExtras } from "./staticUx"; // Live status badge for the in-browser generation runtime (transformers.js): - // idle → loading → webgpu·q4f16 / wasm·q8 → (or error). Polled — the runtime + // idle → loading → webgpu·q8 / wasm·q8 → (or error). Polled — the runtime // reports a plain snapshot and loading happens inside a lazy chunk, so a light // interval is the simplest honest "live update". + // + // The device/dtype actually in use is NAMED here, and a rung the runtime rejected + // (thrown error, or the load-time non-degeneracy check) shows as "· fallback" with + // the rejected rungs in the tooltip. A user on a fallback path must be able to tell: + // the q4f16 defect was bad because it was silent, not because it fell back. const sc = staticExtras(); let info = $state(sc ? sc.staticRuntimeInfo().generation : null); @@ -20,20 +25,30 @@ }); const status = $derived(info?.status ?? "idle"); + const rejected = $derived(info?.rejected ?? []); const text = $derived( status === "ready" && info?.device && info?.dtype - ? `in-browser · ${info.device} · ${info.dtype}` + ? `in-browser · ${info.device} · ${info.dtype}${rejected.length ? " · fallback" : ""}` : status === "loading" ? "in-browser · loading model…" : status === "error" ? "in-browser runtime error" : "in-browser · model loads on first use", ); + const rejectedNote = $derived( + rejected.length + ? ` ${rejected.join(", ")} ${rejected.length === 1 ? "was" : "were"} rejected first — ` + + "either the session failed to build, or its output did not depend on its input " + + "(the load-time check); see the browser console for the exact reason." + : "", + ); const tip = $derived( status === "error" ? (info?.error ?? "the in-browser runtime failed to load") : status === "ready" - ? `generation runs locally via transformers.js (${info?.onnx_repo ?? "ONNX export"}) — real logits, no server` + ? `generation runs locally via transformers.js (${info?.onnx_repo ?? "ONNX export"}) on ` + + `${info?.device}/${info?.dtype}, which passed a load-time check that its logits ` + + `actually depend on the input — real logits, no server.${rejectedNote}` : "generation runs locally in your browser via transformers.js — the ONNX model downloads on first Generate", ); diff --git a/code/frontend/src/lib/dataClient.ts b/code/frontend/src/lib/dataClient.ts index e035049..1662688 100644 --- a/code/frontend/src/lib/dataClient.ts +++ b/code/frontend/src/lib/dataClient.ts @@ -382,6 +382,91 @@ export interface ArchGenerateResult { finish_reason: "eos" | "length"; } +// --- Feature 007: the pretrained arm of the vacancy instrument (contract §8) --- + +export interface ArchVacancyScoreBody { + model_id: string; + /** One passage (the panel's editable excerpt) … */ + passage?: string; + /** … or several, pooled at the token level. Omit both for the shipped default set. */ + passages?: string[]; + p?: number; // default 1.0 — full vacancy, the measured condition + seed?: number; + match_prosody?: boolean; + keep?: string[]; +} + +/** A quantity this stack measured but may not report, with the reason and the fix. */ +export interface ArchVacancyRefusal { + type: string; // the typed-error name the full stack would raise + message: string; +} + +/** Contract §8.1, per variant. Absolutes are `null` where the running dtype has no bound. */ +export interface ArchVacancyStats { + nllPreserved: number | null; + nllAll: number | null; + bitsPerChar: number | null; + nTokens: number; + nPreservedTokens: number; + nChars: number; +} + +export interface ArchVacancyVariant { + id: "english" | "swap" | "nonce"; + pooled: ArchVacancyStats; + preview: string; + /** Present when the absolute NLLs are withheld (quantized stack). */ + refused?: ArchVacancyRefusal; +} + +export interface ArchVacancyDifference { + /** `wrong_content` = swap − english; `unknown_form` = nonce − swap; `total` = their sum. */ + id: "wrong_content" | "unknown_form" | "total"; + label: string; + expr: string; + /** `total` is false: it conflates the two and is never a headline (contract §8.3). */ + headline: boolean; + nats: number | null; + se: number | null; + nPairs: number; + upperBound?: boolean; + note?: string; + /** Stated only where it was MEASURED for the dtype that ran; never invented. */ + quantizationUncertaintyNats?: number; + refused?: ArchVacancyRefusal; +} + +export interface ArchVacancyPassage { + index: number; + nWords: number; + nPreservedWords: number; + variants: Record; +} + +export interface ArchVacancyScoreResult { + model_id: string; + revision?: string; + /** "backend" (torch, fp32) or "static" (transformers.js, quantized ONNX). */ + stack: "backend" | "static"; + dtype: string; + device?: string; + p: number; + seed: number; + match_prosody: boolean; + keep: string[]; + alignment: { mechanism: string; unit: string; verified: boolean; note: string }; + variants: ArchVacancyVariant[]; + /** The English passages exactly as scored (NFC-normalized), so the UI can show them. */ + passages_used: string[]; + differences: ArchVacancyDifference[]; + /** Per-passage rows, or `null` where they are refused (quantized stack). */ + passages: ArchVacancyPassage[] | null; + passagesRefused?: ArchVacancyRefusal; + tiny_arm: { delta_nats: number; exact: boolean; label: string; note: string }; + confound: string; +} + export class ApiError extends Error { type: string; constructor(type: string, message: string) { @@ -657,6 +742,13 @@ export function createClient(opts: ClientOptions = {}) { return request("/api/arch/generate", jsonInit(body)); } + function archVacancyScore( + body: ArchVacancyScoreBody, + signal?: AbortSignal, + ): Promise { + return request("/api/arch/vacancy-score", { ...jsonInit(body), signal }); + } + return { listModels, resolveModel, @@ -682,6 +774,7 @@ export function createClient(opts: ClientOptions = {}) { getArchWeights, getArchTrace, archGenerate, + archVacancyScore, }; } diff --git a/code/frontend/src/lib/geoEngine/index.ts b/code/frontend/src/lib/geoEngine/index.ts index e8a5b78..3d14e8c 100644 --- a/code/frontend/src/lib/geoEngine/index.ts +++ b/code/frontend/src/lib/geoEngine/index.ts @@ -58,8 +58,28 @@ export interface ExportedWeightSet { weights: Record; // tensor name -> base64 of float32-LE bytes sources: Record; setSource: string; + /** + * The word list this set's token ids mean, for the sets that HAVE one of their own + * (`scratch`, `imported`). Absent for `edited` / `finetuned` sets, which keep the + * canonical vocabulary — for those, absence is the correct answer, not a gap. + * + * Omitting it for a set that owns a vocabulary is not a lossy shortcut, it is a + * corruption: the engine would fall back to the canonical tokenizer and `exportBundle` + * would then write a file pairing YOUR weights with Alice in Wonderland's words, under + * a `vocab_sha256` computed over that wrong list — internally consistent, so no + * integrity check could catch it. That is precisely the failure the three digests + * exist to prevent, committed by the writer. `restorePersistedSets` therefore drops a + * payload that lacks a vocabulary it needs rather than restoring it half-right. + */ + vocabWords?: string[]; } +/** + * Weight-set kinds whose token ids mean words of their OWN, not the canonical model's. + * A set of this kind is only usable with its vocabulary beside it. + */ +const SET_SOURCES_WITH_OWN_VOCAB: ReadonlySet = new Set(["scratch", "imported"]); + function b64FromF32(arr: Float32Array): string { const bytes = new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength); let s = ""; @@ -318,10 +338,15 @@ export class GeoEngine { if (!ws) throw notFound(`weights_token '${token}' is unknown (nothing to export)`); const weights: Record = {}; for (const [name, arr] of Object.entries(ws)) weights[name] = b64FromF32(arr); + const vocab = this.vocabs.get(token); return { weights, sources: { ...(this.sourceMaps.get(token) ?? {}) }, setSource: this.setSources.get(token) ?? "edited", + // The word list travels WITH the weights, exactly as the backend's + // `save_weight_set(..., vocab_json=…)` stores it beside them: a scratch or + // imported model's ids mean its own words, so weights alone do not describe it. + ...(vocab ? { vocabWords: [...vocab.words] } : {}), }; } @@ -334,9 +359,20 @@ export class GeoEngine { return false; } if (weightsToken(ws) !== token) return false; // hash mismatch — refuse + const words = payload.vocabWords; + const ownsVocab = SET_SOURCES_WITH_OWN_VOCAB.has(payload.setSource); + if (words !== undefined && (!Array.isArray(words) || words.some((w) => typeof w !== "string"))) { + return false; + } + // A set whose ids mean its own words is not restorable without them. Restoring the + // weights alone would leave `tokenizerFor` falling back to the canonical vocabulary + // and every later read — the sphere's labels, a trace, and above all a SAVED model + // file — would silently describe the wrong words. Refuse, and let the caller drop it. + if (ownsVocab && words === undefined) return false; this.weightSets.set(token, ws); this.sourceMaps.set(token, { ...payload.sources }); this.setSources.set(token, payload.setSource); + if (words !== undefined) this.vocabs.set(token, new GeoTokenizer(words)); return true; } @@ -448,6 +484,19 @@ export class GeoEngine { exportBundle(token?: string | null): GeoModelBundle { const resolved = token && token !== "learned" ? token : this.canonicalToken; const ws = this.resolveWeightSet(resolved); + // `tokenizerFor` falls back to the canonical vocabulary, which is RIGHT for an + // edited or fine-tuned set (those keep the canonical words) and CATASTROPHIC for a + // scratch or imported one: the file would carry your weights under Alice in + // Wonderland's word list, with a `vocab_sha256` computed over that list, so no + // reader could ever detect it. Writing such a file is refused. + if (SET_SOURCES_WITH_OWN_VOCAB.has(this.setSources.get(resolved) ?? "") && !this.vocabs.has(resolved)) { + throw notFound( + `weights_token '${resolved}' has no vocabulary in this session, and its ids mean ` + + "its own words rather than the shipped model's — saving it now would pair these " + + "weights with the wrong word list. Load the model file again (or retrain) so its " + + "vocabulary is present.", + ); + } const vocabJson = JSON.stringify({ format: "geo-tokenizer-v1", specials: { [UNK_TOKEN]: UNK_ID, [EOS_TOKEN]: EOS_ID, [PAD_TOKEN]: PAD_ID }, diff --git a/code/frontend/src/lib/lexEngine/vacancy.ts b/code/frontend/src/lib/lexEngine/vacancy.ts new file mode 100644 index 0000000..b3ffdee --- /dev/null +++ b/code/frontend/src/lib/lexEngine/vacancy.ts @@ -0,0 +1,1499 @@ +/** + * The vacancy transform — field without location, at a controlled rate. + * + * TypeScript half of `specs/007-vacancy-transform-field/architecture.md`, which is the + * normative document: this file implements *that contract*, not the Python module, and + * the Python module implements it too. Feature 006 taught us why — a contract that omits + * one sentence lets two stacks drift for a day without either being "wrong". + * + * What it does: rewrite a corpus in place so that closed-class scaffolding, inflectional + * morphology, punctuation and line structure survive byte for byte, while a controlled + * fraction `p` of open-class STEMS is replaced by phonotactically legal nonce forms that + * carry the stem's syllable count and stress. The result is Carroll's condition — a token + * whose distributional neighbourhood is fully specified by context and whose form carries + * no prior — manufactured on demand rather than borrowed from the 28 words Carroll minted. + * + * The three properties that make a `p`-sweep interpretable, and where they come from: + * + * * NESTING (§4). A stem is vacated iff `u(stem) < p`, and `u` is a hash of + * `(seed, stem)` alone — not of `p`, not of traversal order, not of which other words + * exist. So `{vacated at p} ⊆ {vacated at p'}` whenever `p < p'`. + * * STABILITY (§5). The nonce map is built ONCE over the whole type set in canonical + * (sorted) order, and the map at any `p` is its restriction to `{u < p}`. The source's + * minter built the map lazily while rewriting, so its `used` set — and therefore the + * nonce a word got — depended on `p`. That breaks the stability the source claims for + * itself; §5.2 of the contract is the correction, and it makes stability structural + * rather than hoped for. + * * INJECTIVITY AT EVERY `p` (§5.2 A/B). The check is over assembled, lowercased SURFACE + * forms, not bare nonces, and it forbids a surface form from equalling ANY domain type + * — eligible or not. Both weaker checks were tried and both were wrong; the comments + * on `buildVacancyMap` name the collision each one missed. + * + * All three feed the invariance theorem (§7.3): with `consistent = true` and + * `revealAfter = 0`, the transform is a pure relabelling of the vocabulary, so a + * word-level model trained from scratch sees the identical token id stream and trains + * bit-identically. That is only true if the transform's idea of a word is EXACTLY the + * tokenizer's, which is why `WORD_RE`/`tokenize` are imported from `./vocab` rather than + * re-declared here (departure 1 from the source, whose `[A-Za-z][A-Za-z']*` split + * `good-bye` in two). + * + * The tokenizer lowercases, so the transform must COMMUTE WITH LOWERCASING (§5.7): + * + * lower(transformWord(w)) === transformWord(lower(w)) // normative, and a test + * + * Everything is therefore computed on the lowercased word — stem, suffix, seam test, seam + * hash, assembly — and `matchCase` is applied once at the end to the whole assembled + * surface form. Slicing the suffix case-preserved (as the source does) makes the seam test + * branch on case: `gums -> flels` but `GUMS -> FLESS`, one source type with two surface + * forms, and the theorem is false. + * + * Determinism across the TS/Python seam is bought with three deliberate departures: + * + * * `u = (top64 >> 11) / 2**53`, not `top64 / 2**64`. NOT because the source's + * expression diverges — §4 records that it was measured over 200 006 values and does + * not — but because a 53-bit numerator over 2^53 is exactly representable, so the + * agreement is structural rather than a proof one refactor could invalidate. + * * `random.Random` is replaced by a sha256 counter stream (`bytesFor`), because + * MT19937 seeded from a string is not reproducible in TypeScript. + * * the seam fix and the give-up path are hashes of their inputs rather than draws from + * a shared RNG or counts of a mutable `used` set, so neither depends on order. + * + * The hash is the SYNCHRONOUS pure-JS sha256 of `../geoEngine/hash`; WebCrypto is async + * and this transform has to be callable from a store update. + */ + +import { sha256Hex, utf8Bytes } from "../geoEngine/hash"; +import { dolchBudget } from "./dolch"; +import { WORD_RE, splitLines } from "./vocab"; + +// --- §2.1 the closed class ----------------------------------------------------------- + +/** + * The source's curated closed class, ported verbatim (whitespace-split, lowercased). + * + * The source carries a warning we keep: an earlier version unioned this with short Dolch + * service words, which silently protected content verbs (`run`, `eat`, `see`, `get`, + * `let`, `put`) and understated the vacancy rate. The closed class is THIS LIST ONLY. + */ +export const FUNCTION_WORDS: ReadonlySet = new Set( + `a an the this that these those my your his her its our +their some any all both each every no none i me you he she it we they him them +us who whom whose which what where when why how is am are was were be been being +do does did done have has had having will would shall should can could may might +must not and or but so if then than as of to in on at by for with from into onto +up down out off over under again once here there very too also only just even +still yet ever never always about after before while because though although +unless until since during between among against through above below near far +one two three four five six seven eight nine ten` + .split(/\s+/) + .filter((w) => w.length > 0), +); + +/** `FUNCTION_WORDS ∪ lower(extra)` — the effective keep set (§2.1). */ +export function effectiveKeepSet(extra: Iterable = []): ReadonlySet { + const out = new Set(FUNCTION_WORDS); + for (const w of extra) out.add(w.toLowerCase()); + return out; +} + +// --- §3 suffix splitting ------------------------------------------------------------- + +/** Tried in THIS order; the first match wins. Order is part of the contract. */ +export const SUFFIXES: readonly string[] = [ + "ing", + "edly", + "est", + "ies", + "'s", + "n't", + "ed", + "es", + "er", + "ly", + "s", +]; + +/** + * Never split, whatever the spelling suggests (§3, departure 9). + * + * From the AUDITED copy of the source, not the copy in the zip: without it + * `brother -> broth+er` and `morning -> morn+ing`, which the source itself flags as a + * known artifact. This is a spelling heuristic and not a morphological analyser, so it is + * still wrong outside the list (`ladder -> ladd+er`). That is acceptable — the nonce + * still carries a consistent identity and an inflected-looking surface — but it must be + * documented in the UI rather than quietly tolerated. + */ +export const SPLIT_EXCEPTIONS: ReadonlySet = new Set([ + "brother", + "father", + "mother", + "sister", + "never", + "over", + "under", + "morning", + "giving", + "thing", +]); + +/** + * Split a word into `[stem, suffix]`, preserving case. + * + * A suffix `s` matches iff `lower(word)` ends with it AND `len(word) - len(s) >= 3`. + * The slice is taken from the ORIGINAL word, so `Dog's` -> `["Dog", "'s"]`. + */ +export function stemAndSuffix(word: string): [string, string] { + const lower = word.toLowerCase(); + if (SPLIT_EXCEPTIONS.has(lower)) return [word, ""]; + for (const suffix of SUFFIXES) { + if (lower.endsWith(suffix) && word.length - suffix.length >= 3) { + const cut = word.length - suffix.length; + return [word.slice(0, cut), word.slice(cut)]; + } + } + return [word, ""]; +} + +// --- §2.2 eligibility ---------------------------------------------------------------- + +/** ASCII letters only. Python must use `re.fullmatch(r"[A-Za-z]+", ...)`, NOT + * `str.isalpha()`, which is Unicode-aware and would accept what this rejects. */ +const ASCII_STEM = /^[A-Za-z]+$/; + +/** + * May this stem be vacated? (§2.2) All three must hold: + * + * 1. `lower(stem)` is not in the keep set, + * 2. `stem` is ASCII letters only, + * 3. `stem` is longer than two characters. + * + * Test 2 is what makes hyphens and apostrophes behave, and the three cases both stacks + * must agree on exactly: + * + * * `good-bye` — no suffix matches, the stem is `good-bye`, the hyphen fails test 2, + * so the word is NEVER vacated; + * * `don't` — the `n't` suffix splits it to stem `do`, which fails test 3 (and test 1); + * * `dog's` — the `'s` suffix splits it to stem `dog`, which passes, so the output is + * `'s`. + * + * `keep` is the EFFECTIVE set (`effectiveKeepSet`), not the caller's extras. + */ +export function isEligible(stem: string, keep: ReadonlySet): boolean { + if (keep.has(stem.toLowerCase())) return false; + if (!ASCII_STEM.test(stem)) return false; + return stem.length > 2; +} + +// --- §5.4 the phonotactic tables ----------------------------------------------------- +// +// Ported verbatim from `tiny-seuss/synth/jabberwockify.py`, ORDER SIGNIFICANT: the index +// into each list is what the byte stream selects, so reordering silently changes every +// nonce ever minted. + +/** 47 entries. */ +export const ONSETS: readonly string[] = [ + "b", "br", "bl", "d", "dr", "f", "fl", "fr", "g", "gl", "gr", "h", + "j", "k", "kl", "kr", "l", "m", "n", "p", "pl", "pr", "r", "s", "sk", + "sl", "sm", "sn", "sp", "st", "str", "sw", "t", "tr", "th", "thr", + "v", "w", "wr", "y", "z", "sh", "shr", "ch", "gn", "sc", "sq", +]; + +/** 19 entries. */ +export const NUCLEI: readonly string[] = [ + "a", "e", "i", "o", "u", "ai", "ee", "ea", "oo", "ou", "oa", "ie", + "y", "au", "ur", "ir", "or", "ar", "er", +]; + +/** + * 46 entries, beginning with the empty string. + * + * An early draft of §5.4 said 49. The contract now says 46 and, more usefully, states that + * **the source lists are normative and the counts are commentary** — a nonce is a function + * of these strings and their indices, not of a tally in a document. + */ +export const CODAS: readonly string[] = [ + "", "b", "d", "f", "g", "k", "l", "m", "n", "p", "r", "s", "t", "v", + "z", "sh", "ch", "th", "ck", "ff", "ll", "mp", "nd", "ng", "nk", "nt", + "sk", "sp", "st", "ft", "lt", "lk", "rd", "rk", "rm", "rn", "rt", "ble", + "dle", "gle", "kle", "tle", "mble", "ndle", "ffle", "zzle", +]; + +/** 13 entries — the tail of a non-initial unstressed syllable. */ +export const UNSTRESSED_TAILS: readonly string[] = [ + "y", "le", "er", "ow", "en", "el", "ish", "ous", "id", + "ic", "um", "ent", "ing", +]; + +/** The prefix an INITIAL unstressed syllable draws from. */ +export const UNSTRESSED_PREFIXES: readonly string[] = ["a", "be", "re", "de", "un", "en"]; + +/** + * The reduced coda set for an unstressed syllable. The duplicated empty string doubles + * its weight — keep it (§5.4). + * + * §5.5's enumerated rule has exactly three branches and none of them reaches this table: + * a stressed syllable draws from `CODAS`, an unstressed one emits a whole prefix or tail. + * It is dead in the source too (`_syl(stressed=False)` is never called from `mint`). + * **Do not "fix" this** — the byte stream's list indices must not shift, and drawing from + * it would change every multi-syllable nonce in both stacks. + */ +export const REDUCED_CODAS: readonly string[] = ["", "", "l", "n", "r", "s"]; + +/** Replacement characters for a seam, indexed by a hash of `(stem, suffix)` (§5.7). */ +const SEAM_CHARS = "lnrtk"; + +// --- §6 prosody ---------------------------------------------------------------------- + +/** + * The 61 polysyllables of the Dolch list, hand-set. `1` = stressed, `0` = unstressed. + * Ported verbatim from `tiny-seuss/synth/lexicon.py`. + * + * PROVENANCE — read this before quoting a prosody number. The source describes the table + * as "seeded by rule and then overridden by a hand table", and its own status table lists + * it under *not yet exercised*: "seeded by rule; wants roughly an hour of human + * checking." So we do NOT claim exact prosody and no UI string may. The shipped corpus is + * *The Real Mother Goose*, most of whose types are not Dolch words, so most of them fall + * through to the spelling rule — which is exactly what `stressTableCoverage*` measures, + * and why every prosody statistic must be shown next to it. + */ +export const STRESS_TABLE: ReadonlyMap = new Map([ + ["away", "01"], ["funny", "10"], ["little", "100"], ["yellow", "10"], + ["into", "10"], ["over", "10"], ["pretty", "10"], ["under", "10"], + ["after", "10"], ["again", "01"], ["any", "10"], ["every", "100"], + ["giving", "10"], ["once", "1"], ["open", "10"], + ["always", "100"], ["around", "01"], ["because", "01"], ["before", "01"], + ["seven", "10"], ["eight", "1"], ["myself", "01"], ["never", "10"], + ["only", "10"], ["today", "01"], ["together", "0100"], ["better", "10"], + ["carry", "10"], ["many", "10"], ["upon", "01"], ["very", "100"], + ["apple", "10"], ["baby", "10"], ["birthday", "100"], ["brother", "10"], + ["chicken", "10"], ["children", "100"], ["Christmas", "10"], + ["farmer", "10"], ["flower", "10"], ["garden", "10"], ["good-bye", "01"], + ["horse", "1"], ["kitty", "10"], ["letter", "10"], ["money", "10"], + ["morning", "10"], ["mother", "10"], ["paper", "100"], ["party", "10"], + ["picture", "10"], ["rabbit", "10"], ["robin", "10"], ["squirrel", "1"], + ["table", "10"], ["water", "10"], ["window", "10"], ["Santa Claus", "101"], + ["father", "10"], ["sister", "10"], ["summer", "10"], +]); + +/** Where a stress pattern came from — the honesty of every prosody number (§6.1). */ +export type StressSource = "minted" | "table" | "rule"; + +/** + * Syllable count by spelling rule (§6.2). Fallback only. + * + * The source has a further `if w.endswith("le") ...: pass` branch. It is a literal `pass` + * — dead code — and is NOT ported; the behaviour here is byte-identical to the source's. + */ +export function ruleSyllables(word: string): number { + let w = word.toLowerCase().replace(/^['-]+/, "").replace(/['-]+$/, ""); + w = w.replace(/[^a-z]/g, ""); + if (w.length === 0) return 1; + let n = (w.match(/[aeiouy]+/g) ?? []).length; + if (w.endsWith("e") && n > 1 && !(w.endsWith("le") || w.endsWith("ee") || w.endsWith("ye"))) { + n -= 1; + } + return Math.max(1, n); +} + +/** + * The stress pattern of a word, and where it came from. Lookup order is exactly §6.3: + * + * 1. `minted[lower(word)]` — a form WE minted, whose pattern is intended rather than + * guessed, so prosody scoring on a vacated corpus is exact for the minted forms; + * 2. `STRESS_TABLE[word]` — CASE-SENSITIVE, which is there for `Christmas`; + * 3. `STRESS_TABLE[lower(word)]`; + * 4. the rule. + * + * `minted` is passed explicitly rather than kept in a module-level registry (the source + * used a global `MINTED_STRESS`): a global would leak one build's nonces into the next + * one's syllable checks and make minting depend on call order, which is the class of bug + * §5 exists to remove. Omit it and you get pure spelling+table prosody, which is what + * minting itself uses and what the ORIGINAL corpus should be scored with. + */ +export function stressWithSource( + word: string, + minted?: ReadonlyMap, +): { pattern: string; source: StressSource } { + const lower = word.toLowerCase(); + const mintedPattern = minted?.get(lower); + if (mintedPattern !== undefined) return { pattern: mintedPattern, source: "minted" }; + const exact = STRESS_TABLE.get(word); + if (exact !== undefined) return { pattern: exact, source: "table" }; + const lowered = STRESS_TABLE.get(lower); + if (lowered !== undefined) return { pattern: lowered, source: "table" }; + const n = ruleSyllables(lower); + return { pattern: n === 1 ? "1" : "1" + "0".repeat(n - 1), source: "rule" }; +} + +/** §6.3. */ +export function stress(word: string, minted?: ReadonlyMap): string { + return stressWithSource(word, minted).pattern; +} + +/** §6.3: `syllables(word) := len(stress(word))`. */ +export function syllables(word: string, minted?: ReadonlyMap): number { + return stress(word, minted).length; +} + +/** The repeating feet §6.4 names. */ +export const METER_FEET: Readonly> = { + anapest: "001", + iamb: "01", + trochee: "10", + dactyl: "100", +}; + +/** Raw `WORD_RE` matches, case preserved — `stress`'s case-sensitive step 2 needs them. + * A fresh RegExp per call: the shared `g`-flagged literal carries `lastIndex`. */ +function wordMatches(text: string): string[] { + const re = new RegExp(WORD_RE.source, "g"); + const out: string[] = []; + for (let m = re.exec(text); m !== null; m = re.exec(text)) out.push(m[0]); + return out; +} + +/** + * §6.4: the fraction of syllable positions in the line's concatenated stress string that + * match the repeating foot. `0.0` for a line with no syllables. + */ +export function meterScore( + line: string, + foot: string = "anapest", + minted?: ReadonlyMap, +): number { + const pat = METER_FEET[foot]; + if (pat === undefined) { + throw new Error(`unknown foot ${JSON.stringify(foot)}; expected ${Object.keys(METER_FEET).join(", ")}`); + } + let scan = ""; + for (const w of wordMatches(line)) scan += stress(w, minted); + if (scan.length === 0) return 0; + let hits = 0; + for (let i = 0; i < scan.length; i++) if (scan[i] === pat[i % pat.length]) hits++; + return hits / scan.length; +} + +// --- §4 the vacancy decision --------------------------------------------------------- + +/** sha256 of a UTF-8 string, as bytes. */ +function digestBytes(s: string): Uint8Array { + const hex = sha256Hex(utf8Bytes(s)); + const out = new Uint8Array(32); + for (let i = 0; i < 32; i++) out[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16); + return out; +} + +/** + * Big-endian uint32 from four bytes. The `>>> 0` is REQUIRED: without it JS sign-extends + * and `%` returns a negative index (§5.3). + */ +function u32At(bytes: Uint8Array, offset: number): number { + return ( + ((bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]) >>> 0 + ); +} + +/** + * `u(stem, seed) ∈ [0, 1)` — the vacancy coordinate (§4). A stem is vacated iff `u < p`. + * + * `u` depends on `(seed, lower(stem))` alone, which is what makes the vacated sets NESTED + * as `p` grows, so a `p`-sweep varies the vacancy set and nothing else. + * + * The `>> 11n` is mandatory and is a departure from the source's `top64 / 2**64`: a + * 64-bit integer divided by 2^64 is not exactly representable as a float64, so Python and + * JavaScript can land on different doubles for the same digest and disagree about a word + * at the boundary. Shifting to 53 bits makes the numerator exact in both — and + * `Number()` of a BigInt below 2^53 is itself exact, so no rounding enters here either. + */ +export function vacancyU(stem: string, seed: number): number { + if (!Number.isInteger(seed)) { + throw new Error(`vacancyU: seed must be an integer, got ${seed}`); + } + const hex = sha256Hex(utf8Bytes(`${seed}:${stem.toLowerCase()}`)); + return Number(BigInt("0x" + hex.slice(0, 16)) >> 11n) / 2 ** 53; +} + +// --- §5.3 the deterministic byte stream ---------------------------------------------- + +/** + * `bytesFor(seed, stem, salt, counter) = sha256(f"{seed}:mint:{stem}:{salt}:{counter}")`, + * consumed four bytes at a time as a big-endian uint32 and refilled from the next + * `counter` when exhausted. This replaces `random.Random`, which cannot be reproduced + * across the two languages (departure 3). + */ +class MintStream { + private readonly prefix: string; + private buf: Uint8Array; + private pos = 0; + private counter = 0; + + /** `tag` separates the minting stream from the swap-draw stream of §8.3. It is part of + * the hashed prefix, so the two can never alias however the salts line up. */ + constructor(seed: number, stem: string, salt: number, tag: string = "mint") { + this.prefix = `${seed}:${tag}:${stem}:${salt}`; + this.buf = digestBytes(`${this.prefix}:0`); + } + + nextU32(): number { + if (this.pos + 4 > this.buf.length) { + this.counter += 1; + this.buf = digestBytes(`${this.prefix}:${this.counter}`); + this.pos = 0; + } + const v = u32At(this.buf, this.pos); + this.pos += 4; + return v; + } + + /** `list[nextU32() % len(list)]`. Every list here is shorter than 256, so the modulo + * bias is aesthetic rather than statistical — but both stacks bias IDENTICALLY. */ + choice(items: readonly T[]): T { + return items[this.nextU32() % items.length]; + } +} + +// --- §5.5 the mint loop -------------------------------------------------------------- + +/** Attempt `a >= 400` drops the syllable-count check. */ +const ATTEMPT_DROP_SYLLABLES = 400; +/** Attempt `a >= 800` drops the length check. */ +const ATTEMPT_DROP_LENGTH = 800; +/** Attempt `a >= 1200` raises — it has never happened and if it does we want to know. */ +const ATTEMPT_GIVE_UP = 1200; + +/** Collapse runs: `bbb -> bb`. */ +const RUN_RE = /([bcdfghjklmnpqrstvwxz])\1{2,}/g; + +/** + * Mint one nonce (§5.5). + * + * Depends only on `(seed, key, pattern, forbidden, baseSalt)` — never on `p`, never on + * the document, never on the order words are encountered while rewriting. That is what + * makes a stem's nonce the same at every `p` (§5.6). + * + * TWO COUNTERS, and the distinction is load-bearing (§5.5). The byte stream is keyed on + * `salt = baseSalt + a`, but the quality thresholds are on the ATTEMPT counter `a`, which + * restarts at 0 in every call. Read the other way — thresholds on the absolute salt — a + * re-mint starting at `baseSalt = 1001` would begin with every check already relaxed, so + * the replacement would not be prosody-matched, and round 2 would blow straight past the + * give-up bound and contradict §5.2's "raise after 8 rounds". + * + * The relaxations replace the source's give-up path, which returned + * `syllable + str(len(self.used))` — a count of how many words happened to be minted + * first, and therefore order-dependent (departure 6). + */ +function mintNonce( + key: string, + pattern: string, + seed: number, + forbidden: ReadonlySet, + baseSalt: number, +): { nonce: string; salt: number } { + const nSyl = pattern.length; + for (let a = 0; a < ATTEMPT_GIVE_UP; a++) { + const salt = baseSalt + a; + const stream = new MintStream(seed, key, salt); + let w = ""; + for (let i = 0; i < nSyl; i++) { + if (pattern[i] === "1") { + w += stream.choice(ONSETS) + stream.choice(NUCLEI) + stream.choice(CODAS); + } else if (i === 0) { + w += stream.choice(UNSTRESSED_PREFIXES); + } else { + w += stream.choice(UNSTRESSED_TAILS); + } + } + w = w.replace(RUN_RE, "$1$1"); + if (w.length < 3 && a < ATTEMPT_DROP_LENGTH) continue; + if (forbidden.has(w)) continue; + if (a < ATTEMPT_DROP_SYLLABLES && syllables(w) !== nSyl) continue; + return { nonce: w, salt }; + } + throw new Error( + `vacancy: could not mint a nonce for ${JSON.stringify(key)} (pattern ${pattern}) ` + + `in ${ATTEMPT_GIVE_UP} attempts from base salt ${baseSalt} (§5.5) — this has never ` + + `happened; report it rather than raising the bound`, + ); +} + +// --- §8.3 the swap control ----------------------------------------------------------- + +/** Half-width of the frequency-rank window a swap replacement is drawn from. */ +export const SWAP_WINDOW = 32; +/** The window doubles every this many attempts, up to the whole pool — §5.5's relaxation + * applied to a draw, and necessary because "anything already used" depletes a window. */ +export const SWAP_WIDEN_EVERY = 64; +/** Attempt at which the prosody filter is dropped. */ +export const SWAP_RELAX_PROSODY = 1024; +/** Reaching this many attempts throws. Unlike minting, the pool is finite, so this bound is + * reachable in principle and we want to know if it ever is. */ +export const SWAP_MAX_ATTEMPTS = 4096; + +/** The two minting strategies of §7.1 / §8.3. */ +export type MintStrategy = "nonce" | "swap"; + +/** + * Occurrences per lowercased type — the frequency source the swap control ranks by. + * + * Takes the TOKEN STREAM (`tokenize(text)`), not the type set: a set has no frequencies, and + * `mint = "swap"` needs them. `buildVacancyMap` requires this explicitly and throws without + * it rather than falling back to an alphabetical rank, which would be a frequency match in + * name only — the `avoid`-parameter mistake of §5.2, one level up. + */ +export function typeCounts(tokens: Iterable): Map { + if (typeof tokens === "string") { + throw new Error("typeCounts: expected an iterable of TOKENS, got a string; pass tokenize(text)"); + } + const counts = new Map(); + for (const t of tokens) { + const key = t.toLowerCase(); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return counts; +} + +/** `(-count, type)` ascending — the tie rule `frequencyBudget` already uses, so "frequency + * rank" means one thing in this codebase. */ +function swapKeyBefore( + aCount: number, + aType: string, + bCount: number, + bType: string, +): boolean { + if (aCount !== bCount) return aCount > bCount; // higher count sorts first + return aType < bType; +} + +/** + * The replacement pool of §8.3: the domain's open-class TYPES, by frequency rank. + * + * TYPES, NOT STEMS. The stem set is exactly the set of keys the map assigns, so drawing from + * it would consume the pool exactly and leave an A-collision with nowhere to move. On the + * shipped corpus the pool is 1944 types against 1680 stems, and that slack is what the + * re-draw rounds spend. + */ +export function swapPool( + domain: Iterable, + counts: ReadonlyMap, + keep: ReadonlySet, +): string[] { + const pool = new Set(); + for (const t of domain) { + const lower = t.toLowerCase(); + if (isEligible(stemAndSuffix(lower)[0], keep)) pool.add(lower); + } + return [...pool].sort((a, b) => + swapKeyBefore(counts.get(a) ?? 0, a, counts.get(b) ?? 0, b) ? -1 : 1, + ); +} + +/** + * Where a stem sits in the frequency-ranked pool (§8.3). + * + * NOT `pool.indexOf(stem)`: 375 of the shipped corpus's 1680 eligible stems are not domain + * types at all — `hang` and `gum` reach the map only as the stems of `hanged` and `gums` — so + * a lookup would fail on a fifth of them. The rank is the position the stem's own key would + * take, with its frequency summed over its whole INFLECTIONAL FAMILY: `hang` is as frequent + * as `hanged` and `hanging` make it, which is the frequency a reader of the corpus meets. + * + * Defined as the number of pool entries sorting strictly before the stem's key, so it is a + * plain count and cannot be read two ways; the binary search is only how it is computed. + */ +export function swapRank( + stem: string, + family: Iterable, + pool: readonly string[], + counts: ReadonlyMap, +): number { + let freq = 0; + for (const t of family) freq += counts.get(t) ?? 0; + let lo = 0; + let hi = pool.length; + while (lo < hi) { + const mid = (lo + hi) >> 1; + const other = pool[mid]; + if (swapKeyBefore(counts.get(other) ?? 0, other, freq, stem)) lo = mid + 1; + else hi = mid; + } + return lo; +} + +/** + * Draw one real-word replacement for `stem` (§8.3), as `{ word, salt, forms }` — `forms` + * being the surfaces the stem now owns, one per suffix it occurs with. + * + * Deterministic in `(seed, stem, pool, used, claimed, baseSalt)` and independent of `p`, + * exactly as `mintNonce` is, so §5.6's stability property survives the swap control. + * + * CONDITIONS A AND B₁ ARE ENFORCED HERE, AT DRAW TIME, not only checked afterwards. A + * candidate is rejected when any surface it would produce is already claimed by an earlier + * stem, is an ineligible domain type, or equals the very type it replaces. Checking only + * afterwards costs 29 collision rounds at seed 0 and does not converge inside the 8 §5.2 + * allows: the pool holds inflected types, so a bare stem drawing `years` and a suffixed one + * drawing `year` land on the same surface, and re-drawing one of them walks into the next + * such pair. Enforcing at draw time makes the map correct by construction. + */ +function drawSwap( + stem: string, + seed: number, + matchProsody: boolean, + pool: readonly string[], + rank: ReadonlyMap, + used: ReadonlySet, + suffixes: readonly string[], + claimed: ReadonlySet, + barred: ReadonlySet, + baseSalt: number, +): { word: string; salt: number; forms: string[] } { + const n = pool.length; + if (n === 0) { + throw new Error( + "vacancy: the swap pool is empty — no domain type has an eligible stem, so there is " + + "no real word to draw (architecture.md §8.3)", + ); + } + const r = rank.get(stem); + if (r === undefined) throw new Error(`vacancy: no swap rank for stem ${JSON.stringify(stem)}`); + const pattern = matchProsody ? stress(stem) : null; + const family = new Set(suffixes.map((s) => stem + s)); + for (let a = 0; a < SWAP_MAX_ATTEMPTS; a++) { + const salt = baseSalt + a; + // The doubling is capped at 20 before the min because JavaScript's `<<` takes its shift + // count modulo 32; an uncapped `a / 64` reaches 63 at the give-up bound and Python would + // compute a different width from the same attempt. 32 << 20 already exceeds any pool. + const width = Math.min(SWAP_WINDOW << Math.min(Math.floor(a / SWAP_WIDEN_EVERY), 20), n); + const offset = new MintStream(seed, stem, salt, "swap").nextU32() % (2 * width); + const delta = offset < width ? offset - width : offset - width + 1; + const candidate = pool[(((r + delta) % n) + n) % n]; + if (candidate === stem || used.has(candidate) || family.has(candidate)) continue; + if (pattern !== null && a < SWAP_RELAX_PROSODY && stress(candidate) !== pattern) continue; + const forms = suffixes.map((suffix) => surfaceForm(stem, suffix, candidate, seed)); + if (forms.some((f) => claimed.has(f) || barred.has(f) || family.has(f))) continue; + if (new Set(forms).size !== forms.length) continue; + return { word: candidate, salt, forms }; + } + throw new Error( + `vacancy: could not draw a swap replacement for ${JSON.stringify(stem)} in ` + + `${SWAP_MAX_ATTEMPTS} attempts — the pool is finite, so report this rather than ` + + `raising the bound`, + ); +} + +/** + * §5.7. Re-attaching a suffix can produce a seam (`wee` + `er` -> `weeer`). When the + * nonce's last character equals the suffix's first, replace it with a character chosen by + * a hash of `(stem, suffix)` — deterministic and order-independent, where the source drew + * from a shared RNG (departure 7). + * + * `suffix` and `key` are ALWAYS the lowercased forms. This is the case-commuting invariant + * of §5.7: the seam test branches on `suffix[0]`, so a case-preserved suffix would make + * `gums -> flels` and `GUMS -> FLESS` — one source type, two surface forms, theorem false. + */ +function seamFix(nonce: string, suffix: string, key: string, seed: number): string { + if (suffix.length === 0 || nonce.length === 0) return nonce; + if (nonce[nonce.length - 1] !== suffix[0]) return nonce; + const idx = u32At(digestBytes(`${seed}:seam:${key}:${suffix}`), 0) % SEAM_CHARS.length; + return nonce.slice(0, -1) + SEAM_CHARS[idx]; +} + +/** + * Source's `match_case`: ALL CAPS stays all caps, Titlecase stays titlecase. + * + * §5.7: applied ONCE, to the whole assembled surface form, with the ORIGINAL WHOLE WORD as + * the case source. It is the only step in the transform allowed to look at case, which is + * what makes the transform commute with lowercasing. + */ +function matchCase(src: string, surface: string): string { + if (src.length > 1 && src === src.toUpperCase() && /[A-Za-z]/.test(src)) return surface.toUpperCase(); + if (/^[A-Z]/.test(src)) return surface.slice(0, 1).toUpperCase() + surface.slice(1); + return surface; +} + +/** §1: every output is itself a single, COMPLETE `WORD_RE` match — checked, not assumed. + * If this ever fired, `tokenize(vacate(text))` would not align with `tokenize(text)` and + * the invariance theorem would be quietly false. */ +const WHOLE_WORD_RE = new RegExp(`^(?:${WORD_RE.source})$`); + +function assertWholeWord(output: string, input: string): string { + if (!WHOLE_WORD_RE.test(output)) { + throw new Error( + `vacancy: transform of ${JSON.stringify(input)} produced ${JSON.stringify(output)}, ` + + `which is not a single complete WORD_RE match`, + ); + } + return output; +} + +/** + * The assembled, LOWERCASED surface form of §5.7 — `stem` and `suffix` lowercase, the seam + * fixed, the suffix re-attached. This is the object conditions A and B of §5.2 range over, + * and the only thing `matchCase` is ever applied to. + */ +function surfaceForm(stem: string, suffix: string, nonce: string, seed: number): string { + return seamFix(nonce, suffix, stem, seed) + suffix; +} + +// --- §7.1 parameters ----------------------------------------------------------------- + +/** The knobs of §7.1. Defaults in `DEFAULT_VACANCY_PARAMS`. */ +export interface VacancyParams { + /** Fraction of eligible types vacated. Compared to `u` as given — the UI emits two + * decimal places and both stacks parse it as float64. */ + p: number; + /** Selects both `u` and the nonce assignment. Must be an integer. */ + seed: number; + /** One nonce per source type, corpus-wide. `false` is the source's "inconsistent + * assignment" control: same vacancy rate, no learnable identity, and DELIBERATELY no + * stability property. */ + consistent: boolean; + /** Nonce carries the stem's syllable count and stress. */ + matchProsody: boolean; + /** First N occurrences of a vacated stem keep the English form — a partial location. */ + revealAfter: number; + /** Extra words added to the closed class. */ + keep: readonly string[]; + /** How a replacement is produced (§8.3). `"nonce"` invents a phonotactically legal form; + * `"swap"` draws a REAL English word from the domain's own open-class types by frequency + * rank, so the passage stays equally nonsensical while every form remains a known word + * with ordinary tokenization. That is the control separating "wrong content" from + * "unknown form" in the pretrained arm — and §5.2a proves it can only be injective at + * full vacancy, which is where the pretrained arm measures. */ + mint: MintStrategy; +} + +export const DEFAULT_VACANCY_PARAMS: VacancyParams = { + p: 0, + seed: 0, + consistent: true, + matchProsody: true, + revealAfter: 0, + keep: [], + mint: "nonce", +}; + +/** Fill in §7.1's defaults around a partial setting. */ +export function vacancyParams(partial: Partial = {}): VacancyParams { + return { ...DEFAULT_VACANCY_PARAMS, ...partial }; +} + +// --- §5.2 the map -------------------------------------------------------------------- + +/** The `p`-independent nonce assignment, plus the facts §7.3 requires be verified. */ +export interface VacancyMap { + /** lowercase stem -> nonce, over EVERY eligible stem of the domain. The map at a given + * `p` is this map restricted to `{stem : u(stem) < p}`. */ + mapping: ReadonlyMap; + /** nonce -> intended stress pattern, so prosody scoring on a vacated corpus reflects + * what we built. `vacateText` adds to it in the `consistent = false` control. */ + mintedStress: Map; + /** How many re-mint rounds conditions A/B of §5.2 needed. Reported, not assumed. */ + remintRounds: number; + /** Conditions A and B both hold — §7.3's injectivity, measured at build time and + * therefore true at EVERY `p`, not only at `p = 1`. */ + bijective: boolean; + /** `|image of the domain under the full map|`, reported in the statistics (§10). */ + imageSize: number; + /** Every replacement ever handed out plus the whole domain — INCLUDING nonces a re-mint + * round superseded (§5.8). Stored, never reconstructed as `domain ∪ mapping.values()`, + * which drops the superseded ones (`wak` at seed 7); the `consistent = false` control + * draws against this set, and a form rejected for cause must stay out of circulation. */ + forbidden: ReadonlySet; + /** The lowercased domain (corpus types ∪ budget words). Condition B ranges over it and + * §10's type counts are defined over it. */ + domain: ReadonlySet; + /** Injective at EVERY `p`, or only at full vacancy? `true` for `mint = "nonce"`, where + * condition B keeps every image out of the domain; `false` for `mint = "swap"`, whose + * images ARE domain types — §5.2a proves no `p`-stable swap can do better, and + * `mapVocabWords` refuses the cases where it matters. */ + injectiveAtEveryP: boolean; +} + +/** How many re-mint rounds §5.2 allows before raising. */ +const MAX_REMINT_ROUNDS = 8; + +/** + * The §5.2 domain: `corpus types ∪ the FULL Dolch list`. Every call site uses this rather + * than building the union itself — the asymmetry of Python having such a helper and + * TypeScript not is exactly how two call sites end up constructing the domain two + * different ways. + * + * The full list ALWAYS, never the active budget: the domain must not depend on which + * budget the reader has selected, or switching budgets would re-mint the corpus in front + * of them and the stability the panel is demonstrating would look false. A frequency + * budget needs no special case, since its words are corpus types by construction. + * + * Takes an iterable of TYPES, not a text. The guard is not pedantry in either language: a + * `string` is itself an iterable of characters, so `vacancyDomain(corpusText)` would + * silently yield a domain of single letters — every one of which fails the length test of + * §2.2, giving an empty map and a transform that does nothing, with no error anywhere. + */ +export function vacancyDomain(types: Iterable): string[] { + if (typeof types === "string") { + throw new Error( + "vacancyDomain: expected an iterable of TYPES, got a string. A string iterates " + + "character by character and would yield a domain of single letters. Pass tokenize(text).", + ); + } + const out = new Set(); + for (const t of types) out.add(t.toLowerCase()); + for (const w of dolchBudget("full")) out.add(w.toLowerCase()); + // Sorted so the helper is a pure function with a canonical order. The order does not + // reach the map — `buildVacancyMap` sorts the stems itself — which is asserted. + return [...out].sort(); +} + +/** A domain type decomposed for the surface-form check: both parts lowercase. */ +interface StemSuffixPair { + stem: string; + suffix: string; +} + +/** The full-map (i.e. `p = 1`) image of a lowercased type. */ +function imageOfType(type: string, map: ReadonlyMap, seed: number): string { + const [stem, suffix] = stemAndSuffix(type); + const nonce = map.get(stem); + if (nonce === undefined) return type; + return surfaceForm(stem, suffix, nonce, seed); +} + +/** + * Conditions A and B of §5.2, evaluated over the assembled surface forms. Returns the + * stems that must be re-minted — empty when the map is injective at every `p`. + * + * A. the surface forms are pairwise distinct + * B. no surface form equals a type in `barred`, nor the very type it replaces + * + * `barred` is EVERY domain type under `mint = "nonce"` — condition B as §5.2 states it — and + * only the types that can never be vacated (the ineligible ones) under `mint = "swap"`, whose + * replacements are domain types by construction. §5.2a works out why those are the two + * choices and proves that swap cannot have the stronger one. + * + * The loser of an A-collision is the stem later in canonical (ASCII-ascending) order among + * those involved; the winner keeps its nonce, so a re-mint never cascades (§5.8). A + * B-violation has no winner — the English word is fixed — so the offending stem re-mints. + */ +function injectivityOffenders( + pairs: readonly StemSuffixPair[], + map: ReadonlyMap, + barred: ReadonlySet, + seed: number, +): Set { + const offenders = new Set(); + const bySurface = new Map>(); + for (const { stem, suffix } of pairs) { + const nonce = map.get(stem); + if (nonce === undefined) continue; + const surface = surfaceForm(stem, suffix, nonce, seed); + if (barred.has(surface) || surface === stem + suffix) offenders.add(stem); // B + const bucket = bySurface.get(surface); + if (bucket === undefined) bySurface.set(surface, new Set([stem])); + else bucket.add(stem); + } + for (const [, stems] of bySurface) { + if (stems.size < 2) continue; // A: one stem, one surface — nothing to break the tie + const sorted = [...stems].sort(); + for (let i = 1; i < sorted.length; i++) offenders.add(sorted[i]); + } + return offenders; +} + +/** + * Build the nonce assignment ONCE over the whole type set, in canonical order (§5.2). + * + * domain := { lower(t) for t in types } + * stems := sorted({ stemOf(t) for t in domain if eligible(stemOf(t)) }) + * used := {} + * for stem in stems: # canonical order — never p, never document order + * nonce := mint(stem, seed, matchProsody, forbidden = used ∪ domain) + * used.add(nonce); map[stem] = nonce + * + * `types` must be the UNION of the corpus's type set and the full Dolch list — build it + * with `vacancyDomain`, never by hand. §7.2 pushes budget words through the same + * transform, so a budget word absent from the corpus still needs an image. + * + * THERE IS NO CALLER-SUPPLIED `avoid` PARAMETER. The domain is always avoided, implicitly. + * Both stacks first gave `avoid` a default of empty and left the caller to pass the type + * set; both agreed with each other, so no parity test could catch it — but the map was + * then a function of what the caller remembered to pass. Measured: the same corpus and + * seed give different nonces, and a different `remintRounds`, depending only on whether + * the caller passed the set. Both maps are valid, which is precisely the problem — one + * caller passing it and another not (the panel and the golden fixture, say) is a silent + * divergence with nothing failing. Condition B below already forbids a surface form equal + * to any domain type, so avoiding the domain at mint time is not extra policy, only the + * cheaper route to the same fixed point. Afterwards the map is a pure function of + * `(domain, seed, matchProsody)` — asserted in the tests, through two call paths. + * + * The source accepts an `avoid` parameter and then never passes one, which lets a minted + * form silently merge with an English type (departure 5). We do not repeat that by making + * it optional. + * + * INJECTIVITY IS VERIFIED, NOT ASSUMED, AND AT EVERY `p` (§5.2 / §7.3). Two weaker checks + * were tried first and both were wrong, each for a reason worth keeping written down: + * + * * checking BARE NONCES misses the collision that arrives through the suffix; + * * checking `|image| == |types|` AT `p = 1` ONLY misses it too, because at full vacancy + * every eligible type has moved and no minted form can meet a surviving English word. + * + * The measured example, on the shipped corpus: at `seed = 7` the stem `hang` minted `wak`. + * No corpus type equals `wak`, so a bare-nonce check passes; at `p = 1` `waked` is itself + * vacated, so a full-vacancy check passes; but at `p ∈ {0.25, 0.5}` `hanged` is vacated and + * `waked` is not, so `hanged -> wak + ed = waked` collides with the English `waked` and + * injectivity — and with it §7.3 — fails. + * + * So the check is conditions A and B of `injectivityOffenders`, over assembled surface + * forms, both `p`-independent, which is what makes injectivity hold simultaneously for + * every `p`. Offenders are re-minted in canonical order at salt + * `1000 * round + previousSalt + 1` (round from 1, §5.8); only the loser moves, so a + * re-mint never cascades. Eight rounds and then it raises. + */ +export function buildVacancyMap( + types: Iterable, + params: VacancyParams, + counts?: ReadonlyMap, +): VacancyMap { + const keep = effectiveKeepSet(params.keep); + const swapping = params.mint === "swap"; + if (params.mint !== "nonce" && !swapping) { + throw new Error(`vacancy: unknown mint strategy ${JSON.stringify(params.mint)}`); + } + if (swapping && !params.consistent) { + throw new Error( + "vacancy: mint = 'swap' requires consistent = true — the inconsistent control needs a " + + "fresh type per occurrence and the corpus has 1680 open-class stems against 8202 " + + "vacated tokens, so there is no supply of real words (architecture.md §8.3)", + ); + } + if (swapping && counts === undefined) { + throw new Error( + "vacancy: mint = 'swap' needs the corpus's type counts to rank the replacement pool by " + + "frequency (architecture.md §8.3); pass typeCounts(tokenize(text))", + ); + } + const domain = new Set(); + for (const t of types) domain.add(t.toLowerCase()); + + // `pairs` is one (stem, suffix) per domain type with an eligible stem — the objects + // conditions A and B range over. `stems` is their canonical, ASCII-ascending order. + const pairs: StemSuffixPair[] = []; + const stems = new Set(); + const families = new Map>(); + const suffixesOf = new Map(); + const eligibleTypes = new Set(); + for (const t of [...domain].sort()) { + const [stem, suffix] = stemAndSuffix(t); + if (!isEligible(stem, keep)) continue; + pairs.push({ stem, suffix }); + stems.add(stem); + (families.get(stem) ?? families.set(stem, new Set()).get(stem)!).add(t); + (suffixesOf.get(stem) ?? suffixesOf.set(stem, []).get(stem)!).push(suffix); + eligibleTypes.add(t); + } + const ordered = [...stems].sort(); + + // Condition B's scope: EVERY domain type under `nonce`, and only the types that can never + // be vacated under `swap` — §5.2a's B₁, which is what full-vacancy injectivity needs and + // all a map drawing from the domain can possibly satisfy. + const barred = new Set(); + for (const t of domain) if (!swapping || !eligibleTypes.has(t)) barred.add(t); + + // The domain is forbidden from the start — implicitly, never by caller agreement. The set + // accumulates and is never pruned, so a superseded nonce stays out of circulation (§5.8). + const forbidden = new Set(domain); + const map = new Map(); + const mintedStress = new Map(); + const patterns = new Map(); + const salts = new Map(); + let pool: string[] = []; + const rank = new Map(); + if (swapping) { + pool = swapPool(domain, counts!, keep); + for (const stem of ordered) rank.set(stem, swapRank(stem, families.get(stem)!, pool, counts!)); + const used = new Set(); + const claimedForms = new Set(); + for (const stem of ordered) { + const { word, salt, forms } = drawSwap( + stem, + params.seed, + params.matchProsody, + pool, + rank, + used, + suffixesOf.get(stem)!, + claimedForms, + barred, + 0, + ); + used.add(word); + for (const f of forms) claimedForms.add(f); + forbidden.add(word); + map.set(stem, word); + salts.set(stem, salt); + } + } else { + for (const stem of ordered) { + const pattern = params.matchProsody ? stress(stem) : "1"; + patterns.set(stem, pattern); + const { nonce, salt } = mintNonce(stem, pattern, params.seed, forbidden, 0); + forbidden.add(nonce); + map.set(stem, nonce); + mintedStress.set(nonce, pattern); + salts.set(stem, salt); + } + } + + let remintRounds = 0; + for (;;) { + const offenders = injectivityOffenders(pairs, map, barred, params.seed); + if (offenders.size === 0) break; + remintRounds += 1; + if (remintRounds > MAX_REMINT_ROUNDS) { + throw new Error( + `vacancy: conditions A/B of architecture.md §5.2 still violated after ` + + `${MAX_REMINT_ROUNDS} re-mint rounds (${offenders.size} stems outstanding, e.g. ` + + `${JSON.stringify([...offenders].sort().slice(0, 5))})`, + ); + } + // Canonical order, so the result does not depend on iteration order. + for (const stem of [...offenders].sort()) { + const previousSalt = salts.get(stem); + if (previousSalt === undefined) { + throw new Error(`vacancy: re-mint offender ${JSON.stringify(stem)} was never minted`); + } + const startSalt = 1000 * remintRounds + previousSalt + 1; + if (swapping) { + // A superseded REPLACEMENT returns to the pool, unlike a superseded nonce, which + // stays forbidden forever. The pool is finite (1944 real words against 1680 stems on + // the shipped corpus), so retiring words permanently would starve later rounds; and a + // real word cannot recreate the collision it was rejected for the way a nonce can, + // because that collision was with another stem's surface, which has itself moved. + const others = new Set(); + for (const [s, w] of map) if (s !== stem) others.add(w); + const elsewhere = new Set(); + for (const pair of pairs) { + if (pair.stem === stem) continue; + elsewhere.add(surfaceForm(pair.stem, pair.suffix, map.get(pair.stem)!, params.seed)); + } + const drawn = drawSwap( + stem, + params.seed, + params.matchProsody, + pool, + rank, + others, + suffixesOf.get(stem)!, + elsewhere, + barred, + startSalt, + ); + forbidden.add(drawn.word); + map.set(stem, drawn.word); + salts.set(stem, drawn.salt); + continue; + } + const pattern = patterns.get(stem); + if (pattern === undefined) { + throw new Error(`vacancy: re-mint offender ${JSON.stringify(stem)} was never minted`); + } + const { nonce, salt } = mintNonce(stem, pattern, params.seed, forbidden, startSalt); + // The superseded nonce stays in `forbidden` (it is not handed to anyone else) but + // leaves `mintedStress`, where a stale key would claim a pattern nothing carries. + const superseded = map.get(stem); + if (superseded !== undefined) mintedStress.delete(superseded); + forbidden.add(nonce); + map.set(stem, nonce); + mintedStress.set(nonce, pattern); + salts.set(stem, salt); + } + } + + const image = new Set(); + for (const t of domain) image.add(imageOfType(t, map, params.seed)); + + return { + mapping: map, + mintedStress, + remintRounds, + // MEASURED, not asserted. The loop above exits only when A and B hold, so this is `true` + // whenever it returns — but a hardcoded `true` is a claim about the loop rather than + // about the map, and Python has always computed it. Two stacks reporting the same field + // by two different routes is how a divergence hides. + bijective: image.size === domain.size, + imageSize: image.size, + forbidden, + domain, + injectiveAtEveryP: !swapping, + }; +} + +// --- §5 / §7 the transform ----------------------------------------------------------- + +/** Per-`vacateText` mutable state; nothing here survives the call. */ +interface TransformState { + /** stem -> how many eligible, vacancy-decided occurrences have been seen so far. */ + counts: Map; + /** Growing forbidden set for the `consistent = false` control. */ + forbidden: Set; +} + +/** + * The core rewrite of one `WORD_RE` match. `state` is `undefined` for the order-free + * paths (`mapVocabWords`), which have no occurrence index and therefore no `revealAfter`. + * + * Everything is computed on the LOWERCASED word and `matchCase` is applied once at the + * end, which is what makes `lower(transformWord(w)) === transformWord(lower(w))` (§5.7). + */ +function transformWordWith( + word: string, + vmap: VacancyMap, + params: VacancyParams, + keep: ReadonlySet, + state?: TransformState, +): string { + const [stem, suffix] = stemAndSuffix(word.toLowerCase()); + if (!isEligible(stem, keep)) return word; + if (!(vacancyU(stem, params.seed) < params.p)) return word; + + // 0-based occurrence index of this STEM in document order (§5.8). + let index = 0; + if (state !== undefined) { + index = state.counts.get(stem) ?? 0; + state.counts.set(stem, index + 1); + if (index + 1 <= params.revealAfter) return word; + } else if (params.revealAfter > 0) { + throw new Error( + "vacancy: revealAfter > 0 needs an occurrence order; use vacateText, not mapVocabWords", + ); + } + + let nonce: string; + if (params.consistent) { + const mapped = vmap.mapping.get(stem); + if (mapped === undefined) { + throw new Error( + `vacancy: no nonce for stem ${JSON.stringify(stem)} — the map's domain must include ` + + `every type of the corpus AND every word of the budget (architecture.md §5.2)`, + ); + } + nonce = mapped; + } else { + if (state === undefined) { + throw new Error("vacancy: consistent = false needs an occurrence order; use vacateText"); + } + // The control condition: a fresh type per occurrence, so the vacancy rate is held + // fixed while the learnable identity is destroyed. It has, deliberately, no stability + // property — the nonce is a function of (stem, occurrenceIndex) in document order. + // `#` is not a legal WORD_RE character, so the key can never collide with a stem. + // + // CONDITION B APPLIES HERE TOO (§5.8): the nonce may equal neither a domain type nor + // THE STEM IT REPLACES. The second clause is not implied by the first — a stem need not + // be a type. Measured: at seed 7, `p = 1`, `tak` minted `tak`, so `Taking -> Taking` and + // a token silently failed to vacate. §7.1 denies this control a *stability* property, + // which is about reusing a nonce across occurrences; it does not license a word + // surviving the transform, and a control whose vacancy rate is not the stated rate is + // not a control. Forbidding the stem routes the collision through §5.5's ordinary + // re-mint loop, so the replacement meets the same quality bar as any other nonce. + // The stem is forbidden for THIS mint only, then restored — the set is threaded through + // the whole rewrite, and a stem left in it would forbid that string to every later + // occurrence of every OTHER stem, which condition B does not ask for. + const pattern = params.matchProsody ? stress(stem) : "1"; + const stemWasForbidden = state.forbidden.has(stem); + state.forbidden.add(stem); + let minted; + try { + minted = mintNonce(`${stem}#${index}`, pattern, params.seed, state.forbidden, 0); + } finally { + if (!stemWasForbidden) state.forbidden.delete(stem); + } + nonce = minted.nonce; + state.forbidden.add(nonce); + vmap.mintedStress.set(nonce, pattern); + } + const surface = surfaceForm(stem, suffix, nonce, params.seed); + return assertWholeWord(matchCase(word, surface), word); +} + +/** + * Transform ONE word, in the mapped condition (`consistent = true`, `revealAfter = 0`). + * This is `transformWord` as §1 and §7.2 name it; `vacateText` is this applied to every + * `WORD_RE` match of a text, and `mapVocabWords` is this applied to a budget in order. + */ +export function transformWord(word: string, vmap: VacancyMap, params: VacancyParams): string { + return transformWordWith(word, vmap, params, effectiveKeepSet(params.keep)); +} + +/** + * Rewrite a text in place (§1). Every `WORD_RE` match is replaced by `transformWord`; + * everything else — whitespace, punctuation, digits, line breaks — passes through + * unchanged, byte for byte. Since every output is itself a single complete `WORD_RE` + * match, `tokenize(vacateText(t))` has the same length and ordering as `tokenize(t)`, and + * because line breaks are untouched the ``-per-line rule fires in the same places. + */ +export function vacateText(text: string, vmap: VacancyMap, params: VacancyParams): string { + const keep = effectiveKeepSet(params.keep); + const state: TransformState = { counts: new Map(), forbidden: new Set(vmap.forbidden) }; + // A fresh RegExp per call: the shared `g`-flagged literal carries `lastIndex`. + return text.replace(new RegExp(WORD_RE.source, "g"), (m) => + transformWordWith(m, vmap, params, keep, state), + ); +} + +/** + * §7.2, MAPPED VOCABULARY. Push the budget's word list through the SAME transform, + * PRESERVING ORDER, so `itos_p = SPECIALS ++ mapVocabWords(words, ...)` gives every word + * the id its pre-image had. That, with the map's injectivity, is what makes the token id + * stream identical and training bit-identical (§7.3). + * + * Valid only for the mapped condition (`consistent = true`, `revealAfter = 0`); every + * other condition rebuilds the budget from the vacated corpus instead, and the resulting + * collapse in coverage IS the measurement. + */ +export function mapVocabWords( + words: readonly string[], + vmap: VacancyMap, + params: VacancyParams, +): string[] { + if (!params.consistent) { + throw new Error("vacancy: mapVocabWords requires consistent = true (architecture.md §7.2)"); + } + if (params.revealAfter !== 0) { + throw new Error("vacancy: mapVocabWords requires revealAfter = 0 (architecture.md §7.2)"); + } + if (!vmap.injectiveAtEveryP && params.p > 0 && params.p < 1) { + // §5.2a: swap's images ARE domain types, so at intermediate `p` a vacated type can land + // on an un-vacated one and two budget words would share a row. That is not a defect to be + // re-drawn away — the theorem there shows no `p`-stable swap avoids it — so the mapped + // vocabulary is refused, exactly as it is for the two controls above. + throw new Error( + `vacancy: mint = 'swap' has no mapped vocabulary at p = ${params.p}: its replacements ` + + `are domain types, so a vacated type can collide with an un-vacated one and the map ` + + `is injective only at full vacancy (architecture.md §5.2a). Use p = 0 or p = 1, or ` + + `rebuild the budget from the vacated corpus`, + ); + } + const keep = effectiveKeepSet(params.keep); + return words.map((w) => transformWordWith(w, vmap, params, keep)); +} + +// --- §10 statistics ------------------------------------------------------------------ + +/** + * The statistics contract of §10 — these exact names (camelCase in BOTH stacks, §5.8), + * computed from these definitions. + * + * The counting definitions are pinned because the two implementations first disagreed on + * them: `tokensVacated` agreed to the token while the type counts read 1 922 against + * 1 665, which was a gap in the contract and not a disagreement about the transform. Types + * are counted over the DOMAIN (corpus types ∪ budget words); tokens over the CORPUS. + */ +export interface VacancyStats { + /** Distinct lowercased types in the §5.2 DOMAIN (corpus types ∪ budget words). The + * diagnostic scope: it governs the map and the mapped vocabulary. */ + domainTypesTotal: number; + /** Domain types whose STEM is eligible per §2.2. */ + domainTypesEligible: number; + /** Domain types the MAP would rewrite at this `p` — map membership, not text. The + * domain's 22 Dolch-only words never occur in the corpus, so there is nothing to + * measure for them; this scope is a diagnostic about the map. */ + domainTypesVacated: number; + /** Distinct lowercased types of the CORPUS itself. **This is the scope a panel shows a + * reader**: the domain-only words (`funny`, `squirrel`, `today`, …) are in the budget + * but never appear in the text, so counting them inflates the vacancy rate being + * reported to someone looking at that text. §10 forbids an unprefixed `types*` for + * exactly this reason — the unprefixed name read either way, and the two stacks read + * it differently. */ + corpusTypesTotal: number; + /** Corpus types whose STEM is eligible per §2.2. */ + corpusTypesEligible: number; + /** + * Corpus types MEASURED FROM THE TWO TEXTS: a type counts as vacated iff at least one of + * its occurrences actually changed. + * + * NOT map membership, which is what the golden fixture caught. Under `revealAfter > 0` a + * type whose every occurrence falls inside the reveal window is still listed in the map + * yet has changed nowhere in the text — map membership over-reports it by ~2x on this + * corpus. The readings coincide at `revealAfter = 0`, which is why it took a control + * condition to expose. The text reading is what this number claims to a reader looking at + * "N of M types vacated" about the text in front of them. + */ + corpusTypesVacated: number; + /** Distinct eligible stems — the size of the map. `typesEligible >= stemsTotal` always, + * since inflected forms share a stem. */ + stemsTotal: number; + /** Stems with `u(stem) < p` (departure 11: the source reports `len(self.map)`, which is + * `p`-independent and therefore wrong below full vacancy). */ + stemsVacated: number; + /** Over the CORPUS token stream, not the domain. */ + tokensTotal: number; + tokensVacated: number; + meanSyllablesBefore: number; + meanSyllablesAfter: number; + meanAnapestBefore: number; + meanAnapestAfter: number; + /** The hand table of §6.1 — the honesty number for English words, and the one that must + * appear beside every prosody statistic. */ + stressFromTableBefore: number; + stressFromTableAfter: number; + /** Forms we minted, whose intended pattern we registered. Known by construction but + * ASSERTED rather than verified: §5.5 accepts a candidate on syllable COUNT, so the + * count is checked and the pattern is not. */ + stressFromMintedBefore: number; + stressFromMintedAfter: number; + /** The spelling heuristic of §6.2 — i.e. a guess. */ + stressFromRuleBefore: number; + stressFromRuleAfter: number; + bijective: boolean; + imageSize: number; + remintRounds: number; +} + +function mean(xs: readonly number[]): number { + if (xs.length === 0) return 0; + let s = 0; + for (const x of xs) s += x; + return s / xs.length; +} + +/** Mean anapest over the lines that produce at least one token (§6.4). */ +function meanAnapest(text: string, minted?: ReadonlyMap): number { + const scores: number[] = []; + for (const line of splitLines(text)) { + if (wordMatches(line).length === 0) continue; + scores.push(meterScore(line, "anapest", minted)); + } + return mean(scores); +} + +/** §10's three-way split: token-weighted fractions that sum to 1. */ +function stressSplit( + words: readonly string[], + minted: ReadonlyMap, +): { table: number; mintedFrac: number; rule: number } { + if (words.length === 0) return { table: 0, mintedFrac: 0, rule: 0 }; + let table = 0; + let mintedHits = 0; + for (const w of words) { + const source = stressWithSource(w, minted).source; + if (source === "table") table++; + else if (source === "minted") mintedHits++; + } + const n = words.length; + return { table: table / n, mintedFrac: mintedHits / n, rule: (n - table - mintedHits) / n }; +} + +/** Types whose stem passes §2.2. Shared by both scopes — eligibility is a property of the + * stem alone, so there is only one reading of it to get wrong. */ +function countEligible(types: ReadonlySet, keep: ReadonlySet): number { + let eligible = 0; + for (const t of types) if (isEligible(stemAndSuffix(t)[0], keep)) eligible++; + return eligible; +} + +/** + * `vacated` BY MAP MEMBERSHIP: the types the map would rewrite at this `p`. + * + * This is the DOMAIN scope's reading, and the choice is forced rather than preferred: the + * domain contains the 22 Dolch words that never occur in the corpus, so they have no + * occurrences to measure. Under a text-measured reading the domain scope would silently + * collapse onto the corpus scope and stop being a separate diagnostic. The domain number + * answers "what does the map do", and this computes exactly that. + * + * It is deliberately NOT the corpus scope's reading — see `vacancyStats`, where the two + * readings are named and contrasted rather than left to look like one mechanism. + */ +function countVacatedByMap( + types: ReadonlySet, + vmap: VacancyMap, + params: VacancyParams, + keep: ReadonlySet, +): number { + let vacated = 0; + for (const t of types) { + const [stem, suffix] = stemAndSuffix(t); + if (!isEligible(stem, keep)) continue; + if (!(vacancyU(stem, params.seed) < params.p)) continue; + const nonce = vmap.mapping.get(stem); + if (nonce !== undefined && surfaceForm(stem, suffix, nonce, params.seed) !== t) vacated++; + } + return vacated; +} + +/** + * §10. Both sides are scored with the SAME minted map, so the split is symmetric; on the + * `Before` side `stressFromMinted` comes out 0 because the implicitly-forbidden domain + * keeps every bare nonce off the corpus's type list, and conditions A/B keep every + * assembled surface form off it too. + */ +export function vacancyStats( + original: string, + vacated: string, + vmap: VacancyMap, + params: VacancyParams, +): VacancyStats { + const before = wordMatches(original); + const after = wordMatches(vacated); + if (before.length !== after.length) { + throw new Error( + `vacancy: the transform changed the token count (${before.length} -> ${after.length}); ` + + `architecture.md §1 requires a word-for-word bijection`, + ); + } + + const keep = effectiveKeepSet(params.keep); + const corpusTypes = new Set(before.map((w) => w.toLowerCase())); + + // TWO SCOPES, TWO DELIBERATELY DIFFERENT READINGS — stated here rather than left to be + // inferred, because the golden fixture caught them being silently different mechanisms. + // + // domain: MAP MEMBERSHIP. A diagnostic about the map. The 22 Dolch-only words have no + // occurrences in the text, so a text reading cannot see them at all. + // corpus: MEASURED FROM THE TWO TEXTS. A type counts as vacated iff at least one of its + // occurrences actually changed — which is what the number claims to a reader, + // who is looking at "N of M types vacated" about the text in front of them. + // + // The readings coincide at `revealAfter = 0`, which is why only a control condition + // exposed the difference. At `revealAfter > 0` a type whose every occurrence falls inside + // the reveal window is still in the map but has changed nowhere in the text; counting it + // over-reports by roughly 2x on this corpus. + const domainVacated = countVacatedByMap(vmap.domain, vmap, params, keep); + + let tokensVacated = 0; + const changedTypes = new Set(); + for (let i = 0; i < before.length; i++) { + const was = before[i].toLowerCase(); + if (was === after[i].toLowerCase()) continue; + tokensVacated++; + changedTypes.add(was); + } + + let stemsVacated = 0; + for (const stem of vmap.mapping.keys()) if (vacancyU(stem, params.seed) < params.p) stemsVacated++; + + const splitBefore = stressSplit(before, vmap.mintedStress); + const splitAfter = stressSplit(after, vmap.mintedStress); + + return { + domainTypesTotal: vmap.domain.size, + domainTypesEligible: countEligible(vmap.domain, keep), + domainTypesVacated: domainVacated, + corpusTypesTotal: corpusTypes.size, + corpusTypesEligible: countEligible(corpusTypes, keep), + corpusTypesVacated: changedTypes.size, + stemsTotal: vmap.mapping.size, + stemsVacated, + tokensTotal: before.length, + tokensVacated, + meanSyllablesBefore: mean(before.map((w) => syllables(w, vmap.mintedStress))), + meanSyllablesAfter: mean(after.map((w) => syllables(w, vmap.mintedStress))), + meanAnapestBefore: meanAnapest(original, vmap.mintedStress), + meanAnapestAfter: meanAnapest(vacated, vmap.mintedStress), + stressFromTableBefore: splitBefore.table, + stressFromTableAfter: splitAfter.table, + stressFromMintedBefore: splitBefore.mintedFrac, + stressFromMintedAfter: splitAfter.mintedFrac, + stressFromRuleBefore: splitBefore.rule, + stressFromRuleAfter: splitAfter.rule, + bijective: vmap.bijective, + imageSize: vmap.imageSize, + remintRounds: vmap.remintRounds, + }; +} diff --git a/code/frontend/src/lib/staticClient/arch.ts b/code/frontend/src/lib/staticClient/arch.ts index d60578e..0e0987e 100644 --- a/code/frontend/src/lib/staticClient/arch.ts +++ b/code/frontend/src/lib/staticClient/arch.ts @@ -15,21 +15,41 @@ import type { ArchGraph, ArchTrace, ArchTraceParams, + ArchVacancyDifference, + ArchVacancyRefusal, + ArchVacancyScoreBody, + ArchVacancyScoreResult, + ArchVacancyStats, ArchWeightsData, ArchWeightsParams, TokenizeResult, } from "../dataClient"; +import { WORD_RE, tokenize } from "../lexEngine"; +import { + buildVacancyMap, + typeCounts, + vacancyDomain, + vacancyParams, + vacateText, +} from "../lexEngine/vacancy"; import type { StaticAssets, StaticIndexModel } from "./assets"; +import { preservedTokenIndices, tokenByteSpans, wordSpans, type WordSpan } from "./byteSpans"; import { computeError, invalidParamError, notFoundError, staticModeError } from "./errors"; -import { IDLE_GENERATION_INFO, type ArchRuntime, type RuntimeGenerationInfo, type RuntimeLoader } from "./runtimeTypes"; +import { + IDLE_GENERATION_INFO, + type ArchRuntime, + type RuntimeGenerationInfo, + type RuntimeLoader, + type RuntimeScoredText, +} from "./runtimeTypes"; import { SafetensorsFile, asMatrixShape } from "./safetensors"; const DEFAULT_MAX_CELLS = 4096; // ARCH_WEIGHTS_MAX_CELLS (backend config) const EXACT_CELLS_HARD_CAP = 65536; // bound the number/size of range reads const TRACE_EXPORT_MAX_CONTEXT = 64; // the exporter ran the backend default -/** Community ONNX exports for the curated models (repos verified on the Hub; - * all ship model_q4f16.onnx + a q8 "quantized" file). */ +/** Community ONNX exports for the curated models (repos verified on the Hub; all ship + * the `model_quantized.onnx` the runtime's q8 ladder loads — see transformersRuntime). */ const ONNX_REPOS: Record = { "HuggingFaceTB/SmolLM2-135M-Instruct": "onnx-community/SmolLM2-135M-Instruct-ONNX", "HuggingFaceTB/SmolLM2-360M-Instruct": "onnx-community/SmolLM2-360M-Instruct-ONNX", @@ -101,6 +121,348 @@ export function dequantizeTile( return out; } +// --- the vacancy instrument, pretrained arm (contract §8) -------------------------------- + +/** The three variants, in the order the panel reads them. MIRROR of `VARIANTS` (Python). */ +const VACANCY_VARIANTS = ["english", "swap", "nonce"] as const; +type VacancyVariant = (typeof VACANCY_VARIANTS)[number]; + +/** MIRROR of `MAX_PASSAGES` — each passage costs three real forward passes. */ +const VACANCY_MAX_PASSAGES = 12; +/** MIRROR of `DEFAULT_PASSAGE_WORDS` / `DEFAULT_PASSAGE_COUNT` (the measured shape). */ +const VACANCY_PASSAGE_WORDS = 250; +const VACANCY_PASSAGE_COUNT = 6; +/** + * MIRROR of `FRONT_MATTER_FRACTION`. The shipped book opens with a title page and an + * alphabetical index of first lines — measured to end in block 11 of 63 — and a passage + * cut from that is a column of titles, not English. The digest fixture pins the two + * stacks to the same cut. + */ +const VACANCY_FRONT_MATTER_FRACTION = 0.2; + +/** + * Dtypes whose error against float32 has actually been MEASURED on this contrast + * (§8.3a). q8 is bounded at ≤ 0.054 nats on the pooled difference; q4f16 is not on this + * list because it does not compute — it returns input-independent logits — and no other + * dtype has been measured. Anything absent is refused, never extrapolated onto. + */ +const VACANCY_MEASURED_DTYPES: readonly string[] = ["q8"]; + +/** + * The uncertainty stated beside a pooled difference in static mode. + * + * DERIVED FROM TWO MEASUREMENTS, not chosen: + * + * 1. The independent six-passage study of §8.3a (2 models × 2 seeds, its own passage + * cut and its own swap implementation) bounded the pooled q8-vs-fp32 discrepancy at + * **≤ 0.054 nats** on every contrast, and recommended stating ~2× that. + * 2. This build's own configuration, measured across the two stacks on the SHIPPED + * default passage set with gpt2 — identical tokenization (2754/2858/3810 tokens, + * 847 preserved) in both, so the only difference is the dtype: + * + * swap − english : fp32 0.7166 q8 0.6440 |Δ| = 0.073 + * nonce − english : fp32 0.9892 q8 0.8790 |Δ| = 0.110 + * + * The second exceeds 0.1, so a stated ±0.1 would have been a bound its own first + * comparison broke. 0.2 is ~2× the largest discrepancy actually observed here, in the + * configuration that actually ships. Never quoted to more than one decimal place: that + * is all either measurement supports. If this constant changes, re-run BOTH stacks on + * the default set before changing it — the number is a measurement, not a margin. + */ +const VACANCY_Q8_UNCERTAINTY_NATS = 0.2; + +/** + * Preserved tokens that must be pooled before a q8 number may be shown. The bound above + * was measured on ~700 preserved tokens per condition; a single 250-word passage carries + * ~120, and at that size q8 was wrong by up to 115 % of the passage's own delta. + */ +const VACANCY_MIN_POOLED_PRESERVED = 700; + +/** MIRROR of `TINY_ARM` (Python) — the other half of the 2×2, restated for the panel. */ +const VACANCY_TINY_ARM = { + delta_nats: 0, + exact: true, + label: "the same measurement on a model with no locations", + note: + "For the from-scratch word-level GeoTransformer of the Lexicon Lab, the vacancy " + + "transform is a pure relabelling of the vocabulary: with consistent=true and " + + "revealAfter=0 the token id stream is element-for-element identical, so the training " + + "loss is bit-identical and a word's FORM is worth exactly 0. That is not a rounding " + + "— it is an identity, and it is asserted in that tab, not assumed.", +}; + +/** MIRROR of `UNKNOWN_FORM_NOTE` (Python) — the residual, stated even where the number + * itself is refused: a reader must not have to earn the caveat by being shown a value. */ +const VACANCY_UNKNOWN_FORM_NOTE = + "Nonce forms fragment into more subword tokens than real words do, so this difference " + + "is the cost of an unknown form TOGETHER WITH the cost of a stranger, longer context. " + + "The two are not separable without a tokenizer-level control, so treat this as an " + + "UPPER BOUND on what a word's location was worth — never as pure location."; + +/** MIRROR of `CONFOUND_NOTE` (Python) — §8.4, stated wherever a delta is. */ +const VACANCY_CONFOUND = + "A vacated passage genuinely has higher entropy, so every prediction inside it gets " + + "worse — the scaffolding included. A positive difference is therefore expected, not a " + + "surprise: its MAGNITUDE is the result, and it is only interpretable against the tiny " + + "arm's exact zero."; + +/** + * The passage and its two vacated twins (§8.3), from the SAME TypeScript transform the + * Lexicon Lab runs — so the nonce a stem gets here is the nonce it gets there, for the + * same seed, and the golden fixture that pins TS against Python covers this too. + */ +export function vacancyVariantTexts( + passage: string, + opts: { p: number; seed: number; matchProsody: boolean; keep: readonly string[] }, +): Record { + const tokens = tokenize(passage); + const domain = vacancyDomain(tokens); + const counts = typeCounts(tokens); + const out = { english: passage } as Record; + for (const mint of ["swap", "nonce"] as const) { + // consistent / revealAfter are fixed at the invariance theorem's condition: the tiny + // arm's exact zero holds only there, and this number is only interpretable beside it. + const params = vacancyParams({ + p: opts.p, + seed: opts.seed, + consistent: true, + matchProsody: opts.matchProsody, + revealAfter: 0, + keep: [...opts.keep], + mint, + }); + out[mint] = vacateText(passage, buildVacancyMap(domain, params, counts), params); + } + return out; +} + +/** + * Word spans of the English passage, and the word indices PRESERVED in EVERY variant. + * + * "Preserved" is character identity across all three, which is stronger than "closed + * class": it also covers eligible stems the `u(stem) < p` decision spared. Restricting + * all three NLLs to the same word set is what makes them comparable. + */ +export function preservedWordIndices(texts: Record): { + words: WordSpan[]; + preserved: Set; +} { + const per = {} as Record; + for (const name of VACANCY_VARIANTS) per[name] = wordSpans(texts[name], WORD_RE); + const counts = VACANCY_VARIANTS.map((n) => per[n].length); + if (new Set(counts).size !== 1) { + throw computeError( + "the variants do not have the same number of words, so preserved words cannot be " + + `aligned: ${VACANCY_VARIANTS.map((n, i) => `${n}=${counts[i]}`).join(", ")}`, + ); + } + const preserved = new Set(); + for (const w of per.english) { + if (VACANCY_VARIANTS.every((n) => per[n][w.index].word === w.word)) preserved.add(w.index); + } + return { words: per.english, preserved }; +} + +function mean(values: readonly number[]): number { + if (values.length === 0) { + throw computeError("no tokens to average — the passage has no scored positions"); + } + let sum = 0; + for (const v of values) sum += v; + return sum / values.length; +} + +/** + * §8.1's fields pooled over passages at the TOKEN level — token-weighted, never a mean of + * means, so a passage with twice the tokens carries twice the weight and the pooled + * figure really is the mean surprisal of a preserved token. + * + * The absolutes are `null` here by design: this function only ever runs in the quantized + * static build, where they are refused (§8.3a). The counts are not — a token count is + * exact at any dtype, and it is what shows the reader that nonce forms fragment. + */ +export function pooledStats( + scored: readonly RuntimeScoredText[], + preserved: readonly number[][], +): ArchVacancyStats { + let nTokens = 0; + let nChars = 0; + let nPreserved = 0; + for (let i = 0; i < scored.length; i++) { + nTokens += scored[i].nll.length - 1; // position 0 has no prediction + nChars += scored[i].nChars; + nPreserved += preserved[i].length; + } + return { + nllPreserved: null, + nllAll: null, + bitsPerChar: null, + nTokens, + nPreservedTokens: nPreserved, + nChars, + }; +} + +/** + * `mean(nll_b − nll_a)` over preserved tokens, PAIRED, with its standard error. + * + * The pairing is exact, not an approximation: preserved words are character-identical + * across variants and the curated models' pretokenizers never merge across a word + * boundary, so each preserved word yields the same pieces in every variant. Pairing + * removes the between-token variance — the same function word is compared with itself in + * the other condition — which is what makes a standard error on a ~0.1 nat effect worth + * printing. A length mismatch means the pairing assumption broke, and it refuses. + */ +export function pairedDifference( + a: readonly RuntimeScoredText[], + aPreserved: readonly number[][], + b: readonly RuntimeScoredText[], + bPreserved: readonly number[][], +): { nats: number; se: number; nPairs: number } { + const diffs: number[] = []; + for (let i = 0; i < a.length; i++) { + if (aPreserved[i].length !== bPreserved[i].length) { + throw computeError( + `the variants have ${aPreserved[i].length} and ${bPreserved[i].length} preserved ` + + "tokens, so they cannot be paired", + ); + } + for (let j = 0; j < aPreserved[i].length; j++) { + diffs.push(b[i].nll[bPreserved[i][j]] - a[i].nll[aPreserved[i][j]]); + } + } + const m = mean(diffs); + let se = NaN; + if (diffs.length > 1) { + let ss = 0; + for (const d of diffs) ss += (d - m) * (d - m); + se = Math.sqrt(ss / (diffs.length - 1) / diffs.length); + } + return { nats: m, se, nPairs: diffs.length }; +} + +/** + * Evenly spaced excerpts of the shipped corpus — MIRROR of `default_passages` (Python), + * the configuration the reference numbers of §8.3a were measured in. The first eighth of + * the book is skipped: it is the title page and table of contents, a list of titles + * rather than English. + */ +export function defaultVacancyPassages( + text: string, + count = VACANCY_PASSAGE_COUNT, + words = VACANCY_PASSAGE_WORDS, +): string[] { + const blocks: string[] = []; + let current: string[] = []; + let n = 0; + const counter = new RegExp(WORD_RE.source, "g"); + for (const line of text.split("\n")) { + current.push(line); + n += (line.match(counter) ?? []).length; + if (n >= words) { + blocks.push(current.join("\n").replace(/^\n+|\n+$/g, "")); + current = []; + n = 0; + } + } + if (blocks.length === 0) { + throw computeError("the shipped corpus produced no passage of the requested size"); + } + const start = Math.max(1, Math.round(blocks.length * VACANCY_FRONT_MATTER_FRACTION)); + const step = Math.max(1, Math.floor((blocks.length - start) / Math.max(1, count))); + return Array.from({ length: count }, (_, i) => + blocks[Math.min(start + i * step, blocks.length - 1)], + ); +} + +/** + * Refusal: absolute log-likelihoods, from a quantized model. MEASURED, not cautious. + */ +export const VACANCY_ABSOLUTE_REFUSAL: ArchVacancyRefusal = { + type: "StaticModeError", + message: + "Absolute log-likelihoods are not reportable from a quantized model: q8 shifts " + + "nllPreserved by −0.19 nats on gpt2 and +0.40 on SmolLM2-135M — the sign is not " + + "even stable across models. The pooled DIFFERENCES below survive quantization; " + + "these numbers do not. The full stack reports them at float32.", +}; + +/** Refusal: any single-passage delta, from a quantized model. */ +export const VACANCY_PER_PASSAGE_REFUSAL: ArchVacancyRefusal = { + type: "StaticModeError", + message: + "A per-passage delta is not reportable under q8: the worst measured discrepancy was " + + "0.65 nats, 115 % of that passage's own float32 delta, caused by q8 compressing the " + + "16–18 nat line-initial function words this measurement is precisely about. Only the " + + "pooled figure has a measured bound. Run the full stack for the per-passage table.", +}; + +/** Refusal: `nonce − swap`, the contrast quantization destroys. */ +export const VACANCY_UNKNOWN_FORM_REFUSAL: ArchVacancyRefusal = { + type: "StaticModeError", + message: + "This is the contrast the quantized model destroys, and it is the one the result " + + "rests on. It is a small number — 0.16–0.27 nats across the curated models at " + + "float32, 0.06–0.21 in the independent study — and q8's error on it is 14–23 % of " + + "that, up to 0.28 nats on a single passage, with a sign flip in one passage of six " + + "per model. Measured on this very configuration: float32 says 0.273 for gpt2 and q8 " + + "says 0.235, a 14 % error on the quantity the whole result turns on. The two pooled " + + "numbers shown here do differ by it — that arithmetic is not a measurement, and the " + + "difference is not reportable at this dtype. Run the full stack (uvicorn " + + "llm_geometry.api.app:app), which scores at float32, where ONNX and torch agree to " + + "5.3e-4 nats.", +}; + +/** + * What the quantized static build MAY say about the two differences it computed + * (contract §8.3a, FR-720a). Pure policy, separated from the measurement so it can be + * asserted directly: pooled `swap − english` and `nonce − english` carry a stated, + * MEASURED quantization uncertainty; `nonce − swap` carries a refusal and no number. + * + * The refusal is not squeamishness about a wide error bar. q8's error on `nonce − swap` + * is 14–23 % of an effect whose true value is 0.06–0.21 nats, and it flips sign on one + * passage in six — so a number here would not be imprecise, it would be wrong in a + * direction the reader could not detect. + */ +export function staticVacancyDifferences( + swapMinusEnglish: { nats: number; se: number; nPairs: number }, + nonceMinusEnglish: { nats: number; se: number; nPairs: number }, +): ArchVacancyDifference[] { + return [ + { + id: "wrong_content", + label: "the cost of wrong content", + expr: "nll(swap) − nll(english)", + headline: true, + quantizationUncertaintyNats: VACANCY_Q8_UNCERTAINTY_NATS, + ...swapMinusEnglish, + }, + { + id: "unknown_form", + label: "the cost of unknown form", + expr: "nll(nonce) − nll(swap)", + headline: true, + upperBound: true, + note: VACANCY_UNKNOWN_FORM_NOTE, + nats: null, + se: null, + nPairs: 0, + refused: VACANCY_UNKNOWN_FORM_REFUSAL, + }, + { + id: "total", + label: "both costs together", + expr: "nll(nonce) − nll(english)", + headline: false, + note: + "The sum of the two differences above. It conflates wrong content with unknown " + + "form and is never the headline.", + quantizationUncertaintyNats: VACANCY_Q8_UNCERTAINTY_NATS, + ...nonceMinusEnglish, + }, + ]; +} + export class ArchSection { private readonly safetensors = new Map(); private runtimePromise: Promise | null = null; @@ -340,6 +702,175 @@ export class ArchSection { return rt.tokenize(m.model_id, m.revision, text); } + /** + * The pretrained arm of the vacancy instrument, in the browser (contract §8). + * + * Everything is computed here for real: the three variants come from the TypeScript + * vacancy transform (the same one the Lexicon Lab runs, golden-tested against Python), + * and each is scored by one real forward pass on the community ONNX export. + * + * WHAT IT MAY THEN SAY is narrower than what it computed, and that is the point + * (§8.3a / FR-720a). This build runs a **q8** export, and the measurement of §8.3a + * bounds q8's error only on POOLED differences over several hundred preserved tokens: + * + * - absolute `nllPreserved` is refused — q8 moves it by −0.19 nats on gpt2 and +0.40 + * on SmolLM2-135M, so even its SIGN is not stable across models; + * - per-passage deltas are refused — worst case 0.65 nats, 115 % of that passage's + * own fp32 delta; + * - `nonce − swap` is refused — its true value is 0.06–0.21 nats and q8's error on it + * is 14–23 % pooled with sign flips, so quantization eats exactly the contrast that + * makes the result mean something; + * - pooled `swap − english` and `nonce − english` are reported, with the measured + * `VACANCY_Q8_UNCERTAINTY_NATS` quantization uncertainty stated beside the sampling + * standard error. (This line said "±0.1" until that constant was re-derived on the + * configuration that actually ships and came out at 0.2 — see its own comment. A + * number written twice is a number that drifts, so it is named here rather than + * retyped.) + * + * A dtype with no measured bound is refused outright rather than given a ± copied from + * a different dtype: a stated error bar that was never measured is a fabrication, and + * worse than no number. + */ + async archVacancyScore(body: ArchVacancyScoreBody): Promise { + const m = await this.model(body.model_id); + const repo = ONNX_REPOS[m.model_id]; + if (!repo) { + throw staticModeError( + `No browser (ONNX) export is wired up for ${m.model_id}, so this measurement ` + + "cannot be run on it here — it covers: " + + Object.keys(ONNX_REPOS).join(", ") + + ". Run the full stack (see the README) for other models.", + ); + } + if (body.passage !== undefined && body.passages !== undefined) { + throw invalidParamError( + "send either `passage` (one) or `passages` (several), not both — they would " + + "silently disagree about what was scored", + ); + } + const requested = + body.passages ?? (body.passage !== undefined ? [body.passage] : await this.defaultPassages()); + if (requested.length === 0 || requested.some((t) => !t.trim())) { + throw invalidParamError("every passage must be a non-empty string"); + } + if (requested.length > VACANCY_MAX_PASSAGES) { + throw invalidParamError( + `at most ${VACANCY_MAX_PASSAGES} passages per request, got ${requested.length}; ` + + "each one costs three real forward passes", + ); + } + const p = body.p ?? 1.0; + if (!(p >= 0 && p <= 1)) throw invalidParamError(`p must lie in [0, 1], got ${p}`); + const seed = body.seed ?? 0; + const matchProsody = body.match_prosody ?? true; + const keep = body.keep ?? []; + + // NFC once, up front: every byte span downstream indexes THIS string. Qwen's + // tokenizer normalizes internally and gpt2's/SmolLM2's do not, so without this a + // decomposed character would shift every span after it (§8.2). + const passages = requested.map((t) => t.normalize("NFC")); + const rt = await this.runtime(); + + const scored: Record = { english: [], swap: [], nonce: [] }; + const preservedIdx: Record = { english: [], swap: [], nonce: [] }; + const previews: Record = { english: "", swap: "", nonce: "" }; + for (const passage of passages) { + const texts = vacancyVariantTexts(passage, { p, seed, matchProsody, keep }); + const { words, preserved } = preservedWordIndices(texts); + if (preserved.size === 0) { + throw computeError( + "this passage has no word that survives the transform, so there is no " + + "scaffolding to score. Lower p, or use a passage with closed-class words.", + ); + } + const order = VACANCY_VARIANTS.map((name) => texts[name]); + const results = await rt.scoreTexts(repo, order); + VACANCY_VARIANTS.forEach((name, i) => { + if (!previews[name]) previews[name] = texts[name]; + const variantWords = name === "english" ? words : wordSpans(texts[name], WORD_RE); + const spans = tokenByteSpans(results[i].pieces, texts[name]); + scored[name].push(results[i]); + preservedIdx[name].push( + preservedTokenIndices(spans, variantWords, preserved).filter((j) => j > 0), + ); + }); + } + + const info = rt.info(); + const dtype = info.dtype ?? "unknown"; + const device = info.device ?? "unknown"; + if (!VACANCY_MEASURED_DTYPES.includes(dtype)) { + throw staticModeError( + `This browser loaded the model at dtype "${dtype}", and no error bound has been ` + + "measured for it. Quantization moves absolute log-likelihoods by nats and can " + + "reverse the sign of the effect this panel measures, so stating a number here " + + "would mean inventing an error bar. Run the full stack (uvicorn " + + "llm_geometry.api.app:app), which scores at float32.", + ); + } + + const pooledPreserved = preservedIdx.english.reduce((n, list) => n + list.length, 0); + if (pooledPreserved < VACANCY_MIN_POOLED_PRESERVED) { + throw staticModeError( + `Pooled over ${pooledPreserved} preserved tokens, this is below the ` + + `${VACANCY_MIN_POOLED_PRESERVED} at which q8's error on the pooled difference ` + + "was measured (≤ 0.054 nats). Below it the only honest answer is no number: a " + + "single-passage delta under q8 was wrong by up to 115 % of its own value. Add " + + "more passages, or run the full stack, which scores at float32 and reports " + + "every per-passage number.", + ); + } + + const differences = staticVacancyDifferences( + pairedDifference(scored.english, preservedIdx.english, scored.swap, preservedIdx.swap), + pairedDifference(scored.english, preservedIdx.english, scored.nonce, preservedIdx.nonce), + ); + + return { + model_id: m.model_id, + revision: m.revision, + stack: "static", + dtype, + device, + p, + seed, + match_prosody: matchProsody, + keep: [...keep], + alignment: { + mechanism: "byte-level pieces → UTF-8 byte spans", + unit: "utf8_bytes", + verified: true, + note: + "Token→word attribution is verified at run time by reconstructing the passage " + + "from the token byte spans; a mismatch raises rather than mis-attributing.", + }, + variants: VACANCY_VARIANTS.map((name) => ({ + id: name, + pooled: pooledStats(scored[name], preservedIdx[name]), + preview: previews[name].slice(0, 400), + refused: VACANCY_ABSOLUTE_REFUSAL, + })), + passages_used: passages, + differences, + passages: null, + passagesRefused: VACANCY_PER_PASSAGE_REFUSAL, + tiny_arm: VACANCY_TINY_ARM, + confound: VACANCY_CONFOUND, + }; + } + + /** The six evenly spaced corpus excerpts the measurement of §8.3a used. */ + private async defaultPassages(): Promise { + const corpus = await this.assets.json<{ text?: string }>("lex/corpus.json"); + if (typeof corpus?.text !== "string" || !corpus.text) { + throw staticModeError( + "This build did not export the corpus (static-data/lex/corpus.json), so the " + + "default passage set cannot be cut. Paste a passage, or run the full stack.", + ); + } + return defaultVacancyPassages(corpus.text.normalize("NFC")); + } + /** LIVE generation on the model's community ONNX export. */ async archGenerate(body: ArchGenerateBody): Promise { const m = await this.model(body.model_id); diff --git a/code/frontend/src/lib/staticClient/byteSpans.ts b/code/frontend/src/lib/staticClient/byteSpans.ts new file mode 100644 index 0000000..d43354e --- /dev/null +++ b/code/frontend/src/lib/staticClient/byteSpans.ts @@ -0,0 +1,153 @@ +/** + * Token→word attribution in UTF-8 BYTE coordinates (contract §8.2, FR-718). + * + * The mirror of `llm_geometry/arch/vacancy_score.py`. It is byte-based rather than + * character-based for reasons that were measured, not assumed: + * + * - transformers.js exposes NO token offsets at all (`return_offsets_mapping` is + * ignored; there is not one `offset` key anywhere in the tokenizer's object graph), + * so HF's Python-side offsets are not a mechanism the two stacks can share; + * - decoding tokens one at a time and concatenating emits U+FFFD and destroys the text + * on any split multi-byte character, in BOTH stacks; + * - Python indexes code points where JavaScript indexes UTF-16 units — the two genuinely + * disagree about the same string (31 vs 32 on one probe text), so "character" is not a + * unit a cross-language contract can be written in; + * - HF's own offsets OVERLAP on multi-byte characters, so per-token quantities summed + * over a word would be double-counted. Byte spans are a true partition. + * + * The reconstruction assertion in `tokenByteSpans` is the guard rail: if the concatenated + * pieces are not `utf8(text)` byte for byte, this raises rather than mis-attributing. + */ + +import { computeError } from "./errors"; + +/** GPT-2's unicode→byte table: the inverse of `bytes_to_unicode`. */ +function buildByteDecoder(): Map { + const bs: number[] = []; + for (let b = "!".charCodeAt(0); b <= "~".charCodeAt(0); b++) bs.push(b); + for (let b = 0xa1; b <= 0xac; b++) bs.push(b); + for (let b = 0xae; b <= 0xff; b++) bs.push(b); + const cs = [...bs]; + let n = 0; + for (let b = 0; b < 256; b++) { + if (!bs.includes(b)) { + bs.push(b); + cs.push(256 + n); + n += 1; + } + } + const out = new Map(); + for (let i = 0; i < bs.length; i++) out.set(String.fromCharCode(cs[i]), bs[i]); + return out; +} + +const BYTE_DECODER = buildByteDecoder(); + +export type ByteSpan = readonly [number, number]; + +/** + * The `[start, end)` UTF-8 byte range each token owns, verified by reconstruction. + * + * The spans tile `utf8(text)` exactly once. A token that is a bare continuation byte gets + * a degenerate EMPTY span — the honest answer for it — and still resolves to the right + * word, because its start byte lies strictly inside the character it continues. + */ +export function tokenByteSpans(pieces: readonly string[], text: string): ByteSpan[] { + const spans: ByteSpan[] = []; + const parts: number[] = []; + let cursor = 0; + for (let i = 0; i < pieces.length; i++) { + const piece = pieces[i]; + let width = 0; + for (const ch of piece) { + const b = BYTE_DECODER.get(ch); + if (b === undefined) { + throw computeError( + `token ${i} (${JSON.stringify(piece)}) contains a character outside the ` + + "byte-level BPE table, so its byte width cannot be determined", + ); + } + parts.push(b); + width += 1; + } + spans.push([cursor, cursor + width]); + cursor += width; + } + + const expected = new TextEncoder().encode(text); + let same = parts.length === expected.length; + if (same) { + for (let i = 0; i < expected.length; i++) { + if (parts[i] !== expected[i]) { + same = false; + break; + } + } + } + if (!same) { + throw computeError( + "token→text alignment failed: the concatenated byte-level pieces do not reproduce " + + `the passage (${parts.length} bytes rebuilt vs ${expected.length} expected). ` + + "Refusing to attribute tokens to words rather than mis-attribute them.", + ); + } + return spans; +} + +export interface WordSpan { + index: number; + word: string; + start: number; + end: number; +} + +/** Every `WORD_RE` match of `text`, in UTF-8 byte coordinates. */ +export function wordSpans(text: string, wordRe: RegExp): WordSpan[] { + const re = new RegExp(wordRe.source, wordRe.flags.includes("g") ? wordRe.flags : `${wordRe.flags}g`); + const encoder = new TextEncoder(); + const out: WordSpan[] = []; + let m: RegExpExecArray | null; + let index = 0; + while ((m = re.exec(text)) !== null) { + const start = encoder.encode(text.slice(0, m.index)).length; + out.push({ index, word: m[0], start, end: start + encoder.encode(m[0]).length }); + index += 1; + if (m[0] === "") re.lastIndex += 1; // defensive: never loop on an empty match + } + return out; +} + +/** + * Indices of the tokens belonging to a PRESERVED word. + * + * A token belongs to a word when their byte ranges OVERLAP — not "starts inside". A + * byte-level BPE folds a word's leading space into the word's own token, so the token + * that *is* the function word starts one byte before the word does, and the start rule + * would drop nearly all of them. + * + * A token overlapping both a preserved and a vacated word cannot be attributed, and that + * raises: the curated models' pretokenizers never merge across a word boundary, so if it + * happens the assumption behind this attribution has changed and no number may be + * reported from it. + */ +export function preservedTokenIndices( + spans: readonly ByteSpan[], + words: readonly WordSpan[], + preserved: ReadonlySet, +): number[] { + const out: number[] = []; + for (let i = 0; i < spans.length; i++) { + const [a, b] = spans[i]; + const hits = words.filter((w) => a < w.end && b > w.start); + if (hits.length === 0) continue; // punctuation, whitespace, line breaks + const flags = new Set(hits.map((w) => preserved.has(w.index))); + if (flags.size > 1) { + throw computeError( + `token ${i} spans both a preserved and a vacated word ` + + `(${hits.map((w) => JSON.stringify(w.word)).join(", ")}); refusing to attribute it`, + ); + } + if (flags.has(true)) out.push(i); + } + return out; +} diff --git a/code/frontend/src/lib/staticClient/geo.ts b/code/frontend/src/lib/staticClient/geo.ts index 33b8cd3..f4acb6d 100644 --- a/code/frontend/src/lib/staticClient/geo.ts +++ b/code/frontend/src/lib/staticClient/geo.ts @@ -45,8 +45,14 @@ import type { LocalJobRegistry, ProgressFn } from "./jobs"; // Minted weight sets persist across reloads (red-team static finding #3: the // engine's store is in-memory, so a sessionStorage token would otherwise silently // self-heal back to "learned" after a reload — the backend build persists edits). +// +// A set trained from scratch or loaded from a file also carries its OWN vocabulary, +// and `exportWeightSet` puts it in the payload: weights alone do not describe such a +// model, and restoring them alone made `Save model` write a file pairing them with the +// shipped word list under a matching `vocab_sha256` — an unrejectable wrong file. The +// backend has always stored the two together (`save_weight_set(..., vocab_json=…)`). const MINTED_SETS_KEY = "llm-geometry:static-weight-sets"; -const MINTED_SETS_CAP = 8; // LRU; each entry is ~50 KB of JSON +const MINTED_SETS_CAP = 8; // LRU; each entry is ~50 KB of JSON, ~60 KB with a vocabulary function loadPersistedSets(): Record { try { diff --git a/code/frontend/src/lib/staticClient/index.ts b/code/frontend/src/lib/staticClient/index.ts index dd32700..1c7a986 100644 --- a/code/frontend/src/lib/staticClient/index.ts +++ b/code/frontend/src/lib/staticClient/index.ts @@ -23,6 +23,8 @@ import type { ArchGraph, ArchTrace, ArchTraceParams, + ArchVacancyScoreBody, + ArchVacancyScoreResult, ArchWeightsData, ArchWeightsParams, Client, @@ -208,6 +210,10 @@ export function createStaticClient(opts: StaticClientOptions = {}): StaticClient getArchTrace: (params: ArchTraceParams, _signal?: AbortSignal): Promise => arch.getArchTrace(params), archGenerate: (body: ArchGenerateBody): Promise => arch.archGenerate(body), + archVacancyScore: ( + body: ArchVacancyScoreBody, + _signal?: AbortSignal, + ): Promise => arch.archVacancyScore(body), // Lexicon Lab: budgets, training, generation and spectra all computed in-browser by // lexEngine; only the shipped corpus text and the read-only /spec are precomputed. ...lexClientFrom(lex), diff --git a/code/frontend/src/lib/staticClient/lex.ts b/code/frontend/src/lib/staticClient/lex.ts index 11a27fb..3ac4f20 100644 --- a/code/frontend/src/lib/staticClient/lex.ts +++ b/code/frontend/src/lib/staticClient/lex.ts @@ -87,6 +87,17 @@ import { type SpectrumResult, type WeightSet, } from "../lexEngine"; +import { + buildVacancyMap, + mapVocabWords, + vacancyDomain, + vacancyParams as vacancyParamsWithDefaults, + vacancyStats, + vacateText, + type VacancyMap, + type VacancyParams, + type VacancyStats, +} from "../lexEngine/vacancy"; import { sha256Hex, utf8Bytes } from "../geoEngine/hash"; import type { StaticAssets } from "./assets"; import { computeError, invalidParamError, notFoundError, staticModeError, toApiError } from "./errors"; @@ -217,6 +228,76 @@ export interface LexCoverageResult { words: string[]; } +/** + * Contract §7.1's knobs, on the wire. `snake_case` because that is this API's convention; + * the transform's own TypeScript surface is camelCase in both stacks (§5.8), and + * `vacancyParamsFrom` is the single place the two spellings meet. + */ +export interface LexVacancyParamsBody { + p?: number; + seed?: number; + consistent?: boolean; + match_prosody?: boolean; + reveal_after?: number; + keep?: readonly string[]; +} + +export interface LexVacancyBody extends LexCoverageBody, LexVacancyParamsBody { + preview_chars?: number; +} + +/** §10's statistics, camelCase in both stacks because §10 names them that way. */ +export interface LexVacancyStats { + domainTypesTotal: number; + domainTypesEligible: number; + domainTypesVacated: number; + corpusTypesTotal: number; + corpusTypesEligible: number; + corpusTypesVacated: number; + stemsTotal: number; + stemsVacated: number; + tokensTotal: number; + tokensVacated: number; + meanSyllablesBefore: number; + meanSyllablesAfter: number; + meanAnapestBefore: number; + meanAnapestAfter: number; + stressFromTableBefore: number; + stressFromTableAfter: number; + stressFromMintedBefore: number; + stressFromMintedAfter: number; + stressFromRuleBefore: number; + stressFromRuleAfter: number; + bijective: boolean; + imageSize: number; + remintRounds: number; +} + +export interface LexVacancyResult { + p: number; + seed: number; + consistent: boolean; + match_prosody: boolean; + reveal_after: number; + keep: string[]; + /** Which of §7.2's two rules produced `words`. */ + vocabulary_rule: "mapped" | "rebuilt"; + words: string[]; + budget: { source: string; budget: string; size: number; rows: number; coverage: Coverage }; + corpus: { n_tokens: number; n_distinct: number; n_lines: number; n_chars: number }; + vacancy_stats: LexVacancyStats; + bijective: boolean; + remint_rounds: number; + preview: string; + original_preview: string; + preview_chars: number; + truncated: boolean; + vacated_chars: number; + vacated_sha256: string; + original_chars: number; + original_sha256: string; +} + export interface LexTrainBody extends LexCoverageBody, LexShapeParams { steps?: number; lr?: number; @@ -225,6 +306,8 @@ export interface LexTrainBody extends LexCoverageBody, LexShapeParams { seed?: number; sample_every?: number; base?: string; + /** Feature 007, optional and additive: train on the VACATED corpus (§7.2). */ + vacancy?: LexVacancyParamsBody; } export interface LexTrainRecord { @@ -344,6 +427,7 @@ export interface LexClient { lexSpec(): Promise; lexBudgets(params?: LexShapeParams & { source?: string }): Promise; lexCoverage(body?: LexCoverageBody): Promise; + lexVacancy(body?: LexVacancyBody): Promise; lexTrain(body?: LexTrainBody): Promise; lexSpectrum(params: LexSpectrumParams): Promise; lexGenerate(body: LexGenerateBody): Promise; @@ -357,6 +441,72 @@ export const LEX_BUNDLE_VERSION = 1; /** The most frequent out-of-budget types returned by /coverage. A sample, labelled so. */ const OOV_SAMPLE_SIZE = 24; +/** + * Characters of vacated text returned inline by default, and the ceiling a caller may + * ask for. These MUST equal `routes_lex.py`'s `VACANCY_PREVIEW_CHARS` / + * `VACANCY_PREVIEW_MAX`: the API-parity fixture is generated with the default and + * compared here, so a drift in the default fails a test rather than quietly serving two + * different excerpts in the two modes. + */ +export const VACANCY_PREVIEW_CHARS = 2000; +export const VACANCY_PREVIEW_MAX = 20000; + +/** + * Contract §7.1's knobs out of a wire body, snake_case to camelCase, defaults filled by + * the engine's own `vacancyParams` so this cannot drift from them. + * + * Written as a spread over the engine's defaults rather than as an object literal on + * purpose: the transform's parameter set is still growing (§8.3's swap mint), and a + * literal here would silently drop whatever is added next. + */ +function vacancyParamsFrom(body: LexVacancyParamsBody): VacancyParams { + const keep = body.keep; + if (typeof keep === "string") { + throw invalidParamError( + `keep must be a list of words, not a string (got ${JSON.stringify(keep)}); ` + + "a string would be read letter by letter", + ); + } + if (keep != null && !Array.isArray(keep)) { + throw invalidParamError(`keep must be a list of words, got ${JSON.stringify(keep)}`); + } + const p = asFloat(body.p, "p", 0); + if (!(Number.isFinite(p) && p >= 0 && p <= 1)) { + throw invalidParamError(`p must lie in [0, 1], got ${JSON.stringify(body.p)}`); + } + const revealAfter = asInt(body.reveal_after, "reveal_after", 0); + if (revealAfter < 0) throw invalidParamError(`reveal_after must be >= 0, got ${revealAfter}`); + return vacancyParamsWithDefaults({ + p, + seed: asInt(body.seed, "seed", 0), + consistent: asBool(body.consistent, "consistent", true), + matchProsody: asBool(body.match_prosody, "match_prosody", true), + revealAfter, + keep: keep == null ? [] : keep.map(String), + }); +} + +/** + * The backend runs the whole response through `jsonable_6sig`, so the counts pass through + * as integers and every measured fraction is rounded to 6 significant digits. Doing the + * same here is what lets the two payloads be compared field for field. + */ +function roundVacancyStats(stats: VacancyStats): LexVacancyStats { + return { + ...stats, + meanSyllablesBefore: sig6(stats.meanSyllablesBefore, "meanSyllablesBefore"), + meanSyllablesAfter: sig6(stats.meanSyllablesAfter, "meanSyllablesAfter"), + meanAnapestBefore: sig6(stats.meanAnapestBefore, "meanAnapestBefore"), + meanAnapestAfter: sig6(stats.meanAnapestAfter, "meanAnapestAfter"), + stressFromTableBefore: sig6(stats.stressFromTableBefore, "stressFromTableBefore"), + stressFromTableAfter: sig6(stats.stressFromTableAfter, "stressFromTableAfter"), + stressFromMintedBefore: sig6(stats.stressFromMintedBefore, "stressFromMintedBefore"), + stressFromMintedAfter: sig6(stats.stressFromMintedAfter, "stressFromMintedAfter"), + stressFromRuleBefore: sig6(stats.stressFromRuleBefore, "stressFromRuleBefore"), + stressFromRuleAfter: sig6(stats.stressFromRuleAfter, "stressFromRuleAfter"), + }; +} + // --- transport parity ------------------------------------------------------------------ /** @@ -842,6 +992,101 @@ export class LexSection { }; } + // --- the vacancy transform (feature 007) ------------------------------------------- + + /** + * POST /api/lex/vacancy — computed here, LIVE, not refused. + * + * The Lexicon Lab is browser-side in both modes, and the transform is pure string work + * over a corpus this build already ships, so there is nothing here a static page cannot + * do for real. `lexEngine/vacancy.ts` and `llm_geometry/lex/vacancy.py` implement the + * same normative document, and `tests/unit/staticVacancy.test.ts` pins this method's + * whole response against what the real FastAPI route returned for the same request — + * the statistics AND the sha256 of the entire vacated corpus, which is the parity this + * feature rests on. + * + * The response is an excerpt plus a digest for the same reason the backend's is: the + * corpus is ~86 kB and a `p` sweep would otherwise move megabytes to show a screenful. + */ + async lexVacancy(body: LexVacancyBody = {}): Promise { + const original = await this.textSource(body); + const params = vacancyParamsFrom(body); + + const previewChars = asInt(body.preview_chars, "preview_chars", VACANCY_PREVIEW_CHARS); + if (!(previewChars >= 0 && previewChars <= VACANCY_PREVIEW_MAX)) { + throw invalidParamError( + `preview_chars must be in 0..${VACANCY_PREVIEW_MAX}, got ${previewChars}. ` + + "The whole vacated corpus is never returned; `vacated_sha256` identifies it, " + + "and /api/lex/train vacates in place so the text never needs a round trip.", + ); + } + + const vmap = buildVacancyMap(vacancyDomain(tokenize(original)), params); + const vacated = vacateText(original, vmap, params); + const { vocab, rule } = this.vacancyVocab(body, params, vmap, original, vacated); + const stats = vacancyStats(original, vacated, vmap, params); + + return { + p: sig6(params.p, "p"), + seed: params.seed, + consistent: params.consistent, + match_prosody: params.matchProsody, + reveal_after: params.revealAfter, + keep: [...params.keep].map(String).sort(), + vocabulary_rule: rule, + words: [...vocab.words], + budget: { + source: vocab.source, + budget: vocab.budgetName, + size: vocab.budgetSize, + rows: vocab.rows, + coverage: this.roundCoverage(vocab.coverage(vacated)), + }, + corpus: corpusStats(vacated), + vacancy_stats: roundVacancyStats(stats), + bijective: stats.bijective, + remint_rounds: stats.remintRounds, + preview: vacated.slice(0, previewChars), + original_preview: original.slice(0, previewChars), + preview_chars: previewChars, + truncated: vacated.length > previewChars, + vacated_chars: vacated.length, + vacated_sha256: sha256Hex(utf8Bytes(vacated)), + original_chars: original.length, + original_sha256: sha256Hex(utf8Bytes(original)), + }; + } + + /** + * §7.2's two rules, and which one applies. + * + * **Mapped** (`consistent`, `revealAfter = 0`): resolve the budget against the ENGLISH + * corpus, then push its word list through the same `transformWord`, preserving order. + * The map is injective, so every word keeps the id its pre-image had — that is the + * invariance theorem of §7.3, and it is why a mapped run's loss is bit-identical. + * + * **Rebuilt** (everything else): a source type no longer has a single image, so the + * budget is rebuilt from the vacated corpus by the tab's normal rule and coverage + * collapses. The collapse is not a defect, it is the measurement (FR-715). + */ + private vacancyVocab( + body: LexCoverageBody, + params: VacancyParams, + vmap: VacancyMap, + original: string, + vacated: string, + ): { vocab: LexVocab; rule: "mapped" | "rebuilt" } { + if (!params.consistent || params.revealAfter !== 0) { + return { vocab: this.resolveBudgetSync(body, vacated), rule: "rebuilt" }; + } + const english = this.resolveBudgetSync(body, original); + const mapped = mapVocabWords([...english.words], vmap, params); + return { + vocab: new LexVocab(mapped, english.source, english.budgetName), + rule: "mapped", + }; + } + // --- training --------------------------------------------------------------------- /** @@ -851,7 +1096,27 @@ export class LexSection { * are 200-style cache hits, exactly like the backend's content-hash single-flight. */ async lexTrain(body: LexTrainBody = {}): Promise { - const text = await this.textSource(body); + const originalText = await this.textSource(body); + + // Feature 007, optional and additive: absent, everything below is exactly what it was + // before the transform existed. Present, the model trains on the VACATED corpus under + // the vocabulary §7.2 assigns it. Vacating here rather than in the caller matches the + // backend, where `/api/lex/vacancy` deliberately returns only an excerpt. + let text = originalText; + let vacParams: VacancyParams | null = null; + let vmap: VacancyMap | null = null; + if (body.vacancy != null) { + if (typeof body.vacancy !== "object" || Array.isArray(body.vacancy)) { + throw invalidParamError( + "vacancy must be an object of the transform's parameters " + + "(p, seed, consistent, match_prosody, reveal_after, keep), got " + + JSON.stringify(body.vacancy), + ); + } + vacParams = vacancyParamsFrom(body.vacancy); + vmap = buildVacancyMap(vacancyDomain(tokenize(originalText)), vacParams); + text = vacateText(originalText, vmap, vacParams); + } const steps = asInt(body.steps, "steps", DEFAULT_STEPS); if (!(steps >= 1 && steps <= MAX_STEPS)) { @@ -883,6 +1148,9 @@ export class LexSection { cfg = base.cfg; vocab = base.vocab; initialWeights = base.model.weights; + } else if (vacParams !== null && vmap !== null) { + vocab = this.vacancyVocab(body, vacParams, vmap, originalText, text).vocab; + cfg = configFrom(body, vocab.rows); } else { vocab = await this.resolveBudget(body, text); cfg = configFrom(body, vocab.rows); @@ -893,6 +1161,9 @@ export class LexSection { cfg, words: vocab.words, source: vocab.source, + // Redundant with (text, words) today, and in the key anyway so that a knob added to + // the transform later cannot land on a cache entry made before it existed. + vacancy: vacParams, base: body.base ?? null, steps, lr, @@ -1292,6 +1563,15 @@ export class LexSection { } private async resolveBudget(body: LexCoverageBody, text: string): Promise { + return this.resolveBudgetSync(body, text); + } + + /** + * The same resolution, without the promise. `resolveBudget` is `async` for its callers' + * convenience and never awaits anything; the vacancy path resolves a budget twice + * against two different texts (§7.2) and reads better without the ceremony. + */ + private resolveBudgetSync(body: LexCoverageBody, text: string): LexVocab { const source = this.assertSource(body.source ?? DEFAULT_BUDGET_SOURCE); const budget = String(body.budget ?? DEFAULT_BUDGET); if (!(DOLCH_ORDER as readonly string[]).includes(budget)) { @@ -1461,6 +1741,7 @@ export function lexClientFrom(section: LexSection): LexClient { lexSpec: () => section.lexSpec(), lexBudgets: (params) => section.lexBudgets(params), lexCoverage: (body) => section.lexCoverage(body), + lexVacancy: (body) => section.lexVacancy(body), lexTrain: (body) => section.lexTrain(body), lexSpectrum: (params) => section.lexSpectrum(params), lexGenerate: (body) => section.lexGenerate(body), diff --git a/code/frontend/src/lib/staticClient/logitsSanity.ts b/code/frontend/src/lib/staticClient/logitsSanity.ts new file mode 100644 index 0000000..7d3d09c --- /dev/null +++ b/code/frontend/src/lib/staticClient/logitsSanity.ts @@ -0,0 +1,78 @@ +/** + * The non-degeneracy invariant every in-browser inference session must satisfy + * before the app is allowed to report numbers from it. + * + * WHY THIS EXISTS. The app used to ask for `q4f16` first. On any machine whose + * browser exposes a WebGPU adapter with `shader-f16`, that session BUILDS + * SUCCESSFULLY on this onnxruntime-web build and then returns logits that carry no + * information about the input at all — measured in a real browser (Chrome 150, + * Apple Metal-3): + * + * onnx-community/gpt2-ONNX webgpu/q4f16 → every row of [1,T,V] + * bit-identical; greedy decode ",,,,,,,,,," + * onnx-community/SmolLM2-135M-…-ONNX webgpu/q4f16 → every logit exactly 0; + * every NLL = ln(49152) = 10.80267 + * + * Nothing throws, so an exception-only fallback ladder is structurally incapable of + * catching it. The invariant below is what "the session works" actually means, and it + * is asserted once, at load, in the same spirit as the Geometry Lab's training gates + * (final loss, coverage uniformity, field directional entropy): a property the system + * must satisfy, checked explicitly, failing loudly. + * + * THE INVARIANT, stated once. A causal LM's output must DEPEND ON ITS INPUT: run a + * prompt of T ≥ 2 distinct tokens through one forward pass and the next-token + * distribution at the last position must differ from the one at the first position. + * A model whose rows are identical — all-zero logits being one such case, not a + * separate rule — has told you nothing, whatever its perplexity would have been. + * + * The threshold is deliberately far below any real model's separation and far above + * float noise: healthy sessions measure 20–92 nats of separation on the probe prompt + * (gpt2 webgpu/q8: 91.99; SmolLM2-135M webgpu/q8: 36.97; Qwen2.5-0.5B webgpu/q8: + * 20.27), degenerate ones measure exactly 0. + */ + +/** Minimum L∞ separation, in logit units, between the first and last position. */ +export const MIN_ROW_SEPARATION = 1e-3; + +/** + * L∞ distance between the first and last row of a flat [1, T, V] logits buffer. + * `NaN` if any compared entry is not finite — an all-NaN session is degenerate too, + * and NaN fails the `>` comparison in {@link assertNonDegenerateLogits} by itself. + */ +export function rowSeparation(logits: ArrayLike, seqLen: number, vocab: number): number { + if (seqLen < 2) { + throw new Error(`rowSeparation needs at least 2 positions, got ${seqLen}`); + } + const last = (seqLen - 1) * vocab; + let sep = 0; + for (let i = 0; i < vocab; i++) { + const a = logits[i]; + const b = logits[last + i]; + if (!Number.isFinite(a) || !Number.isFinite(b)) return NaN; + const d = Math.abs(a - b); + if (d > sep) sep = d; + } + return sep; +} + +/** + * Throw unless the session's output depends on its input. `label` names the + * device/dtype under test so the message says which configuration is broken. + */ +export function assertNonDegenerateLogits( + logits: ArrayLike, + seqLen: number, + vocab: number, + label: string, +): number { + const sep = rowSeparation(logits, seqLen, vocab); + if (!(sep > MIN_ROW_SEPARATION)) { + throw new Error( + `${label} produced degenerate logits: the next-token distribution at the last ` + + `position differs from the first by ${Number.isNaN(sep) ? "NaN" : sep.toExponential(3)} ` + + `(needs > ${MIN_ROW_SEPARATION}), i.e. the model's output does not depend on its input. ` + + `This session cannot be trusted and was rejected.`, + ); + } + return sep; +} diff --git a/code/frontend/src/lib/staticClient/runtimeTypes.ts b/code/frontend/src/lib/staticClient/runtimeTypes.ts index f405d76..6aff435 100644 --- a/code/frontend/src/lib/staticClient/runtimeTypes.ts +++ b/code/frontend/src/lib/staticClient/runtimeTypes.ts @@ -6,13 +6,52 @@ import type { ArchGenerateBody, ArchGenerateResult, TokenizeResult } from "../dataClient"; +/** Devices the ladder in transformersRuntime may select. */ +export type RuntimeDevice = "webgpu" | "wasm"; +/** + * Quantizations the ladder may select. `q4f16` is deliberately NOT here: it builds a + * session and returns input-independent logits on WebGPU — see logitsSanity.ts. + */ +export type RuntimeDtype = "q8"; + +/** + * Every dtype the fp16-ACTIVATION defect covers. Measured in a real browser on a real + * Apple Metal-3 adapter (Chrome 150 / Chromium 148, transformers.js 4.2.0, + * onnxruntime-web 1.26.0-dev): `q4f16` and `fp16` build a session and then return + * logits identical at every position for gpt2 (greedy ",,,,,,,,,,") and identically + * ZERO for SmolLM2-135M and SmolLM2-360M (empty generation; every NLL = ln V). `q4` + * (4-bit weights, fp32 activations), `q8` and `fp32` are correct on the same adapter, + * so the defect is the fp16 activation path, not 4-bit weights. + */ +export const FP16_ACTIVATION_DTYPES = ["q4f16", "fp16"] as const; + +/** + * Load order for the in-browser runtime. Both rungs read the SAME + * `model_quantized.onnx`, so a rejected WebGPU rung costs no second download — and q8 + * is the smallest quantization verified correct here: in every curated ONNX repo + * `model_q4.onnx` is LARGER than `model_quantized.onnx` (gpt2 498 vs 280 MB; + * SmolLM2-135M 181 vs 136; SmolLM2-360M 386 vs 363; Qwen2.5-0.5B 786 vs 512), so the + * smaller `q4f16` was the only download saving on offer, and it does not work. + */ +export const RUNTIME_LADDER: readonly { device: RuntimeDevice; dtype: RuntimeDtype }[] = [ + { device: "webgpu", dtype: "q8" }, + { device: "wasm", dtype: "q8" }, +]; + export interface RuntimeGenerationInfo { status: "idle" | "loading" | "ready" | "error"; - device: "webgpu" | "wasm" | null; - dtype: "q4f16" | "q8" | null; + device: RuntimeDevice | null; + dtype: RuntimeDtype | null; model_id: string | null; // the HF model whose ONNX export is loaded onnx_repo: string | null; error: string | null; + /** + * `device/dtype` rungs that were built but REJECTED — by an exception or by the + * load-time non-degeneracy check — before the reported one was accepted. Non-empty + * means the user is on a fallback path, and the badge says so: the unforgivable + * part of the q4f16 defect was that a wrong configuration was invisible. + */ + rejected: string[]; } export interface StaticRuntimeInfo { @@ -20,14 +59,35 @@ export interface StaticRuntimeInfo { generation: RuntimeGenerationInfo; } +/** One text scored by ONE real teacher-forced forward pass (contract §8.1). */ +export interface RuntimeScoredText { + /** Byte-level pieces, in order — the input to the UTF-8 span algorithm (§8.2). */ + pieces: string[]; + /** + * Per-token negative log-likelihood in nats: `nll[i]` is the cost of predicting token + * `i` given tokens `< i`. Position 0 has no prediction and is `NaN`, never 0 — a zero + * there would read as a perfectly predicted first token. + */ + nll: number[]; + /** Characters of the scored text, for `bitsPerChar`. */ + nChars: number; +} + /** The surface the lazily-imported transformersRuntime module implements. */ export interface ArchRuntime { info(): RuntimeGenerationInfo; /** Live tokenization from the pinned original-repo tokenizer files. */ tokenize(modelId: string, revision: string, text: string): Promise; - /** Live generation (webgpu q4f16 → wasm q8 fallback) with real per-token probs. */ + /** Live generation (webgpu q8 → wasm q8 fallback) with real per-token probs. */ /** The ONNX mirror is resolved at `main` — see transformersRuntime's header. */ generate(body: ArchGenerateBody, onnxRepo: string): Promise; + /** + * Per-token NLL for each text, one real forward pass each, plus the byte-level pieces + * the caller needs to attribute those tokens to words. No special tokens and no chat + * template: the passage is scored exactly as written, so variants of it differ by the + * transform and by nothing else. + */ + scoreTexts(onnxRepo: string, texts: readonly string[]): Promise; } export type RuntimeLoader = () => Promise; @@ -39,4 +99,5 @@ export const IDLE_GENERATION_INFO: RuntimeGenerationInfo = { model_id: null, onnx_repo: null, error: null, + rejected: [], }; diff --git a/code/frontend/src/lib/staticClient/transformersRuntime.ts b/code/frontend/src/lib/staticClient/transformersRuntime.ts index 95b099b..d1594bb 100644 --- a/code/frontend/src/lib/staticClient/transformersRuntime.ts +++ b/code/frontend/src/lib/staticClient/transformersRuntime.ts @@ -7,7 +7,7 @@ * Honesty contract (FR-203): * - Tokenization uses the ORIGINAL model repo's tokenizer files at the pinned * revision from meta.json — real BPE, no vendored copies. - * - Generation runs the model's community ONNX export (webgpu/q4f16, falling + * - Generation runs the model's community ONNX export (webgpu/q8, falling * back to wasm/q8) and reports per-token probabilities computed from REAL * logits via one teacher-forced forward pass over prompt+reply — the same * quantities the backend reports (chosen-token prob under the temperature @@ -33,7 +33,14 @@ import { import type { ArchGenerateBody, ArchGenerateResult, ArchGeneratedToken, TokenizeResult } from "../dataClient"; import { computeError, invalidParamError } from "./errors"; -import { IDLE_GENERATION_INFO, type ArchRuntime, type RuntimeGenerationInfo } from "./runtimeTypes"; +import { assertNonDegenerateLogits } from "./logitsSanity"; +import { + IDLE_GENERATION_INFO, + RUNTIME_LADDER, + type ArchRuntime, + type RuntimeGenerationInfo, + type RuntimeScoredText, +} from "./runtimeTypes"; const MAX_NEW_TOKENS_LIMIT = 128; // ARCH_MAX_NEW_TOKENS (backend config) const TOPK = 5; @@ -77,48 +84,72 @@ async function webgpuUsable(): Promise { const adapter = (await gpu.requestAdapter()) as { features?: { has(name: string): boolean }; } | null; - // q4f16 needs f16 shaders; without them WASM/q8 is the honest fallback. + // `shader-f16` is NOT needed by the q8 path — it is kept as the marker of a + // HARDWARE adapter. Measured across Chromium configurations on this stack: the + // real Apple Metal-3 adapter advertises it, the software (SwiftShader) adapter + // does not, and plain headless Chromium has no adapter at all. Software WebGPU + // is slower than the WASM backend, so it is not worth preferring. return adapter != null && adapter.features?.has("shader-f16") === true; } catch { return false; } } +// One forward pass over a handful of tokens — the whole load-time check. +const SELF_CHECK_PROMPT = "The capital of France is Paris. The capital of Germany is"; + +/** + * The load-time correctness gate (see logitsSanity.ts for why a thrown-exception + * ladder is not enough). Throws if the freshly built session's next-token + * distribution does not depend on its input. + */ +async function selfCheck(p: TextGenerationPipeline, label: string): Promise { + const ids = p.tokenizer.encode(SELF_CHECK_PROMPT, { add_special_tokens: false }); + if (ids.length < 2) { + throw new Error(`${label}: tokenizer returned ${ids.length} tokens for the self-check prompt`); + } + const out = (await (p.model as unknown as (o: Record) => Promise<{ logits: Tensor }>)({ + input_ids: idsTensor(ids), + attention_mask: new Tensor("int64", BigInt64Array.from(ids, () => 1n), [1, ids.length]), + })) as { logits: Tensor }; + const [, seqLen, vocab] = out.logits.dims as number[]; + try { + return assertNonDegenerateLogits(out.logits.data as Float32Array, seqLen, vocab, label); + } finally { + (out.logits as unknown as { dispose?: () => void }).dispose?.(); + } +} + async function getPipeline(onnxRepo: string): Promise { if (pipelineCache?.key === onnxRepo) return pipelineCache.promise; const promise = (async () => { generationInfo = { + ...IDLE_GENERATION_INFO, status: "loading", - device: null, - dtype: null, - model_id: null, onnx_repo: onnxRepo, - error: null, - }; - const tryLoad = async (device: "webgpu" | "wasm", dtype: "q4f16" | "q8") => { - generationInfo = { ...generationInfo, status: "loading", device, dtype }; - const p = (await pipeline("text-generation", onnxRepo, { - dtype, - device, - })) as TextGenerationPipeline; - generationInfo = { ...generationInfo, status: "ready", device, dtype }; - return p; }; - if (await webgpuUsable()) { + const skipWebgpu = !(await webgpuUsable()); + const rejected: string[] = []; + let lastError = ""; + for (const { device, dtype } of RUNTIME_LADDER) { + const label = `${device}/${dtype}`; + if (device === "webgpu" && skipWebgpu) continue; + generationInfo = { ...generationInfo, status: "loading", device, dtype, rejected: [...rejected] }; try { - return await tryLoad("webgpu", "q4f16"); + const p = (await pipeline("text-generation", onnxRepo, { dtype, device })) as TextGenerationPipeline; + // "The session constructed" is not evidence that the model works. + await selfCheck(p, `${onnxRepo} on ${label}`); + generationInfo = { ...generationInfo, status: "ready", device, dtype, rejected: [...rejected] }; + return p; } catch (e) { - // Degradation ladder: WebGPU failed → retry on WASM before giving up. - console.warn(`[staticClient] webgpu/q4f16 load failed for ${onnxRepo}; falling back to wasm/q8:`, e); + lastError = e instanceof Error ? e.message : String(e); + rejected.push(label); + console.warn(`[staticClient] rejected ${label} for ${onnxRepo}: ${lastError}`); } } - try { - return await tryLoad("wasm", "q8"); - } catch (e) { - const msg = e instanceof Error ? e.message : String(e); - generationInfo = { ...generationInfo, status: "error", error: msg }; - throw computeError(`Could not load the in-browser model ${onnxRepo}: ${msg}`); - } + const msg = lastError || "no usable device/dtype for this browser"; + generationInfo = { ...generationInfo, status: "error", error: msg, rejected: [...rejected] }; + throw computeError(`Could not load the in-browser model ${onnxRepo}: ${msg}`); })(); pipelineCache = { key: onnxRepo, promise }; promise.catch(() => { @@ -295,6 +326,78 @@ async function generateImpl( }; } +/** + * Byte-level pieces for a text — the strings the UTF-8 span algorithm decodes (§8.2). + * + * `_tokenizer` is the underlying `tokenizers` object; `encode(...).tokens` is the only + * place transformers.js 4.x surfaces the raw pieces (there is no offsets API at all). + * It is a private-ish field, so it is checked here and the failure is loud: the + * alternative — decoding tokens one at a time — provably corrupts multi-byte characters. + */ +function byteLevelPieces(tokenizer: PreTrainedTokenizer, text: string): string[] { + const inner = (tokenizer as unknown as { _tokenizer?: { encode(t: string): { tokens?: string[] } } })._tokenizer; + const tokens = inner?.encode(text)?.tokens; + if (!Array.isArray(tokens)) { + throw computeError( + "this build of @huggingface/transformers does not expose byte-level token pieces " + + "(tokenizer._tokenizer.encode(text).tokens), so tokens cannot be attributed to " + + "words. Refusing to guess — per-token decoding corrupts multi-byte characters.", + ); + } + return tokens; +} + +async function scoreTextsImpl( + onnxRepo: string, + texts: readonly string[], +): Promise { + if (texts.length === 0) throw invalidParamError("scoreTexts needs at least one text"); + const generator = await getPipeline(onnxRepo); + const tokenizer = generator.tokenizer; + const model = generator.model; + const out: RuntimeScoredText[] = []; + for (const text of texts) { + const ids = tokenizer.encode(text, { add_special_tokens: false }); + if (ids.length < 2) { + throw invalidParamError( + `a passage must tokenize to at least 2 tokens to be scored, got ${ids.length}`, + ); + } + const pieces = byteLevelPieces(tokenizer, text); + if (pieces.length !== ids.length) { + throw computeError( + `the tokenizer returned ${ids.length} ids but ${pieces.length} byte-level pieces`, + ); + } + const result = (await ( + model as unknown as (o: Record) => Promise<{ logits: Tensor }> + )({ + input_ids: idsTensor(ids), + attention_mask: new Tensor("int64", BigInt64Array.from(ids, () => 1n), [1, ids.length]), + })) as { logits: Tensor }; + const [, seqLen, vocab] = result.logits.dims as number[]; + if (seqLen !== ids.length) { + throw computeError(`the forward pass returned ${seqLen} positions for ${ids.length} tokens`); + } + const data = result.logits.data as Float32Array; + const nll = new Array(ids.length).fill(NaN); + for (let t = 0; t + 1 < ids.length; t++) { + const row = data.subarray(t * vocab, (t + 1) * vocab); + // log softmax at the target, computed stably — there is no log_softmax helper on + // a transformers.js tensor, and materializing probabilities for a 50k vocabulary + // at every position would be needlessly expensive. + let max = -Infinity; + for (let i = 0; i < row.length; i++) if (row[i] > max) max = row[i]; + let sum = 0; + for (let i = 0; i < row.length; i++) sum += Math.exp(row[i] - max); + nll[t + 1] = -(row[ids[t + 1]] - max - Math.log(sum)); + } + (result.logits as unknown as { dispose?: () => void }).dispose?.(); + out.push({ pieces, nll, nChars: text.length }); + } + return out; +} + export const runtime: ArchRuntime = { info: () => ({ ...generationInfo }), @@ -308,4 +411,6 @@ export const runtime: ArchRuntime = { }, generate: generateImpl, + + scoreTexts: scoreTextsImpl, }; diff --git a/code/frontend/src/viz/arch/ArchChat.svelte b/code/frontend/src/viz/arch/ArchChat.svelte index 37128cd..df95c88 100644 --- a/code/frontend/src/viz/arch/ArchChat.svelte +++ b/code/frontend/src/viz/arch/ArchChat.svelte @@ -112,7 +112,8 @@
{#if STATIC_MODE} + the real device/dtype ladder (webgpu·q8 → wasm·q8) as it loads, and marks + the badge "fallback" when a rung was rejected. --> {/if} {#if chatTemplate === false} diff --git a/code/frontend/src/viz/arch/ArchitectureExplorer.svelte b/code/frontend/src/viz/arch/ArchitectureExplorer.svelte index 6d73b06..3b40b2c 100644 --- a/code/frontend/src/viz/arch/ArchitectureExplorer.svelte +++ b/code/frontend/src/viz/arch/ArchitectureExplorer.svelte @@ -21,6 +21,7 @@ import { view } from "../../lib/stores"; import ArchInspector from "./ArchInspector.svelte"; import ArchTracePanel from "./ArchTracePanel.svelte"; + import VacancyScorePanel from "./VacancyScorePanel.svelte"; import { evictArchGraph, fetchArchGraph, formatCount, plainError } from "./archShared"; import { STATIC_MODE, isStaticMiss, staticExtras } from "../../lib/staticUx"; import type { TraceIndexEntry } from "../../lib/staticClient/arch"; @@ -487,6 +488,11 @@ onHighlight={(id) => (highlightId = id)} onRetry={() => traceArgs && runTrace(traceArgs.m, traceArgs.p, traceArgs.sp)} /> + + + diff --git a/code/frontend/src/viz/info/InfoTab.svelte b/code/frontend/src/viz/info/InfoTab.svelte index 893e0b2..0a06532 100644 --- a/code/frontend/src/viz/info/InfoTab.svelte +++ b/code/frontend/src/viz/info/InfoTab.svelte @@ -6,7 +6,10 @@ // what you can manipulate, and — as importantly — what is NOT claimed. Every number // and equation here is transcribed from the code it describes: // geo/model.py, geo/fields.py, geo/config.py, geo/bundle.py, geo/scratch.py, - // arch/{graph,trace,generate}.py, config.py. + // arch/{graph,trace,generate,vacancy_score}.py, config.py, + // lex/{dolch,vacancy}.py + lib/lexEngine/vacancy.ts (the vacancy section's counts are + // what `POST /api/lex/vacancy` reports on the shipped corpus), and + // lib/staticClient/arch.ts (what the static build may and may not say). // If you change one of those, change the matching sentence here; the e2e docs test // pins the values that are cheapest to let drift. @@ -16,6 +19,7 @@ { id: "arch", label: "Architecture Explorer" }, { id: "geo", label: "Geometry Lab" }, { id: "lex", label: "Lexicon Lab" }, + { id: "vacancy", label: "Vacancy transform" }, { id: "real", label: "What's real" }, { id: "limits", label: "Known limits" }, { id: "refs", label: "Source & references" }, @@ -614,10 +618,12 @@ So nothing here reproduces its curves. Every number in this tab is computed live from a model that actually trained in your browser. Three of its instruments were deliberately left out rather than shipped broken: its meter score, which does not measure meter (the line - “and I do not like green eggs and ham” scores 0.333 against a nonsense corpus's 0.346); - its constrained decoder, which can emit fused non-words and is unnecessary here anyway; and its - nonce-word “vacancy” experiment, which needs a parameter-matched control that does not yet - exist. + “and I do not like green eggs and ham” scores 0.333 against a nonsense corpus's 0.346), + and its constrained decoder, which can emit fused non-words and is unnecessary here anyway. + Its third instrument — the nonce-word vacancy transform — was held back for want of a + parameter-matched control, and then shipped once it became clear that the control is the + design: under the mapped condition the transform preserves the vocabulary exactly. It has its + own section below, including the four properties its original implementation claims and breaks.

Finally, on the name: this is not a Dr. Seuss model. His work is under copyright and is @@ -626,6 +632,273 @@ model is therefore not a Seuss pastiche.”

+ +

The vacancy transform

+ +

+ The two labs above each hold one thing fixed and move another. This instrument moves a third, + and it spans both tabs. It rewrites a corpus so that its syntax, its inflection, its + punctuation and its line structure survive byte for byte, while a controlled fraction + p of the open-class stems is replaced by invented forms carrying the same + syllable count and stress. What is left is the condition Jabberwocky puts a reader in — + complete scaffolding, vacant content — manufactured at a rate you choose, on any text. +

+ +

The 2×2 it fills in, and what each arm measures

+

+ A token can carry two independent things. Its location is whatever the form itself has + already earned — you have met the string before, and something about it is yours. Its + field is everything the surrounding context fixes about it. Vacancy manufactures the + cell where the field is fully supplied and the location is gone: +

+ +
+ + + + + + + + + + + + + + + + +
no fieldfield supplied
no location(i) random init, no data + (iii) vacancy — nonce form, full syntactic support. This instrument. +
location(ii) minting at a hub centroid(iv) ordinary word learning
+
+

+ Two arms measure that cell, and they give different answers because their models differ in + exactly the way the 2×2 is about. The Lexicon Lab's word-level model is trained from + scratch and has no locations at all — its entire lexicon is a table of embedding rows, and it + never sees a letter — so for it the transform is a pure relabelling and the answer is an + exact zero. The Architecture Explorer's pretrained model does have locations, so + its answer is not zero. Neither number means anything alone; the pair is the result. +

+ +

What is preserved, and what is replaced

+ +
+ words are found with the tokenizer's OWN regex   [A-Za-z]+(?:['-][A-Za-z]+)*
+ u(stem) = (first 8 bytes of sha256(“seed:stem”) as a uint64 ≫ 11) / 2⁵³
+ vacate the stem   iff   u(stem) < p +
+

+ Preserved, byte for byte: everything that is not a word match — whitespace, punctuation, + digits, line breaks; the closed class, which is a curated function-word list and deliberately + not a Dolch budget (unioning the two silently protects content verbs like + run and eat and understates the vacancy rate); inflectional suffixes, + since the stem is vacated and the suffix re-attached, so dog's becomes + <nonce>'s; and anything failing the eligibility test — good-bye + contains a hyphen, so it never moves. Replaced: eligible stems, and only stems. +

+

+ That the transform finds words with the tokenizer's regular expression rather than one + written beside it is load-bearing rather than tidy: the theorem below is false the moment the + transform's idea of a word differs from the trainer's. On the shipped corpus the map's domain + is 2,233 types — the corpus's own 2,211 plus the full Dolch list, so that + switching budgets cannot re-mint the text under the reader — of which 1,944 are + eligible, sharing 1,680 distinct stems. At p = 1 that rewrites + 8,202 of the corpus's 16,000 word tokens; the rest are closed class, too short, + or not ASCII letters. +

+ +

Nesting and stability — which the original implementation breaks

+

+ Two properties make a p-sweep interpretable. Nesting: since + u is a function of (seed, stem) alone, the set of stems vacated at + p is a subset of the set vacated at any larger p. Stability: a + stem's replacement is the same string at every p at which it is vacated, because + the map is built once over the whole type set in canonical order, before any + p is chosen. +

+

+ The source project claims both and has neither, and the reasons are worth naming because they + are the kind of bug a test suite does not see. Its map is built lazily while rewriting + and guards uniqueness with a growing used set, so which of two colliding stems has + to retry depends on which was reached first — and therefore on p. Its give-up path + returns a syllable plus len(used), a count of how many words happened to be minted + earlier. Its seam fix (wee + er → weeer) draws from a + shared RNG, so it too depends on order. And its injectivity is assumed: it accepts an + avoid parameter and never passes one, so a minted form can silently merge with a + real English word. Here the map is a pure function of the domain, the seed and the prosody + setting; the seam fix is a hash of (stem, suffix); and injectivity is + verified over assembled surface forms and re-minted on collision. The Lexicon Lab's + ribbon shows both properties cell by cell — the classification comes from the real map, so a + broken one would show as a reverted cell rather than as a paragraph that stopped being true. +

+ +

The invariance theorem, and why the zero is the finding

+

+ With one nonce per source type and no partial reveal, and with the budget's word list pushed + through the same transform in the same order: +

+ +
+ tokenStream(vacate(C, p), V_p)  =  tokenStream(C, V)   element for element
+ for every p, every seed, every budget, and either prosody setting +
+

+ Three facts make it true. The transform is a bijection on word occurrences preserving order + and line structure, so the <eos>-per-line rule fires in the same places. The + type map is injective on the whole domain — verified at build time over assembled surface + forms, and therefore at every p at once, not merely at full vacancy. And because + the budget is mapped in order, every word keeps the id its pre-image had and out-of-budget + types still land out of budget, so <unk> appears in exactly the same + positions. The corollary is that training is bit-identical: the model's configuration + depends on the vocabulary only through its row count, which does not move. +

+

+ So the headline of the tiny arm is an exact zero, and the exact zero is the result — not a + chart that failed to render. Three of the knobs — p, seed, + match prosody — are invisible to a word-level model trained from scratch, + and only the controls that break type identity can move a loss. Read plainly: for this model + class, all of a word's meaning is field and none of it is form. The Lexicon Lab checks it + rather than asserting it, in two tiers — the id streams are compared element for element on + every control change, and a button trains twice and reports max |Δloss|, which is + 0 and is printed as 0, never as “≈ 0”. A non-zero there would be a + defect, and the panel says so instead of rounding it away. +

+

+ What the theorem does not prove is that form is worthless in general. It proves that a + model whose lexicon is a table of rows has no channel through which a form could matter. + A model with subword tokens has that channel — which is why the second arm exists. +

+ +

The swap control, and the decomposition

+

+ Vacating the content words of a passage changes three things at once: the forms become unknown, + they fragment into many subword tokens, and the passage stops meaning anything. Only the first + is “location”, and no caveat can separate them — but a control can. Swap runs the same + transform with the replacement drawn as a real English word: from the corpus's own + open-class types, matched on frequency rank. The passage is then exactly as nonsensical while + every form remains a word the model knows and the tokenizer segments normally. So: +

+ +
+ nll(swap)  −  nll(english)  =  the cost of wrong content
+ nll(nonce)  −  nll(swap)   =  the cost of unknown form +
+

+ Both are means over the tokens of preserved words only — the closed-class scaffolding, + which is character-identical in all three variants, so the comparison is the same function word + against itself. Their sum, nll(nonce) − nll(english), is never the headline: it + credits the cost of nonsense to the cost of an unknown word. +

+

+ “Cost of unknown form” is an upper bound, not a measurement of location. Nonce forms + fragment into more subword tokens than real words do, so that difference carries the cost of an + unknown form together with the cost of a longer, stranger context. The two are not + separable without a tokenizer-level control, which this instrument does not have — so the + number bounds what a word's location was worth rather than equalling it, and the panel says so + beside the number instead of at the bottom of the page. +

+

+ Swap draws its replacements from a finite pool — 1,944 eligible domain types against the + map's 1,680 stems — so the tail of the canonical order draws from what is left and its + frequency match degrades there; and a source type carrying a suffix may receive an already + inflected replacement. Both are consequences of drawing from a real vocabulary, and both are + stated rather than smoothed over. +

+

+ A swap map is injective only at p = 0 and p = 1, and that is a + theorem rather than an implementation limit. Its images are domain words, so at an + intermediate p a vacated word can land on one that has not been vacated yet. This + cannot be engineered away: a map that is stable in p and whose images are domain + types would, if it were injective at every p, have to be a bijection carrying each + nested vacated set onto itself — hence the identity. Measured on the shipped corpus, swap loses + 244 / 322 / 233 image slots at p = 0.25 / 0.5 / 0.75, and 0 at both + endpoints. The engine therefore refuses the mapped vocabulary in between, with a typed + error naming the theorem, and the panel shows you that refusal rather than clamping the slider + or quietly substituting a nonce map. Full vacancy — where swap is a bijection of the + domain and the invariance theorem holds for it exactly as for nonce — is where the pretrained + arm scores, so the control loses nothing it exists for. The inconsistent-assignment condition + is refused under swap for a different and equally countable reason: it needs a fresh type per + occurrence, and 1,680 stems cannot cover 8,202 vacated tokens. +

+ +

The stress table, stated honestly

+

+ When match prosody is on, a nonce is built to carry its stem's syllable count and stress + pattern — which requires knowing the stem's stress. That comes from a hand table of + 61 entries covering the polysyllables of the Dolch list, and the table's own provenance + is: seeded by rule and then never checked by a human. The source's status page says it + “wants roughly an hour of human checking”, and that hour has not happened here either. The + table covers 5.1% of this corpus's tokens; everything else falls through to a spelling + heuristic that counts vowel groups. So every prosody number on these pages is indicative, + not exact, and none is ever shown without the three-way split beside it — how much of the + stress came from the hand table, how much from forms we minted ourselves (known by + construction, but asserted rather than verified: the minter checks syllable count, not + pattern), and how much from the rule, which is a guess. +

+

+ The suffix splitter is a spelling heuristic too, not a morphological analyser: it is right on + its exception list and wrong outside it (ladder → ladd + + er). That is tolerable — the nonce still carries a consistent identity and an + inflected-looking surface — and it is said here rather than absorbed quietly. +

+

+ One rule this project holds itself to, and the reason the numbers above are the ones they are: + the source document reports its own prosody figures on its corpus, which we do not have. + None of its numbers is transcribed anywhere here. Every figure on this page was measured + on The Real Mother Goose, by the code that ships. +

+ +

What is refused, and where

+

+ The full stack scores in float32 and reports everything. The static build runs a quantized ONNX + export in your browser, and quantization moves absolute log-likelihoods by tenths of a nat — + in a direction that is not even the same across two models. It may therefore state a number + only where an error bound has actually been measured for the dtype it ran, and it refuses the + rest by name rather than printing a value with a plausible-looking margin: +

+
    +
  • + Absolute nllPreserved: refused. Quantization shifts it by tenths of a + nat, with the sign varying by model, so the number would say more about the export than about + the passage. +
    • +
  • + nll(nonce) − nll(swap): refused. It is the small difference — the + interesting one — and quantization's error on it reaches a fifth of its own size, with sign + flips. A contrast that quantization eats is not a contrast. +
    • +
  • + Per-passage rows: refused. Pooled differences cancel; a single passage's does not, + and one measured case was wrong by more than its own value. +
    • +
  • + A pool below 700 preserved tokens: refused — that is the size at which the bound was + measured, and below it the honest answer is no number. +
    • +
  • + Any dtype without a measured bound: refused outright. A stated ± that was never + measured is a fabricated error bar, which is worse than no number at all. +
    • +
  • + What it does report — pooled swap − english and + nonce − english — carries ±0.2 nats of quantization uncertainty stated + beside the sampling standard error, quoted to one decimal place because that is all the + measurement supports. +
    • +
+

+ One coverage gap belongs here rather than in a commit message. The browser runs + webgpu/q8 where a GPU is available and wasm/q8 otherwise, and every + session is gated at load by a non-degeneracy check — a causal model's output must depend on its + input — because a dtype this app once tried first built a working-looking session and returned + input-independent logits. That gate and the ladder are unit-tested, and the WebGPU path is + verified end to end on a real GPU. But GitHub's runners have no GPU, so CI only ever + exercises the WASM rung: the WebGPU path is checked on a developer machine, not in + continuous integration. +

+

What's real, and where it runs

This page is deployed as a static site with no Python behind it, so it is worth being precise @@ -692,6 +965,27 @@ budgets against text it cannot identify + + The vacancy transform itself + + Real, in your browser, in both modes. The transform, the map, the statistics and + the invariance check are the TypeScript half of the same contract the Python package + implements, pinned to it by a golden fixture: the map's every stem→nonce pair at two + seeds, the vacated text, the statistics, and the token-id-stream digests, compared + exactly for strings and ids. The backend's /api/lex/vacancy exists for + parity and for callers outside the tab + + + + The pretrained arm (vacancy scoring) + real, in PyTorch at float32 — every number, including per-passage rows + + real, in your browser via transformers.js + ONNX at q8, but only the + quantities with a measured error bound: the pooled differences, with ±0.2 nats of + quantization uncertainty stated. Absolute NLL, per-passage rows and + nonce − swap are refused by name — see the section above + + Architecture: weight matrices from the loaded model @@ -743,6 +1037,26 @@ and every spectrum statistic are pinned to ≤ 1e-5 against the Python implementation by a golden test, but whole-run training equality with a Python run is not claimed. +

  • + The stress table is unverified. 61 hand-set entries, seeded by rule and never checked + by a human, covering 5.1% of this corpus's tokens. Every prosody statistic is therefore + indicative rather than exact, and is shown with the three-way split that says so. +
    • +
  • + The swap control is injective only at p = 0 and p = 1, which + is a theorem about maps whose images are domain words, not a defect. In between, the mapped + vocabulary is refused with a typed error instead of being computed with two words on one row. +
    • +
  • + The static build refuses most of the pretrained arm's numbers, and states ±0.2 nats of + measured quantization uncertainty on the ones it does report. For absolute NLL, per-passage + rows or nonce − swap, run the full stack, which scores at float32. +
    • +
  • + The WebGPU path is not covered by CI. It is verified end to end on a real GPU on a + developer machine; GitHub's runners have none, so continuous integration exercises the WASM + rung, the load-time non-degeneracy gate and the dtype ladder, but never the GPU one. +
  • A model trained in the Lexicon Lab lives in that tab and nowhere else. There is no account and no server-side checkpoint, so closing the page ends the model unless you save the @@ -783,6 +1097,12 @@ Blanche Fisher Wright: the Lexicon Lab's corpus. It is committed to the repository whole, header and licence footer intact, and trimmed to its body only when it is used.
    • +
  • + + Project Gutenberg ebook #12 — Through the Looking-Glass (Lewis Carroll, 1871), + whose “Jabberwocky” is the condition the vacancy transform manufactures on demand: + every function word and every inflection in place, every content stem vacant. +
  • E. W. Dolch, A Basic Sight Vocabulary, The Elementary School Journal 36(6):456–460, 1936, @@ -992,6 +1312,15 @@ .tbl td.span { background: rgba(91, 224, 176, 0.06); } + /* The one cell of the 2×2 this instrument occupies: marked in the table rather than + described underneath it, so the claim and the picture cannot drift apart. */ + .tbl td.cell-mark { + background: rgba(110, 168, 254, 0.09); + box-shadow: inset 0 0 0 1px rgba(110, 168, 254, 0.35); + } + .tbl td.cell-mark b { + color: var(--accent); + } .tbl.notation td:nth-child(odd) { white-space: nowrap; width: 1%; diff --git a/code/frontend/src/viz/lex/LexiconLab.svelte b/code/frontend/src/viz/lex/LexiconLab.svelte index 113c3b2..8b55800 100644 --- a/code/frontend/src/viz/lex/LexiconLab.svelte +++ b/code/frontend/src/viz/lex/LexiconLab.svelte @@ -55,11 +55,23 @@ paramCount, randomBaselineSpectrum, spectrum, + tokenize, type BudgetSource, type Coverage, type LexConfig, type SpectrumResult, } from "../../lib/lexEngine"; + import { + buildVacancyMap, + mapVocabWords, + typeCounts, + vacancyDomain, + vacancyParams, + vacateText, + type MintStrategy, + type VacancyMap, + type VacancyParams, + } from "../../lib/lexEngine/vacancy"; import { sha256Hex, utf8Bytes } from "../../lib/geoEngine/hash"; import Explain from "../../lib/Explain.svelte"; import { view } from "../../lib/stores"; @@ -73,6 +85,7 @@ import SpectrumPanel from "./SpectrumPanel.svelte"; import TokenCloud from "./TokenCloud.svelte"; import TrainPanel from "./TrainPanel.svelte"; + import VacancyPanel from "./VacancyPanel.svelte"; /** The build-time corpus export described in the header comment. */ interface CorpusAsset { @@ -110,6 +123,21 @@ let tied = $state(DEFAULT_TIED); let dropout = $state(DEFAULT_DROPOUT); + /** + * The vacancy controls (feature 007). They live here because the transform changes the + * CORPUS every panel below reads: the budget is measured against the vacated text, the + * trainer is fed the vacated text, and the vocabulary is either mapped through the same + * transform or rebuilt from it. `VacancyPanel` renders them and calls back — it owns no + * state of its own beyond which window of the corpus it is showing. + */ + type VacancyCondition = "consistent" | "inconsistent" | "reveal"; + let vacP = $state(0); + let vacSeed = $state(0); + let vacCondition = $state("consistent"); + let vacRevealAfter = $state(1); + let vacProsody = $state(true); + let vacMint = $state("nonce"); + /** * What a training run produced, or null while nothing has been trained at the CURRENT * shape. Its vocabulary travels with it (FR-619): fine-tuning and generation both use @@ -170,13 +198,127 @@ } } + // ---- derived: the vacancy transform (feature 007) ------------------------------------ + + const vacParams = $derived( + vacancyParams({ + p: vacP, + seed: vacSeed, + consistent: vacCondition !== "inconsistent", + matchProsody: vacProsody, + revealAfter: vacCondition === "reveal" ? vacRevealAfter : 0, + mint: vacMint, + }), + ); + + /** + * The replacement assignment, built ONCE over the domain (corpus types ∪ the full Dolch + * list) in canonical order — contract §5.2. It reads `vacSeed`, `vacProsody` and the + * minting strategy, and deliberately NOT `vacP`: the map is `p`-independent, which is what + * makes a stem's replacement the same string at every `p` where it is vacated, and building + * it inside a derived that also read `p` would silently re-mint the whole corpus on every + * tick of the slider — visibly breaking the stability the panel exists to demonstrate. + * + * `consistent` IS passed, even though the map itself does not depend on it, because the + * engine refuses `mint = "swap"` under the inconsistent control (contract §8.3: 1 680 + * open-class stems against 8 202 vacated tokens, so there is no supply of real words to + * mint a fresh one per occurrence). Deciding that here would be re-deriving a rule the + * engine owns; asking the engine and CARRYING its refusal is the honest form. Nothing is + * substituted for a refused map — `vacRefusal` renders the engine's own sentence and the + * panels below have no vocabulary, which is exactly the state the reader is in. + */ + const vacBuild = $derived.by<{ map: VacancyMap | null; refusal: string }>(() => { + if (!corpus) return { map: null, refusal: "" }; + const tokens = tokenize(corpus.text); + try { + return { + map: buildVacancyMap( + vacancyDomain(tokens), + vacancyParams({ + seed: vacSeed, + matchProsody: vacProsody, + mint: vacMint, + consistent: vacParams.consistent, + }), + // The swap control ranks its replacement pool by corpus frequency, so it needs the + // TOKEN STREAM's counts; `nonce` never looks at them. + vacMint === "swap" ? typeCounts(tokens) : undefined, + ), + refusal: "", + }; + } catch (e) { + return { map: null, refusal: e instanceof Error ? e.message : String(e) }; + } + }); + const vacMap = $derived(vacBuild.map); + + /** The corpus every panel below measures, trains on and generates from. */ + const vacatedText = $derived.by(() => + corpus && vacMap ? vacateText(corpus.text, vacMap, vacParams) : "", + ); + const vacated = $derived(vacP > 0); + /** `consistent = true` and `revealAfter = 0` — the only condition §7.2 maps the budget in. */ + const vacMapped = $derived(vacParams.consistent && vacParams.revealAfter === 0); + // ---- derived: vocabulary, coverage, parameter count -------------------------------- - const vocab = $derived.by(() => + /** `V` — the budget resolved against the UNTRANSFORMED corpus, the theorem's reference. */ + const baseVocab = $derived.by(() => corpus ? buildVocab(budgetSource, budgetName, corpus.text) : null, ); + + /** + * `V_p`, by the two rules of contract §7.2. + * + * MAPPED (`consistent`, no reveal): the budget's word list is pushed through the same + * `transformWord`, PRESERVING ORDER, so every word keeps the id its pre-image had. That, + * with the map's injectivity, is what makes the token id stream identical. + * + * REBUILT (every other condition): the budget is rebuilt from the vacated corpus by the + * tab's normal rule. Coverage then collapses — and the collapse is the measurement, not + * a failure to be papered over. + */ + const vocabResult = $derived.by<{ vocab: LexVocab | null; refusal: string }>(() => { + if (vacBuild.refusal) return { vocab: null, refusal: vacBuild.refusal }; + if (!corpus || !baseVocab) return { vocab: null, refusal: "" }; + if (!vacated) return { vocab: baseVocab, refusal: "" }; + if (vacMapped && vacMap) { + try { + return { + vocab: new LexVocab( + mapVocabWords(baseVocab.words, vacMap, vacParams), + baseVocab.source, + baseVocab.budgetName, + ), + refusal: "", + }; + } catch (e) { + // Contract §5.2a, and the one refusal a reader will actually meet: `mint = "swap"` + // draws its replacements FROM the domain, so at an intermediate `p` a vacated type + // can land on one that has not moved and two budget words would share an embedding + // row. The theorem there proves no `p`-stable swap avoids it, so this is not a defect + // to re-draw away. The engine's sentence is carried up verbatim and NOTHING is put in + // its place: rebuilding the budget from the vacated corpus here would be a different + // measurement (§7.2's rebuilt rule belongs to the control conditions) wearing the + // mapped condition's label. + return { vocab: null, refusal: e instanceof Error ? e.message : String(e) }; + } + } + return { vocab: buildVocab(budgetSource, budgetName, vacatedText), refusal: "" }; + }); + const vocab = $derived(vocabResult.vocab); + /** The engine's own words for a configuration it declines, or `""`. Never paraphrased. */ + const vacRefusal = $derived(vocabResult.refusal); const coverage = $derived.by(() => - vocab && corpus ? vocab.coverage(corpus.text) : null, + vocab && corpus ? vocab.coverage(vacatedText) : null, + ); + /** What the trainer, the sampler and the geometry are describing. */ + const activeCorpusLabel = $derived( + corpus + ? vacated + ? `${corpus.label} · vacated p=${vacP.toFixed(2)}, seed ${vacSeed}, ${vacCondition}, ${vacMint}` + : corpus.label + : "", ); const nParams = $derived(vocab ? paramCount(vocab.rows, dModel, nLayers, ctx, tied) : 0); @@ -449,10 +591,16 @@ one matrix counted twice.
  • - Not shipped: nonce-word minting (it needs a parameter-matched control that - does not exist), and the meter/rhyme "fingerprint" (the source's meter score does - not measure meter — under its scheme every word's stress pattern begins the same - way, so the score converges to the template's own density whatever you feed it). + Shipped, corrected: nonce-word minting — the vacancy panel above. Feature 006 + deferred it for want of a parameter-matched control; the control turned out to be the + design, since under the mapped condition the transform preserves the vocabulary + exactly. Four properties the source claims for itself are broken by its own + implementation and fixed here (its map is built lazily while rewriting, so a nonce + depends on p; its give-up path and its seam fix are order-dependent; and + injectivity is assumed rather than verified). + Not shipped: the meter/rhyme "fingerprint" (the source's meter score does not + measure meter — under its scheme every word's stress pattern begins the same way, so + the score converges to the template's own density whatever you feed it).
  • Browser and Python run the same recipe, held to ≤1e-5 on the forward pass, @@ -490,7 +638,7 @@ budget={budgetName} {vocab} {coverage} - corpusLabel={corpus?.label ?? ""} + corpusLabel={activeCorpusLabel} onSource={(s) => (budgetSource = s as BudgetSource)} onBudget={(b) => (budgetName = b)} /> @@ -516,13 +664,35 @@
    • +
    + (vacP = v)} + onSeed={(v) => (vacSeed = v)} + onCondition={(c) => (vacCondition = c as VacancyCondition)} + onRevealAfter={(v) => (vacRevealAfter = v)} + onProsody={(v) => (vacProsody = v)} + onMint={(m) => (vacMint = m as MintStrategy)} + /> +
    +
    + /** + * The vacancy transform, live on the corpus this tab trains on (feature 007, ui.md §1). + * + * What it is for: a word can carry meaning two ways. Its FORM can be known — you have + * seen `crow` before and something about the string is already yours — or its meaning + * can be entirely FIELD, fixed by the company it keeps. This panel manufactures the + * second condition on demand: closed-class scaffolding, inflection, punctuation and + * line structure survive byte for byte, while a controlled fraction `p` of open-class + * STEMS is replaced by phonotactically legal nonce forms carrying the stem's syllable + * count and stress. + * + * Everything below is computed here, in this browser, by `lib/lexEngine/vacancy.ts` — + * the TypeScript half of `specs/007-vacancy-transform-field/architecture.md`, which the + * Python backend implements too. No number in the prose is retyped: every one of them + * is read out of `vacancyStats` or out of a source constant, so changing the constant + * changes the sentence. + * + * THREE THINGS THIS PANEL HAS TO MAKE VISIBLE, and how: + * + * * NESTING and STABILITY (FR-711) — the ribbon, not the prose. `u(stem)` is a hash of + * `(seed, stem)` alone, so `{vacated at p} ⊆ {vacated at p'}` for `p < p'`; and the + * map is built ONCE over the whole type set in canonical order, so a stem's nonce is + * the same string at every `p` where it is vacated. The ribbon shows both by showing + * the same eligible stems at five values of `p` side by side. + * * THE INVARIANCE THEOREM (FR-714) — as a live computation in two tiers. The instant + * tier compares the two token id streams element for element on every control + * change; the on-demand tier really trains twice and subtracts the loss curves. In + * a condition that breaks the theorem both tiers show the real, broken result. There + * is no hard-coded tick anywhere in this file. + * * THAT THE NULL IS THE FINDING (§1.6) — an exact zero drawn as a flat line looks + * like a broken chart. It is stated as the result it is, in words, next to the + * measurement. + * * WHAT THE ENGINE REFUSES, AND WHY (§5.2a) — `mint = "swap"` draws its replacements + * from the domain, so it is injective only at `p ∈ {0, 1}`; at an intermediate `p` + * the mapped vocabulary is refused with a typed error. That error is rendered here + * VERBATIM and nothing is computed in its place: the `p` slider is never clamped, + * no fallback to `nonce` is performed, and the injectivity counter beside the mint + * control reports the collisions as a measured number at the current `p` — so the + * reader sees the theorem rather than a control that quietly disagrees with them. + * + * PROSODY HONESTY (FR-712 / SC-708): the stress table is rule-seeded and unverified, so + * no prosody statistic is ever rendered without the three-way stress split beside it and + * the sentence saying what fraction of this corpus's tokens the hand table actually + * covers — itself a measured number, `stressFromTable*`. + * + * The type counts shown are `corpusTypes*`, never `domainTypes*` (contract §10): the + * domain adds budget words that never appear in the text, and counting words the reader + * cannot see inflates the vacancy rate they are being shown. + */ + import { onDestroy } from "svelte"; + + import { + LexVocab, + UNK_ID, + WORD_RE, + hasWord, + splitLines, + tokenStream, + tokenize, + trainInWorker, + } from "../../lib/lexEngine"; + import { + FUNCTION_WORDS, + STRESS_TABLE, + SUFFIXES, + effectiveKeepSet, + isEligible, + stemAndSuffix, + transformWord, + vacancyStats, + vacancyU, + type VacancyMap, + type VacancyParams, + } from "../../lib/lexEngine/vacancy"; + import Explain from "../../lib/Explain.svelte"; + import Progress from "../../lib/Progress.svelte"; + import { view } from "../../lib/stores"; + + interface Props { + /** The untransformed corpus — the `p = 0` reference for every comparison here. */ + corpusText: string; + /** `vacateText(corpusText, map, params)`, computed by the tab (it also trains on it). */ + vacatedText: string; + /** The `p`-independent nonce assignment. Null only before the corpus has loaded. */ + map: VacancyMap | null; + params: VacancyParams; + /** `V` — the budget resolved against the untransformed corpus. */ + baseVocab: LexVocab | null; + /** `V_p` — mapped in the mapped condition, rebuilt from the vacated corpus otherwise. */ + vocab: LexVocab | null; + /** "consistent" | "inconsistent" | "reveal" — `params` encodes it, this names it. */ + condition: string; + /** Kept even while the condition is not `reveal`, so the input does not lose its value. */ + revealAfter: number; + /** "nonce" | "swap" (contract §8.3). */ + mint: string; + /** + * The engine's own sentence for a configuration it declines, or `""`. The tab does not + * paraphrase it and does not compute anything in its place — see the refusal card below, + * and `LexiconLab`'s `vocabResult` for why substituting a rebuilt budget would be worse + * than showing nothing. + */ + refusal: string; + onP: (v: number) => void; + onSeed: (v: number) => void; + onCondition: (v: string) => void; + onRevealAfter: (v: number) => void; + onProsody: (v: boolean) => void; + onMint: (v: string) => void; + } + let { + corpusText, + vacatedText, + map, + params, + baseVocab, + vocab, + condition, + revealAfter, + mint, + refusal, + onP, + onSeed, + onCondition, + onRevealAfter, + onProsody, + onMint, + }: Props = $props(); + + const CONDITIONS = [ + { + id: "consistent", + label: "consistent", + title: + "One nonce per source type, corpus-wide. The mapped condition — the invariance theorem holds here.", + }, + { + id: "inconsistent", + label: "inconsistent", + title: + "A fresh nonce for every OCCURRENCE. Same vacancy rate, no learnable identity — the source's control.", + }, + { + id: "reveal", + label: "partial reveal", + title: + "The first N occurrences of a vacated stem keep their English form, so the type is split in two.", + }, + ]; + const MINTS = [ + { + id: "nonce", + label: "nonce", + title: + "Replace the stem with a phonotactically legal invented form. Its images are outside the domain, so the map is injective at every p.", + }, + { + id: "swap", + label: "swap", + title: + "Draw a real, frequency-rank-matched English word instead (contract §8.3) — the control that separates wrong content from unknown form. Its images ARE domain words, so it is injective only at p = 0 or p = 1 (§5.2a).", + }, + ]; + + /** See BudgetPanel: arrows must move an ARIA radiogroup's selection and focus together. */ + function segKey(e: KeyboardEvent, apply: (value: string) => void): void { + const dir = + e.key === "ArrowRight" || e.key === "ArrowDown" + ? 1 + : e.key === "ArrowLeft" || e.key === "ArrowUp" + ? -1 + : 0; + if (dir === 0) return; + e.preventDefault(); + const group = e.currentTarget as HTMLElement; + const radios = Array.from( + group.querySelectorAll('[role="radio"]:not([disabled])'), + ); + if (radios.length === 0) return; + const cur = radios.findIndex((r) => r.getAttribute("aria-checked") === "true"); + const next = radios[((cur < 0 ? 0 : cur) + dir + radios.length) % radios.length]; + next.focus(); + apply(next.dataset.value ?? ""); + } + + // ---- statistics (contract §10) ------------------------------------------------------- + + const keep = $derived(effectiveKeepSet(params.keep)); + const stats = $derived.by(() => + map && corpusText.length > 0 ? vacancyStats(corpusText, vacatedText, map, params) : null, + ); + /** The mapped condition — the only one in which §7.3's theorem is claimed to hold. */ + const mapped = $derived(params.consistent && params.revealAfter === 0); + + const pct = (x: number) => `${(x * 100).toFixed(1)}%`; + const n = (x: number) => x.toLocaleString(); + + // ---- injectivity at the CURRENT p, measured rather than claimed (contract §5.2a) ------ + + /** + * Push every type of the map's domain through the transform at the current `p` and count + * how many distinct surface forms come back. `|domain| − |images|` is the number of rows + * two source types would have to share, so it is 0 exactly when the relabelling is a + * relabelling. + * + * This is the measurement, not a restatement of the flag: `map.injectiveAtEveryP` says + * which regime a map is in, and this says what the regime costs HERE, at the `p` on the + * slider. Under `nonce` it is 0 at every `p` — condition B keeps every image out of the + * domain — and under `swap` it is 0 at the two endpoints and positive in between, which is + * §5.2a's theorem happening in front of the reader rather than being asserted at them. + * + * Only defined in the mapped condition, because `transformWord` is the order-free path: the + * inconsistent control and partial reveal need an occurrence index and rebuild the budget + * from the vacated corpus anyway, so injectivity of a type map is not what they claim. + */ + const images = $derived.by(() => { + if (!map || !mapped) return null; + const seen = new Set(); + for (const t of map.domain) seen.add(transformWord(t, map, params).toLowerCase()); + return { types: map.domain.size, distinct: seen.size, lost: map.domain.size - seen.size }; + }); + + // ---- the corpus view (ui.md §1.2 — the doc's Figure 5, live) ------------------------- + + const WINDOW_LINES = 40; + + /** + * The shipped corpus opens with 618 token-producing lines of front matter — a title page, + * a list of rhymes, and an index of first lines — before `LITTLE BO-PEEP` starts the verse + * at line 619. Opening the reader on a table of contents makes the transform look like it + * is rewriting an index, which is the least interesting thing it does. + * + * Pinned rather than detected, deliberately. Every general rule tried here picks the wrong + * boundary: the index of first lines has verse-length lines (median 6 tokens), so a + * line-length heuristic lands on line 320, still inside the front matter. Separating an + * index from a stanza needs to know the book. The corpus is committed and digest-verified, + * so a constant is the honest way to say that, and `vacancy.spec.ts` asserts the default + * page really opens on Bo-Peep. + * + * It applies ONLY to the shipped corpus. Pasted text and HuggingFace datasets have no front + * matter, so they open at line 1. + */ + const SHIPPED_BODY_LINE = 619; + const SHIPPED_LINE_COUNT = 3071; + + let windowIndex = $state(0); + /** Once the reader pages, their choice wins over `defaultWindow`. */ + let userPaged = $state(false); + + /** + * Page by `delta`, reading the CURRENT window before taking ownership of it. + * + * The order matters and is not obvious: setting `userPaged` first makes `win` fall straight + * back to `windowIndex`, which is still 0 while the default is in force, so the first click + * would jump to line 1 instead of stepping. Capture, then assign. + */ + function page(delta: number): void { + const from = win; + windowIndex = from + delta; + userPaged = true; + } + + type Seg = { text: string; cls: "gap" | "kept" | "open" | "minted" }; + + /** Raw `WORD_RE` matches with their offsets, case preserved. */ + function wordsOf(line: string): { text: string; at: number }[] { + const re = new RegExp(WORD_RE.source, "g"); + const out: { text: string; at: number }[] = []; + for (let m = re.exec(line); m !== null; m = re.exec(line)) out.push({ text: m[0], at: m.index }); + return out; + } + + /** + * One line, split into coloured runs. The classification comes from the REAL map — the + * eligibility test of §2.2 and the transform's own output — never from a hand-annotated + * list (FR-711). A word is `minted` iff the transform actually changed it, which is what + * makes `revealAfter` and the inconsistent control show up here honestly rather than + * being predicted from `u` alone. + */ + function segmentsOf(orig: string, vac: string): Seg[] { + const before = wordsOf(orig); + const after = wordsOf(vac); + const segs: Seg[] = []; + let cursor = 0; + for (let i = 0; i < before.length; i++) { + const w = before[i]; + if (w.at > cursor) segs.push({ text: orig.slice(cursor, w.at), cls: "gap" }); + const out = after[i]?.text ?? w.text; + const [stem] = stemAndSuffix(w.text.toLowerCase()); + const cls: Seg["cls"] = + out.toLowerCase() !== w.text.toLowerCase() + ? "minted" + : isEligible(stem, keep) + ? "open" + : "kept"; + segs.push({ text: out, cls }); + cursor = w.at + w.text.length; + } + if (cursor < orig.length) segs.push({ text: orig.slice(cursor), cls: "gap" }); + return segs; + } + + const origLines = $derived(splitLines(corpusText)); + const vacLines = $derived(splitLines(vacatedText)); + /** Indices of the token-producing lines — the ones `tokenStream` turns into training data. */ + const wordLines = $derived.by(() => { + const out: number[] = []; + for (let i = 0; i < origLines.length; i++) if (hasWord(origLines[i])) out.push(i); + return out; + }); + const nWindows = $derived(Math.max(1, Math.ceil(wordLines.length / WINDOW_LINES))); + /** + * Where the view opens. The shipped corpus is recognised by its line count — it is + * digest-verified upstream, so this cannot be some other book of the same length — and + * opens on the verse; anything else opens at line 1. + */ + const defaultWindow = $derived( + wordLines.length === SHIPPED_LINE_COUNT ? Math.floor(SHIPPED_BODY_LINE / WINDOW_LINES) : 0, + ); + /** Clamped rather than reset by an effect: a shorter corpus must not strand the view. */ + const win = $derived( + Math.min(Math.max(0, userPaged ? windowIndex : defaultWindow), nWindows - 1), + ); + const shown = $derived.by(() => { + const start = win * WINDOW_LINES; + return wordLines.slice(start, start + WINDOW_LINES).map((i) => ({ + i, + segs: segmentsOf(origLines[i], vacLines[i] ?? origLines[i]), + })); + }); + + // ---- the nesting ribbon (ui.md §1.3) ------------------------------------------------- + + const P_CELLS = [0, 0.25, 0.5, 0.75, 1] as const; + const RIBBON_ROWS = 8; + + /** Eligible stems the reader can actually find in the text above. */ + const corpusStems = $derived.by(() => { + const out = new Set(); + for (const t of new Set(tokenize(corpusText))) { + const [stem] = stemAndSuffix(t); + if (isEligible(stem, keep)) out.add(stem); + } + return out; + }); + + /** + * ~8 stems spanning the `u` range, each shown at five values of `p`. Chosen by rank in + * `u` rather than by hand, so the row set is a function of the corpus and the seed. + */ + const ribbon = $derived.by(() => { + if (!map) return []; + const rows = [...map.mapping.keys()] + .filter((s) => corpusStems.has(s)) + .map((stem) => ({ stem, u: vacancyU(stem, params.seed), nonce: map.mapping.get(stem) ?? stem })) + .sort((a, b) => a.u - b.u || (a.stem < b.stem ? -1 : 1)); + if (rows.length === 0) return []; + const picked: typeof rows = []; + for (let k = 0; k < RIBBON_ROWS; k++) { + const idx = Math.round((k * (rows.length - 1)) / (RIBBON_ROWS - 1)); + const row = rows[Math.min(idx, rows.length - 1)]; + if (!picked.some((r) => r.stem === row.stem)) picked.push(row); + } + return picked.map((r) => ({ + ...r, + cells: P_CELLS.map((pc) => ({ p: pc, vacated: r.u < pc, form: r.u < pc ? r.nonce : r.stem })), + })); + }); + + // ---- tier 1: the instant invariance check (ui.md §1.5) ------------------------------- + + /** + * §7.3, computed rather than asserted, on every control change: encode the original + * corpus under `V` and the vacated corpus under `V_p`, and compare the id streams + * element for element. `tokenStream` is the SAME function the trainer feeds on, so this + * is the object the theorem is about and not a proxy for it. + */ + const streams = $derived.by(() => { + if (!baseVocab || !vocab || corpusText.length === 0) return null; + const a = tokenStream(corpusText, baseVocab); + const b = tokenStream(vacatedText, vocab); + const common = Math.min(a.length, b.length); + let differ = Math.abs(a.length - b.length); + for (let i = 0; i < common; i++) if (a[i] !== b[i]) differ++; + const unk = (xs: number[]) => + xs.length === 0 ? 0 : xs.reduce((c, x) => c + (x === UNK_ID ? 1 : 0), 0) / xs.length; + return { + compared: Math.max(a.length, b.length), + differ, + identical: differ === 0, + unkBefore: unk(a), + unkAfter: unk(b), + }; + }); + + // ---- tier 2: the on-demand training demonstration (ui.md §1.5) ----------------------- + + /** Small on purpose: the point is the comparison, not the final loss. Two runs, not three. */ + const DEMO_STEPS = 40; + const DEMO_SEED = 0; + const DEMO_DIMS = { dModel: 16, nLayers: 1, nHeads: 2, ctx: 32, dropout: 0 }; + + let demoBusy = $state(false); + let demoFraction = $state(0); + let demoMessage = $state(""); + let demoError = $state(""); + let demo = $state<{ + atP: number; + conditionLabel: string; + a: number[]; + b: number[]; + maxDelta: number; + lengthsAgree: boolean; + finalA: number; + finalB: number; + valA: number; + valB: number; + expectedZero: boolean; + elapsedMs: number; + } | null>(null); + let demoAbort: AbortController | null = null; + + onDestroy(() => demoAbort?.abort()); + + async function trainOnce( + text: string, + v: LexVocab, + label: string, + base: number, + signal: AbortSignal, + ): Promise<{ history: number[]; finalLoss: number; valLoss: number }> { + const res = await trainInWorker( + { + ...DEMO_DIMS, + vocab: { words: v.words, source: v.source, budgetName: v.budgetName }, + text, + steps: DEMO_STEPS, + seed: DEMO_SEED, + sampleEvery: DEMO_STEPS, + signal, + }, + (prog) => { + demoFraction = base + (prog.step / Math.max(1, prog.totalSteps)) * 0.5; + demoMessage = `${label} · step ${prog.step}/${prog.totalSteps} · loss ${prog.loss.toFixed(3)}`; + }, + ); + return { + history: res.model.history.map((h) => h.loss), + finalLoss: res.finalLoss, + valLoss: res.valLoss, + }; + } + + /** + * Two real training runs — `p = 0` under `V`, and the current `p` under `V_p` — with the + * same seed, the same dimensions and the same step count, then `max |Δloss|` between the + * two curves. Under the mapped condition the theorem says this is EXACTLY zero, and the + * UI reports whatever it actually is: a non-zero here is a bug, not a rounding artifact. + */ + async function runDemo(): Promise { + if (!baseVocab || !vocab) return; + demoAbort?.abort(); + const ctrl = new AbortController(); + demoAbort = ctrl; + demoBusy = true; + demoError = ""; + demo = null; + demoFraction = 0; + demoMessage = ""; + const started = Date.now(); + try { + const runA = await trainOnce(corpusText, baseVocab, "p = 0", 0, ctrl.signal); + const runB = await trainOnce( + vacatedText, + vocab, + `p = ${params.p.toFixed(2)}`, + 0.5, + ctrl.signal, + ); + const common = Math.min(runA.history.length, runB.history.length); + let maxDelta = 0; + for (let i = 0; i < common; i++) { + const d = Math.abs(runA.history[i] - runB.history[i]); + if (d > maxDelta) maxDelta = d; + } + demo = { + atP: params.p, + conditionLabel: condition, + a: runA.history, + b: runB.history, + maxDelta, + lengthsAgree: runA.history.length === runB.history.length, + finalA: runA.finalLoss, + finalB: runB.finalLoss, + valA: runA.valLoss, + valB: runB.valLoss, + expectedZero: mapped, + elapsedMs: Date.now() - started, + }; + demoFraction = 1; + demoMessage = ""; + } catch (e) { + demoError = e instanceof Error ? e.message : String(e); + } finally { + demoBusy = false; + if (demoAbort === ctrl) demoAbort = null; + } + } + + function stopDemo(): void { + demoAbort?.abort(); + demoAbort = null; + demoBusy = false; + } + + // The two loss curves, drawn on one pair of axes so an exact overlap is what an exact + // zero LOOKS like: the dashed curve sits on the solid one for its whole length. + const CW = 320; + const CH = 96; + const demoCurves = $derived.by(() => { + if (!demo || demo.a.length < 2 || demo.b.length < 2) return null; + const all = [...demo.a, ...demo.b]; + const hi = Math.max(...all); + const lo = Math.min(...all); + const span = hi - lo || 1; + const path = (xs: number[]) => + xs + .map( + (y, i) => + `${((i / Math.max(1, xs.length - 1)) * CW).toFixed(1)},${(CH - ((y - lo) / span) * CH).toFixed(1)}`, + ) + .join(" "); + return { a: path(demo.a), b: path(demo.b), hi, lo }; + }); + + +
    +
    +

    Vacancy — field without location

    + rewrite the corpus so a word's form tells you nothing +
    + +

    + A word can mean something to you two ways: because you know the form — you have met + crow before, and the string itself already carries a prior — or because the + field around it fixes what it must be. This control separates them. Raising + p replaces that fraction of eligible open-class stems with invented + forms, while closed-class scaffolding, inflectional suffixes, punctuation and line breaks + survive byte for byte. Everything below is measured on the corpus this tab actually trains + on, in this browser, by the same transform the Python backend runs. +

    + + +
    + + + + +
    + condition +
    segKey(e, onCondition)} + > + {#each CONDITIONS as c (c.id)} + + {/each} +
    +
    + + {#if condition === "reveal"} + + {/if} + + + +
    + mint +
    segKey(e, onMint)} + > + {#each MINTS as m (m.id)} + + {/each} +
    +
    +
    + +
    +

    + nonce invents the replacement. swap draws a real English word instead — + from this corpus's own open-class types, matched on frequency rank (contract §8.3). The + passage is left exactly as nonsensical either way, but under swap every form is a + word the reader, and a pretrained tokenizer, already knows. That difference is the whole + control: it separates the cost of wrong content from the cost of an + unknown form, which is the measurement the + makes on a model that has forms it knows. +

    +

    + A swap map is injective only at p = 0 and p = 1. That is a + theorem, not a rough edge. + Swap's replacements are drawn from the domain, so at an intermediate + p a vacated word can land on a word that has not been vacated yet, and two + source types share one row. Contract §5.2a proves that no non-trivial map can avoid this: + one that is stable in p and whose images are domain words is injective at + every p only if it is the identity. So the engine refuses the mapped + vocabulary in between rather than handing the trainer a vocabulary with a duplicated row, + and this panel shows you the refusal instead of quietly moving the slider for you. Full + vacancy — p = 1, where swap is a bijection of the domain — is the + configuration the pretrained arm actually measures at, so nothing the control exists for + is lost. +

    + {#if images} +

    0} + data-testid="lex-vacancy-injectivity" + > + Measured just now at p = {params.p.toFixed(2)}, {mint}: + {n(images.distinct)} distinct images from {n(images.types)} domain types — + {n(images.lost)} + lost image {images.lost === 1 ? "slot" : "slots"}. + {#if images.lost === 0} + Every type still has a row of its own, so the relabelling is a relabelling. + {:else} + Each lost slot is two source types on one row — exactly what §5.2a says must happen + here, counted rather than described. + {/if} +

    + {/if} +
    + + {#if refusal} +
    +

    + Refused, in the engine's own words: + {refusal} +

    +

    + Nothing has been substituted for it: there is no vocabulary, so the budget counters, the + trainer and the invariance check below have nothing to report — which is the honest state + of this configuration, not a rendering failure. Rebuilding the budget from the vacated + corpus would produce numbers, but they would be a different measurement (contract §7.2 + gives that rule to the control conditions) wearing the mapped condition's label. +

    +
    + + + + +
    +
    + {/if} + + +
    +
    + the corpus, transformed +
    + + + token-producing lines {n(win * WINDOW_LINES + 1)}–{n( + Math.min((win + 1) * WINDOW_LINES, wordLines.length), + )} of {n(wordLines.length)} + + +
    +
    + + +
    + {#each shown as line (line.i)} +
    + {#each line.segs as seg, k (k)}{seg.text}{/each} +
    + {/each} +
    +

    + closed class, preserved + open class, not yet vacated + minted + + — the classes come from the real map: a word is minted when the transform + actually changed it, open when its stem passes the eligibility test of + contract §2.2 but u(stem) ≥ p, and preserved when the stem is in + the closed class or fails eligibility (too short, or not ASCII letters — which is why + good-bye never moves). Nothing here is annotated by hand. The corpus opens + with its own table of contents; page forward for verse. + +

    +
    + + +
    + nesting & stability +
    + + + + + + {#each P_CELLS as pc (pc)} + + {/each} + + + + {#each ribbon as row (row.stem)} + + + + {#each row.cells as cell (cell.p)} + + {/each} + + {/each} + +
    stemup = {pc.toFixed(2)}
    {row.stem}{row.u.toFixed(3)}{cell.form}
    +
    +

    + Read each row left to right. Nesting: once a cell turns minted it never reverts — + a stem is vacated iff u(stem) < p, and u is a hash of + (seed, stem) alone, so the vacated sets are nested as p grows. + Stability: the minted string is the same string in every later cell — the + nonce map is built once over the whole type set in canonical order, so it does not depend + on p, on document order, or on which other words exist. The source project's + minter built its map lazily while rewriting and had neither property; contract §5.2 is the + correction. Rows are the eligible stems of this corpus at eight evenly spaced ranks of + u; a highlighted row is one that is vacated at the current + p = {params.p.toFixed(2)}. + {#if !mapped} + + The corpus above is in the {condition === "reveal" ? "partial reveal" : condition} + condition, which deliberately does not use this map for every occurrence — that control + exists precisely to destroy the identity the ribbon is showing. + + {/if} +

    +
    + + + {#if stats} +
    +
    + types vacated + + {n(stats.corpusTypesVacated)}/ {n(stats.corpusTypesEligible)} + + + distinct corpus types whose form changed, out of the eligible ones — + corpus scope, not the map's domain, which adds + {n(stats.domainTypesEligible - stats.corpusTypesEligible)} budget words that never appear + in the text + +
    +
    + tokens vacated + + {n(stats.tokensVacated)}/ {n(stats.tokensTotal)} + + {pct(stats.tokensVacated / Math.max(1, stats.tokensTotal))} of the corpus's word occurrences +
    +
    + stems vacated + {n(stats.stemsVacated)}/ {n(stats.stemsTotal)} + + the map's size is every eligible stem; a stem vacates when u < p + +
    +
    + map + + {#if !stats.bijective} + NOT injective + {:else if map?.injectiveAtEveryP} + injective ✓ + {:else} + injective at p = 0, 1 + {/if} + + + {#if map?.injectiveAtEveryP} + verified over assembled surface forms at every p, not assumed + {:else} + verified at full vacancy — swap's replacements are domain words, so contract + §5.2a bounds injectivity to p = 0 and p = 1, and the count + beside the mint control says what it costs in between + {/if} + · {n(stats.remintRounds)} re-mint {stats.remintRounds === 1 ? "round" : "rounds"} · + image {n(stats.imageSize)} + +
    +
    + +
    +
    + prosody, before → after +

    + mean syllables + {stats.meanSyllablesBefore.toFixed(3)} → {stats.meanSyllablesAfter.toFixed(3)} +

    +

    + mean anapest + {stats.meanAnapestBefore.toFixed(3)} → {stats.meanAnapestAfter.toFixed(3)} +

    +
    +
    + where each token's stress came from + + + + + + + + + + + + + + + + + + + + + + + + + +
    sourcebeforeafter
    hand table{pct(stats.stressFromTableBefore)}{pct(stats.stressFromTableAfter)}
    minted{pct(stats.stressFromMintedBefore)}{pct(stats.stressFromMintedAfter)}
    spelling rule{pct(stats.stressFromRuleBefore)}{pct(stats.stressFromRuleAfter)}
    +
    +

    + Read those two numbers only together with this table. The stress table is + {STRESS_TABLE.size} hand-set entries, seeded by rule and never checked by a + human, and it covers {pct(stats.stressFromTableBefore)} of this corpus's tokens + before the transform and {pct(stats.stressFromTableAfter)} after. Everything else + is either a form we minted — whose pattern we chose, and whose syllable + count is checked while its pattern is only asserted — or the spelling heuristic + of contract §6.2, which is a guess. So the prosody statistics are + indicative, not exact. +

    +
    + {/if} + + + {#if streams} +
    + {#if streams.identical} + token id streams identical + · {n(streams.compared)} ids compared, element for element, just now. The vacated corpus + under V_p encodes to exactly the token stream the original corpus encodes to + under V, so the trainer cannot tell them apart. + <unk> rate {pct(streams.unkBefore)} → {pct(streams.unkAfter)}. + {:else} + token id streams differ + · {n(streams.differ)} of {n(streams.compared)} positions. + <unk> rate {pct(streams.unkBefore)} → {pct(streams.unkAfter)}. This is + the real result, not a warning: {condition === "inconsistent" + ? "the inconsistent control mints a fresh form per occurrence, so the vacated corpus has types the budget cannot express and the theorem's premise is gone" + : condition === "reveal" + ? `partial reveal splits every vacated stem into two types — the first ${revealAfter} occurrences keep their English form — so the vocabulary is rebuilt from the vacated corpus and the ids move` + : "the vocabulary was rebuilt rather than mapped, so ids no longer correspond"}. + {/if} +
    + {/if} + + +
    +

    + The headline result here is an exact zero, and that is the finding — not a broken + chart. + For a word-level model trained from scratch, the mapped condition is a + pure relabelling of the vocabulary: the map is injective, the budget's words are + pushed through the same transform in the same order, so every word keeps the embedding row + its pre-image had. The model is provably blind to p, to + seed and to match prosody. Read plainly: for this model class, + all of a word's meaning is field and none of it is form. +

    +

    + The number that is not zero belongs to a model that has forms it already knows — a + real pretrained transformer, scored on a passage and its vacated twin. That measurement is + this feature's pretrained arm and it lives in the + . The two numbers are the point of the pair: the same transform, worth exactly nothing to + one model and something measurable to the other. +

    +
    + + +
    + the invariance theorem, trained +
    + + {#if demoBusy} + + {/if} + + two real runs · {DEMO_STEPS} steps · d={DEMO_DIMS.dModel}, {DEMO_DIMS.nLayers} layer, + ctx {DEMO_DIMS.ctx} · same seed {DEMO_SEED}, same hyperparameters + +
    + + {#if demoBusy} + + {/if} + + {#if demoError} +
    {demoError}
    + {/if} + + {#if demo} +
    +
    + + {#if demoCurves} + + + {/if} + +
    +

    + p = 0 + p = {demo.atP.toFixed(2)} ({demo.conditionLabel}) + — training loss in nats, {n(demo.a.length)} steps each, in + {(demo.elapsedMs / 1000).toFixed(1)}s +

    +

    + {#if !demo.lengthsAgree} + The two runs produced different numbers of loss points + ({n(demo.a.length)} vs {n(demo.b.length)}) — the curves are not comparable. + {:else if demo.maxDelta === 0} + max |Δloss| = 0 — exactly zero, over every one of the {n(demo.a.length)} steps. + {#if demo.expectedZero} + Not "≈ 0", not "within tolerance": the two runs are the same computation on the same + token ids, so they are bit-identical. Final loss + {demo.finalA.toFixed(6)} in both, held-out {demo.valA.toFixed(6)} in both. + {:else} + This condition is not expected to be invariant, so a zero here means the controls + happen to leave the token stream unchanged — check p. + {/if} + {:else} + max |Δloss| = {demo.maxDelta.toExponential(3)} · + final {demo.finalA.toFixed(4)} vs {demo.finalB.toFixed(4)} · held-out + {demo.valA.toFixed(4)} vs {demo.valB.toFixed(4)}. + {#if demo.expectedZero} + This should have been exactly 0. The mapped condition is a pure relabelling, + so any difference at all is a defect in the transform, the vocabulary mapping or the + trainer — report it rather than rounding it away. + {:else} + That is the expected direction: this condition breaks type identity, so the model + really is seeing a different corpus. + {/if} + {/if} +

    +
    + {/if} +
    + +

    + The corpus every panel below now uses is the vacated one, so the budget's coverage + counters, the training run, the samples and the embedding geometry all respond to + p together. In the mapped condition the budget is pushed through the same + transform in the same order and keeps its size; in the control conditions it is rebuilt from + the vacated text instead, and the collapse in coverage is the measurement. +

    + + +

    + A word is split into stem + suffix by a spelling heuristic (contract §3): + the first match among the {SUFFIXES.length} suffixes + {SUFFIXES.join(" · ")} wins, tried in that order, and only when at least + three characters would remain. A stem may be vacated when all three of §2.2 hold: it is + not one of the + {FUNCTION_WORDS.size} closed-class words, it is ASCII letters only, and it is + longer than two characters. So don't splits to the stem do and + never moves; good-bye fails the ASCII test and never moves; + dog's becomes <nonce>'s. +

    +

    + Every output is itself a single complete tokenizer match — checked at run time, not + assumed — so tokenize(vacated) has the same length and ordering as + tokenize(original), and because line breaks are untouched the + <eos>-per-line rule fires in exactly the same places. That is what the + invariance theorem rests on, and it is why the transform's idea of a word is the + tokenizer's own regular expression rather than a second one written beside it. +

    +

    + The heuristic is not a morphological analyser and is wrong outside its exception list — + ladder splits to ladd + er. That is tolerable (the nonce still + carries a consistent identity and an inflected-looking surface) but it is a known + artifact, and it is stated here rather than quietly absorbed. +

    +     + + +

    + With consistent = true and revealAfter = 0, for every + p, seed, budget and setting of match prosody, the + token id stream of the vacated corpus under the mapped vocabulary equals the token id + stream of the original under the original vocabulary, element for element. Three facts + make it true: the transform is a bijection on word occurrences preserving order and line + structure; the type map is injective on the union of the corpus's types and the budget's + words, verified over assembled surface forms at map-build time and therefore at every + p at once; and the budget is pushed through the same transform in the same + order, so out-of-budget types still land out of budget and <unk> + appears in exactly the same places. +

    +

    + Injectivity is verified, not assumed. Two weaker checks were tried first and both + were wrong: checking bare nonces misses a collision that arrives through the suffix, and + checking the image size at p = 1 only misses it too, because at full vacancy + every eligible type has moved and no minted form can meet a surviving English word. The + collision exists only at intermediate p, which is exactly where a reader + sweeping this slider spends their time. +

    +

    + The swap control cannot have that property, and the argument is three lines. Suppose + a map is stable in p and every image is a domain word — both true of swap by + construction. If it were injective at every p it would be a bijection of the + domain onto itself, and would therefore carry each vacated set + V_p onto itself; but the V_p grow one stem family at a time as + p rises, so such a bijection fixes every family, i.e. it is the identity. + Hence no non-trivial swap is injective in between, and the counter above reports how many + rows collide at the p you are on. At p = 1 the un-vacated set is + exactly the ineligible types, swap is a bijection of the domain, and the theorem holds for + it exactly as it does for nonce — which is the check that the control is + implemented correctly, and the configuration the pretrained arm scores at. +

    +

    + What it does not prove: that form is worthless in general. It proves that a + word-level model whose entire lexicon is a table of embedding rows has no channel through + which a form could matter — it never sees the characters. A model with subword tokens has + that channel, which is why the pretrained arm exists and why its answer is not zero. +

    +     +
    + + diff --git a/code/frontend/tests/e2e/archVacancy.spec.ts b/code/frontend/tests/e2e/archVacancy.spec.ts new file mode 100644 index 0000000..92e3fb5 --- /dev/null +++ b/code/frontend/tests/e2e/archVacancy.spec.ts @@ -0,0 +1,114 @@ +import { expect, test } from "@playwright/test"; + +/** + * The pretrained arm of the vacancy instrument, driven against the REAL backend + * (feature 007, contract §8; SC-707/707a/707b). + * + * Two runs: the pooled default set (which resolves the small effect and lets the + * measured ordering be asserted), then a single short passage, which must NOT be + * presented as if it resolved anything. + */ + +/** A short passage: ~60 preserved tokens, far too few for a 0.1-nat effect. */ +const PASSAGE = [ + "Hey diddle diddle, the cat and the fiddle,", + "The cow jumped over the moon;", + "The little dog laughed to see such sport,", + "And the dish ran away with the spoon.", + "Little Jack Horner sat in a corner,", + "Eating a Christmas pie;", + "He put in his thumb, and pulled out a plum,", + "And said, What a good boy am I!", +].join("\n"); + +test("scores a passage and reports the decomposition, never the conflated total", async ({ + page, +}) => { + test.setTimeout(300_000); + await page.goto("/"); + await page.getByTestId("tab-architecture").click(); + const panel = page.getByTestId("arch-vacancy"); + await panel.scrollIntoViewIfNeeded(); + await expect(panel).toBeVisible(); + + // The default: the six shipped corpus excerpts, pooled — the configuration the + // reference numbers were measured in. Pooling matters here rather than being a + // nicety: the "unknown form" effect is a tenth of a nat, and one short passage + // cannot resolve its SIGN, let alone its size. + await page.getByTestId("arch-vac-run").click(); + + // Real download + three forward passes on a cold cache. + await expect(page.getByTestId("arch-vac-table")).toBeVisible({ timeout: 280_000 }); + await expect(page.getByTestId("arch-vac-error")).toHaveCount(0); + + // The three variants, each with its own real statistics. + for (const v of ["english", "swap", "nonce"]) { + await expect(page.getByTestId(`arch-vac-row-${v}`)).toBeVisible(); + } + + // The two LABELLED differences, in words, with real numbers. + const wrong = page.getByTestId("arch-vac-wrong_content"); + const form = page.getByTestId("arch-vac-unknown_form"); + await expect(wrong).toContainText(/\d\.\d{3}/); + await expect(form).toContainText(/\d\.\d{3}/); + await expect(panel).toContainText("the cost of wrong content"); + await expect(panel).toContainText("the cost of unknown form"); + + // nll(nonce) − nll(english) appears ONLY as the small, explicitly-labelled sum. + const total = page.getByTestId("arch-vac-total"); + await expect(total).toContainText("nll(nonce) − nll(english)"); + await expect(total).toContainText("conflates"); + + // The tiny arm's exact 0 is beside it — that juxtaposition IS the 2×2 (FR-719). + await expect(page.getByTestId("arch-vac-tiny-arm")).toContainText("0"); + await expect(page.getByTestId("arch-vac-tiny-arm")).toContainText("exactly"); + + // The honesty block: the residual, the confound, the alignment mechanism. + const honesty = page.getByTestId("arch-vac-honesty"); + await expect(honesty).toContainText("UPPER BOUND"); + await expect(honesty).toContainText("higher entropy"); + await expect(honesty).toContainText("UTF-8 byte spans"); + + // The measured ordering: wrong content costs more than unknown form (SC-707b). + const value = async (testid: string): Promise => { + const text = (await page.getByTestId(testid).innerText()).match(/-?\d+\.\d{3}/); + if (!text) throw new Error(`no number rendered in ${testid}`); + return Number(text[0]); + }; + const wrongNats = await value("arch-vac-wrong_content"); + const formNats = await value("arch-vac-unknown_form"); + expect(wrongNats).toBeGreaterThan(0); + expect(formNats).toBeGreaterThan(0); + expect(wrongNats).toBeGreaterThan(formNats); + + await panel.screenshot({ + path: "tests/e2e/__screenshots__/arch-vacancy-score.png", + }); +}); + +test("a passage too short to resolve the small effect says so instead of concluding", async ({ + page, +}) => { + test.setTimeout(300_000); + await page.goto("/"); + await page.getByTestId("tab-architecture").click(); + const panel = page.getByTestId("arch-vacancy"); + await panel.scrollIntoViewIfNeeded(); + + await page.getByTestId("arch-vac-defaults").uncheck(); + await page.getByTestId("arch-vac-passage").fill(PASSAGE); + await page.getByTestId("arch-vac-run").click(); + await expect(page.getByTestId("arch-vac-table")).toBeVisible({ timeout: 280_000 }); + await expect(page.getByTestId("arch-vac-error")).toHaveCount(0); + + // ~60 preserved tokens: the standard error swamps a tenth of a nat, and the verdict + // must say that rather than reading a conclusion off the sign. + const pairs = Number( + (await page.getByTestId("arch-vac-unknown_form-err").innerText()).match( + /([\d,]+) paired tokens/, + )?.[1]?.replace(/,/g, "") ?? "0", + ); + expect(pairs).toBeGreaterThan(0); + expect(pairs).toBeLessThan(300); + await expect(page.getByTestId("arch-vac-verdict")).toContainText("does not resolve"); +}); diff --git a/code/frontend/tests/e2e/docs.spec.ts b/code/frontend/tests/e2e/docs.spec.ts index cba6f23..ed4336e 100644 --- a/code/frontend/tests/e2e/docs.spec.ts +++ b/code/frontend/tests/e2e/docs.spec.ts @@ -1,5 +1,11 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + import { expect, test, type Page } from "@playwright/test"; +const SRC = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "src"); + // The Info tab and the in-tab explainers, against the REAL backend. // // The point of this file is not "does the text render" — it is that the documentation @@ -23,6 +29,7 @@ test.describe("Info tab", () => { "The Architecture Explorer", "The Geometry Lab", "The Lexicon Lab", + "The vacancy transform", "What's real, and where it runs", "Known limits", "Source & references", @@ -282,3 +289,141 @@ test.describe("Lexicon Lab documentation", () => { await expect(info).toContainText("10607"); }); }); + +test.describe("vacancy transform documentation", () => { + // Feature 007's numbers, pinned by feature 005's rule: read the fact from the running + // system — the real transform, the real panel, the real constant in the source — and then + // assert the sentence still agrees with it. Nothing in the Info tab's vacancy section is a + // number someone typed and nobody checks again. + // + // The source document's own prosody figures are ITS numbers on a corpus we do not have, + // and are transcribed nowhere; every figure asserted here is measured on Mother Goose. + + const en = (x: number): string => x.toLocaleString("en-US"); + + test("the documented counts are what the transform really produces", async ({ + page, + request, + }) => { + // p = 1, seed 0 on the shipped corpus — the configuration the prose quotes. + const res = await request.post("/api/lex/vacancy", { data: { p: 1, seed: 0 } }); + expect(res.ok(), await res.text()).toBe(true); + const body = await res.json(); + const s = body.vacancy_stats; + + // Full vacancy means these identities hold; the sentence about 8,202 tokens is only + // true while they do, so they are asserted rather than assumed. + expect(s.stemsVacated).toBe(s.stemsTotal); + expect(s.corpusTypesVacated).toBe(s.corpusTypesEligible); + expect(body.bijective).toBe(true); + + await openInfo(page); + const info = page.getByTestId("info-view"); + // Domain scope governs the map; corpus scope is what a reader can see in the text. The + // prose states both, labelled — an unprefixed "types" is forbidden (contract §10). + for (const value of [ + s.domainTypesTotal, // 2,233 = corpus types ∪ the FULL Dolch list + s.corpusTypesTotal, // 2,211 of them are the corpus's own + s.domainTypesEligible, // 1,944 eligible — also the size of the swap pool + s.stemsTotal, // 1,680 distinct stems, i.e. the size of the map + s.tokensVacated, // 8,202 rewritten word occurrences at p = 1 + s.tokensTotal, // out of 16,000 + ]) { + await expect(info, `the prose no longer states ${en(value)}`).toContainText(en(value)); + } + + // The honesty number beside every prosody statistic: what fraction of this corpus's + // tokens the unverified hand table actually covers. + await expect(info).toContainText(`${(s.stressFromTableBefore * 100).toFixed(1)}%`); + }); + + test("the documented swap collisions are the ones the engine measures", async ({ page }) => { + // Contract §5.2a as a number rather than as an adjective. `swap` draws its replacements + // FROM the domain, so at an intermediate p a vacated type lands on one that has not + // moved. The prose states the measured triple; this reads it off the running panel, so a + // change to the map, the pool or the tie rule fails here instead of quietly making the + // documentation wrong. + await page.goto("/#lexicon"); + await expect(page.getByTestId("lex-vacancy")).toBeVisible({ timeout: 30_000 }); + await page + .getByTestId("lex-vacancy-mint") + .getByRole("radio", { name: "swap", exact: true }) + .click(); + + const slider = page.getByTestId("lex-vacancy-p"); + const lost = page.getByTestId("lex-vacancy-lost-slots"); + const measured: string[] = []; + for (const p of ["0.25", "0.5", "0.75"]) { + await slider.fill(p); + await slider.dispatchEvent("input"); + await expect(page.getByTestId("lex-vacancy")).toContainText(`p = ${Number(p).toFixed(2)}`); + // The refusal is SHOWN, not worked around: no clamped p, no silent fall back to nonce. + await expect(page.getByTestId("lex-vacancy-refusal")).toBeVisible(); + await expect(page.getByTestId("lex-vacancy-refusal-message")).toContainText("§5.2a"); + measured.push((await lost.innerText()).trim()); + } + + // …and 0 at full vacancy, where swap IS a bijection of the domain and the invariance + // theorem holds for it exactly as it does for nonce. + await slider.fill("1"); + await slider.dispatchEvent("input"); + await expect(lost).toHaveText("0"); + await expect(page.getByTestId("lex-vacancy-refusal")).toHaveCount(0); + await expect(page.getByTestId("lex-vacancy-invariance-verdict")).toHaveText(/identical/i); + + await openInfo(page); + await expect(page.getByTestId("info-view")).toContainText(measured.join(" / ")); + }); + + test("the documented stress-table size is the table the engine actually has", async ({ + page, + }) => { + await page.goto("/#lexicon"); + await expect(page.getByTestId("lex-vacancy")).toBeVisible({ timeout: 30_000 }); + const honesty = await page.getByTestId("lex-vacancy-prosody-honesty").innerText(); + const entries = honesty.match(/(\d+) hand-set entries/)?.[1]; + expect(entries, `no entry count in: ${honesty}`).toBeDefined(); + + await openInfo(page); + await expect(page.getByTestId("info-view")).toContainText(`hand table of ${entries} entries`); + }); + + test("the documented static-mode limits are the constants the static client enforces", async ({ + page, + }) => { + // These two live in the static client rather than behind an endpoint, so the fact is + // read from the source that enforces it. A stated ± that was never measured is a + // fabricated error bar — which is exactly why the number must not be retyped. + const src = readFileSync(path.join(SRC, "lib", "staticClient", "arch.ts"), "utf8"); + const uncertainty = src.match(/VACANCY_Q8_UNCERTAINTY_NATS = ([\d.]+)/)?.[1]; + const floor = src.match(/VACANCY_MIN_POOLED_PRESERVED = (\d+)/)?.[1]; + expect(uncertainty, "VACANCY_Q8_UNCERTAINTY_NATS is gone or renamed").toBeDefined(); + expect(floor, "VACANCY_MIN_POOLED_PRESERVED is gone or renamed").toBeDefined(); + + await openInfo(page); + const info = page.getByTestId("info-view"); + await expect(info).toContainText(`±${uncertainty} nats`); + await expect(info).toContainText(`${floor} preserved tokens`); + }); + + test("the caveats that make the numbers readable are all present", async ({ page }) => { + // Each of these is a claim the instrument would be dishonest without, so each is + // asserted rather than left to survive an edit by luck. + await openInfo(page); + const info = page.getByTestId("info-view"); + // The decomposition, and that its second term is a BOUND rather than a measurement. + await expect(info).toContainText("the cost of wrong content"); + await expect(info).toContainText("the cost of unknown form"); + await expect(info).toContainText(/upper bound/i); + // What the static build refuses, by name. + await expect(info).toContainText("nonce − swap"); + await expect(info).toContainText(/Per-passage rows: refused/i); + await expect(info).toContainText(/dtype without a measured bound: refused/i); + // The coverage gap that belongs in the documentation rather than in a commit message. + // `\s+` rather than a space: a regex matcher sees the raw textContent, newlines and + // source indentation included, so a rewrapped paragraph must not fail this. + await expect(info).toContainText(/CI only ever\s+exercises the WASM rung/i); + // The exact zero, framed as the finding rather than as a missing curve. + await expect(info).toContainText(/exact zero is the result/i); + }); +}); diff --git a/code/frontend/tests/e2e/static.spec.ts b/code/frontend/tests/e2e/static.spec.ts index e1235f7..832efef 100644 --- a/code/frontend/tests/e2e/static.spec.ts +++ b/code/frontend/tests/e2e/static.spec.ts @@ -151,6 +151,70 @@ test("geometry lab runs fully live in-browser (engine, edits, worker fine-tune)" }); }); +/** + * A model trained here has a vocabulary of its OWN — its token id 17 is not the shipped + * model's token id 17 — and the static build keeps the model across a reload by + * persisting the minted weight set to sessionStorage. It used to persist the WEIGHTS + * ONLY, so after a reload the engine fell back to the shipped tokenizer and `Save model` + * wrote a `.llmgeo.json` pairing these weights with Alice in Wonderland's word list, + * hashing THAT list into `vocab_sha256`. The file was internally consistent, so no + * reader on either side could reject it: save → reload → save silently changed which + * words the model file described. The full stack was never affected — the Python + * backend stores the vocabulary beside the weights (`save_weight_set(..., vocab_json=)`). + * + * One epoch is enough: this is about what the file SAYS, not about the loss. + */ +test("a model trained here keeps its own vocabulary across a reload (save → reload → save)", async ({ + page, +}) => { + test.setTimeout(600_000); + await page.goto(`${BASE}#geometry`); + await ready(page, "geo-view", 60_000); + + const corpus = readFileSync( + path.join( + path.dirname(fileURLToPath(import.meta.url)), + "../../../backend/src/llm_geometry/lex/data/real-mother-goose.txt", + ), + "utf8", + ); + await page.getByTestId("geo-train-src-paste").click(); + await page.getByTestId("geo-train-text").fill(corpus); + await expect(page.getByTestId("geo-train-stats")).toContainText(/enough to fill/); + const epochs = page.getByTestId("geo-train-epochs"); + await epochs.fill("1"); + await epochs.dispatchEvent("input"); + await page.getByTestId("geo-train-run").click(); + await expect(page.getByTestId("geo-train-result")).toBeVisible({ timeout: 480_000 }); + + const saveBundle = async (): Promise<{ weights_token: string; vocab: string }> => { + const dl = page.waitForEvent("download", { timeout: 120_000 }); + await page.getByTestId("geo-save-model").click(); + await expect(page.getByTestId("geo-io-error")).toHaveCount(0); + const stream = await (await dl).createReadStream(); + const chunks: Buffer[] = []; + for await (const c of stream) chunks.push(c as Buffer); + return JSON.parse(Buffer.concat(chunks).toString("utf8")) as { + weights_token: string; + vocab: string; + }; + }; + + const before = await saveBundle(); + const wordsBefore = (JSON.parse(before.vocab) as { words: string[] }).words; + // It really is a vocabulary of its own, not the shipped one. + expect(wordsBefore).not.toContain("alice"); + + await page.reload(); + await ready(page, "geo-view", 60_000); + const after = await saveBundle(); + expect(after.weights_token).toBe(before.weights_token); + expect( + (JSON.parse(after.vocab) as { words: string[] }).words, + "the saved model file's vocabulary changed across a reload", + ).toEqual(wordsBefore); +}); + // --------------------------------------------------------------------------------- // [c] Architecture Explorer — precomputed graph/traces, LIVE weight windows (US-2) // --------------------------------------------------------------------------------- @@ -277,12 +341,62 @@ test("real in-browser generation on the smallest model", async ({ page }) => { const tok = page.getByTestId("arch-reply").locator(".tok").first(); await expect(tok).toBeAttached(); await expect(tok).toHaveAttribute("aria-label", /%/); - // the runtime ladder settled on a real device/dtype pair + // The runtime ladder settled on a device/dtype pair that PASSED the load-time + // non-degeneracy check (transformersRuntime.selfCheck). Both rungs read the same + // model_quantized.onnx; only the execution provider differs. await expect(page.getByTestId("static-runtime-badge")).toContainText( - /webgpu · q4f16|wasm · q8/, + /(webgpu|wasm) · q8/, ); await page.screenshot({ path: "tests/e2e/__screenshots__/static-generation.png", fullPage: true, }); }); + +// --------------------------------------------------------------------------------- +// [g] The vacancy instrument's pretrained arm — computed live, reported NARROWLY +// (feature 007, contract §8.3a / FR-720a / SC-707b). The browser runs a quantized +// export, so it may state only what was MEASURED for that dtype: pooled +// `swap − english` and `nonce − english` with the measured ±0.1 nats, and nothing +// else. This drives the built site and asserts it does one or the other — never a +// bare number. + +test("the vacancy panel reports only what q8 has a measured bound for", async ({ page }) => { + test.setTimeout(600_000); + await page.goto(BASE); + await page.getByTestId("tab-architecture").click(); + const panel = page.getByTestId("arch-vacancy"); + await panel.scrollIntoViewIfNeeded(); + await page.getByTestId("arch-vac-run").click(); + + // Real ONNX download + 18 real forward passes (6 pooled excerpts × 3 variants). + await expect(page.getByTestId("arch-vac-table")).toBeVisible({ timeout: 560_000 }); + await expect(page.getByTestId("arch-vac-error")).toHaveCount(0); + + // Refused: nonce − swap, by name, with the reason and the command that would fix it. + const refusal = page.getByTestId("arch-vac-refused-unknown_form"); + await expect(refusal).toBeVisible(); + await expect(refusal).toContainText("sign flip"); + await expect(refusal).toContainText("uvicorn"); + await expect(page.getByTestId("arch-vac-unknown_form")).toHaveCount(0); + + // Refused: the absolute NLLs and every per-passage row. + await expect(page.getByTestId("arch-vac-refused-absolute")).toBeVisible(); + await expect(page.getByTestId("arch-vac-refused-passages")).toBeVisible(); + const english = page.getByTestId("arch-vac-row-english"); + await expect(english).toContainText("—"); + + // Reported: the two pooled differences, each with the MEASURED quantization ±. + const wrong = page.getByTestId("arch-vac-wrong_content"); + await expect(wrong).toContainText(/\d\.\d{3}/); + await expect(page.getByTestId("arch-vac-wrong_content-err")).toContainText("0.2 (quantization, measured)"); + + // The residual caveat is stated even where the number is refused: a reader must not + // have to earn the caveat by being shown a value. + await expect(page.getByTestId("arch-vac-honesty")).toContainText("UPPER BOUND"); + await expect(page.getByTestId("arch-vac-honesty")).toContainText("higher entropy"); + + await panel.screenshot({ + path: "tests/e2e/__screenshots__/static-arch-vacancy.png", + }); +}); diff --git a/code/frontend/tests/e2e/vacancy.spec.ts b/code/frontend/tests/e2e/vacancy.spec.ts new file mode 100644 index 0000000..668118d --- /dev/null +++ b/code/frontend/tests/e2e/vacancy.spec.ts @@ -0,0 +1,395 @@ +import { expect, test, type Page } from "@playwright/test"; + +/** + * The vacancy panel of the Lexicon Lab (feature 007, `ui.md` §1), driven as a visitor + * drives it. + * + * These tests exist for the claims this panel would be worthless without, and every one + * of them is checked against the LIVE computation rather than against a string the panel + * could print unconditionally: + * + * * the transform really acts on the corpus, and moving `p` really changes it (FR-710); + * * nesting and stability are VISIBLE — a cell that turns minted never reverts, and the + * minted string is the same string in every later column (FR-711); + * * the instant invariance check reports identical id streams under `consistent` AND a + * real difference under `inconsistent` (FR-714 — a hard-coded tick would pass the + * first assertion and fail the second, which is the point of testing both); + * * the on-demand demonstration trains twice and reports max |Δloss| = exactly 0; + * * the tab still trains on the vacated corpus, at any budget (FR-713/FR-716). + * + * Every test that exercises a run watches for `pageerror` and console errors: a duplicate + * -key crash once shipped to the live site because nothing here was watching. + */ + +const LEX = "#lexicon"; + +/** Fail on any client-side exception or console error raised during the test. */ +function watchErrors(page: Page): string[] { + const errors: string[] = []; + page.on("pageerror", (e) => errors.push(String(e))); + page.on("console", (m) => { + if (m.type() === "error") errors.push(m.text()); + }); + return errors; +} + +/** Move the `p` slider and wait for the synchronous re-derivation to land. */ +async function setP(page: Page, value: string): Promise { + const slider = page.getByTestId("lex-vacancy-p"); + await slider.fill(value); + await slider.dispatchEvent("input"); + await expect(page.getByTestId("lex-vacancy")).toContainText(`p = ${Number(value).toFixed(2)}`); +} + +test.beforeEach(async ({ page }) => { + await page.goto(`/${LEX}`); + await expect(page.getByTestId("lex-view")).toBeVisible({ timeout: 30_000 }); + await expect(page.getByTestId("lex-corpus-error")).toHaveCount(0); + await expect(page.getByTestId("lex-vacancy")).toBeVisible({ timeout: 30_000 }); +}); + +test("the panel renders with its controls, corpus view, ribbon and statistics", async ({ + page, +}) => { + const errors = watchErrors(page); + for (const id of [ + "lex-vacancy-p", + "lex-vacancy-seed", + "lex-vacancy-condition", + "lex-vacancy-prosody", + "lex-vacancy-mint", + "lex-vacancy-corpus", + "lex-vacancy-legend", + "lex-vacancy-ribbon", + "lex-vacancy-stats", + "lex-vacancy-invariance", + "lex-vacancy-framing", + ]) { + await expect(page.getByTestId(id), `missing ${id}`).toBeVisible(); + } + + // The corpus view is the real corpus, not a placeholder. + const corpus = await page.getByTestId("lex-vacancy-corpus").innerText(); + expect(corpus.trim().length).toBeGreaterThan(200); + + // §1.4: corpus scope, never domain scope. `domainTypes*` must never reach a reader. + const stats = page.getByTestId("lex-vacancy-stats"); + await expect(stats).toContainText("types vacated"); + await expect(stats).not.toContainText(/domainTypes/i); + await expect(page.getByTestId("lex-vacancy-bijective")).toContainText("injective"); + + // §1.4 / SC-708: no prosody number without the three-way split and the caveat beside it. + await expect(page.getByTestId("lex-vacancy-prosody-stats")).toContainText("mean anapest"); + const split = page.getByTestId("lex-vacancy-stress-split"); + await expect(split).toContainText("hand table"); + await expect(split).toContainText("minted"); + await expect(split).toContainText("spelling rule"); + await expect(page.getByTestId("lex-vacancy-prosody-honesty")).toContainText( + /seeded by rule and never checked/i, + ); + + // §1.6: the null is framed as the finding, with the pretrained arm named. + await expect(page.getByTestId("lex-vacancy-framing")).toContainText(/exact zero/i); + await expect(page.getByTestId("lex-vacancy-framing")).toContainText(/Architecture Explorer/); + + expect(errors, `client-side errors: ${errors.join(" | ")}`).toEqual([]); +}); + +test("moving p rewrites the corpus, and the statistics move with it", async ({ page }) => { + const errors = watchErrors(page); + const corpus = page.getByTestId("lex-vacancy-corpus"); + const tokens = page.getByTestId("lex-vacancy-tokens"); + const types = page.getByTestId("lex-vacancy-types"); + + // At p = 0 nothing is vacated — that is the definition, not a claim about rendering. + await expect(tokens).toContainText(/^0\b/); + const at0 = await corpus.innerText(); + + await setP(page, "0.5"); + const at50 = await corpus.innerText(); + expect(at50).not.toBe(at0); + const tokens50 = await tokens.innerText(); + const types50 = await types.innerText(); + expect(Number(tokens50.split("/")[0].replace(/[^\d]/g, ""))).toBeGreaterThan(0); + + await setP(page, "1"); + const at100 = await corpus.innerText(); + expect(at100).not.toBe(at50); + // Monotone: more of the corpus is vacated at p = 1 than at p = 0.5 (nesting, in counts). + const num = (s: string) => Number(s.split("/")[0].replace(/[^\d]/g, "")); + expect(num(await tokens.innerText())).toBeGreaterThan(num(tokens50)); + expect(num(await types.innerText())).toBeGreaterThan(num(types50)); + + // At full vacancy every eligible type has moved: the two halves of the ratio agree. + const full = await types.innerText(); + const [vacated, eligible] = full.split("/").map((s) => Number(s.replace(/[^\d]/g, ""))); + expect(vacated).toBe(eligible); + + expect(errors, `client-side errors: ${errors.join(" | ")}`).toEqual([]); +}); + +test("the ribbon shows nesting and stability, cell by cell (FR-711)", async ({ page }) => { + const errors = watchErrors(page); + const rows = page.getByTestId("lex-vacancy-ribbon").locator("tbody tr"); + await expect(rows.first()).toBeVisible(); + const count = await rows.count(); + expect(count).toBeGreaterThanOrEqual(4); + + let sawFlip = false; + for (let r = 0; r < count; r++) { + const cells = rows.nth(r).locator("td.cell"); + expect(await cells.count()).toBe(5); + const classes: string[] = []; + const forms: string[] = []; + for (let c = 0; c < 5; c++) { + classes.push((await cells.nth(c).getAttribute("class")) ?? ""); + forms.push((await cells.nth(c).innerText()).trim()); + } + const minted = classes.map((c) => c.includes("minted")); + + // NESTING — once minted, never reverts as p grows. + let flipped = false; + for (let c = 0; c < 5; c++) { + if (minted[c]) flipped = true; + else expect(flipped, `row ${r} reverted at column ${c}: ${forms.join(" | ")}`).toBe(false); + } + // STABILITY — every minted cell in a row carries the SAME string. + const mintedForms = forms.filter((_, c) => minted[c]); + if (mintedForms.length > 0) { + expect(new Set(mintedForms).size, `row ${r} minted forms: ${mintedForms.join(" | ")}`).toBe(1); + // ...and it is not the English stem it replaced. + expect(mintedForms[0]).not.toBe(forms[0]); + } + if (mintedForms.length > 0 && mintedForms.length < 5) sawFlip = true; + } + // The rows span the u range, so at least one flips somewhere in the middle rather than + // every row being all-English or all-minted. + expect(sawFlip).toBe(true); + + // p = 0 is never vacated and p = 1 always is, so the first and last columns are the + // extremes of the demonstration. + await expect(rows.first().locator("td.cell").first()).toHaveClass(/open/); + await expect(rows.first().locator("td.cell").last()).toHaveClass(/minted/); + + await expect(page.getByTestId("lex-vacancy-ribbon-caption")).toContainText(/Nesting/); + await expect(page.getByTestId("lex-vacancy-ribbon-caption")).toContainText(/Stability/); + + expect(errors, `client-side errors: ${errors.join(" | ")}`).toEqual([]); +}); + +test("the instant invariance check is LIVE: identical under consistent, broken under inconsistent", async ({ + page, +}) => { + const errors = watchErrors(page); + const verdict = page.getByTestId("lex-vacancy-invariance-verdict"); + const box = page.getByTestId("lex-vacancy-invariance"); + + await setP(page, "0.6"); + // The theorem: the mapped condition encodes to the identical id stream. + await expect(verdict).toHaveText(/identical/i); + await expect(box).toContainText(/ids compared/); + const compared = Number( + (await box.innerText()).match(/([\d,]+) ids compared/)?.[1].replace(/,/g, "") ?? "0", + ); + expect(compared).toBeGreaterThan(1000); + + // ...and the control conditions really break it. A hard-coded tick passes above and + // fails here, which is exactly why both halves are asserted in one test. + await page + .getByTestId("lex-vacancy-condition") + .getByRole("radio", { name: "inconsistent", exact: true }) + .click(); + await expect(verdict).toHaveText(/differ/i); + const differing = Number( + (await box.innerText()).match(/([\d,]+) of [\d,]+ positions/)?.[1].replace(/,/g, "") ?? "0", + ); + expect(differing).toBeGreaterThan(0); + // Coverage collapses: a fresh type per occurrence cannot be in any fixed budget. + const unk = (await box.innerText()).match(/rate ([\d.]+)% → ([\d.]+)%/); + expect(unk, `no rates in: ${await box.innerText()}`).not.toBeNull(); + expect(Number(unk![2])).toBeGreaterThan(Number(unk![1])); + + // Partial reveal splits every vacated type in two, and breaks it too. + await page + .getByTestId("lex-vacancy-condition") + .getByRole("radio", { name: "partial reveal", exact: true }) + .click(); + await expect(verdict).toHaveText(/differ/i); + + // Back to the mapped condition and it is identical again — the check tracks the state. + await page + .getByTestId("lex-vacancy-condition") + .getByRole("radio", { name: "consistent", exact: true }) + .click(); + await expect(verdict).toHaveText(/identical/i); + + expect(errors, `client-side errors: ${errors.join(" | ")}`).toEqual([]); +}); + +test("changing the seed re-mints the corpus but keeps the theorem (SC-702/SC-703)", async ({ + page, +}) => { + const errors = watchErrors(page); + await setP(page, "0.7"); + const before = await page.getByTestId("lex-vacancy-corpus").innerText(); + + await page.getByTestId("lex-vacancy-seed").fill("7"); + await page.getByTestId("lex-vacancy-seed").dispatchEvent("input"); + await expect + .poll(async () => page.getByTestId("lex-vacancy-corpus").innerText(), { timeout: 30_000 }) + .not.toBe(before); + + await expect(page.getByTestId("lex-vacancy-invariance-verdict")).toHaveText(/identical/i); + await expect(page.getByTestId("lex-vacancy-bijective")).toContainText("injective"); + + expect(errors, `client-side errors: ${errors.join(" | ")}`).toEqual([]); +}); + +test("the on-demand demonstration trains twice and reports max |Δloss| = 0 (FR-714)", async ({ + page, +}) => { + test.setTimeout(300_000); + const errors = watchErrors(page); + + await setP(page, "0.75"); + await page.getByTestId("lex-vacancy-demo-run").click(); + + const delta = page.getByTestId("lex-vacancy-demo-delta"); + await expect(delta).toBeVisible({ timeout: 240_000 }); + // Reported as an exact 0 — not "≈0", not rounded away. §1.6 is a hard requirement. + await expect(delta).toContainText("max |Δloss| = 0"); + await expect(delta).not.toContainText(/should have been exactly 0/); + await expect(page.getByTestId("lex-vacancy-demo-result")).toBeVisible(); + + expect(errors, `client-side errors: ${errors.join(" | ")}`).toEqual([]); +}); + +test("the tab trains on the VACATED corpus, at a non-default budget (FR-713/FR-716)", async ({ + page, +}) => { + test.setTimeout(300_000); + const errors = watchErrors(page); + + // Compose with the tab's own controls: a frequency budget, a small model, a real run. + await page.getByTestId("lex-budget-source").getByRole("radio", { name: /corpus top-N/i }).click(); + await setP(page, "0.8"); + + // The budget's coverage is measured against the VACATED text, so it is live before the + // first gradient step — that is the whole argument of the budget panel, preserved here. + await expect(page.getByTestId("lex-coverage-tokens")).toHaveText(/^\d+(\.\d+)?%$/); + await expect(page.getByTestId("lex-train-active-corpus")).toContainText(/vacated p=0\.80/); + + await page.getByTestId("lex-dmodel").getByRole("radio", { name: "16" }).click(); + // A multiple of the default sampleEvery (50): the periodic sampler and the final sample + // then land on the same step, which is the case that once threw `each_key_duplicate`. + await page.getByTestId("lex-steps").fill("100"); + await page.getByTestId("lex-train-run").click(); + + const done = page.getByTestId("lex-train-done"); + await expect(done).toBeVisible({ timeout: 240_000 }); + const nums = [...(await done.innerText()).matchAll(/(\d+\.\d+)/g)].map((m) => Number(m[1])); + expect(nums.length).toBeGreaterThanOrEqual(2); + expect(Math.min(...nums)).toBeLessThan(Math.max(...nums)); + await expect(page.getByTestId("lex-active-model")).toContainText(/vacated p=0\.80/); + + expect(errors, `client-side errors: ${errors.join(" | ")}`).toEqual([]); +}); + +test("the swap control runs, and refuses the p it cannot support (FR-719a, §5.2a)", async ({ + page, +}) => { + const errors = watchErrors(page); + const corpus = page.getByTestId("lex-vacancy-corpus"); + const lost = page.getByTestId("lex-vacancy-lost-slots"); + const mint = page.getByTestId("lex-vacancy-mint"); + + // The control is live, not decoration: switching it rewrites the corpus with real English + // words drawn by frequency rank instead of invented ones. + await setP(page, "1"); + const withNonce = await corpus.innerText(); + await expect(lost).toHaveText("0"); + await mint.getByRole("radio", { name: "swap", exact: true }).click(); + await expect(mint.getByRole("radio", { name: "swap", exact: true })).toHaveAttribute( + "aria-checked", + "true", + ); + const withSwap = await corpus.innerText(); + expect(withSwap).not.toBe(withNonce); + + // At FULL vacancy swap is a bijection of the domain, so the theorem holds for it exactly + // as it does for nonce — that is the check that the control is implemented correctly. + await expect(lost).toHaveText("0"); + await expect(page.getByTestId("lex-vacancy-invariance-verdict")).toHaveText(/identical/i); + await expect(page.getByTestId("lex-vacancy-refusal")).toHaveCount(0); + + // In between it CANNOT be injective (contract §5.2a). The panel must show the engine's + // refusal and the measured cost — never a clamped p, never a silent nonce map. + await setP(page, "0.5"); + const refusal = page.getByTestId("lex-vacancy-refusal"); + await expect(refusal).toBeVisible(); + await expect(page.getByTestId("lex-vacancy-refusal-message")).toContainText("§5.2a"); + await expect(page.getByTestId("lex-vacancy-refusal-message")).toContainText("swap"); + expect(Number((await lost.innerText()).replace(/[^\d]/g, ""))).toBeGreaterThan(0); + // The slider really is still where the reader put it, and nothing was computed in place + // of the refused vocabulary. + await expect(page.getByTestId("lex-vacancy")).toContainText("p = 0.50"); + await expect(mint.getByRole("radio", { name: "swap", exact: true })).toHaveAttribute( + "aria-checked", + "true", + ); + await expect(page.getByTestId("lex-vacancy-invariance")).toHaveCount(0); + await expect(page.getByTestId("lex-vacancy-demo-run")).toBeDisabled(); + + // The way out is offered explicitly and works. + await page.getByTestId("lex-vacancy-refusal-p1").click(); + await expect(refusal).toHaveCount(0); + await expect(page.getByTestId("lex-vacancy")).toContainText("p = 1.00"); + await expect(page.getByTestId("lex-vacancy-invariance-verdict")).toHaveText(/identical/i); + + // The inconsistent control is refused under swap for a countable reason: it needs a fresh + // type per occurrence and the corpus has no supply of real words at that rate. + await page + .getByTestId("lex-vacancy-condition") + .getByRole("radio", { name: "inconsistent", exact: true }) + .click(); + await expect(refusal).toBeVisible(); + await expect(page.getByTestId("lex-vacancy-refusal-message")).toContainText(/consistent/); + + expect(errors, `client-side errors: ${errors.join(" | ")}`).toEqual([]); +}); + +test("no horizontal page overflow at 390px with the panel present", async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto(`/${LEX}`); + await expect(page.getByTestId("lex-vacancy")).toBeVisible({ timeout: 30_000 }); + await setP(page, "0.65"); + const overflow = await page.evaluate( + () => document.documentElement.scrollWidth - document.documentElement.clientWidth, + ); + expect(overflow).toBeLessThanOrEqual(0); +}); + +test("the corpus view opens on the verse, not on the book's table of contents", async ({ + page, +}) => { + const errors = watchErrors(page); + + // The shipped corpus carries 618 token-producing lines of front matter -- a title page, a + // list of rhymes, and an index of first lines -- before LITTLE BO-PEEP starts the verse at + // line 619. Landing there makes the transform look like it rewrites an index, which is the + // least interesting thing it does. No general rule separates the two honestly: the index of + // first lines has verse-length lines, so a line-length heuristic stops inside the front + // matter. The panel therefore pins the boundary, and this is the assertion that it is right. + const windowLabel = page.getByTestId("lex-vacancy-window"); + await expect(windowLabel).toContainText("601"); + + // At p = 0 the corpus is untransformed, so the real words must be on screen. + const corpus = page.getByTestId("lex-vacancy-corpus"); + await expect(corpus).toContainText("Bo-Peep"); + + // Paging back must still work, and must reach the front matter it deliberately skipped. + await page.getByTestId("lex-vacancy-prev").click(); + await expect(windowLabel).toContainText("561"); + + expect(errors, `console errors: ${errors.join(" | ")}`).toEqual([]); +}); diff --git a/code/frontend/tests/e2e/webgpu.spec.ts b/code/frontend/tests/e2e/webgpu.spec.ts new file mode 100644 index 0000000..7371dc0 --- /dev/null +++ b/code/frontend/tests/e2e/webgpu.spec.ts @@ -0,0 +1,150 @@ +import { expect, test, type Page } from "@playwright/test"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; + +/** + * The WebGPU regression suite — the coverage gap that let a broken dtype ship. + * + * Plain headless Chromium exposes NO WebGPU adapter (`requestAdapter()` resolves to + * `null`), so every other e2e project silently exercises the WASM rung only. That is + * why `webgpu/q4f16` — which BUILDS a session and then returns logits identical at + * every position — reached the deployed site: nothing threw, and CI never ran the code + * path. This project launches Chromium with `--enable-unsafe-webgpu`, which does + * surface the machine's real adapter, and asserts the property the defect violated: + * the model's next-token distribution must DEPEND ON POSITION. + * + * Where there is no adapter, the test SKIPS with an explicit reason — it must never + * pass vacuously by falling through to WASM. + * + * READ THIS BEFORE TRUSTING A GREEN CI RUN. GitHub-hosted runners have no GPU, and the + * software adapter Chromium offers there (SwiftShader) advertises no `shader-f16`, so + * this test SKIPS in CI and the WebGPU path is verified only on a developer machine + * with a real GPU. That is a named, deliberate gap, not coverage. What CI does always + * verify: tests/unit/logitsSanity.test.ts (the invariant and the dtype ladder) and + * tests/e2e/static.spec.ts (a real session built and passed through the same + * load-time gate, on the WASM rung). + */ + +const BASE = "/llm-geometry/"; +const DATA = path.join(path.dirname(fileURLToPath(import.meta.url)), "../../public/static-data"); + +interface StaticIndex { + arch_models: { model_id: string; slug: string }[]; +} +const index = JSON.parse(readFileSync(path.join(DATA, "index.json"), "utf8")) as StaticIndex; + +// The smallest curated model this export ships: both are known-degenerate under +// q4f16 on WebGPU and known-correct under q8, so either discriminates. +const model = + index.arch_models.find((m) => m.model_id === "HuggingFaceTB/SmolLM2-135M-Instruct") ?? + index.arch_models.find((m) => m.model_id === "gpt2") ?? + index.arch_models[0]; + +interface AdapterProbe { + hasNavigatorGpu: boolean; + adapter: null | { vendor?: string; architecture?: string; shaderF16: boolean }; +} + +async function probeAdapter(page: Page): Promise { + return page.evaluate(async () => { + const gpu = (navigator as unknown as { gpu?: { requestAdapter(): Promise } }).gpu; + if (!gpu) return { hasNavigatorGpu: false, adapter: null }; + const a = (await gpu.requestAdapter()) as null | { + features: { has(n: string): boolean }; + info?: { vendor?: string; architecture?: string }; + }; + if (!a) return { hasNavigatorGpu: true, adapter: null }; + return { + hasNavigatorGpu: true, + adapter: { + vendor: a.info?.vendor, + architecture: a.info?.architecture, + shaderF16: a.features.has("shader-f16"), + }, + }; + }); +} + +test("in-browser generation on a REAL WebGPU adapter is not degenerate", async ({ page }) => { + test.setTimeout(600_000); + await page.goto(BASE); + + const probe = await probeAdapter(page); + // A loud skip, never a vacuous pass: say exactly what was missing. + const why = !probe.hasNavigatorGpu + ? "navigator.gpu is undefined in this browser build" + : probe.adapter === null + ? "navigator.gpu.requestAdapter() resolved to null — this environment has no WebGPU adapter " + + "(the usual case for Linux CI runners and for headless Chromium without --enable-unsafe-webgpu)" + : !probe.adapter.shaderF16 + ? `the only adapter is ${probe.adapter.vendor}/${probe.adapter.architecture}, which lacks ` + + "shader-f16 — the app treats that as a software adapter and takes the WASM rung, so this " + + "test cannot observe the WebGPU path" + : ""; + if (why) { + console.warn( + `\n!!! SKIPPING THE WEBGPU REGRESSION TEST — THE WEBGPU PATH IS UNVERIFIED IN THIS RUN.\n` + + `!!! Reason: ${why}.\n` + + `!!! This is expected on GitHub-hosted runners, which have no GPU. A green CI run\n` + + `!!! therefore does NOT mean the WebGPU path was checked: run it on a machine with a\n` + + `!!! real GPU — npx playwright test --project webgpu\n`, + ); + test.skip( + true, + `WEBGPU PATH UNVERIFIED (expected on GPU-less CI runners): ${why}. ` + + "Run `npx playwright test --project webgpu` on a machine with a GPU.", + ); + } + console.log( + `[webgpu] real adapter: ${probe.adapter?.vendor}/${probe.adapter?.architecture} (shader-f16)`, + ); + + await page.getByTestId("tab-architecture").click(); + await expect(page.locator('[data-testid^="diagram-node-"]').first()).toBeVisible({ + timeout: 60_000, + }); + if (model.model_id !== (await page.getByTestId("arch-model-select").inputValue())) { + await page.getByTestId("arch-model-select").selectOption(model.model_id); + await expect(page.getByTestId("arch-model-status")).toHaveText("ok", { timeout: 60_000 }); + } + await page.getByTestId("arch-prompt").fill("The capital of France is Paris. The capital of Germany is"); + await page.getByTestId("arch-generate").click(); + + // Real ONNX download (100+ MB cold) + real sampling on the GPU. + await expect(page.getByTestId("arch-reply")).not.toBeEmpty({ timeout: 480_000 }); + + // (1) The WebGPU path really was the one exercised — otherwise everything below + // would be a WASM result wearing a WebGPU test's name. + const badge = page.getByTestId("static-runtime-badge"); + await expect(badge).toContainText("webgpu"); + // (2) …on a dtype the runtime verified at load time. q4f16 is gone; if it ever + // comes back, this pins the failure to the dtype rather than to the symptom. + await expect(badge).toContainText("webgpu · q8"); + + // (3) THE DEFECT ITSELF, as a user sees it. Under webgpu/q4f16 every row of the + // [1,T,V] logits was bit-identical, so the tab reported the SAME top-5 at every + // generated position. A model whose distribution does not move with the context + // has told the user nothing. Measured on this page, 64 generated tokens: + // webgpu/q4f16 (pre-fix) → 2 distinct top-5 lists out of 64 + // webgpu/q8 (post-fix) → 64 distinct top-5 lists out of 64 + // so "most positions differ" separates them by the whole range. Requiring ALL to + // differ would be a fair statement too, but it is a claim about the model rather + // than about the runtime, and a legitimate repeat would then fail the suite. + const labels = await page.getByTestId("arch-reply").locator(".tok").evaluateAll((els) => + els.map((e) => e.getAttribute("aria-label") ?? ""), + ); + expect(labels.length, "generation produced too few tokens to compare positions").toBeGreaterThan(8); + const topk = labels + .map((l) => l.slice(l.indexOf("top-5:"))) + .filter((s) => s.startsWith("top-5:")); + expect(topk.length).toBe(labels.length); + expect( + new Set(topk).size, + `only ${new Set(topk).size} of ${topk.length} generated positions reported a DIFFERENT ` + + "top-5 — the logits barely depend on the context, which is exactly the webgpu/q4f16 " + + "failure this test exists to catch", + ).toBeGreaterThan(topk.length / 2); + + await page.screenshot({ path: "tests/e2e/__screenshots__/webgpu-generation.png", fullPage: true }); +}); diff --git a/code/frontend/tests/fixtures/arch-vacancy-passages.json b/code/frontend/tests/fixtures/arch-vacancy-passages.json new file mode 100644 index 0000000..2ba3597 --- /dev/null +++ b/code/frontend/tests/fixtures/arch-vacancy-passages.json @@ -0,0 +1,50 @@ +{ + "note": "Digests of the default passage set of contract §8.3a. Written by scripts/export_arch_vacancy_golden.py from the real corpus; asserted against the browser's own cut in tests/unit/archVacancy.test.ts.", + "corpus_sha256": "d514f0fd2cd40967eb6cf35b140a6cddc11200126e07d76603fae3f88bf1e0ab", + "count": 6, + "words_per_passage": 250, + "passages": [ + { + "index": 0, + "sha256": "31e1b77674aa4b0b3c599ddbfbfb33bb437716d2147cec7455d36fb9431c0452", + "n_words": 252, + "n_chars": 1396, + "head": "\"She is lamed, leaping over a stile.\" \"Alack! and I must ke" + }, + { + "index": 1, + "sha256": "74d2d5a416328bb587c6767bdb5152f28b0d2e7f75414f33b9ff71138758f360", + "n_words": 255, + "n_chars": 1413, + "head": "ABOUT THE BUSH About the bush, Willie, About the beehiv" + }, + { + "index": 2, + "sha256": "b4dc966fca010cc20a3a0473159206452252e2766d7538bb9b5422025ed119b6", + "n_words": 251, + "n_chars": 1285, + "head": "In a velvet coat, He kissed a maid And gave her a groat. The" + }, + { + "index": 3, + "sha256": "3f0bb57d5f59ef4a2f964c89287ef7cf1cd9d748416d8b7389c7bd57bf947be5", + "n_words": 256, + "n_chars": 1370, + "head": "That kissed the maiden all forlorn, That milked the cow with" + }, + { + "index": 4, + "sha256": "56237a4fd99b8c3cfc8ca3a8833e52929a21c544da94b89f75a643d2991eb114", + "n_words": 254, + "n_chars": 1410, + "head": "Saturday's child works hard for its living, But the child th" + }, + { + "index": 5, + "sha256": "0f12a7366b1baa64d4bce99978c273200b98a7395a0e1238ef32b3c98f55af25", + "n_words": 253, + "n_chars": 1433, + "head": "And ne'er went up again. PETER PIPER Peter Piper pick" + } + ] +} diff --git a/code/frontend/tests/fixtures/vacancy-api-golden.json b/code/frontend/tests/fixtures/vacancy-api-golden.json new file mode 100644 index 0000000..07e3439 --- /dev/null +++ b/code/frontend/tests/fixtures/vacancy-api-golden.json @@ -0,0 +1,1170 @@ +{ + "format": "vacancy-api-golden-v1", + "generated": "2026-08-04", + "git_sha": "7dc491808cea5dc07b4b397e8ca50d217586165a", + "command": "python scripts/export_vacancy_api_golden.py", + "source": "POST /api/lex/vacancy on the real FastAPI app with the real committed corpus, through fastapi.testclient \u2014 real route, real transform, no mocks", + "contract": "specs/002-interactive-model-explorer/contracts/api.md", + "tolerance": 0.0, + "endpoint": "/api/lex/vacancy", + "defaults": { + "preview_chars": 2000, + "preview_max": 20000 + }, + "encoding": "exactly what the route serves: every float already rounded to 6 significant digits by api/encoding.py::jsonable_6sig, which staticClient/lex.ts::sig6 reproduces, so every field compares EXACTLY. `vacated_sha256` is over the UTF-8 bytes of the WHOLE vacated corpus \u2014 86 kB pinned in 64 characters.", + "corpus": { + "path": "code/backend/src/llm_geometry/lex/data/real-mother-goose.txt", + "sha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c", + "chars": 86408 + }, + "cases": [ + { + "label": "p1-seed7-pre_primer", + "request": { + "p": 1.0, + "seed": 7, + "source": "dolch", + "budget": "pre_primer" + }, + "response": { + "p": 1.0, + "seed": 7, + "consistent": true, + "match_prosody": true, + "reveal_after": 0, + "keep": [], + "vocabulary_rule": "mapped", + "words": [ + "a", + "and", + "unpryng", + "gnoud", + "brermp", + "can", + "vieck", + "down", + "scyb", + "for", + "vezzleum", + "go", + "sorp", + "here", + "i", + "in", + "is", + "it", + "scerrk", + "shooventy", + "hieff", + "shream", + "me", + "my", + "not", + "one", + "blurt", + "smust", + "maik", + "plys", + "sqourk", + "the", + "three", + "to", + "two", + "up", + "we", + "where", + "frarder", + "you" + ], + "budget": { + "source": "dolch", + "budget": "pre_primer", + "size": 40, + "rows": 44, + "coverage": { + "total_tokens": 16000, + "in_budget_tokens": 4457, + "distinct_types": 2211, + "oov_types": 2172, + "total_lines": 3071, + "whole_lines_in_budget": 5, + "token_coverage": 0.278562, + "unk_rate": 0.721437 + } + }, + "corpus": { + "n_tokens": 16000, + "n_distinct": 2211, + "n_lines": 3075, + "n_chars": 96334 + }, + "vacancy_stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1944, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1922, + "stemsTotal": 1680, + "stemsVacated": 1680, + "tokensTotal": 16000, + "tokensVacated": 8202, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.29113, + "meanAnapestBefore": 0.320932, + "meanAnapestAfter": 0.321422, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.012, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.404062, + "stressFromRuleBefore": 0.948562, + "stressFromRuleAfter": 0.583937, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "bijective": true, + "remint_rounds": 1, + "preview": " THE KRARRD\n SORISH KLOALK\n\n _Wermishened by_\nPirv Smausker Chouth\n\n1916\n\n\n\nA SHIRK OF THE SORFES\n\nShooventy Bo-Peep\nShooventy Skurth Brermp\nYurnk\nThe Desp\nZeller\nYeebys and Spoults\nA Smeapicumen Strieft\nSaun Shrurll and Her Florll\nThree Broarmidy on the Skach\nKroots Ploank\nThe Kir Fliespous Under a Drier\nTweedle-Dum and Tweedle-Dee\nOh Wousk!\nKir Sorish Kloalk\nShooventy Scerrking Blarrn\nPat-a-Cake\nFrerkic and the Treark\nTrirmpy Klauvent\nA Gielkishously Strieft\nPlief\nSmoav to St. Choosts\nSwaimum Slorms Churg Frazzleener\nYarmid Bebly\nFrocks\nVieck Out to Blurt\nIf Sqastes Were Skornes\nTo Joskent\nKir Rirchs to Chait\nTrirmpy and Shruffum\nA Tun and a Kor\nHere Swoafs My Hang\nThe Flermer Sum\nTwo Snourns\nBuk Over Buk\nShiffel Fringow\nWhen Driedle Thrirl Was Tarl\nKraiger\nThe Friming Groask\nSwaishowing Jubing\nHush-a-Bye\nVurlk Frock\nThree Heach Sked of Snoordic\nThe Breafter of Plaimle\nShooventy Slorshly Floshums\nHock Unpryng, Hock Unpryng\nNotum Drier\nPussy-Cat and Hysp\nThe Strirlts\nVyg Merrties\nTrysles\nGnaugumicer\nJust Streash Me\nBlurt Slorms\nHeigh-Ho, the Struftous Strang\nDREAND\nA Trurndel and Skeek\nSporrdelic Kroots\nThe Tun in Our Loarn\nStroudle Strerbleel\nFor Every Dryndel\nKurle Chuz\nBlyp Vurp Vud\nAbout the Trut\nSee-Saw\nRobin-a-Bobbin\nBlirlt Kron\nFliken Bliernous\nThree Smard Porff\nFive Spoults\nA Shooventy Tun\nGerrmen Seacker\nGnoozzleel Gnoozzleel Woording\nNiernel Chyf\nZaukleishing Slorms\nThe Shoal Sum\nThe Soot\nA Vemer\nSmarms Tairmic\nCurly-Locks\nRezle Spercking\nOne, Two, Three\nThe Stursk and the Thrirl\nMorster I Have\nLarls\nShall We Go A-Shearing?\nJarler, Jarler, Smarnker\nKir Sorish Cherngle\nThe Kault and the Sum\nBrermp Snid Skurth\nWhy May Not I Sceerm Plonker?\nPlief Hoth\nPlief Shirck\nHush-a-Bye\nRarbenouns\nThe Grud in the Hurs\nHush-a-Bye\nSweandleel Gneffleic\nStrorskish Faly\nPlief and Choan\nThe Zorngidel\nThourt to Your Drong\nOne Kegent Flerrmel Dreapent\nTrirmpy Scoort and Shooventy Blirlt\nYurnk\nThe Kir Fliespous from Wyff\nFlith and Threrns\nThe Trirmpys\nThe Kir Tun\nT'Othe", + "original_preview": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\nA Melancholy Song\nJack\nGoing to St. Ives\nThirty Days Hath September\nBaby Dolly\nBees\nCome Out to Play\nIf Wishes Were Horses\nTo Market\nOld Chairs to Mend\nRobin and Richard\nA Man and a Maid\nHere Goes My Lord\nThe Clever Hen\nTwo Birds\nLeg Over Leg\nLucy Locket\nWhen Jenny Wren Was Young\nBarber\nThe Flying Pig\nSolomon Grundy\nHush-a-Bye\nBurnie Bee\nThree Wise Men of Gotham\nThe Hunter of Reigate\nLittle Polly Flinders\nRide Away, Ride Away\nPippen Hill\nPussy-Cat and Queen\nThe Winds\nClap Handies\nChristmas\nElizabeth\nJust Like Me\nPlay Days\nHeigh-Ho, the Carrion Crow\nABC\nA Needle and Thread\nBanbury Cross\nThe Man in Our Town\nGeorgy Porgy\nFor Every Evil\nCushy Cow\nWee Willie Winkie\nAbout the Bush\nSee-Saw\nRobin-a-Bobbin\nJohn Smith\nSimple Simon\nThree Blind Mice\nFive Toes\nA Little Man\nDoctor Foster\nDiddle Diddle Dumpling\nJerry Hall\nLengthening Days\nThe Black Hen\nThe Mist\nA Candle\nMiss Muffet\nCurly-Locks\nHumpty Dumpty\nOne, Two, Three\nThe Dove and the Wren\nMaster I Have\nPins\nShall We Go A-Shearing?\nGoosey, Goosey, Gander\nOld Mother Hubbard\nThe Cock and the Hen\nBlue Bell Boy\nWhy May Not I Love Johnny?\nJack Jelf\nJack Sprat\nHush-a-Bye\nDaffodils\nThe Girl in the Lane\nHush-a-Bye\nNancy Dawson\nHandy Pandy\nJack and Jill\nThe Alphabet\nDance to Your Daddie\nOne Misty Moisty Morning\nRobin Hood and Little John\nRain\nThe Old Woman from France\nTeeth and Gums\nThe Robins\nThe Old Man\nT'Other Little Tune\nMy Kitten\nIf All the Seas Were One Sea\nPancake Day\nA Plum Pudding\nForehead, Eyes, Cheeks, Nose, etc.\nTwo Pigeons\nA Sure Test\nLock and Key\nThe Lion and the Unicorn\nThe Merchants of London\nI Had a Little Husband\nTo Babylon\n", + "preview_chars": 2000, + "truncated": true, + "vacated_chars": 96334, + "vacated_sha256": "263b8dd3733dbf4802a2bc4c74fb905388062952e45931e4ebcd561cbd4e76df", + "original_chars": 86408, + "original_sha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c" + } + }, + { + "label": "p035-seed0-full", + "request": { + "p": 0.35, + "seed": 0, + "source": "dolch", + "budget": "full", + "preview_chars": 400 + }, + "response": { + "p": 0.35, + "seed": 0, + "consistent": true, + "match_prosody": true, + "reveal_after": 0, + "keep": [], + "vocabulary_rule": "mapped", + "words": [ + "a", + "and", + "away", + "hirm", + "gea", + "can", + "wrais", + "down", + "wif", + "for", + "funny", + "go", + "help", + "here", + "i", + "in", + "is", + "it", + "byll", + "little", + "snout", + "make", + "me", + "my", + "not", + "one", + "play", + "red", + "run", + "grurst", + "vault", + "the", + "three", + "to", + "two", + "up", + "we", + "where", + "yellow", + "you", + "all", + "am", + "are", + "at", + "ate", + "be", + "black", + "bor", + "but", + "came", + "did", + "do", + "ben", + "four", + "get", + "good", + "have", + "he", + "into", + "kliff", + "must", + "new", + "no", + "now", + "on", + "our", + "out", + "please", + "pretty", + "glaimp", + "scard", + "derck", + "say", + "she", + "so", + "dreer", + "that", + "there", + "they", + "tharcks", + "too", + "under", + "lirnk", + "was", + "well", + "smen", + "what", + "white", + "who", + "will", + "with", + "smoarn", + "after", + "again", + "an", + "any", + "as", + "gnurnk", + "by", + "could", + "every", + "fly", + "from", + "give", + "myck", + "had", + "has", + "her", + "him", + "his", + "how", + "just", + "know", + "sorr", + "live", + "may", + "of", + "garr", + "once", + "open", + "over", + "put", + "sirs", + "some", + "stop", + "take", + "glault", + "them", + "then", + "slurrk", + "walk", + "were", + "when", + "always", + "asqeaf", + "because", + "been", + "before", + "best", + "both", + "buy", + "call", + "cold", + "does", + "don't", + "yourm", + "first", + "five", + "staim", + "gave", + "voonks", + "green", + "its", + "brorrk", + "wieckel", + "off", + "or", + "mirmp", + "read", + "lound", + "sing", + "sit", + "sleep", + "tell", + "their", + "these", + "those", + "adroust", + "us", + "kleall", + "very", + "hirn", + "which", + "why", + "wish", + "work", + "would", + "write", + "your", + "about", + "storsher", + "bring", + "carry", + "zaul", + "cut", + "done", + "draw", + "drink", + "eight", + "griesp", + "far", + "dil", + "got", + "grow", + "hold", + "hot", + "hurt", + "if", + "keep", + "kind", + "wourm", + "light", + "long", + "much", + "begnirsk", + "never", + "only", + "own", + "pick", + "seven", + "shall", + "hirk", + "six", + "small", + "start", + "ten", + "refon", + "together", + "gliest", + "warm", + "apple", + "baby", + "back", + "ball", + "vert", + "runk", + "bell", + "zel", + "birthday", + "boat", + "mylk", + "boy", + "bread", + "zemer", + "scoog", + "wrarsp", + "cat", + "scielk", + "chicken", + "kliesowle", + "christmas", + "coat", + "corn", + "cow", + "strirm", + "scarrt", + "doll", + "sposk", + "fav", + "naird", + "plouch", + "farm", + "farmer", + "father", + "plerv", + "skaust", + "fish", + "loll", + "droorner", + "game", + "garden", + "girl", + "good-bye", + "grass", + "ground", + "thaint", + "head", + "prornt", + "mout", + "horse", + "keaf", + "kitty", + "baimp", + "thrirrmer", + "woosp", + "men", + "milk", + "money", + "noarish", + "wielkow", + "name", + "nest", + "night", + "paper", + "party", + "freaching", + "pig", + "spylous", + "rain", + "ring", + "sqorlous", + "school", + "seed", + "gloas", + "shoe", + "sister", + "snow", + "song", + "straull", + "stick", + "brap", + "sun", + "table", + "praisk", + "time", + "hor", + "toy", + "tree", + "watch", + "water", + "way", + "wind", + "window", + "wood" + ], + "budget": { + "source": "dolch", + "budget": "full", + "size": 314, + "rows": 318, + "coverage": { + "total_tokens": 16000, + "in_budget_tokens": 9726, + "distinct_types": 2211, + "oov_types": 1919, + "total_lines": 3071, + "whole_lines_in_budget": 215, + "token_coverage": 0.607875, + "unk_rate": 0.392125 + } + }, + "corpus": { + "n_tokens": 16000, + "n_distinct": 2211, + "n_lines": 3075, + "n_chars": 90204 + }, + "vacancy_stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 683, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 672, + "stemsTotal": 1680, + "stemsVacated": 587, + "tokensTotal": 16000, + "tokensVacated": 3152, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.29006, + "meanAnapestBefore": 0.320932, + "meanAnapestAfter": 0.320887, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0416875, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.154875, + "stressFromRuleBefore": 0.948562, + "stressFromRuleAfter": 0.803438, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "bijective": true, + "remint_rounds": 0, + "preview": " THE PRIRLK\n WIELKOW BRARSP\n\n _Illustrated by_\nBlanche Fisher Trurrk\n\n1916\n\n\n\nA LIST OF THE SHRENGES\n\nLittle Bo-Peep\nLittle Boy Gea\nRain\nThe Clock\nWinter\nTitows and Toes\nA Squrvididish Song\nStruv Vard and Her Cat\nThree Kliesowle on the Ice\nDroalls Brem\nThe Garr Plaispy Under a Prornt\nTweedle-Dum and Tweedle-Dee\nOh Grirn!\nGarr Wielkow Brarsp\nLittle Bylling Joan\nPat-a-Cake\nMoney and the Mar", + "original_preview": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\n", + "preview_chars": 400, + "truncated": true, + "vacated_chars": 90204, + "vacated_sha256": "f70d3febc5552d9e194b980a8811cb16fccc9d6d011049e574aca3fa6cf39d3a", + "original_chars": 86408, + "original_sha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c" + } + }, + { + "label": "p0-seed0-pre_primer", + "request": { + "p": 0.0, + "seed": 0, + "source": "dolch", + "budget": "pre_primer", + "preview_chars": 400 + }, + "response": { + "p": 0.0, + "seed": 0, + "consistent": true, + "match_prosody": true, + "reveal_after": 0, + "keep": [], + "vocabulary_rule": "mapped", + "words": [ + "a", + "and", + "away", + "big", + "blue", + "can", + "come", + "down", + "find", + "for", + "funny", + "go", + "help", + "here", + "i", + "in", + "is", + "it", + "jump", + "little", + "look", + "make", + "me", + "my", + "not", + "one", + "play", + "red", + "run", + "said", + "see", + "the", + "three", + "to", + "two", + "up", + "we", + "where", + "yellow", + "you" + ], + "budget": { + "source": "dolch", + "budget": "pre_primer", + "size": 40, + "rows": 44, + "coverage": { + "total_tokens": 16000, + "in_budget_tokens": 4457, + "distinct_types": 2211, + "oov_types": 2172, + "total_lines": 3071, + "whole_lines_in_budget": 5, + "token_coverage": 0.278562, + "unk_rate": 0.721437 + } + }, + "corpus": { + "n_tokens": 16000, + "n_distinct": 2211, + "n_lines": 3075, + "n_chars": 86408 + }, + "vacancy_stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 0, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 0, + "stemsTotal": 1680, + "stemsVacated": 0, + "tokensTotal": 16000, + "tokensVacated": 0, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.29, + "meanAnapestBefore": 0.320932, + "meanAnapestAfter": 0.320932, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0514375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.0, + "stressFromRuleBefore": 0.948562, + "stressFromRuleAfter": 0.948562, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "bijective": true, + "remint_rounds": 0, + "preview": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\n", + "original_preview": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\n", + "preview_chars": 400, + "truncated": true, + "vacated_chars": 86408, + "vacated_sha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c", + "original_chars": 86408, + "original_sha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c" + } + }, + { + "label": "control-inconsistent", + "request": { + "p": 0.5, + "seed": 0, + "consistent": false, + "source": "dolch", + "budget": "primer" + }, + "response": { + "p": 0.5, + "seed": 0, + "consistent": false, + "match_prosody": true, + "reveal_after": 0, + "keep": [], + "vocabulary_rule": "rebuilt", + "words": [ + "a", + "and", + "away", + "big", + "blue", + "can", + "come", + "down", + "find", + "for", + "funny", + "go", + "help", + "here", + "i", + "in", + "is", + "it", + "jump", + "little", + "look", + "make", + "me", + "my", + "not", + "one", + "play", + "red", + "run", + "said", + "see", + "the", + "three", + "to", + "two", + "up", + "we", + "where", + "yellow", + "you", + "all", + "am", + "are", + "at", + "ate", + "be", + "black", + "brown", + "but", + "came", + "did", + "do", + "eat", + "four", + "get", + "good", + "have", + "he", + "into", + "like", + "must", + "new", + "no", + "now", + "on", + "our", + "out", + "please", + "pretty", + "ran", + "ride", + "saw", + "say", + "she", + "so", + "soon", + "that", + "there", + "they", + "this", + "too", + "under", + "want", + "was", + "well", + "went", + "what", + "white", + "who", + "will", + "with", + "yes" + ], + "budget": { + "source": "dolch", + "budget": "primer", + "size": 92, + "rows": 96, + "coverage": { + "total_tokens": 16000, + "in_budget_tokens": 5863, + "distinct_types": 5727, + "oov_types": 5659, + "total_lines": 3071, + "whole_lines_in_budget": 9, + "token_coverage": 0.366438, + "unk_rate": 0.633563 + } + }, + "corpus": { + "n_tokens": 16000, + "n_distinct": 5727, + "n_lines": 3075, + "n_chars": 92804 + }, + "vacancy_stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 966, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 954, + "stemsTotal": 1680, + "stemsVacated": 833, + "tokensTotal": 16000, + "tokensVacated": 4470, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.28994, + "meanAnapestBefore": 0.320932, + "meanAnapestAfter": 0.321837, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.023375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.220625, + "stressFromRuleBefore": 0.948562, + "stressFromRuleAfter": 0.756, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "bijective": true, + "remint_rounds": 0, + "preview": " THE GLAIRN\n SCIRRDER PLERNT\n\n _Illustrated by_\nBlanche Thursper Styn\n\n1916\n\n\n\nA HURF OF THE YORCKES\n\nWrerkenle Bo-Peep\nFlorlkishle Boy Poun\nRain\nThe Clock\nWinter\nNietens and Toes\nA Gnuntingenle Jai\nSculk Mursh and Her Cat\nThree Thoochousish on the Ice\nNefs Gask\nThe Soamp Therltle Under a Gloang\nTweedle-Dum and Tweedle-Dee\nOh Steb!\nSwouck Felkid Rarp\nLainowid Yersking Joan\nPat-a-Cake\nMoney and the Klerck\nYiftel Smombleer\nA Melancholy Striesp\nJack\nSkamp to St. Ives\nPlorntel Wroors Strarg Joffleinger\nBaby Dolly\nGnesps\nPlieft Out to Play\nIf Wishes Were Brouses\nTo Fleevic\nBluch Garrms to Mend\nTrembley and Richard\nA Shrerrt and a Treeck\nHere Grends My Lord\nThe Clever Hen\nTwo Volts\nStoon Over Shorlt\nLucy Locket\nWhen Jenny Slarz Was Wroult\nBarber\nThe Flying Pig\nSolomon Zaibish\nHush-a-Bye\nBurnie Stairm\nThree Smooff Men of Gotham\nThe Skoster of Reigate\nBlerningous Polly Purnishs\nSpurmp Away, Foum Away\nGnydleid Fronk\nPussy-Cat and Lorch\nThe Klorlks\nClap Scusies\nChristmas\nElizabeth\nJust Bief Me\nPlay Brarrts\nHeigh-Ho, the Zirkleel Crow\nABC\nA Needle and Thread\nBanbury Wirs\nThe Weav in Our Town\nKrymbleing Maurner\nFor Every Skorrking\nCushy Cow\nSlailt Threart Winkie\nAbout the Broonk\nSee-Saw\nRobin-a-Bobbin\nFraung Kych\nTything Smaispent\nThree Blan Brock\nFive Toes\nA Spampelle Zurst\nDoctor Foster\nDiddle Diddle Vurrting\nLiden Prars\nFlirnkering Zoolls\nThe Black Hen\nThe Grerl\nA Rospent\nBoms Muffet\nCurly-Locks\nZeavel Thirving\nOne, Two, Three\nThe Dove and the Broark\nMaster I Have\nPins\nShall We Go A-Shearing?\nGoosey, Goosey, Sweacker\nSpousp Glaikleic Wrirrle\nThe Cock and the Hen\nBlep Bell Boy\nWhy May Not I Skerch Johnny?\nJack Jelf\nJack Prask\nHush-a-Bye\nThrieshishents\nThe Girl in the Firv\nHush-a-Bye\nWroving Dawson\nFiekleer Miepel\nJack and Jill\nThe Alphabet\nDance to Your Daddie\nOne Kruckish Scorstic Grerger\nKykent Scyg and Flyfterent Floul\nRain\nThe Blien Freezzleum from Churnd\nTeeth and Gums\nThe Woskishs\nThe Scooz Mar\nT'Other Niempicid Tune\nMy Kitten\nIf All the Snirns Were One Shrost\n", + "original_preview": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\nA Melancholy Song\nJack\nGoing to St. Ives\nThirty Days Hath September\nBaby Dolly\nBees\nCome Out to Play\nIf Wishes Were Horses\nTo Market\nOld Chairs to Mend\nRobin and Richard\nA Man and a Maid\nHere Goes My Lord\nThe Clever Hen\nTwo Birds\nLeg Over Leg\nLucy Locket\nWhen Jenny Wren Was Young\nBarber\nThe Flying Pig\nSolomon Grundy\nHush-a-Bye\nBurnie Bee\nThree Wise Men of Gotham\nThe Hunter of Reigate\nLittle Polly Flinders\nRide Away, Ride Away\nPippen Hill\nPussy-Cat and Queen\nThe Winds\nClap Handies\nChristmas\nElizabeth\nJust Like Me\nPlay Days\nHeigh-Ho, the Carrion Crow\nABC\nA Needle and Thread\nBanbury Cross\nThe Man in Our Town\nGeorgy Porgy\nFor Every Evil\nCushy Cow\nWee Willie Winkie\nAbout the Bush\nSee-Saw\nRobin-a-Bobbin\nJohn Smith\nSimple Simon\nThree Blind Mice\nFive Toes\nA Little Man\nDoctor Foster\nDiddle Diddle Dumpling\nJerry Hall\nLengthening Days\nThe Black Hen\nThe Mist\nA Candle\nMiss Muffet\nCurly-Locks\nHumpty Dumpty\nOne, Two, Three\nThe Dove and the Wren\nMaster I Have\nPins\nShall We Go A-Shearing?\nGoosey, Goosey, Gander\nOld Mother Hubbard\nThe Cock and the Hen\nBlue Bell Boy\nWhy May Not I Love Johnny?\nJack Jelf\nJack Sprat\nHush-a-Bye\nDaffodils\nThe Girl in the Lane\nHush-a-Bye\nNancy Dawson\nHandy Pandy\nJack and Jill\nThe Alphabet\nDance to Your Daddie\nOne Misty Moisty Morning\nRobin Hood and Little John\nRain\nThe Old Woman from France\nTeeth and Gums\nThe Robins\nThe Old Man\nT'Other Little Tune\nMy Kitten\nIf All the Seas Were One Sea\nPancake Day\nA Plum Pudding\nForehead, Eyes, Cheeks, Nose, etc.\nTwo Pigeons\nA Sure Test\nLock and Key\nThe Lion and the Unicorn\nThe Merchants of London\nI Had a Little Husband\nTo Babylon\n", + "preview_chars": 2000, + "truncated": true, + "vacated_chars": 92804, + "vacated_sha256": "c2f9e95fe83f464d54c6f0ae85e6d3df3cd6dd304614212c5cebe950ce000a7a", + "original_chars": 86408, + "original_sha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c" + } + }, + { + "label": "control-reveal-after-2", + "request": { + "p": 0.5, + "seed": 0, + "reveal_after": 2, + "source": "dolch", + "budget": "primer", + "preview_chars": 400 + }, + "response": { + "p": 0.5, + "seed": 0, + "consistent": true, + "match_prosody": true, + "reveal_after": 2, + "keep": [], + "vocabulary_rule": "rebuilt", + "words": [ + "a", + "and", + "away", + "big", + "blue", + "can", + "come", + "down", + "find", + "for", + "funny", + "go", + "help", + "here", + "i", + "in", + "is", + "it", + "jump", + "little", + "look", + "make", + "me", + "my", + "not", + "one", + "play", + "red", + "run", + "said", + "see", + "the", + "three", + "to", + "two", + "up", + "we", + "where", + "yellow", + "you", + "all", + "am", + "are", + "at", + "ate", + "be", + "black", + "brown", + "but", + "came", + "did", + "do", + "eat", + "four", + "get", + "good", + "have", + "he", + "into", + "like", + "must", + "new", + "no", + "now", + "on", + "our", + "out", + "please", + "pretty", + "ran", + "ride", + "saw", + "say", + "she", + "so", + "soon", + "that", + "there", + "they", + "this", + "too", + "under", + "want", + "was", + "well", + "went", + "what", + "white", + "who", + "will", + "with", + "yes" + ], + "budget": { + "source": "dolch", + "budget": "primer", + "size": 92, + "rows": 96, + "coverage": { + "total_tokens": 16000, + "in_budget_tokens": 5903, + "distinct_types": 2591, + "oov_types": 2502, + "total_lines": 3071, + "whole_lines_in_budget": 14, + "token_coverage": 0.368937, + "unk_rate": 0.631063 + } + }, + "corpus": { + "n_tokens": 16000, + "n_distinct": 2591, + "n_lines": 3075, + "n_chars": 90859 + }, + "vacancy_stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 966, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 467, + "stemsTotal": 1680, + "stemsVacated": 833, + "tokensTotal": 16000, + "tokensVacated": 3128, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.28981, + "meanAnapestBefore": 0.320932, + "meanAnapestAfter": 0.321139, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0256875, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.160125, + "stressFromRuleBefore": 0.948562, + "stressFromRuleAfter": 0.814187, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "bijective": true, + "remint_rounds": 0, + "preview": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nSkoufenty Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbrea", + "original_preview": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\n", + "preview_chars": 400, + "truncated": true, + "vacated_chars": 90859, + "vacated_sha256": "eaab90a02f74a23b0ec5cb5de79fcd1adc8a227a683f43b4f6a5cafd40d3250d", + "original_chars": 86408, + "original_sha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c" + } + }, + { + "label": "p07-seed0-frequency100", + "request": { + "p": 0.7, + "seed": 0, + "match_prosody": false, + "source": "frequency", + "budget": "full", + "size": 100, + "preview_chars": 400 + }, + "response": { + "p": 0.7, + "seed": 0, + "consistent": true, + "match_prosody": false, + "reveal_after": 0, + "keep": [], + "vocabulary_rule": "mapped", + "words": [ + "the", + "and", + "a", + "to", + "i", + "strek", + "of", + "was", + "in", + "my", + "he", + "garr", + "you", + "his", + "she", + "all", + "that", + "is", + "as", + "with", + "on", + "there", + "for", + "her", + "it", + "woosp", + "smen", + "plaisp", + "me", + "what", + "be", + "when", + "an", + "had", + "if", + "tharcks", + "do", + "so", + "dance", + "they", + "up", + "not", + "will", + "jack", + "one", + "wrais", + "squz", + "have", + "shall", + "were", + "baby", + "your", + "him", + "sqorl", + "go", + "two", + "strorn", + "them", + "grieg", + "but", + "down", + "out", + "who", + "over", + "i'll", + "we", + "boy", + "streesh", + "three", + "grurst", + "are", + "bist", + "oh", + "some", + "at", + "brorrk", + "by", + "keaf", + "again", + "can", + "fers", + "mout", + "traiz", + "strirm", + "myck", + "no", + "pussy-cat", + "vault", + "floosh", + "scarrt", + "failt", + "sqirz", + "cock", + "may", + "scard", + "yart", + "shoe", + "how", + "kliff", + "thream" + ], + "budget": { + "source": "frequency", + "budget": "top100", + "size": 100, + "rows": 104, + "coverage": { + "total_tokens": 16000, + "in_budget_tokens": 8378, + "distinct_types": 2211, + "oov_types": 2111, + "total_lines": 3071, + "whole_lines_in_budget": 87, + "token_coverage": 0.523625, + "unk_rate": 0.476375 + } + }, + "corpus": { + "n_tokens": 16000, + "n_distinct": 2211, + "n_lines": 3075, + "n_chars": 89070 + }, + "vacancy_stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1354, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1337, + "stemsTotal": 1680, + "stemsVacated": 1167, + "tokensTotal": 16000, + "tokensVacated": 6094, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.15469, + "meanAnapestBefore": 0.320932, + "meanAnapestAfter": 0.286373, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.017375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.30325, + "stressFromRuleBefore": 0.948562, + "stressFromRuleAfter": 0.679375, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "bijective": true, + "remint_rounds": 1, + "preview": " THE PRIRLK\n WIELK BRARSP\n\n _Trirmed by_\nBlanche Neker Trurrk\n\n1916\n\n\n\nA SORP OF THE SHRENGES\n\nStrek Bo-Peep\nStrek Boy Gea\nRain\nThe Snuch\nWinter\nTits and Smurlks\nA Squrv Shyf\nStruv Vard and Her Cat\nThree Klies on the Ice\nDroalls Brem\nThe Garr Plaisp Under a Prornt\nTweedle-Dum and Tweedle-Dee\nOh Grirn!\nGarr Wielk Brarsp\nStrek Bylling Boub\nPat-a-Cake\nNied and the Meez\nSqorl Buff\nA Melanchol", + "original_preview": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\n", + "preview_chars": 400, + "truncated": true, + "vacated_chars": 89070, + "vacated_sha256": "4e19fffc82f8ca95bbab20ecc1c8a900d5339c01a98017d644a848d7290a6720", + "original_chars": 86408, + "original_sha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c" + } + } + ] +} diff --git a/code/frontend/tests/fixtures/vacancy-golden.json b/code/frontend/tests/fixtures/vacancy-golden.json new file mode 100644 index 0000000..9ea746e --- /dev/null +++ b/code/frontend/tests/fixtures/vacancy-golden.json @@ -0,0 +1,9959 @@ +{ + "format": "vacancy-golden-v2", + "generated": "2026-08-04", + "git_sha": "1b4704bf76847b582b159a6bea59940053536ff5", + "command": "python scripts/export_vacancy_golden.py", + "source": "llm_geometry.lex.vacancy run directly on the committed corpus \u2014 real code, real text, no mocks", + "contract": "specs/007-vacancy-transform-field/architecture.md", + "tolerance": 1e-12, + "python_version": "3.10.12", + "encoding": "every value is plain JSON; floats are shortest-round-trip in both languages, so `u` and the digests compare EXACTLY. Only the prosody means (meanSyllables*, meanAnapest*, stressFrom*) use `tolerance`. Digests are sha256 hex: `vacatedSha256` over the vacated corpus's UTF-8 bytes, `mappingSha256` over `stem\\tnonce\\n` in ASCII-ascending stem order, `idStream.digest` over the ids joined by ','.", + "corpus": { + "path": "code/backend/src/llm_geometry/lex/data/real-mother-goose.txt", + "note": "the Gutenberg body, trimmed exactly as lex/corpus.py trims it", + "sha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c", + "chars": 86408, + "tokens": 16000, + "corpusTypes": 2211, + "domainSize": 2233, + "budget": "dolch/full", + "budgetSize": 314 + }, + "stems": [ + { + "stem": "little", + "eligible": true, + "stemOf": "little", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.4828060925071693, + "7": 0.3611914557373461 + } + }, + { + "stem": "pretty", + "eligible": true, + "stemOf": "pretty", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.5944985085782986, + "7": 0.02829824691001792 + } + }, + { + "stem": "run", + "eligible": true, + "stemOf": "run", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.9988690574630219, + "7": 0.25923238473134436 + } + }, + { + "stem": "eat", + "eligible": true, + "stemOf": "eat", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.2034756841049038, + "7": 0.47634909208816056 + } + }, + { + "stem": "jump", + "eligible": true, + "stemOf": "jump", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.32941508632239547, + "7": 0.4093928965558291 + } + }, + { + "stem": "away", + "eligible": true, + "stemOf": "away", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.5602299313055585, + "7": 0.07218694485389798 + } + }, + { + "stem": "squirrel", + "eligible": true, + "stemOf": "squirrel", + "suffixOf": "", + "inDomain": true, + "inCorpus": false, + "inDolchFull": true, + "u": { + "0": 0.2076388363273125, + "7": 0.7610203331870103 + } + }, + { + "stem": "funny", + "eligible": true, + "stemOf": "funny", + "suffixOf": "", + "inDomain": true, + "inCorpus": false, + "inDolchFull": true, + "u": { + "0": 0.6518292753602092, + "7": 0.6938234082642826 + } + }, + { + "stem": "today", + "eligible": true, + "stemOf": "today", + "suffixOf": "", + "inDomain": true, + "inCorpus": false, + "inDolchFull": true, + "u": { + "0": 0.15662376856742133, + "7": 0.1756106962923738 + } + }, + { + "stem": "gum", + "eligible": true, + "stemOf": "gum", + "suffixOf": "", + "inDomain": false, + "inCorpus": false, + "inDolchFull": false, + "u": { + "0": 0.5896218925388563, + "7": 0.32158711739792134 + } + }, + { + "stem": "hang", + "eligible": true, + "stemOf": "hang", + "suffixOf": "", + "inDomain": false, + "inCorpus": false, + "inDolchFull": false, + "u": { + "0": 0.8319684963602456, + "7": 0.028958449446448986 + } + }, + { + "stem": "crown", + "eligible": true, + "stemOf": "crown", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": false, + "u": { + "0": 0.5645006323718861, + "7": 0.054891945717030155 + } + }, + { + "stem": "candlestick", + "eligible": true, + "stemOf": "candlestick", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": false, + "u": { + "0": 0.49440045796571275, + "7": 0.6484023755642774 + } + }, + { + "stem": "crooked", + "eligible": true, + "stemOf": "crook", + "suffixOf": "ed", + "inDomain": true, + "inCorpus": true, + "inDolchFull": false, + "u": { + "0": 0.10062333822562819, + "7": 0.901052283718315 + } + }, + { + "stem": "diddle", + "eligible": true, + "stemOf": "diddle", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": false, + "u": { + "0": 0.9059459641083152, + "7": 0.02325572582037938 + } + }, + { + "stem": "moon", + "eligible": true, + "stemOf": "moon", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": false, + "u": { + "0": 0.5703623945090363, + "7": 0.09610913417591349 + } + }, + { + "stem": "pussy", + "eligible": true, + "stemOf": "pussy", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": false, + "u": { + "0": 0.6841963950822295, + "7": 0.4793409816004338 + } + }, + { + "stem": "goose", + "eligible": true, + "stemOf": "goose", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": false, + "u": { + "0": 0.06968076981078941, + "7": 0.14621038126691366 + } + }, + { + "stem": "the", + "eligible": false, + "stemOf": "the", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.9987736588011006, + "7": 0.12747272849684788 + } + }, + { + "stem": "and", + "eligible": false, + "stemOf": "and", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.13688090633946592, + "7": 0.1651719984927117 + } + }, + { + "stem": "you", + "eligible": false, + "stemOf": "you", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.6402246951088357, + "7": 0.6976193800611918 + } + }, + { + "stem": "not", + "eligible": false, + "stemOf": "not", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.48839430801913464, + "7": 0.713033334566615 + } + }, + { + "stem": "ox", + "eligible": false, + "stemOf": "ox", + "suffixOf": "", + "inDomain": false, + "inCorpus": false, + "inDolchFull": false, + "u": { + "0": 0.39576990348181396, + "7": 0.4290536413580559 + } + }, + { + "stem": "good-bye", + "eligible": false, + "stemOf": "good-bye", + "suffixOf": "", + "inDomain": true, + "inCorpus": true, + "inDolchFull": true, + "u": { + "0": 0.24653910052796724, + "7": 0.6036603536638073 + } + } + ], + "maps": [ + { + "label": "seed0", + "seed": 0, + "matchProsody": true, + "mint": "nonce", + "injectiveAtEveryP": true, + "remintRounds": 0, + "bijective": true, + "imageSize": 2233, + "domainSize": 2233, + "mappingSize": 1680, + "mappingSha256": "7ba5235f4b848f548abe1c7bdcdb5ebecaa5d1d0c93be321e540705339d313aa", + "mapping": { + "abc": "plerp", + "abroad": "rigleic", + "adieu": "speegleow", + "admittance": "gryvelle", + "advice": "gnurkish", + "afraid": "meernous", + "aft": "swaiz", + "age": "shoaft", + "agree": "sceskum", + "ain": "vark", + "air": "terrk", + "alabone": "reetleentish", + "alack": "paisker", + "ale": "groopish", + "alehouse": "kaurkerer", + "alive": "chorntent", + "alley": "fryzy", + "almanac": "threrntidid", + "alone": "trorsken", + "along": "torffy", + "alphabet": "waungentum", + "alphabetical": "nursherentousent", + "alway": "shryshid", + "amble": "floanking", + "ampersand": "jotousum", + "angry": "graurish", + "ann": "grauth", + "announce": "frauzous", + "anoth": "virlting", + "answer": "snygous", + "appl": "threeng", + "apple": "hoadent", + "appoint": "waullel", + "april": "gneefting", + "arm": "threeg", + "around": "asqeaf", + "arrow": "jielty", + "art": "slausp", + "ask": "gnurnk", + "asleep": "koagle", + "ate": "shamp", + "aught": "deaz", + "aunt": "threch", + "away": "bepaip", + "awhile": "shrytleumous", + "awoke": "throlid", + "axe": "krerr", + "aye": "zesk", + "baa": "giesp", + "babby": "strirskow", + "baby": "starndleid", + "babylon": "soozery", + "bachelor": "stontentid", + "back": "fiep", + "backward": "strisid", + "bacon": "jirntum", + "bad": "groob", + "bade": "snorg", + "bag": "rirst", + "bailey": "poartel", + "bak": "brat", + "bake": "churlt", + "baker": "thorrmow", + "ball": "fliev", + "balloon": "larskle", + "banbury": "spaukleerish", + "bandy": "froarkum", + "barb": "zerrm", + "barber": "soagen", + "bare": "strarnt", + "bark": "byl", + "barley": "shouzent", + "barm": "sausp", + "barn": "fuz", + "barrel": "kleembleid", + "basket": "loaffent", + "bat": "ploaft", + "battle": "fleezent", + "bead": "gnid", + "bean": "soosp", + "bear": "vert", + "beat": "swauv", + "beaten": "teetleow", + "bed": "runk", + "bedtime": "blyndum", + "bee": "purch", + "beef": "strarm", + "beehive": "threashy", + "beer": "gnaif", + "beetle": "gliedum", + "began": "klumic", + "beggar": "wickid", + "begin": "pooltel", + "beginn": "nushid", + "begun": "gneelous", + "behind": "kredleer", + "believe": "zurffum", + "bell": "floosh", + "belleisle": "toankingic", + "bend": "moush", + "bent": "garlt", + "berr": "plarn", + "bes": "chyth", + "beside": "fedic", + "bessy": "skilish", + "best": "trerft", + "betsy": "fieftent", + "bett": "storsh", + "betty": "snyrment", + "betwixt": "sopy", + "bid": "geaft", + "big": "hirm", + "bigg": "root", + "bil": "sqeaf", + "bill": "churff", + "bind": "weam", + "bird": "zel", + "birthday": "gragingen", + "bit": "jarmp", + "bite": "gneap", + "bitt": "sqormp", + "bitten": "thoogen", + "black": "spair", + "blackbird": "stoavum", + "blacksmith": "scorrning", + "blanche": "zoot", + "ble": "shriev", + "bleak": "thriev", + "bleat": "streang", + "bles": "scersk", + "blessing": "gailkow", + "blind": "kros", + "blis": "scairm", + "blithe": "vit", + "blood": "deesp", + "blow": "skiv", + "blue": "gea", + "blush": "stoock", + "boat": "klirlt", + "bob": "sleech", + "bobby": "zaspic", + "body": "wrurbel", + "boggen": "sceasow", + "boil": "wrurv", + "boldero": "skeetleumic", + "bolin": "blouser", + "bombay": "purbel", + "bone": "chorng", + "bonn": "slyrt", + "bonnet": "porging", + "bonnie": "girv", + "bonny": "lobleow", + "book": "hoolk", + "born": "stysk", + "bottle": "husker", + "bough": "gyrk", + "bought": "marsh", + "bounc": "trud", + "bow": "bro", + "bowl": "shrors", + "box": "mylk", + "boy": "sim", + "bramble": "praiffleel", + "brandy": "zorlty", + "bras": "krarsp", + "bray": "glird", + "bread": "bloal", + "break": "throuck", + "breech": "krond", + "brew": "floff", + "bri": "grierm", + "brickbat": "spandleum", + "bride": "scyll", + "bridge": "skeaff", + "bridle": "vougleish", + "bright": "striff", + "bring": "shies", + "bristol": "rorstent", + "broke": "therth", + "broken": "shoampy", + "brook": "dreab", + "broom": "thround", + "broth": "waulk", + "brother": "zemer", + "brought": "pal", + "brown": "bor", + "buckl": "kauff", + "buckle": "lurdleer", + "build": "stroaft", + "built": "verv", + "bull": "mieff", + "bullet": "sondid", + "bullfinch": "larbleum", + "bump": "skarrd", + "bumpety": "sqantishid", + "bun": "nerr", + "bunch": "blorr", + "bunt": "gorlk", + "buri": "snaipent", + "burial": "blarty", + "burnie": "strooff", + "burnt": "shurnt", + "bush": "zoaft", + "butt": "mouft", + "button": "smeagleish", + "buy": "smirrm", + "bye": "klyrn", + "cabin": "klounky", + "caesar": "jeebow", + "cage": "wailk", + "cake": "scoog", + "calf": "vynt", + "call": "scoord", + "came": "strorn", + "canary": "drieperen", + "candl": "praulk", + "candle": "blaizzleous", + "candlestick": "vormenent", + "candy": "bloornum", + "cannot": "woulle", + "cant": "sloum", + "cap": "hooff", + "captain": "jeerkle", + "car": "wrarsp", + "care": "scarsh", + "carr": "glis", + "carri": "smenking", + "carrion": "borffleic", + "carry": "grirrning", + "carv": "shaisk", + "cast": "spersp", + "cat": "chooz", + "catch": "strusk", + "caught": "thrarn", + "ceas": "shealk", + "ceil": "striev", + "cellar": "smimpent", + "chain": "chilk", + "chair": "scielk", + "chamb": "leab", + "champ": "vook", + "chanc": "port", + "charley": "smantic", + "che": "seent", + "cheek": "narm", + "cheese": "klend", + "cherry": "kailking", + "chicken": "woastish", + "chid": "yarm", + "chief": "bleas", + "child": "raurt", + "children": "kliesowle", + "chimney": "klouzel", + "chin": "shreab", + "chirp": "neant", + "chirrup": "sweertous", + "choice": "smorrt", + "choose": "vees", + "chopp": "krea", + "christen": "wrirrkous", + "christma": "chashish", + "cinder": "shrespid", + "city": "streafic", + "clap": "slolk", + "claw": "yurrk", + "clay": "griend", + "clean": "zaul", + "clear": "boung", + "clergyman": "woampelous", + "clerk": "spoump", + "clev": "laum", + "cloak": "droav", + "clock": "snuch", + "cloth": "geaff", + "clothe": "spirnk", + "cloudy": "skerffleid", + "coachman": "priling", + "coal": "gnirnd", + "coat": "haun", + "cobbler": "drermbleish", + "cobweb": "zoumpy", + "cock": "kert", + "codlin": "profter", + "coffee": "flothen", + "coffin": "zautow", + "cold": "krorlk", + "cole": "skinden", + "collar": "grylker", + "colt": "threng", + "com": "threrth", + "comb": "slaurd", + "come": "wrais", + "comfit": "preangum", + "comical": "brermbleerel", + "comin": "dirtent", + "command": "chaudleid", + "compare": "torfing", + "compliment": "deatleousing", + "consid": "wausid", + "consider": "trympenen", + "contrary": "driermentish", + "contrive": "kentel", + "coo": "derd", + "copp": "shurd", + "coral": "shunden", + "corn": "troas", + "corner": "theesken", + "cost": "straish", + "cottage": "wrorndleid", + "count": "dealt", + "court": "sqirst", + "cov": "sqeft", + "cover": "snery", + "cow": "zast", + "crackabone": "dumpousen", + "cradle": "maitleum", + "cream": "spish", + "creep": "trarrm", + "crept": "thork", + "cri": "slooll", + "croak": "pien", + "crook": "rieg", + "cros": "droall", + "crow": "shraurd", + "crown": "strearm", + "crumb": "chooch", + "crumpl": "klyn", + "crusoe": "threrrk", + "cry": "mem", + "cup": "drarg", + "cupboard": "jiendid", + "cur": "mird", + "curd": "friff", + "curl": "sperth", + "currant": "klordleous", + "curtsy": "zerrmen", + "cushion": "laky", + "cushy": "rindleing", + "custard": "zearmish", + "cut": "roft", + "cutery": "breftidic", + "daddie": "wrerng", + "daddy": "peedleow", + "daff": "swarrk", + "daffodil": "greckenten", + "dainti": "karpid", + "dainty": "yeardel", + "dairy": "driery", + "dam": "skarnd", + "dame": "struv", + "danc": "gnyz", + "dance": "gleert", + "dang": "kush", + "dapple": "sceembleum", + "dare": "rout", + "dark": "broab", + "darlington": "throusentous", + "dat": "bymp", + "daught": "fyrn", + "daughter": "gnaickish", + "daw": "shrurn", + "dawson": "byndleum", + "day": "strirm", + "dead": "sqylt", + "dear": "grirn", + "deary": "storckic", + "death": "skoav", + "deceit": "druskic", + "decide": "scontous", + "deck": "rurk", + "declar": "vyble", + "ded": "gnaird", + "dee": "flooz", + "deed": "swoot", + "deep": "bloonk", + "delight": "bident", + "delve": "roult", + "derby": "bierdic", + "determin": "heerkousid", + "dew": "porch", + "diamond": "skeftent", + "dick": "nyth", + "dickery": "siefouser", + "dickory": "smeandleowy", + "dicky": "squrching", + "diddle": "kobleel", + "die": "maus", + "died": "fraisp", + "diet": "blink", + "difficult": "skeampyle", + "dig": "plook", + "dill": "smurm", + "ding": "neez", + "dinkety": "stiekidel", + "dinn": "fab", + "dirty": "floten", + "dish": "lyck", + "dishy": "strilish", + "displeas": "groatleel", + "ditch": "cherth", + "division": "strelkishic", + "dob": "glunt", + "dock": "sci", + "doctor": "groolkid", + "doe": "blush", + "doff": "troand", + "dog": "scarrt", + "dol": "frailk", + "doll": "thrus", + "dollar": "baullel", + "dolly": "plournish", + "don": "dud", + "dong": "klish", + "donkey": "wryrkle", + "doo": "grirsp", + "doodle": "prorbum", + "door": "sposk", + "dost": "drausp", + "doth": "wroush", + "doubt": "gnop", + "dov": "laung", + "dove": "shousp", + "downstair": "zoobic", + "dozen": "snorchous", + "drak": "gryrm", + "drake": "yyft", + "draw": "brarm", + "dream": "nysk", + "dreamt": "borrd", + "dreary": "swieken", + "dres": "wrych", + "dress": "skoup", + "drink": "blaurt", + "driv": "seech", + "drive": "druck", + "dropp": "girst", + "drove": "karp", + "drown": "sqirnt", + "drum": "shrorr", + "drumm": "snoorm", + "dry": "strailk", + "duck": "fav", + "dumpl": "kors", + "dumpling": "nafent", + "dumpty": "wroutleer", + "dun": "dirr", + "durst": "woaf", + "dusty": "freeffing", + "dwell": "hiz", + "dwelt": "moult", + "ear": "smyst", + "earth": "zount", + "east": "slelk", + "eat": "ben", + "egg": "naird", + "eighteen": "veevle", + "eith": "yarck", + "eleven": "smooleler", + "elizabeth": "kigleentishish", + "ell": "bars", + "else": "baug", + "elspeth": "stetleen", + "empty": "hoovous", + "end": "dind", + "england": "gergleid", + "enough": "gnale", + "equal": "brouffum", + "espi": "krirskum", + "etc": "trorrt", + "etticoat": "shorrticous", + "evermore": "gnardentent", + "everyone": "blauzzleerer", + "evil": "snongel", + "except": "thrertent", + "exet": "skoffleous", + "eye": "plouch", + "face": "cheeck", + "fail": "tharv", + "fair": "swamp", + "fall": "griesp", + "fan": "roam", + "fare": "slonk", + "farm": "shon", + "farmer": "thaifter", + "farth": "kleank", + "farthing": "sqerskle", + "fast": "yourm", + "fat": "smoan", + "father": "gniembleid", + "fear": "hoump", + "feast": "strirff", + "feath": "dad", + "feather": "fikleum", + "february": "kroolleric", + "fed": "zieft", + "feed": "klurnd", + "feet": "plerv", + "fell": "thark", + "fellow": "thysky", + "fetch": "scief", + "fiddl": "pep", + "fiddle": "chortid", + "fiddler": "blerrnow", + "fie": "shraich", + "field": "sork", + "fife": "grorsh", + "fifteen": "roungous", + "fight": "krork", + "fill": "skirff", + "fin": "choort", + "find": "wif", + "fine": "graik", + "fing": "shouf", + "finger": "titow", + "fir": "swark", + "fire": "skaust", + "first": "spind", + "fish": "nek", + "fishy": "squrrming", + "fit": "geed", + "flame": "josh", + "flapp": "jair", + "fleece": "chum", + "fleet": "theend", + "flew": "tuck", + "flinder": "gyming", + "flock": "chiech", + "floor": "loll", + "flour": "wand", + "flow": "droorn", + "flower": "swatleel", + "flung": "shrurch", + "flute": "naik", + "fly": "foob", + "fol": "shroum", + "folk": "poall", + "fond": "benk", + "fool": "fraf", + "foot": "braim", + "footman": "shoufent", + "forc": "stoad", + "forehead": "doakleousle", + "foreman": "voullery", + "forev": "swarger", + "forgot": "swormid", + "forlorn": "zarrum", + "forth": "scok", + "fortune": "gnoudleent", + "forward": "prarngous", + "fost": "derb", + "fought": "breerd", + "found": "staim", + "fourpence": "bienic", + "fourteen": "wrauntum", + "fourth": "thrilt", + "france": "grou", + "fred": "shoord", + "freeze": "rirch", + "fret": "wrorsh", + "friday": "sqeaffic", + "frighten": "krorging", + "frosty": "pleengous", + "fruit": "droon", + "fruiterer": "freafentish", + "frump": "sloaft", + "frumpaty": "frurlerel", + "full": "dil", + "fun": "baur", + "funny": "krunel", + "gai": "dralt", + "gall": "sceart", + "gallant": "snairtid", + "gallop": "strooben", + "gamberal": "yoangidle", + "game": "prourt", + "gand": "drauft", + "gang": "thraist", + "gap": "naull", + "garden": "snorgous", + "garter": "thykleer", + "gate": "shooth", + "gather": "sqircking", + "gave": "kysh", + "gay": "chaush", + "geese": "sqalt", + "gent": "maug", + "gentle": "swumpum", + "gentleman": "thrirdleelen", + "gentlemen": "wrunderum", + "georgy": "rintow", + "get": "nack", + "gett": "gyl", + "giblet": "sherbic", + "gil": "swaik", + "girl": "fers", + "give": "wirp", + "giving": "kraiffley", + "gloucest": "smaubing", + "goat": "throof", + "gobbl": "stieff", + "gobble": "shospic", + "god": "snarnt", + "goe": "voonk", + "goest": "poud", + "going": "myck", + "gold": "churz", + "goldfinch": "plaillid", + "gone": "rieng", + "good": "drurrk", + "goose": "brarsp", + "goosey": "strertow", + "got": "gneak", + "gotham": "tharffow", + "gown": "plienk", + "grace": "pryg", + "grandmoth": "zeambleing", + "gras": "thamp", + "grave": "wouft", + "gravel": "frurtley", + "gray": "keang", + "great": "glaut", + "greedy": "hauzen", + "green": "hav", + "greenwood": "toudid", + "grew": "ferrt", + "griev": "zarst", + "grim": "hunk", + "grin": "soun", + "groat": "yud", + "grocer": "drombleous", + "ground": "drout", + "grow": "mirng", + "gruel": "choash", + "grundy": "wreeffid", + "guinea": "gutleic", + "gum": "fles", + "gun": "swoog", + "hair": "flers", + "half": "wrerl", + "halfpence": "yengish", + "halfpenny": "morspely", + "hall": "drind", + "hame": "kroorm", + "hand": "thaint", + "handkerchief": "gliermicing", + "handsome": "rirbleing", + "handy": "gifter", + "hang": "pruch", + "happen": "koding", + "hard": "drouth", + "hare": "yab", + "hark": "jark", + "harm": "skaid", + "harrow": "sqouben", + "hart": "mauk", + "hat": "noug", + "hath": "spar", + "hatter": "virltid", + "hawk": "thoult", + "hay": "gees", + "haystack": "krirbley", + "hea": "smirnd", + "head": "daf", + "healthy": "flelling", + "hear": "drurch", + "heard": "jyn", + "heart": "zys", + "hearty": "veastow", + "heav": "sqoord", + "hector": "trargel", + "heel": "turnk", + "heighty": "derltum", + "helen": "brysting", + "help": "gurf", + "hem": "kroan", + "hen": "grerz", + "hero": "preerny", + "herring": "shreerkous", + "hey": "sterl", + "hickery": "putleenten", + "hickety": "neeverous", + "hickory": "kreardingic", + "hid": "dur", + "hide": "charlt", + "higgledy": "shysicish", + "high": "sciesp", + "highnes": "jearmen", + "highway": "fraisish", + "hill": "prornt", + "hillock": "fogleid", + "himself": "thrarffleow", + "hire": "flieff", + "hobble": "fleezzleic", + "hog": "deat", + "hold": "shoof", + "hole": "swyrder", + "holiday": "yombleingish", + "home": "mout", + "hon": "chend", + "honey": "shrarrdish", + "honor": "reaskic", + "hood": "braist", + "hop": "trev", + "hope": "zoall", + "hopp": "gloob", + "horn": "yyb", + "horrid": "fernkic", + "hors": "sciern", + "horse": "thorb", + "horseshoe": "fiengid", + "hose": "bolt", + "hosier": "stroontous", + "hot": "beaff", + "hound": "dom", + "hour": "vorrm", + "house": "keaf", + "housetop": "nirkousid", + "hubbard": "kyfow", + "huff": "glust", + "humpty": "yoackish", + "hundr": "spausp", + "hung": "shyl", + "hunt": "froch", + "hurry": "veackous", + "hurt": "flarsp", + "husband": "brarfting", + "hush": "thrarll", + "huzza": "pusher", + "ice": "karn", + "icicle": "sqoufficy", + "ifs": "smork", + "ill": "lelk", + "illustrat": "trirmouser", + "inde": "snurt", + "ink": "wolk", + "instead": "drorsish", + "intery": "tykishle", + "iron": "skaftid", + "ive": "shrer", + "jack": "rirrm", + "jacky": "throorming", + "jag": "skoop", + "jam": "scirmp", + "jelf": "gernt", + "jen": "heeth", + "jenny": "klorzy", + "jerry": "brystic", + "jig": "weeng", + "jiggety": "snithumy", + "jill": "skir", + "jingle": "rirtish", + "joan": "boub", + "jog": "stirrt", + "john": "starll", + "johnny": "ferther", + "joke": "spirch", + "jol": "firth", + "joy": "baull", + "joyou": "trom", + "july": "frurltish", + "jump": "byll", + "june": "nuff", + "keep": "slirnd", + "ken": "skeark", + "kept": "leesh", + "kettl": "hult", + "kettle": "prerbing", + "key": "snaup", + "kilkenny": "thoashenel", + "kill": "sist", + "kind": "ryst", + "king": "scurg", + "kingdom": "birzing", + "kirk": "keet", + "kis": "snip", + "kiss": "slurch", + "kit": "zeand", + "kitchen": "brurndid", + "kite": "frord", + "kitten": "loric", + "kitty": "yooker", + "knave": "sek", + "kne": "gnean", + "knee": "glol", + "knife": "shroank", + "knight": "verg", + "knock": "slaik", + "know": "stoonk", + "kyloe": "graunt", + "lad": "glurnd", + "ladd": "parlt", + "laddie": "kleat", + "laden": "slaffel", + "lady": "failten", + "ladybird": "slealerer", + "lag": "trear", + "laid": "swaib", + "lal": "speen", + "lam": "hormp", + "lamb": "swoob", + "lan": "gink", + "land": "seer", + "lane": "frech", + "lard": "kauv", + "lark": "nieck", + "lass": "glusk", + "last": "thrurs", + "latch": "braulk", + "late": "goash", + "laugh": "wourm", + "lauk": "fraud", + "lay": "gnarp", + "lea": "thrurft", + "lead": "skiesk", + "lean": "swurch", + "leap": "spu", + "least": "slarrd", + "leath": "nauck", + "leav": "sperrd", + "leave": "skack", + "led": "slaind", + "lee": "plealt", + "leed": "gaust", + "left": "riemp", + "leg": "baimp", + "lend": "squrp", + "lengthen": "dreandic", + "lent": "lirrd", + "les": "flerp", + "let": "sorr", + "lett": "thrirrm", + "lick": "trerv", + "lie": "baurd", + "life": "furlk", + "lift": "steeng", + "light": "drernt", + "like": "kliff", + "lin": "tiell", + "linen": "drorfty", + "linnet": "progleing", + "lion": "yorsh", + "list": "sorp", + "little": "skoufenty", + "littleman": "bloarnicy", + "liv": "byrn", + "live": "sqoov", + "load": "gnurk", + "lock": "groask", + "locket": "narndleel", + "lol": "grais", + "london": "brouzzleel", + "long": "prurnd", + "longman": "sceelous", + "look": "snout", + "lord": "slirz", + "lost": "trirnt", + "loud": "peard", + "lov": "neask", + "love": "traiz", + "low": "vierd", + "luck": "briet", + "lucy": "sytid", + "lump": "rearn", + "lumpety": "glerskicow", + "mad": "smiend", + "made": "brorrk", + "maid": "bist", + "maiden": "wopum", + "main": "kroun", + "maintain": "skurchish", + "mak": "visp", + "make": "zynk", + "malt": "speelk", + "mamma": "forting", + "mammie": "groad", + "mammy": "shrotley", + "man": "woosp", + "many": "wieckel", + "march": "glound", + "mare": "meez", + "margaret": "stourkelum", + "margery": "stirtherent", + "mark": "sceech", + "market": "taisent", + "marr": "krirth", + "marri": "brarlling", + "marry": "threrndic", + "martin": "klarnkid", + "mary": "shielkic", + "mast": "shroov", + "master": "niecker", + "match": "stroalt", + "matt": "wet", + "maybe": "fleest", + "mayor": "souf", + "meadow": "sqiernle", + "meal": "nausk", + "mean": "wroorm", + "meat": "learn", + "meet": "prurng", + "melancho": "torsentent", + "men": "bleark", + "mend": "gniet", + "merchant": "sniembleen", + "mercy": "bearden", + "merri": "wienow", + "merry": "grarrmum", + "merrymen": "plurndleentid", + "met": "hurrt", + "mew": "spird", + "mice": "stirch", + "mickle": "smoanky", + "middle": "vallum", + "mil": "shrarb", + "mild": "snoan", + "mile": "blarndleel", + "milk": "smierm", + "mill": "norm", + "mind": "tarmp", + "mine": "reath", + "mintery": "deedleicent", + "minute": "threrment", + "mire": "wrant", + "mis": "voack", + "mischievou": "viskelel", + "miss": "bernd", + "mist": "girp", + "mistaken": "bubleentum", + "mistres": "vythent", + "misty": "loorker", + "moisty": "maultid", + "mol": "gnen", + "monday": "glerndel", + "money": "niedum", + "monkey": "smeeshid", + "monstrou": "spundic", + "moon": "smorft", + "moppet": "zarllel", + "more": "straug", + "morn": "krelt", + "morning": "noarish", + "mortal": "sqeecken", + "mother": "wielkow", + "motion": "preentic", + "mourn": "gnirf", + "mouse": "cheab", + "mouth": "frerch", + "mov": "lult", + "move": "spoub", + "mow": "shrorst", + "mrs": "shrim", + "much": "friv", + "muffet": "frilous", + "mulberry": "zoumperid", + "multiplication": "krallentingelen", + "music": "kroulent", + "muskidun": "threrckentent", + "mutton": "blerbid", + "myself": "begnirsk", + "nag": "stroo", + "nail": "vurl", + "nam": "sloaz", + "name": "leamp", + "nan": "trais", + "nancy": "kukish", + "nanny": "bienten", + "narrow": "chauzzleous", + "nasty": "swirdent", + "naught": "krurs", + "naughty": "thorsper", + "nay": "riff", + "neary": "strauskid", + "neat": "prarr", + "neck": "smalk", + "needl": "gnorsp", + "needle": "rydleent", + "neighbor": "thraithel", + "neith": "wrosh", + "nest": "jursk", + "new": "swilt", + "next": "goaz", + "nibble": "chaungen", + "nice": "kriet", + "niggledy": "shrirbleleel", + "night": "strorr", + "nightgown": "smeenkic", + "nimble": "scaingum", + "nineteen": "taugentid", + "nob": "groorn", + "nobleman": "wimowous", + "nobody": "streluming", + "nodd": "krip", + "noise": "kloaz", + "noon": "snorrm", + "nor": "furst", + "north": "flarng", + "norwich": "narthish", + "nose": "borsp", + "notch": "derll", + "note": "shoock", + "noth": "swiest", + "novemb": "sliegish", + "now": "firch", + "oak": "froang", + "often": "jorchle", + "old": "garr", + "ope": "biesk", + "open": "shrimbleish", + "orange": "gnoothous", + "organ": "jormple", + "oth": "thirnt", + "oven": "brotleing", + "owe": "toorm", + "owl": "glarft", + "own": "wrarck", + "packet": "snikleent", + "pail": "fiert", + "pair": "pront", + "pan": "kem", + "pancake": "moamen", + "pandy": "smorntid", + "pantry": "thryltum", + "pap": "shrarlk", + "papa": "sluntous", + "parent": "trierdent", + "parlor": "vatid", + "parrot": "snarndent", + "parson": "tryspid", + "party": "swerffish", + "pas": "snurk", + "pat": "sheaff", + "patch": "brem", + "pay": "pir", + "peace": "brerch", + "peaceable": "jeelicow", + "peacock": "zourmer", + "pear": "gerst", + "pease": "var", + "peck": "krees", + "pedlar": "strolous", + "peep": "thirz", + "pen": "snirz", + "penny": "prumbleow", + "people": "slutleing", + "pepper": "krurben", + "perhap": "deaser", + "pet": "slul", + "peter": "friertous", + "petticoat": "lealkiden", + "physician": "vermenen", + "piccadil": "lainingel", + "pick": "slor", + "pickety": "streeskenid", + "pickl": "skeer", + "picture": "freaching", + "pie": "praunk", + "piece": "ziend", + "pieman": "flurzzleid", + "pig": "frab", + "pigeon": "weether", + "piggledy": "throafumen", + "pin": "smiesp", + "pinch": "flaull", + "pint": "wrong", + "piou": "rish", + "pip": "stielk", + "pipe": "lait", + "piper": "theaking", + "pippen": "stugish", + "pitch": "muft", + "plac": "wrath", + "plain": "fraiz", + "plast": "fraun", + "plat": "krurnt", + "plate": "frousk", + "platt": "jers", + "play": "naill", + "playfellow": "sterskument", + "playmat": "flimpid", + "please": "shour", + "plenty": "wirum", + "plum": "sqount", + "pocket": "yople", + "pocketful": "brorkleelous", + "point": "spaig", + "poker": "mupy", + "pol": "hermp", + "poll": "sqear", + "pony": "trofen", + "pooh": "bliemp", + "poor": "vell", + "poppety": "wrirthider", + "porgy": "rirtow", + "porridge": "snoozzleum", + "porring": "gnombleid", + "pos": "norng", + "posses": "trainter", + "pot": "snaft", + "potato": "braindleeling", + "pound": "stenk", + "powd": "tert", + "practice": "slardum", + "pray": "sharm", + "prayer": "spieth", + "pretti": "noopen", + "pretty": "streeshum", + "pri": "foom", + "prick": "bleam", + "prince": "frarn", + "princes": "frourking", + "prithee": "krerdish", + "prod": "foll", + "promis": "skeembleer", + "proper": "krernkum", + "protector": "bondleumel", + "proud": "mouf", + "psalm": "hurs", + "pudd": "yark", + "pudding": "worthid", + "puddle": "plumum", + "pull": "mirmp", + "pumpkin": "kroompen", + "pussy": "flyble", + "put": "wrird", + "puzzle": "sniefle", + "quack": "starn", + "quarrel": "droomum", + "queen": "snaurd", + "quick": "thooff", + "quiet": "chaub", + "quite": "grerrk", + "rabbit": "spylous", + "rac": "glaist", + "rag": "shrolk", + "rage": "starg", + "rain": "skarf", + "ram": "wrorv", + "ran": "glaimp", + "rapp": "sqift", + "rare": "gnaup", + "rat": "roorm", + "rattle": "tirpid", + "raven": "klourkent", + "raw": "slind", + "reach": "smoost", + "read": "hilk", + "ready": "boumbleel", + "real": "prirlk", + "reason": "yengous", + "receive": "streegid", + "red": "tang", + "redbreast": "buffow", + "reel": "geasp", + "reigate": "treatle", + "remedy": "glaibleishle", + "repartee": "briengerle", + "repli": "frozzleish", + "request": "roothing", + "resolv": "voanger", + "rest": "flirsp", + "return": "churngel", + "rhym": "shreng", + "rhyme": "shaurn", + "ribbon": "gleagent", + "rice": "thrirk", + "rich": "smolk", + "richard": "choothous", + "rid": "rerrd", + "riddle": "bloadleen", + "ride": "scard", + "rig": "stirrd", + "right": "lound", + "ring": "frorth", + "ringman": "kleampid", + "rise": "slasp", + "riv": "toak", + "roast": "pard", + "rob": "bail", + "robber": "shodent", + "robert": "sworrmen", + "robin": "sqorlous", + "robinson": "purrtingow", + "rock": "wuck", + "rode": "braib", + "rog": "blierk", + "roll": "wreang", + "rook": "shurng", + "room": "gnoank", + "root": "sqaik", + "ros": "truf", + "rosy": "nekish", + "rough": "stint", + "round": "sirs", + "row": "giert", + "rule": "bloandleing", + "run": "jev", + "runn": "nart", + "rush": "krul", + "rye": "skorsk", + "sabbath": "darbel", + "sack": "vurk", + "saddle": "shraisel", + "safe": "durk", + "sage": "lai", + "sago": "sweming", + "said": "grurst", + "sail": "throt", + "sailor": "snoashy", + "salt": "marm", + "sam": "greeb", + "same": "jach", + "sang": "plish", + "sat": "brerck", + "saturday": "frirbleenter", + "saw": "derck", + "say": "gleerd", + "scar": "kroalk", + "scarce": "prirr", + "scarlet": "baupum", + "scholar": "bipid", + "school": "snirll", + "schoolroom": "lirish", + "scotch": "straunk", + "scratch": "stunt", + "scuttle": "skeeffleid", + "sea": "throop", + "seal": "thailk", + "seam": "skoand", + "seasonable": "squrvididish", + "second": "gnory", + "see": "vault", + "seed": "gnynd", + "seek": "syl", + "seen": "zylk", + "seldom": "wrorkous", + "selfsame": "brurchent", + "sell": "barmp", + "sempster": "zeallic", + "send": "strarst", + "sent": "spos", + "septemb": "lurnent", + "serv": "flert", + "servant": "speambleing", + "serve": "rault", + "set": "kroord", + "seventeen": "lierenting", + "sew": "gnep", + "shaftoe": "glirff", + "shake": "sart", + "shalt": "snooz", + "shape": "rarst", + "shave": "shorll", + "shaven": "spoachent", + "shed": "wurr", + "sheep": "gloas", + "shelf": "krisk", + "shell": "proull", + "shepherdes": "krureren", + "shilling": "flearmle", + "shin": "fryrt", + "shine": "barv", + "ship": "sparv", + "shiv": "stool", + "sho": "wroasp", + "shod": "tounk", + "shoe": "keek", + "shook": "shorrt", + "shoot": "spor", + "shop": "prorff", + "shoreditch": "thraurkinging", + "shorn": "barp", + "short": "strarft", + "shot": "bloat", + "show": "hirk", + "shower": "wrychle", + "shroud": "smyk", + "shut": "gauth", + "sick": "glo", + "side": "shoff", + "siege": "scerrn", + "sieve": "strarnd", + "sigh": "swir", + "silk": "ref", + "sill": "narl", + "silv": "woost", + "simon": "kraugleing", + "simple": "slirrdent", + "sing": "sqirz", + "single": "moover", + "sir": "pich", + "sister": "lisic", + "sit": "blausk", + "sitt": "thrork", + "sixpence": "swarrtel", + "sixteen": "gouffish", + "skin": "shraur", + "skipp": "lerm", + "sky": "sparnk", + "slash": "thrarv", + "slat": "weak", + "slatherum": "varspousle", + "sleep": "dryrd", + "sleepy": "spaurtish", + "slend": "nomp", + "slice": "glirnt", + "slid": "trith", + "slipper": "sweertent", + "slitherum": "fliskument", + "slow": "hys", + "sly": "rai", + "small": "grirll", + "smile": "plairnic", + "smith": "sqoor", + "smok": "wroalk", + "snail": "lard", + "snap": "griern", + "snapp": "plysh", + "sneez": "leech", + "sneeze": "gna", + "sniff": "fursk", + "snipe": "lysh", + "snook": "swood", + "snow": "thyrk", + "snuff": "dorg", + "sobb": "strurck", + "soft": "hirmp", + "sold": "krai", + "solomon": "hokleering", + "someth": "sqembleic", + "son": "teest", + "song": "shyf", + "soon": "dreer", + "sore": "toa", + "sorrow": "wrooskum", + "sorrowful": "taichentous", + "soul": "drort", + "sound": "grooll", + "south": "skoub", + "sow": "naick", + "spade": "shoar", + "spain": "sqoach", + "sparrow": "dymbleum", + "speak": "froor", + "spell": "terz", + "spend": "pesk", + "spent": "wrurz", + "spice": "swouft", + "spid": "firm", + "spilt": "gniesk", + "spin": "wroorn", + "spinn": "blaurn", + "spit": "stauff", + "splash": "thurll", + "splish": "prusk", + "spoil": "toand", + "spoke": "noonk", + "spoon": "drirrm", + "sport": "prirsh", + "spr": "shairm", + "sprat": "flear", + "spright": "thraiff", + "spry": "boulk", + "spun": "karr", + "squirrel": "straull", + "staff": "trailk", + "stair": "toach", + "stand": "writ", + "star": "surb", + "start": "chys", + "stay": "klarll", + "steal": "gnursh", + "steel": "serst", + "stepney": "bloffing", + "stepp": "treck", + "stew": "yeech", + "stick": "sturk", + "stiff": "thirt", + "stile": "serskum", + "stingy": "zirrting", + "stock": "kroust", + "stole": "wrorlking", + "stone": "scoach", + "stood": "neell", + "stop": "skint", + "stopp": "shroar", + "storm": "hoark", + "story": "smirmbleing", + "stout": "toft", + "str": "glam", + "straight": "klyth", + "straightway": "krirspent", + "strang": "yesp", + "strange": "floo", + "straw": "graund", + "strawberr": "rymbleel", + "stray": "smieft", + "street": "brap", + "strife": "threasp", + "strong": "snirlt", + "strow": "sump", + "struck": "plorth", + "stump": "goart", + "stumpaty": "herfferous", + "such": "gouv", + "sue": "droont", + "sugar": "neelken", + "sukey": "keagow", + "sulky": "sauffleish", + "summ": "prack", + "summer": "proartow", + "sun": "sheert", + "sunday": "skeellent", + "sung": "prerg", + "sunshine": "threrrnum", + "sunshiny": "strietowen", + "sup": "swuck", + "supp": "snirg", + "suppose": "sliezic", + "sure": "nyf", + "surprise": "theantel", + "surrey": "bliesple", + "swan": "shrert", + "swarm": "siesp", + "sweep": "sqa", + "sweet": "kraurm", + "swim": "joust", + "swimm": "fraisk", + "swine": "sli", + "swoon": "snog", + "sword": "frork", + "swore": "ploll", + "swum": "glorrn", + "table": "lirstow", + "tack": "swoarm", + "taffy": "sporster", + "tail": "smoav", + "tailor": "bleartle", + "tak": "tiv", + "take": "voan", + "tale": "swierdish", + "talk": "skon", + "talkative": "sheethicy", + "tapp": "zoun", + "tar": "gind", + "tarr": "freamp", + "tarry": "swailtic", + "tart": "hip", + "taste": "gryrt", + "tatter": "sceser", + "taught": "plourd", + "tavern": "rarger", + "tea": "throu", + "tear": "feag", + "tee": "shorr", + "teeth": "smin", + "tell": "fyrk", + "test": "strormp", + "thank": "glault", + "thee": "briep", + "thi": "tharck", + "thick": "sterff", + "thief": "wreeb", + "thigh": "sqish", + "thing": "praisk", + "think": "slurrk", + "third": "garck", + "thirteen": "yorfle", + "thirty": "shervy", + "thistle": "zutleer", + "thorn": "yauft", + "thou": "sqarg", + "thought": "rid", + "thousand": "mootous", + "thread": "naif", + "threescore": "bloutleel", + "threw": "griel", + "thrive": "drig", + "throat": "fik", + "throw": "zolt", + "thrush": "choaz", + "thu": "glirg", + "thumb": "voop", + "thumbkin": "daivous", + "thump": "traft", + "thumpaty": "bambleidel", + "thursday": "jithen", + "thy": "broust", + "thyself": "wychic", + "tick": "scielt", + "tickl": "theck", + "tie": "gousk", + "tied": "nunt", + "til": "durm", + "till": "tielt", + "tim": "gnu", + "time": "chot", + "tinker": "wrorlty", + "tip": "gnosh", + "tipple": "noakleent", + "tir": "bauft", + "tis": "bal", + "tisha": "sqampum", + "tittlemouse": "jirftinger", + "tobago": "flibleumel", + "today": "refon", + "toe": "smurlk", + "togeth": "slubleic", + "told": "leaz", + "toll": "krelk", + "tom": "turnd", + "tommy": "chounkic", + "tong": "wrark", + "tongu": "plillum", + "tongue": "hyk", + "took": "heack", + "top": "hor", + "torch": "yib", + "torn": "maump", + "tos": "snyk", + "toss": "snault", + "touch": "derl", + "town": "sheemp", + "toy": "skaurm", + "tramp": "blarff", + "trap": "snoum", + "tre": "sqiem", + "tree": "jaust", + "trencher": "flydow", + "tri": "perll", + "trick": "chirnt", + "trip": "shirrm", + "tripe": "zorlt", + "tripp": "drach", + "trot": "vard", + "trott": "voum", + "troubl": "zursk", + "trouble": "throber", + "trow": "niep", + "trowel": "launging", + "true": "plieg", + "try": "gliest", + "tuck": "zeamp", + "tuesday": "piting", + "tuffet": "stroaftic", + "tumbl": "swod", + "tune": "spest", + "turn": "pruff", + "turnip": "smeekic", + "twa": "fid", + "twaddle": "zuntic", + "twee": "wriz", + "tweedle": "flulic", + "twelve": "purrk", + "twenty": "grokleow", + "twiddle": "scoalkow", + "twig": "chiff", + "twill": "plont", + "twitchett": "spyrdic", + "twopence": "braunder", + "undertaker": "plarltereling", + "unicorn": "prinkousent", + "unto": "chauny", + "upon": "adroust", + "upstair": "noormen", + "upward": "sheable", + "use": "kleall", + "used": "gornkent", + "ush": "theern", + "vale": "thrithel", + "vast": "thraind", + "velvet": "drersking", + "venture": "droankic", + "vex": "hurnd", + "vexation": "shraidleleen", + "victual": "stoothing", + "vinegar": "kleazentow", + "visit": "gnirlky", + "visitor": "fautleicid", + "vow": "druch", + "wag": "blars", + "waggl": "wift", + "wail": "hich", + "wainscot": "strordish", + "wait": "furll", + "wak": "fraunk", + "wake": "barlk", + "walk": "tyl", + "wall": "shryrd", + "walnut": "sheenting", + "wand": "sherrd", + "want": "lirnk", + "ware": "braift", + "warm": "blorm", + "wash": "hirn", + "wat": "kriez", + "watch": "streed", + "watt": "noaz", + "way": "thream", + "wealthy": "yousken", + "wear": "teang", + "weath": "snorrn", + "wed": "frir", + "wedd": "voock", + "wednesday": "grauftering", + "wee": "klirrm", + "weed": "skoard", + "week": "thys", + "weep": "taug", + "welcome": "teeby", + "well": "slug", + "welshman": "ludleid", + "went": "smen", + "west": "smyrn", + "whale": "gryngish", + "whatev": "sworstish", + "wheel": "vooff", + "wheelbarrow": "sleringid", + "whenev": "klyffley", + "wherev": "varshing", + "whey": "fon", + "whip": "gneerd", + "whipp": "tyb", + "whistle": "warffle", + "white": "tim", + "whitechapel": "shirmbleenicent", + "whith": "vealt", + "whoop": "bouf", + "wife": "poord", + "wig": "frysk", + "wil": "krurrk", + "wild": "mest", + "wildernes": "sunkenter", + "wilkin": "flobleid", + "willie": "lar", + "wilt": "bleat", + "wind": "kliez", + "window": "kirper", + "wine": "smauft", + "wing": "shront", + "winkie": "yeath", + "winkle": "flontic", + "wint": "swap", + "winter": "thieffleel", + "wip": "staurn", + "wipe": "wrail", + "wire": "strirll", + "wise": "nyst", + "wish": "floath", + "within": "kirrmer", + "without": "sestle", + "wiv": "skat", + "woe": "jeef", + "woman": "plaispy", + "women": "shreeftous", + "wond": "meard", + "wondrou": "friekow", + "woo": "farl", + "wood": "daus", + "woodbin": "strorlid", + "wool": "maunt", + "word": "vousk", + "work": "prys", + "world": "waum", + "worm": "brez", + "worri": "thraindy", + "worry": "plaurle", + "worse": "sul", + "worth": "sleark", + "wrap": "grurnk", + "wren": "frez", + "wright": "trurrk", + "write": "jorsh", + "wrote": "lourk", + "yard": "spyrd", + "yea": "zont", + "year": "wrylt", + "yellow": "ryckle", + "yes": "smoarn", + "yon": "spielt", + "young": "vaull" + } + }, + { + "label": "seed7", + "seed": 7, + "matchProsody": true, + "mint": "nonce", + "injectiveAtEveryP": true, + "remintRounds": 1, + "bijective": true, + "imageSize": 2233, + "domainSize": 2233, + "mappingSize": 1680, + "mappingSha256": "65efb80b712a63d434bc8c9ecc1266dc93197eca5c3938ebd5f820c8da82fbbd", + "mapping": { + "abc": "dreand", + "abroad": "greshum", + "adieu": "sieskic", + "admittance": "glaizzleleing", + "advice": "streaskish", + "afraid": "charpic", + "aft": "sqoug", + "age": "wrerch", + "agree": "pleastow", + "ain": "trean", + "air": "spirrm", + "alabone": "sqimbleumic", + "alack": "kletleum", + "ale": "weashic", + "alehouse": "shopening", + "alive": "sqoubous", + "alley": "vaping", + "almanac": "woalousic", + "alone": "swiechen", + "along": "yurngish", + "alphabet": "zorngidel", + "alphabetical": "spirspousingidid", + "alway": "korzzleel", + "amble": "zurlten", + "ampersand": "skearkenel", + "angry": "skeazzley", + "ann": "starb", + "announce": "warnken", + "anoth": "bloaffic", + "answer": "kruthum", + "appl": "gnirrd", + "apple": "chyffen", + "appoint": "prordleid", + "april": "scithum", + "arm": "scerl", + "around": "dehoorm", + "arrow": "jirkleing", + "art": "snorg", + "ask": "spylt", + "asleep": "jauthel", + "ate": "yorz", + "aught": "thraur", + "aunt": "smamp", + "away": "unpryng", + "awhile": "dronkelous", + "awoke": "yyllum", + "axe": "smask", + "aye": "floont", + "baa": "noark", + "babby": "rirftow", + "baby": "yarmid", + "babylon": "wagelel", + "bachelor": "goumpider", + "back": "harlk", + "backward": "stryllow", + "bacon": "birrder", + "bad": "flealk", + "bade": "smirft", + "bag": "smork", + "bailey": "blirchle", + "bak": "toomp", + "bake": "blarch", + "baker": "gazzley", + "ball": "glorll", + "balloon": "breanken", + "banbury": "sporrdelic", + "bandy": "scumpow", + "barb": "kraig", + "barber": "snernding", + "bare": "threalt", + "bark": "blup", + "barley": "rarmous", + "barm": "shos", + "barn": "surt", + "barrel": "chorffic", + "basket": "jarffleel", + "bat": "glurng", + "battle": "vieskle", + "bead": "kermp", + "bean": "ziest", + "bear": "wreer", + "beat": "goarm", + "beaten": "scardy", + "bed": "krout", + "bedtime": "grurrdum", + "bee": "frock", + "beef": "flormp", + "beehive": "froobleing", + "beer": "skorch", + "beetle": "sluly", + "began": "zauspum", + "beggar": "jiezing", + "begin": "smoothow", + "beginn": "tebic", + "begun": "soamen", + "behind": "waigen", + "believe": "wrurnen", + "bell": "snid", + "belleisle": "thrurmering", + "bend": "pleek", + "bent": "stirg", + "berr": "jeesh", + "bes": "slaurm", + "beside": "fokleish", + "bessy": "brornid", + "best": "wyp", + "betsy": "dryzzleous", + "bett": "blymp", + "betty": "spouspent", + "betwixt": "karrmle", + "bid": "lof", + "big": "gnoud", + "bigg": "grirrt", + "bil": "scuf", + "bill": "sneand", + "bind": "shaift", + "bird": "snourn", + "birthday": "frienenid", + "bit": "saus", + "bite": "vyng", + "bitt": "griest", + "bitten": "squrckel", + "black": "shoal", + "blackbird": "wrauffleic", + "blacksmith": "drirchic", + "blanche": "pirv", + "ble": "karsh", + "bleak": "foont", + "bleat": "traik", + "bles": "pirft", + "blessing": "trordleish", + "blind": "smard", + "blis": "wraish", + "blithe": "haish", + "blood": "prurnt", + "blow": "zeff", + "blue": "brermp", + "blush": "breer", + "boat": "strarrk", + "bob": "burd", + "bobby": "joazer", + "body": "grungel", + "boggen": "sturging", + "boil": "scosh", + "boldero": "scorrkowing", + "bolin": "blertow", + "bombay": "derbleent", + "bone": "flysk", + "bonn": "tharm", + "bonnet": "chympous", + "bonnie": "spersp", + "bonny": "jenging", + "book": "leen", + "born": "blirl", + "bottle": "storty", + "bough": "froo", + "bought": "swurmp", + "bounc": "glork", + "bow": "sporsh", + "bowl": "derf", + "box": "strylk", + "boy": "skurth", + "bramble": "gneendleer", + "brandy": "thisow", + "bras": "slaur", + "bray": "skoack", + "bread": "sporp", + "break": "dryz", + "breech": "snor", + "brew": "berck", + "bri": "bud", + "brickbat": "draindic", + "bride": "vusk", + "bridge": "thrish", + "bridle": "spaiker", + "bright": "sault", + "bring": "blorp", + "bristol": "strooning", + "broke": "dais", + "broken": "fauffleid", + "brook": "kril", + "broom": "slauth", + "broth": "garnk", + "brother": "blarzic", + "brought": "pryrd", + "brown": "sheed", + "buckl": "lurrm", + "buckle": "kobel", + "build": "lirm", + "built": "pausk", + "bull": "teab", + "bullet": "skongic", + "bullfinch": "mootleish", + "bump": "swulk", + "bumpety": "graitleicen", + "bun": "blaurm", + "bunch": "mers", + "bunt": "doun", + "buri": "shrarrmy", + "burial": "shrergley", + "burnie": "vurlk", + "burnt": "frailk", + "bush": "trut", + "butt": "scoub", + "button": "skirlkic", + "buy": "snab", + "bye": "vank", + "cabin": "yeelkent", + "caesar": "strorndid", + "cage": "koat", + "cake": "sqoam", + "calf": "skyth", + "call": "floall", + "came": "slursk", + "canary": "gathenish", + "candl": "scarsk", + "candle": "vemer", + "candlestick": "strooftishish", + "candy": "trouftid", + "cannot": "rorking", + "cant": "kraik", + "cap": "mard", + "captain": "gruftic", + "car": "shrank", + "care": "pirt", + "carr": "doonk", + "carri": "gerffleous", + "carrion": "struftous", + "carry": "keartle", + "carv": "droulk", + "cast": "pirff", + "cat": "florll", + "catch": "yorp", + "caught": "chaz", + "ceas": "speb", + "ceil": "broad", + "cellar": "firnkish", + "chain": "pround", + "chair": "rirch", + "chamb": "troost", + "champ": "zarlt", + "chanc": "skair", + "charley": "kreaffleing", + "che": "fornd", + "cheek": "krirs", + "cheese": "koarm", + "cherry": "flester", + "chicken": "pleelish", + "chid": "jan", + "chief": "smorf", + "child": "grauk", + "children": "broarmidy", + "chimney": "sqipum", + "chin": "ploa", + "chirp": "mop", + "chirrup": "wooffing", + "choice": "grol", + "choose": "shrerch", + "chopp": "brauff", + "christen": "jaichish", + "christma": "trysle", + "cinder": "houtous", + "city": "smuckle", + "clap": "vyg", + "claw": "smout", + "clay": "verm", + "clean": "pieff", + "clear": "brailt", + "clergyman": "wraifterid", + "clerk": "swuch", + "clev": "flerm", + "cloak": "weft", + "clock": "desp", + "cloth": "wraurd", + "clothe": "bur", + "cloudy": "beekum", + "coachman": "smykel", + "coal": "klorrm", + "coat": "dik", + "cobbler": "thrylkow", + "cobweb": "kargle", + "cock": "kault", + "codlin": "glaundic", + "coffee": "tykic", + "coffin": "birchish", + "cold": "sceeft", + "cole": "rarking", + "collar": "pealkum", + "colt": "gleend", + "com": "herth", + "comb": "stolk", + "come": "vieck", + "comfit": "stoshent", + "comical": "wrooshenten", + "comin": "sqaikley", + "command": "torllen", + "compare": "trickow", + "compliment": "wrolinger", + "consid": "broagid", + "consider": "boadicen", + "contrary": "slerrdening", + "contrive": "snantish", + "coo": "perl", + "copp": "shelk", + "coral": "spyffleing", + "corn": "tront", + "corner": "grunden", + "cost": "loask", + "cottage": "kraiffow", + "count": "hisp", + "court": "glieck", + "cov": "draig", + "cover": "prousing", + "cow": "chuz", + "crackabone": "frarlkelle", + "cradle": "sambleing", + "cream": "rail", + "creep": "moad", + "crept": "sloong", + "cri": "geel", + "croak": "slees", + "crook": "pruf", + "cros": "kroot", + "crow": "strang", + "crown": "trif", + "crumb": "trerft", + "crumpl": "weell", + "crusoe": "drarth", + "cry": "voll", + "cup": "korm", + "cupboard": "songel", + "cur": "slorrt", + "curd": "foarm", + "curl": "brurrn", + "currant": "staundous", + "curtsy": "grarfum", + "cushion": "flurzum", + "cushy": "kurle", + "custard": "syky", + "cut": "tornk", + "cutery": "baumbleishle", + "daddie": "drong", + "daddy": "noorning", + "daff": "streaz", + "daffodil": "rarbenous", + "dainti": "swoaffley", + "dainty": "herchous", + "dairy": "slairkow", + "dam": "serf", + "dame": "saun", + "danc": "gneesp", + "dance": "thourt", + "dang": "snick", + "dapple": "daiffleing", + "dare": "zir", + "dark": "zourt", + "darlington": "jaugleerow", + "dat": "pauch", + "daught": "weak", + "daughter": "slieskous", + "daw": "thriest", + "dawson": "gneffleic", + "day": "slorm", + "dead": "shoaff", + "dear": "wousk", + "deary": "sqausle", + "death": "smauft", + "deceit": "zoorming", + "decide": "shiembley", + "deck": "fryst", + "declar": "slemple", + "ded": "tul", + "dee": "slarlt", + "deed": "jount", + "deep": "fryrm", + "delight": "shroanten", + "delve": "gnup", + "derby": "storrow", + "determin": "sairkinger", + "dew": "thriff", + "diamond": "smezum", + "dick": "krear", + "dickery": "flogleentum", + "dickory": "wrormpishing", + "dicky": "chading", + "diddle": "gnoozzleel", + "die": "blaill", + "died": "prib", + "diet": "glaush", + "difficult": "vurngenous", + "dig": "klyr", + "dill": "blun", + "ding": "seamp", + "dinkety": "frusowent", + "dinn": "plov", + "dirty": "ploany", + "dish": "draish", + "dishy": "spemic", + "displeas": "goordid", + "ditch": "plauch", + "division": "skeembleuming", + "dob": "floust", + "dock": "gnaik", + "doctor": "gerrmen", + "doe": "kar", + "doff": "smeav", + "dog": "swech", + "dol": "beb", + "doll": "syrn", + "dollar": "moble", + "dolly": "vorvic", + "don": "joung", + "dong": "lernt", + "donkey": "glaumen", + "doo": "hurk", + "doodle": "snirsping", + "door": "koan", + "dost": "drirt", + "doth": "mip", + "doubt": "koock", + "dov": "thorrk", + "dove": "stursk", + "downstair": "koafty", + "dozen": "yerpish", + "drak": "preerm", + "drake": "gienk", + "draw": "jies", + "dream": "toog", + "dreamt": "spormp", + "dreary": "prerrming", + "dres": "sturr", + "dress": "flarg", + "drink": "vorll", + "driv": "slorff", + "drive": "blaz", + "dropp": "drauz", + "drove": "roaff", + "drown": "vynk", + "drum": "flousp", + "drumm": "spaunk", + "dry": "zuth", + "duck": "storv", + "dumpl": "woord", + "dumpling": "kroavid", + "dumpty": "spercking", + "dun": "pairk", + "durst": "zarsh", + "dusty": "baffic", + "dwell": "kyng", + "dwelt": "weeng", + "ear": "slau", + "earth": "plarlk", + "east": "prorsk", + "eat": "drult", + "egg": "freest", + "eighteen": "raurmle", + "eith": "wrouch", + "eleven": "jutleingish", + "elizabeth": "gnaugumicer", + "ell": "shrish", + "else": "woug", + "elspeth": "sealkel", + "empty": "breatic", + "end": "klernk", + "england": "snoolkle", + "enough": "sceemy", + "equal": "prilty", + "espi": "nursty", + "etc": "drosh", + "etticoat": "gnerstumous", + "evermore": "slilticow", + "everyone": "warmpenty", + "evil": "dryndel", + "except": "wrermow", + "exet": "jeezle", + "eye": "cherth", + "face": "flilk", + "fail": "sloap", + "fair": "wab", + "fall": "krong", + "fan": "soth", + "fare": "swauch", + "farm": "sqornt", + "farmer": "pormpum", + "farth": "strorff", + "farthing": "jirel", + "fast": "healt", + "fat": "swip", + "father": "heackish", + "fear": "meaz", + "feast": "scorb", + "feath": "breev", + "feather": "drylow", + "february": "riskenic", + "fed": "throck", + "feed": "snait", + "feet": "klys", + "fell": "zis", + "fellow": "skaspum", + "fetch": "wroolk", + "fiddl": "shrorrn", + "fiddle": "lirter", + "fiddler": "gnoony", + "fie": "sperch", + "field": "terch", + "fife": "blooz", + "fifteen": "veesple", + "fight": "shreaz", + "fill": "tys", + "fin": "zoz", + "find": "scyb", + "fine": "thoand", + "fing": "meeb", + "finger": "yeeby", + "fir": "louv", + "fire": "zaiff", + "first": "glarb", + "fish": "smausk", + "fishy": "glotic", + "fit": "yoat", + "flame": "swursh", + "flapp": "parv", + "fleece": "blooff", + "fleet": "sweard", + "flew": "flun", + "flinder": "floshum", + "flock": "jeel", + "floor": "throash", + "flour": "gym", + "flow": "hysk", + "flower": "proofel", + "flung": "paisk", + "flute": "scoak", + "fly": "frim", + "fol": "grat", + "folk": "tornd", + "fond": "thrirs", + "fool": "serck", + "foot": "maz", + "footman": "heaning", + "forc": "starn", + "forehead": "zirsherid", + "foreman": "thurndleowy", + "forev": "squrrting", + "forgot": "deething", + "forlorn": "wraspous", + "forth": "thrilt", + "fortune": "froallel", + "forward": "frondleid", + "fost": "seack", + "fought": "terff", + "found": "smeasp", + "fourpence": "vyrnid", + "fourteen": "teagid", + "fourth": "frarf", + "france": "wyff", + "fred": "zurk", + "freeze": "drien", + "fret": "hes", + "friday": "sqoaffent", + "frighten": "throrllid", + "frosty": "zoortum", + "fruit": "gnaiff", + "fruiterer": "shouvery", + "frump": "draurd", + "frumpaty": "prandenic", + "full": "koth", + "fun": "syd", + "funny": "vezzleum", + "gai": "slerth", + "gall": "brift", + "gallant": "charnic", + "gallop": "verkow", + "gamberal": "maumening", + "game": "skysp", + "gand": "smarnk", + "gang": "frouk", + "gap": "haind", + "garden": "hathle", + "garter": "smoundish", + "gate": "kloash", + "gather": "scaispous", + "gave": "sherrt", + "gay": "shreech", + "geese": "graurt", + "gent": "spolk", + "gentle": "hauchow", + "gentleman": "slautleisher", + "gentlemen": "mergishic", + "georgy": "stroudle", + "get": "plarrk", + "gett": "kirn", + "giblet": "scoackle", + "gil": "thorv", + "girl": "grud", + "give": "kieb", + "giving": "flumbleid", + "gloucest": "gillow", + "goat": "jak", + "gobbl": "glorsh", + "gobble": "plorrmic", + "god": "shourk", + "goe": "swoaf", + "goest": "gaush", + "going": "smoav", + "gold": "warnt", + "goldfinch": "greabley", + "gone": "kleech", + "good": "naith", + "goose": "kloalk", + "goosey": "jarler", + "got": "sqaik", + "gotham": "snoordic", + "gown": "krof", + "grace": "yirl", + "grandmoth": "prachid", + "gras": "weash", + "grave": "cheab", + "gravel": "sweaftish", + "gray": "freall", + "great": "roord", + "greedy": "kleffic", + "green": "marrt", + "greenwood": "zouzzleish", + "grew": "hurck", + "griev": "fied", + "grim": "graz", + "grin": "gurll", + "groat": "floov", + "grocer": "threltent", + "ground": "treen", + "grow": "pest", + "gruel": "sweeb", + "grundy": "jubing", + "guinea": "nully", + "gum": "thrern", + "gun": "striff", + "hair": "woav", + "half": "raill", + "halfpence": "zooftel", + "halfpenny": "skurshentle", + "hall": "chyf", + "hame": "roub", + "hand": "merrt", + "handkerchief": "dirnkousy", + "handsome": "stroady", + "handy": "strorskish", + "hang": "smeeg", + "happen": "glootleid", + "hard": "trirlt", + "hare": "shroasp", + "hark": "snom", + "harm": "jiern", + "harrow": "jailter", + "hart": "smyf", + "hat": "biz", + "hath": "churg", + "hatter": "mospel", + "hawk": "hernt", + "hay": "seesk", + "haystack": "strarndleing", + "hea": "zaith", + "head": "gnorll", + "healthy": "prerndel", + "hear": "slonk", + "heard": "thernt", + "heart": "hift", + "hearty": "prarrent", + "heav": "plaunt", + "hector": "skevel", + "heel": "varr", + "heighty": "frougish", + "helen": "spiethy", + "help": "sorp", + "hem": "paff", + "hen": "sum", + "hero": "gaintous", + "herring": "sqygous", + "hey": "wurrd", + "hickery": "therrdenic", + "hickety": "kreaftentish", + "hickory": "spiepelish", + "hid": "thrien", + "hide": "thust", + "higgledy": "wymbleeric", + "high": "greaf", + "highnes": "hirnkum", + "highway": "locker", + "hill": "drier", + "hillock": "gnoulkic", + "himself": "hoaren", + "hire": "frorm", + "hobble": "skurrking", + "hog": "smir", + "hold": "thairn", + "hole": "prarfer", + "holiday": "glinumy", + "home": "jeerm", + "hon": "skoost", + "honey": "saker", + "honor": "ploudleic", + "hood": "scoort", + "hop": "jauf", + "hope": "preesk", + "hopp": "klyf", + "horn": "smum", + "horrid": "zirous", + "hors": "skorn", + "horse": "pleast", + "horseshoe": "sweckous", + "hose": "yof", + "hosier": "bridleent", + "hot": "jeart", + "hound": "flon", + "hour": "skesh", + "house": "kleank", + "housetop": "boshumen", + "hubbard": "cherngle", + "huff": "sqarck", + "humpty": "rezle", + "hundr": "brir", + "hung": "blerd", + "hunt": "breaft", + "hurry": "stoaffum", + "hurt": "shrooz", + "husband": "swarffleow", + "hush": "korz", + "huzza": "byrmer", + "ice": "skach", + "icicle": "broasperel", + "ifs": "snarm", + "ill": "starch", + "illustrat": "wermishen", + "inde": "fruk", + "ink": "wread", + "instead": "florching", + "intery": "sqoopicum", + "iron": "smouspic", + "ive": "choost", + "jack": "plief", + "jacky": "scaunger", + "jag": "jourm", + "jam": "swaill", + "jelf": "hoth", + "jen": "prath", + "jenny": "driedle", + "jerry": "niernel", + "jig": "birt", + "jiggety": "spooffelum", + "jill": "choan", + "jingle": "sidleum", + "joan": "blarrn", + "jog": "shrauz", + "john": "blirlt", + "johnny": "plonker", + "joke": "stush", + "jol": "kloamp", + "joy": "kiek", + "joyou": "tounk", + "july": "skarkous", + "jump": "scerrk", + "june": "narck", + "keep": "spirk", + "ken": "gnug", + "kept": "sarsh", + "kettl": "gliep", + "kettle": "lochle", + "key": "smerst", + "kilkenny": "steetelent", + "kill": "bais", + "kind": "glar", + "king": "sperlt", + "kingdom": "slerle", + "kirk": "snerll", + "kis": "sqa", + "kiss": "grirv", + "kit": "smoaft", + "kitchen": "mogleum", + "kite": "drorsk", + "kitten": "chample", + "kitty": "lerter", + "knave": "sliv", + "kne": "ske", + "knee": "snornd", + "knife": "farv", + "knight": "slet", + "knock": "swoard", + "know": "yarn", + "kyloe": "starm", + "lad": "baurm", + "ladd": "swe", + "laddie": "wroog", + "laden": "trurser", + "lady": "licky", + "ladybird": "birfterum", + "lag": "pruck", + "laid": "gyrd", + "lal": "slorll", + "lam": "kleelt", + "lamb": "loosk", + "lan": "zirst", + "land": "smuz", + "lane": "hurs", + "lard": "taurk", + "lark": "meesp", + "lass": "frick", + "last": "gek", + "latch": "klirrt", + "late": "brich", + "laugh": "blernk", + "lauk": "paump", + "lay": "cherm", + "lea": "dryff", + "lead": "gliez", + "lean": "raink", + "leap": "prieft", + "least": "choart", + "leath": "gnach", + "leav": "brap", + "leave": "klet", + "led": "shrach", + "lee": "snoord", + "leed": "lood", + "left": "paind", + "leg": "buk", + "lend": "choast", + "lengthen": "zaukleish", + "lent": "cherd", + "les": "swyt", + "let": "sqolk", + "lett": "kroat", + "lick": "tit", + "lie": "swoub", + "life": "rorn", + "lift": "baurd", + "light": "smoag", + "like": "streash", + "lin": "berrd", + "linen": "youlking", + "linnet": "primel", + "lion": "slus", + "list": "shirk", + "little": "shooventy", + "littleman": "smerowid", + "liv": "strorth", + "live": "braz", + "load": "streart", + "lock": "ploort", + "locket": "fringow", + "lol": "gliek", + "london": "zoukel", + "long": "prirnd", + "longman": "ditleer", + "look": "hieff", + "lord": "hang", + "lost": "prarch", + "loud": "drenk", + "lov": "ners", + "love": "sceerm", + "low": "skisk", + "luck": "mauv", + "lucy": "shiffel", + "lump": "sqieck", + "lumpety": "zeengowen", + "mad": "merst", + "made": "jerr", + "maid": "kor", + "maiden": "choapic", + "main": "slorn", + "maintain": "shegleent", + "mak": "gark", + "make": "shream", + "malt": "choth", + "mamma": "spouple", + "mammie": "frarp", + "mammy": "shreefid", + "man": "tun", + "many": "thruskish", + "march": "brirr", + "mare": "treark", + "margaret": "kookowish", + "margery": "wrerskidle", + "mark": "chooft", + "market": "joskent", + "marr": "slysk", + "marri": "glythow", + "marry": "tarskent", + "martin": "scoongle", + "mary": "lorrow", + "mast": "morst", + "master": "bleangle", + "match": "spyrt", + "matt": "snoonk", + "maybe": "klorl", + "mayor": "barsh", + "meadow": "wurkish", + "meal": "tirp", + "mean": "thirnk", + "meat": "graip", + "meet": "daink", + "melancho": "gielkishous", + "men": "sked", + "mend": "chait", + "merchant": "scopel", + "mercy": "gerrmow", + "merri": "brauvent", + "merry": "shordum", + "merrymen": "lormousum", + "met": "rylt", + "mew": "deen", + "mice": "porff", + "mickle": "gleetleic", + "middle": "klaullum", + "mil": "klouk", + "mild": "blarp", + "mile": "shofent", + "milk": "klarb", + "mill": "hoork", + "mind": "burb", + "mine": "siet", + "mintery": "sqeambleidous", + "minute": "paring", + "mire": "blirn", + "mis": "smarm", + "mischievou": "wenkelous", + "miss": "tharg", + "mist": "soot", + "mistaken": "ferdleowy", + "mistres": "thryllow", + "misty": "kegent", + "moisty": "flerrmel", + "mol": "sqauz", + "monday": "swaichow", + "money": "frerkic", + "monkey": "floushing", + "monstrou": "klozow", + "moon": "glet", + "moppet": "jealling", + "more": "trait", + "morn": "skerf", + "morning": "dreapent", + "mortal": "chyler", + "mother": "sorish", + "motion": "glipic", + "mourn": "gniv", + "mouse": "plault", + "mouth": "lerft", + "mov": "vaiz", + "move": "voum", + "mow": "deal", + "mrs": "fraug", + "much": "thees", + "muffet": "tairmic", + "mulberry": "glooltumous", + "multiplication": "pleffentenicel", + "music": "liser", + "muskidun": "firmpidid", + "mutton": "swooftent", + "myself": "engrirt", + "nag": "zymp", + "nail": "kryk", + "nam": "bleev", + "name": "sorm", + "nan": "thoask", + "nancy": "sweandleel", + "nanny": "yorpow", + "narrow": "krouskish", + "nasty": "dermbleel", + "naught": "snorv", + "naughty": "gooskow", + "nay": "smoomp", + "neary": "pirvy", + "neat": "beach", + "neck": "shon", + "needl": "smelk", + "needle": "trurndel", + "neighbor": "glolkid", + "neith": "mirp", + "nest": "yyb", + "new": "moonk", + "next": "swaump", + "nibble": "strurchy", + "nice": "jirst", + "niggledy": "speachicum", + "night": "mourd", + "nightgown": "gearten", + "nimble": "kirbleing", + "nineteen": "pirmbleleel", + "nob": "kroark", + "nobleman": "flypishish", + "nobody": "skidleishel", + "nodd": "wrell", + "noise": "bornd", + "noon": "broack", + "nor": "sursh", + "north": "stroolt", + "norwich": "thruchow", + "nose": "flaus", + "notch": "foos", + "note": "stoack", + "noth": "chorsh", + "novemb": "snarmbleer", + "now": "hick", + "oak": "nol", + "often": "slaunky", + "old": "kir", + "ope": "frusp", + "open": "greebleous", + "orange": "darmpid", + "organ": "gierdel", + "oth": "plerz", + "oven": "bradleent", + "owe": "gnurk", + "owl": "yoad", + "own": "broolt", + "packet": "snirndent", + "pail": "wrert", + "pair": "flirt", + "pan": "broun", + "pancake": "krierdous", + "pandy": "faly", + "pantry": "junker", + "pap": "shoag", + "papa": "saibleow", + "parent": "droozzleid", + "parlor": "jiezzleent", + "parrot": "choallous", + "parson": "loaffleum", + "party": "pokic", + "pas": "bleend", + "pat": "chais", + "patch": "ploank", + "pay": "lerv", + "peace": "chaurk", + "peaceable": "dorgelow", + "peacock": "tharrmel", + "pear": "jarll", + "pease": "kielk", + "peck": "snealk", + "pedlar": "wipy", + "peep": "strol", + "pen": "prai", + "penny": "triethle", + "people": "storthum", + "pepper": "shroalkle", + "perhap": "wreazum", + "pet": "zard", + "peter": "grondleid", + "petticoat": "proasenic", + "physician": "feashentle", + "piccadil": "skurzerum", + "pick": "smoarn", + "pickety": "glerntishow", + "pickl": "chav", + "picture": "thribic", + "pie": "stirl", + "piece": "gloall", + "pieman": "furder", + "pig": "groask", + "pigeon": "friedid", + "piggledy": "gneangentent", + "pin": "larl", + "pinch": "spoart", + "pint": "troam", + "piou": "spiez", + "pip": "tong", + "pipe": "klang", + "piper": "hoakleel", + "pippen": "notum", + "pitch": "thrairm", + "plac": "squrrn", + "plain": "furd", + "plast": "swoust", + "plat": "stryr", + "plate": "park", + "platt": "scaik", + "play": "blurt", + "playfellow": "flozishing", + "playmat": "fleenish", + "please": "klyn", + "plenty": "foornish", + "plum": "friest", + "pocket": "frailow", + "pocketful": "kaillousous", + "point": "wrurrn", + "poker": "bleagleow", + "pol": "slorsh", + "poll": "drorlk", + "pony": "shrupic", + "pooh": "wrorrm", + "poor": "smied", + "poppety": "trorthenish", + "porgy": "strerbleel", + "porridge": "plospow", + "porring": "blymbleum", + "pos": "purz", + "posses": "flurtous", + "pot": "scis", + "potato": "siendelen", + "pound": "cheed", + "powd": "lied", + "practice": "flarkleer", + "pray": "vurrt", + "prayer": "pol", + "pretti": "leelkent", + "pretty": "mienle", + "pri": "klees", + "prick": "voarn", + "prince": "scord", + "princes": "shreller", + "prithee": "roulous", + "prod": "strerst", + "promis": "dryking", + "proper": "swoulkish", + "protector": "prukleelow", + "proud": "neelk", + "psalm": "skarsk", + "pudd": "chorn", + "pudding": "sirndent", + "puddle": "poormish", + "pull": "briel", + "pumpkin": "charllid", + "pussy": "frarsish", + "put": "smaiff", + "puzzle": "prerlen", + "quack": "skoong", + "quarrel": "glorspy", + "queen": "hysp", + "quick": "fouff", + "quiet": "swaub", + "quite": "traup", + "rabbit": "greableow", + "rac": "moath", + "rag": "shouv", + "rage": "zift", + "rain": "yurnk", + "ram": "chousp", + "ran": "krurff", + "rapp": "roth", + "rare": "feen", + "rat": "shrerd", + "rattle": "drearnid", + "raven": "joobid", + "raw": "plount", + "reach": "sqerll", + "read": "swynk", + "ready": "glocker", + "real": "krarrd", + "reason": "speeken", + "receive": "snoaften", + "red": "smust", + "redbreast": "klauvent", + "reel": "streath", + "reigate": "plaimle", + "remedy": "gnoadleishel", + "repartee": "goofering", + "repli": "zoospen", + "request": "diven", + "resolv": "lindleum", + "rest": "fleark", + "return": "rockum", + "rhym": "sorf", + "rhyme": "nech", + "ribbon": "loudow", + "rice": "pynk", + "rich": "sceag", + "richard": "shruffum", + "rid": "swaint", + "riddle": "virlkel", + "ride": "hock", + "rig": "yunt", + "right": "tirk", + "ring": "drient", + "ringman": "glirrow", + "rise": "kelt", + "riv": "gnarst", + "roast": "syl", + "rob": "poag", + "robber": "slirnen", + "robert": "molkel", + "robin": "trirmpy", + "robinson": "fiegower", + "rock": "birf", + "rode": "gnait", + "rog": "plailk", + "roll": "frirnd", + "rook": "froash", + "room": "staus", + "root": "vut", + "ros": "rech", + "rosy": "sporpy", + "rough": "pront", + "round": "yaik", + "row": "thersk", + "rule": "hubow", + "run": "maik", + "runn": "shairn", + "rush": "waill", + "rye": "praib", + "sabbath": "chirrdid", + "sack": "ploand", + "saddle": "fieffleous", + "safe": "sqart", + "sage": "spoz", + "sago": "gnoumbleel", + "said": "plys", + "sail": "jarrk", + "sailor": "deashish", + "salt": "sount", + "sam": "brieth", + "same": "snurlk", + "sang": "theeff", + "sat": "werrm", + "saturday": "struluming", + "saw": "smauck", + "say": "soask", + "scar": "pord", + "scarce": "feek", + "scarlet": "wrackid", + "scholar": "swauntish", + "school": "glaull", + "schoolroom": "klierden", + "scotch": "klirnt", + "scratch": "hit", + "scuttle": "shraurder", + "sea": "sqyd", + "seal": "fraurn", + "seam": "taup", + "seasonable": "smeapicumen", + "second": "girffent", + "see": "sqourk", + "seed": "stord", + "seek": "borth", + "seen": "sqeev", + "seldom": "chirfic", + "selfsame": "glurpent", + "sell": "snarng", + "sempster": "gendic", + "send": "smaish", + "sent": "flirrt", + "septemb": "frazzleen", + "serv": "kraisp", + "servant": "pinkic", + "serve": "sorr", + "set": "teant", + "seventeen": "chyskishle", + "sew": "verst", + "shaftoe": "rert", + "shake": "golt", + "shalt": "smymp", + "shape": "skorrk", + "shave": "greef", + "shaven": "gerrtent", + "shed": "kirth", + "sheep": "saung", + "shelf": "snerk", + "shell": "soorm", + "shepherdes": "pirkleishum", + "shilling": "swengous", + "shin": "kourt", + "shine": "gnaf", + "ship": "sep", + "shiv": "hyft", + "sho": "pend", + "shod": "wrorst", + "shoe": "skurd", + "shook": "kleert", + "shoot": "dreeb", + "shop": "gly", + "shoreditch": "koakishel", + "shorn": "kloolk", + "short": "woank", + "shot": "heam", + "show": "strup", + "shower": "serffleing", + "shroud": "doap", + "shut": "jant", + "sick": "yaip", + "side": "sleth", + "siege": "yosk", + "sieve": "blailt", + "sigh": "strend", + "silk": "shrouk", + "sill": "dierm", + "silv": "glev", + "simon": "bliernous", + "simple": "fliken", + "sing": "scount", + "single": "taizen", + "sir": "wich", + "sister": "neshen", + "sit": "kolk", + "sitt": "noaz", + "sixpence": "rarngow", + "sixteen": "smeegleen", + "skin": "horlk", + "skipp": "zailt", + "sky": "swaisk", + "slash": "sco", + "slat": "saim", + "slatherum": "grillele", + "sleep": "klaump", + "sleepy": "thieder", + "slend": "gnard", + "slice": "gron", + "slid": "drurt", + "slipper": "snoaskent", + "slitherum": "bryningish", + "slow": "rirft", + "sly": "fliff", + "small": "gnarnk", + "smile": "shroasen", + "smith": "kron", + "smok": "tep", + "snail": "pist", + "snap": "thesh", + "snapp": "grerrk", + "sneez": "stoos", + "sneeze": "sorrd", + "sniff": "maim", + "snipe": "larrd", + "snook": "slauff", + "snow": "fraing", + "snuff": "smaut", + "sobb": "glyn", + "soft": "shurd", + "sold": "zourm", + "solomon": "swaishowing", + "someth": "haiving", + "son": "krirck", + "song": "strieft", + "soon": "thrif", + "sore": "teed", + "sorrow": "jaiskel", + "sorrowful": "klozidous", + "soul": "snesp", + "sound": "blaump", + "south": "breand", + "sow": "jief", + "spade": "werrt", + "spain": "klysp", + "sparrow": "kroogley", + "speak": "jym", + "spell": "stirsk", + "spend": "staum", + "spent": "krieff", + "spice": "flyk", + "spid": "grest", + "spilt": "zorb", + "spin": "kersp", + "spinn": "snieg", + "spit": "sqaig", + "splash": "flerk", + "splish": "snul", + "spoil": "gnerg", + "spoke": "nirs", + "spoon": "yiell", + "sport": "frirsp", + "spr": "sqob", + "sprat": "shirck", + "spright": "sharb", + "spry": "slooz", + "spun": "nim", + "squirrel": "tielk", + "staff": "sciet", + "stair": "rauk", + "stand": "garll", + "star": "shraip", + "start": "roump", + "stay": "plith", + "steal": "sqierd", + "steel": "seert", + "stepney": "parkleid", + "stepp": "spask", + "stew": "wourm", + "stick": "sqerlk", + "stiff": "veesk", + "stile": "gumid", + "stingy": "breekleen", + "stock": "drurnk", + "stole": "cheempen", + "stone": "zailk", + "stood": "parrd", + "stop": "shrornk", + "stopp": "tirs", + "storm": "sob", + "story": "drousow", + "stout": "zoush", + "str": "hout", + "straight": "rersk", + "straightway": "treasle", + "strang": "streesp", + "strange": "sluf", + "straw": "wird", + "strawberr": "shreffous", + "stray": "plerth", + "street": "reeng", + "strife": "sqark", + "strong": "resk", + "strow": "pluv", + "struck": "glurg", + "stump": "noull", + "stumpaty": "sporzowen", + "such": "haib", + "sue": "skas", + "sugar": "snarrment", + "sukey": "fryshish", + "sulky": "prirlkic", + "summ": "pyg", + "summer": "sqiezzleen", + "sun": "gurt", + "sunday": "brorble", + "sung": "shout", + "sunshine": "glaisent", + "sunshiny": "briffleident", + "sup": "smoont", + "supp": "dyrm", + "suppose": "nyrmous", + "sure": "grirk", + "surprise": "shoableent", + "surrey": "shermic", + "swan": "krorrd", + "swarm": "morsk", + "sweep": "kroft", + "sweet": "starng", + "swim": "gat", + "swimm": "liend", + "swine": "sqeant", + "swoon": "troart", + "sword": "dierd", + "swore": "threesp", + "swum": "chirv", + "table": "spyffent", + "tack": "ploll", + "taffy": "teaffer", + "tail": "stoand", + "tailor": "therer", + "tak": "staill", + "take": "gnauf", + "tale": "foundleent", + "talk": "plirck", + "talkative": "haurument", + "tapp": "chir", + "tar": "gosk", + "tarr": "chym", + "tarry": "strerkleen", + "tart": "froat", + "taste": "smuff", + "tatter": "thrairtic", + "taught": "churs", + "tavern": "saishy", + "tea": "krounk", + "tear": "baulk", + "tee": "norch", + "teeth": "flith", + "tell": "lien", + "test": "skain", + "thank": "skyrd", + "thee": "shoork", + "thi": "jeest", + "thick": "herrk", + "thief": "swuth", + "thigh": "ploosh", + "thing": "vyv", + "think": "shree", + "third": "shroap", + "thirteen": "wroomel", + "thirty": "swaimum", + "thistle": "streakleous", + "thorn": "teas", + "thou": "zus", + "thought": "strosk", + "thousand": "prumy", + "thread": "skeek", + "threescore": "veandleid", + "threw": "flaith", + "thrive": "shrem", + "throat": "wrylk", + "throw": "bloach", + "thrush": "rad", + "thu": "shroosk", + "thumb": "shron", + "thumbkin": "thyzent", + "thump": "frurn", + "thumpaty": "toaberow", + "thursday": "yauskish", + "thy": "pree", + "thyself": "rainkous", + "tick": "shealt", + "tickl": "struk", + "tie": "bor", + "tied": "wrosk", + "til": "durg", + "till": "groar", + "tim": "blaift", + "time": "vurm", + "tinker": "bloasow", + "tip": "thilt", + "tipple": "verntous", + "tir": "seenk", + "tis": "porp", + "tisha": "miffleish", + "tittlemouse": "woaffidic", + "tobago": "brirthowent", + "today": "rekurd", + "toe": "spoult", + "togeth": "weegish", + "told": "blel", + "toll": "skymp", + "tom": "tard", + "tommy": "flugen", + "tong": "gauck", + "tongu": "moaffleid", + "tongue": "smuth", + "took": "plorz", + "top": "shrealt", + "torch": "krirl", + "torn": "hiek", + "tos": "snairk", + "toss": "krind", + "touch": "yerll", + "town": "loarn", + "toy": "plall", + "tramp": "tirz", + "trap": "soar", + "tre": "faing", + "tree": "sur", + "trencher": "squrlent", + "tri": "jyd", + "trick": "weer", + "trip": "shoort", + "tripe": "lurll", + "tripp": "bauk", + "trot": "shrurll", + "trott": "gliert", + "troubl": "sloaft", + "trouble": "chygleow", + "trow": "jich", + "trowel": "woabum", + "true": "sweat", + "try": "sout", + "tuck": "rarrd", + "tuesday": "doakel", + "tuffet": "krylkish", + "tumbl": "varn", + "tune": "slirrm", + "turn": "groul", + "turnip": "foakleing", + "twa": "stors", + "twaddle": "dooskic", + "twee": "kreav", + "tweedle": "rundle", + "twelve": "slynk", + "twenty": "bliebleing", + "twiddle": "glorffent", + "twig": "threark", + "twill": "trurl", + "twitchett": "speegle", + "twopence": "chauvish", + "undertaker": "wroartidely", + "unicorn": "pirthidel", + "unto": "kreelous", + "upon": "enswirrm", + "upstair": "stroopel", + "upward": "jaurdow", + "use": "tulk", + "used": "bronken", + "ush": "nunt", + "vale": "daiften", + "vast": "triev", + "velvet": "moafen", + "venture": "smoornow", + "vex": "slulk", + "vexation": "klefinging", + "victual": "shoukleel", + "vinegar": "hearkower", + "visit": "posken", + "visitor": "shugleering", + "vow": "shosp", + "wag": "froug", + "waggl": "slin", + "wail": "hyd", + "wainscot": "thoazum", + "wait": "koud", + "wak": "busp", + "wake": "shorrd", + "walk": "fap", + "wall": "ront", + "walnut": "prarling", + "wand": "jeash", + "want": "shoulk", + "ware": "yaust", + "warm": "krend", + "wash": "choll", + "wat": "strorn", + "watch": "sniz", + "watt": "trarlk", + "way": "smaun", + "wealthy": "kliertum", + "wear": "feasp", + "weath": "glim", + "wed": "grarsp", + "wedd": "zeasp", + "wednesday": "sqilkousel", + "wee": "blyp", + "weed": "shread", + "week": "shroord", + "weep": "nath", + "welcome": "krerdy", + "well": "swat", + "welshman": "riespel", + "went": "smaud", + "west": "sond", + "whale": "sqeandleous", + "whatev": "draivish", + "wheel": "slan", + "wheelbarrow": "shrounkicous", + "whenev": "varfen", + "wherev": "fendley", + "whey": "breang", + "whip": "sperck", + "whipp": "kroop", + "whistle": "skauzzleing", + "white": "wrourk", + "whitechapel": "flerrderowish", + "whith": "swurz", + "whoop": "wreak", + "wife": "grerng", + "wig": "zarch", + "wil": "brynd", + "wild": "dauz", + "wildernes": "skeanentous", + "wilkin": "dadleel", + "willie": "vurp", + "wilt": "brisk", + "wind": "strirlt", + "window": "chazzleic", + "wine": "snurrk", + "wing": "poul", + "winkie": "vud", + "winkle": "grospic", + "wint": "zell", + "winter": "smarltous", + "wip": "swirrk", + "wipe": "glirll", + "wire": "purr", + "wise": "heach", + "wish": "sqast", + "within": "wrurbleel", + "without": "tauftel", + "wiv": "soarn", + "woe": "striech", + "woman": "fliespous", + "women": "drarrmous", + "wond": "waim", + "wondrou": "losting", + "woo": "kamp", + "wood": "piez", + "woodbin": "torrding", + "wool": "sirff", + "word": "wreav", + "work": "dryck", + "world": "shyck", + "worm": "klech", + "worri": "klierkent", + "worry": "flartent", + "worse": "prom", + "worth": "yeed", + "wrap": "sqerr", + "wren": "thrirl", + "wright": "chouth", + "write": "font", + "wrote": "thairm", + "yard": "bliem", + "yea": "glaif", + "year": "dalk", + "yellow": "frarder", + "yes": "shraurd", + "yon": "dyl", + "young": "tarl" + } + }, + { + "label": "seed0-noprosody", + "seed": 0, + "matchProsody": false, + "mint": "nonce", + "injectiveAtEveryP": true, + "remintRounds": 1, + "bijective": true, + "imageSize": 2233, + "domainSize": 2233, + "mappingSize": 1680, + "mappingSha256": "37d7dc1902993851b4696a68dcaf73612de37efefdd216d68d72ec896f215044", + "mapping": null, + "sampleNonces": { + "little": "strek", + "pretty": "streesh", + "run": "horlk", + "eat": "ben", + "jump": "byll", + "away": "squz", + "squirrel": "straull", + "funny": "krun", + "today": "krurrd", + "gum": "fles", + "hang": "pruch", + "crown": "strearm", + "candlestick": "vorm", + "crooked": null, + "diddle": "norrn", + "moon": "smorft", + "pussy": "flyb", + "goose": "brarsp", + "the": null, + "and": null, + "you": null, + "not": null, + "ox": null, + "good-bye": null + } + }, + { + "label": "swap-seed0", + "seed": 0, + "matchProsody": true, + "mint": "swap", + "injectiveAtEveryP": false, + "remintRounds": 0, + "bijective": true, + "imageSize": 2233, + "domainSize": 2233, + "mappingSize": 1680, + "mappingSha256": "76f6cfc226e27896b5a5fbb52200f427bf774898f271205146cb3b4e76d3e5ef", + "mapping": null, + "sampleNonces": { + "little": "birthday", + "pretty": "mary", + "run": "sneeze", + "eat": "say", + "jump": "tail", + "away": "upon", + "squirrel": "works", + "funny": "picture", + "today": "pricked", + "gum": "fish", + "hang": "mouth", + "crown": "harm", + "candlestick": "clergyman", + "crooked": null, + "diddle": "mother", + "moon": "bird", + "pussy": "crooked", + "goose": "snow", + "the": null, + "and": null, + "you": null, + "not": null, + "ox": null, + "good-bye": null + } + }, + { + "label": "swap-seed0-noprosody", + "seed": 0, + "matchProsody": false, + "mint": "swap", + "injectiveAtEveryP": false, + "remintRounds": 0, + "bijective": true, + "imageSize": 2233, + "domainSize": 2233, + "mappingSize": 1680, + "mappingSha256": "502bff84d42bbac52660c487d563c91634c4ef144943cfa799a5daa9a01d7eef", + "mapping": null, + "sampleNonces": { + "little": "wrap", + "pretty": "this", + "run": "full", + "eat": "say", + "jump": "lived", + "away": "said", + "squirrel": "works", + "funny": "bear", + "today": "ask", + "gum": "frumpaty", + "hang": "misty", + "crown": "asked", + "candlestick": "cobwebs", + "crooked": null, + "diddle": "days", + "moon": "bird", + "pussy": "crooked", + "goose": "snow", + "the": null, + "and": null, + "you": null, + "not": null, + "ox": null, + "good-bye": null + } + }, + { + "label": "swap-seed7", + "seed": 7, + "matchProsody": true, + "mint": "swap", + "injectiveAtEveryP": false, + "remintRounds": 0, + "bijective": true, + "imageSize": 2233, + "domainSize": 2233, + "mappingSize": 1680, + "mappingSha256": "41b49d5e6a9d0276f80ffddd318550cde0dc4961bb005e03bb700b10e20ac5a6", + "mapping": null, + "sampleNonces": { + "little": "birthday", + "pretty": "woman", + "run": "more", + "eat": "play", + "jump": "men", + "away": "upon", + "squirrel": "wool", + "funny": "wished", + "today": "away", + "gum": "kill", + "hang": "fetch", + "crown": "dish", + "candlestick": "clergyman", + "crooked": null, + "diddle": "jenny", + "moon": "dame", + "pussy": "taffy", + "goose": "dove", + "the": null, + "and": null, + "you": null, + "not": null, + "ox": null, + "good-bye": null + } + }, + { + "label": "swap-seed7-noprosody", + "seed": 7, + "matchProsody": false, + "mint": "swap", + "injectiveAtEveryP": false, + "remintRounds": 0, + "bijective": true, + "imageSize": 2233, + "domainSize": 2233, + "mappingSize": 1680, + "mappingSha256": "9b198a8df6b34eaef07438910be80c39f7d2fb5d2793f325257808af8c8f29c5", + "mapping": null, + "sampleNonces": { + "little": "doll", + "pretty": "woman", + "run": "water", + "eat": "play", + "jump": "men", + "away": "shoe", + "squirrel": "away", + "funny": "wished", + "today": "chicken", + "gum": "hung", + "hang": "fetch", + "crown": "hubbard", + "candlestick": "cellar", + "crooked": null, + "diddle": "crow", + "moon": "dame", + "pussy": "taffy", + "goose": "crumpled", + "the": null, + "and": null, + "you": null, + "not": null, + "ox": null, + "good-bye": null + } + }, + { + "label": "control-inconsistent", + "seed": 0, + "matchProsody": true, + "mint": "nonce", + "injectiveAtEveryP": true, + "remintRounds": 0, + "bijective": true, + "imageSize": 2233, + "domainSize": 2233, + "mappingSize": 1680, + "mappingSha256": "7ba5235f4b848f548abe1c7bdcdb5ebecaa5d1d0c93be321e540705339d313aa", + "mapping": null, + "sampleNonces": { + "little": "skoufenty", + "pretty": "streeshum", + "run": "jev", + "eat": "ben", + "jump": "byll", + "away": "bepaip", + "squirrel": "straull", + "funny": "krunel", + "today": "refon", + "gum": "fles", + "hang": "pruch", + "crown": "strearm", + "candlestick": "vormenent", + "crooked": null, + "diddle": "kobleel", + "moon": "smorft", + "pussy": "flyble", + "goose": "brarsp", + "the": null, + "and": null, + "you": null, + "not": null, + "ox": null, + "good-bye": null + } + }, + { + "label": "control-inconsistent-seed7", + "seed": 7, + "matchProsody": true, + "mint": "nonce", + "injectiveAtEveryP": true, + "remintRounds": 1, + "bijective": true, + "imageSize": 2233, + "domainSize": 2233, + "mappingSize": 1680, + "mappingSha256": "65efb80b712a63d434bc8c9ecc1266dc93197eca5c3938ebd5f820c8da82fbbd", + "mapping": null, + "sampleNonces": { + "little": "shooventy", + "pretty": "mienle", + "run": "maik", + "eat": "drult", + "jump": "scerrk", + "away": "unpryng", + "squirrel": "tielk", + "funny": "vezzleum", + "today": "rekurd", + "gum": "thrern", + "hang": "smeeg", + "crown": "trif", + "candlestick": "strooftishish", + "crooked": null, + "diddle": "gnoozzleel", + "moon": "glet", + "pussy": "frarsish", + "goose": "kloalk", + "the": null, + "and": null, + "you": null, + "not": null, + "ox": null, + "good-bye": null + } + }, + { + "label": "control-reveal-after-2", + "seed": 0, + "matchProsody": true, + "mint": "nonce", + "injectiveAtEveryP": true, + "remintRounds": 0, + "bijective": true, + "imageSize": 2233, + "domainSize": 2233, + "mappingSize": 1680, + "mappingSha256": "7ba5235f4b848f548abe1c7bdcdb5ebecaa5d1d0c93be321e540705339d313aa", + "mapping": null, + "sampleNonces": { + "little": "skoufenty", + "pretty": "streeshum", + "run": "jev", + "eat": "ben", + "jump": "byll", + "away": "bepaip", + "squirrel": "straull", + "funny": "krunel", + "today": "refon", + "gum": "fles", + "hang": "pruch", + "crown": "strearm", + "candlestick": "vormenent", + "crooked": null, + "diddle": "kobleel", + "moon": "smorft", + "pussy": "flyble", + "goose": "brarsp", + "the": null, + "and": null, + "you": null, + "not": null, + "ox": null, + "good-bye": null + } + } + ], + "cases": [ + { + "label": "seed0-p0.0", + "map": "seed0", + "params": { + "p": 0.0, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\n", + "vacatedSha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c", + "vacatedChars": 86408, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 0, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 0, + "stemsTotal": 1680, + "stemsVacated": 0, + "tokensTotal": 16000, + "tokensVacated": 0, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.29, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3209324983421379, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0514375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.0, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.9485625, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "little", + "pretty": "pretty", + "run": "run", + "eat": "eat", + "jump": "jump", + "away": "away", + "squirrel": "squirrel", + "funny": "funny", + "today": "today", + "gum": "gum", + "hang": "hang", + "crown": "crown", + "candlestick": "candlestick", + "crooked": "crooked", + "diddle": "diddle", + "moon": "moon", + "pussy": "pussy", + "goose": "goose", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "2ba43046604884d431a9b8a503ae906a703235c578f7d588b877302c95b74983" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed0-p0.25", + "map": "seed0", + "params": { + "p": 0.25, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE REAL\n WIELKOW BRARSP\n\n _Illustrated by_\nBlanche Fisher Trurrk\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Gea\nRain\nThe Clock\nWinter\nFingers and Toes\nA Squrvididish Song\nStruv Vard and Her Cat\nThree Kliesowle on the Ice\nDroalls Brem\nThe Garr Plaispy Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Grirn!\nGarr Wielkow Brarsp\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nSqo", + "vacatedSha256": "6598ab550c09c2301cca7c38f8a57e3f1e385bb0a0f4648c14d8505cc0e5ad39", + "vacatedChars": 88987, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 469, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 461, + "stemsTotal": 1680, + "stemsVacated": 401, + "tokensTotal": 16000, + "tokensVacated": 2217, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.2901875, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.32098427911681016, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0424375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.1136875, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.843875, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "little", + "pretty": "pretty", + "run": "run", + "eat": "ben", + "jump": "jump", + "away": "away", + "squirrel": "straull", + "funny": "funny", + "today": "refon", + "gum": "gum", + "hang": "hang", + "crown": "crown", + "candlestick": "candlestick", + "crooked": "rieged", + "diddle": "diddle", + "moon": "moon", + "pussy": "pussy", + "goose": "brarsp", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "3db5b2fa17cc7e36128536eaae02a6551a2f0b898ce85b864f5f3a6941ab46aa" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed0-p0.35", + "map": "seed0", + "params": { + "p": 0.35, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE PRIRLK\n WIELKOW BRARSP\n\n _Illustrated by_\nBlanche Fisher Trurrk\n\n1916\n\n\n\nA LIST OF THE SHRENGES\n\nLittle Bo-Peep\nLittle Boy Gea\nRain\nThe Clock\nWinter\nTitows and Toes\nA Squrvididish Song\nStruv Vard and Her Cat\nThree Kliesowle on the Ice\nDroalls Brem\nThe Garr Plaispy Under a Prornt\nTweedle-Dum and Tweedle-Dee\nOh Grirn!\nGarr Wielkow Brarsp\nLittle Bylling Joan\nPat-a-Cake\nMoney and the Mar", + "vacatedSha256": "f70d3febc5552d9e194b980a8811cb16fccc9d6d011049e574aca3fa6cf39d3a", + "vacatedChars": 90204, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 683, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 672, + "stemsTotal": 1680, + "stemsVacated": 587, + "tokensTotal": 16000, + "tokensVacated": 3152, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.2900625, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.320887366369295, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0416875, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.154875, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.8034375, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "little", + "pretty": "pretty", + "run": "run", + "eat": "ben", + "jump": "byll", + "away": "away", + "squirrel": "straull", + "funny": "funny", + "today": "refon", + "gum": "gum", + "hang": "hang", + "crown": "crown", + "candlestick": "candlestick", + "crooked": "rieged", + "diddle": "diddle", + "moon": "moon", + "pussy": "pussy", + "goose": "brarsp", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "a5b250c2e8fbb8adc825b98cb79f1f7160e347d303fb1677299b7398f363b4dd" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed0-p0.5", + "map": "seed0", + "params": { + "p": 0.5, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE PRIRLK\n WIELKOW BRARSP\n\n _Illustrated by_\nBlanche Neker Trurrk\n\n1916\n\n\n\nA SORP OF THE SHRENGES\n\nSkoufenty Bo-Peep\nSkoufenty Boy Gea\nRain\nThe Clock\nWinter\nTitows and Toes\nA Squrvididish Shyf\nStruv Vard and Her Cat\nThree Kliesowle on the Ice\nDroalls Brem\nThe Garr Plaispy Under a Prornt\nTweedle-Dum and Tweedle-Dee\nOh Grirn!\nGarr Wielkow Brarsp\nSkoufenty Bylling Joan\nPat-a-Cake\nMoney and", + "vacatedSha256": "d1613771bcc7f998b4b74ddb90b06cfaf1699488fb0df82988cf2d7d25508e74", + "vacatedChars": 92340, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 966, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 954, + "stemsTotal": 1680, + "stemsVacated": 833, + "tokensTotal": 16000, + "tokensVacated": 4470, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.28975, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.32123047935415144, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.023375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.2205, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.756125, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "skoufenty", + "pretty": "pretty", + "run": "run", + "eat": "ben", + "jump": "byll", + "away": "away", + "squirrel": "straull", + "funny": "funny", + "today": "refon", + "gum": "gum", + "hang": "hang", + "crown": "crown", + "candlestick": "vormenent", + "crooked": "rieged", + "diddle": "diddle", + "moon": "moon", + "pussy": "pussy", + "goose": "brarsp", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "3806792a17a00aa9328207aeefc7399508d0de0fcfc15372eb560dc82e5bb7ff" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed0-p0.7", + "map": "seed0", + "params": { + "p": 0.7, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE PRIRLK\n WIELKOW BRARSP\n\n _Trirmousered by_\nBlanche Neker Trurrk\n\n1916\n\n\n\nA SORP OF THE SHRENGES\n\nSkoufenty Bo-Peep\nSkoufenty Boy Gea\nRain\nThe Snuch\nWinter\nTitows and Smurlks\nA Squrvididish Shyf\nStruv Vard and Her Cat\nThree Kliesowle on the Ice\nDroalls Brem\nThe Garr Plaispy Under a Prornt\nTweedle-Dum and Tweedle-Dee\nOh Grirn!\nGarr Wielkow Brarsp\nSkoufenty Bylling Boub\nPat-a-Cake\nNiedu", + "vacatedSha256": "940e9468e7fa9d1ce59dea8e134704f9245fe829a89745687889eb97bf5b2731", + "vacatedChars": 94252, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1354, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1337, + "stemsTotal": 1680, + "stemsVacated": 1167, + "tokensTotal": 16000, + "tokensVacated": 6094, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.2896875, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.32100389418780717, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.017375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.30325, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.679375, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "skoufenty", + "pretty": "streeshum", + "run": "run", + "eat": "ben", + "jump": "byll", + "away": "bepaip", + "squirrel": "straull", + "funny": "krunel", + "today": "refon", + "gum": "fles", + "hang": "hang", + "crown": "strearm", + "candlestick": "vormenent", + "crooked": "rieged", + "diddle": "diddle", + "moon": "smorft", + "pussy": "flyble", + "goose": "brarsp", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "f7511cad9d20af366bcc81253d75d70025b443db601adb21ccfc9300580c5266" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed0-p0.75", + "map": "seed0", + "params": { + "p": 0.75, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE PRIRLK\n WIELKOW BRARSP\n\n _Trirmousered by_\nZoot Neker Trurrk\n\n1916\n\n\n\nA SORP OF THE SHRENGES\n\nSkoufenty Bo-Peep\nSkoufenty Boy Gea\nRain\nThe Snuch\nWinter\nTitows and Smurlks\nA Squrvididish Shyf\nStruv Vard and Her Cat\nThree Kliesowle on the Ice\nDroalls Brem\nThe Garr Plaispy Under a Prornt\nTweedle-Dum and Tweedle-Dee\nOh Grirn!\nGarr Wielkow Brarsp\nSkoufenty Bylling Boub\nPat-a-Cake\nNiedum a", + "vacatedSha256": "92e04ff4355517b4f75aaa319a004ff4fc4e139bd3fc01f719529dbbc126ffbb", + "vacatedChars": 94493, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1448, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1430, + "stemsTotal": 1680, + "stemsVacated": 1247, + "tokensTotal": 16000, + "tokensVacated": 6412, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.2898125, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3208627892274251, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0164375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.3180625, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.6655, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "skoufenty", + "pretty": "streeshum", + "run": "run", + "eat": "ben", + "jump": "byll", + "away": "bepaip", + "squirrel": "straull", + "funny": "krunel", + "today": "refon", + "gum": "fles", + "hang": "hang", + "crown": "strearm", + "candlestick": "vormenent", + "crooked": "rieged", + "diddle": "diddle", + "moon": "smorft", + "pussy": "flyble", + "goose": "brarsp", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "b88bd63106535d3a146a6dea89205f573b8a0db3fd8e835cb5b42d6260bc111a" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed0-p1.0", + "map": "seed0", + "params": { + "p": 1.0, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE PRIRLK\n WIELKOW BRARSP\n\n _Trirmousered by_\nZoot Neker Trurrk\n\n1916\n\n\n\nA SORP OF THE SHRENGES\n\nSkoufenty Bo-Peep\nSkoufenty Sim Gea\nSkarf\nThe Snuch\nSwaper\nTitows and Smurlks\nA Squrvididish Shyf\nStruv Vard and Her Chooz\nThree Kliesowle on the Karn\nDroalls Brem\nThe Garr Plaispy Under a Prornt\nTweedle-Dum and Tweedle-Dee\nOh Grirn!\nGarr Wielkow Brarsp\nSkoufenty Bylling Boub\nPat-a-Cake\nNied", + "vacatedSha256": "b3ff676c578e5afa50dfd19ae4c0eff65e548f01de38eb87374e815d3bf89ba8", + "vacatedChars": 96883, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1944, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1922, + "stemsTotal": 1680, + "stemsVacated": 1680, + "tokensTotal": 16000, + "tokensVacated": 8202, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.2909375, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.32064071852342685, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.012, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.4040625, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.5839375, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "skoufenty", + "pretty": "streeshum", + "run": "jev", + "eat": "ben", + "jump": "byll", + "away": "bepaip", + "squirrel": "straull", + "funny": "krunel", + "today": "refon", + "gum": "fles", + "hang": "pruch", + "crown": "strearm", + "candlestick": "vormenent", + "crooked": "rieged", + "diddle": "kobleel", + "moon": "smorft", + "pussy": "flyble", + "goose": "brarsp", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "c85505db78d27d0680a5a1a54016b6342b89cde49b545acebe2673d4a81b4998" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed7-p0.0", + "map": "seed7", + "params": { + "p": 0.0, + "seed": 7, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\n", + "vacatedSha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c", + "vacatedChars": 86408, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 0, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 0, + "stemsTotal": 1680, + "stemsVacated": 0, + "tokensTotal": 16000, + "tokensVacated": 0, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.29, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3209324983421379, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0514375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.0, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.9485625, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "stemForms": { + "little": "little", + "pretty": "pretty", + "run": "run", + "eat": "eat", + "jump": "jump", + "away": "away", + "squirrel": "squirrel", + "funny": "funny", + "today": "today", + "gum": "gum", + "hang": "hang", + "crown": "crown", + "candlestick": "candlestick", + "crooked": "crooked", + "diddle": "diddle", + "moon": "moon", + "pussy": "pussy", + "goose": "goose", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "2ba43046604884d431a9b8a503ae906a703235c578f7d588b877302c95b74983" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed7-p0.25", + "map": "seed7", + "params": { + "p": 0.25, + "seed": 7, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE KRARRD\n MOTHER KLOALK\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA SHIRK OF THE SORFES\n\nLittle Bo-Peep\nLittle Boy Blue\nYurnk\nThe Clock\nZeller\nFingers and Toes\nA Seasonable Strieft\nSaun Trot and Her Florll\nThree Children on the Ice\nKroots Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Wousk!\nOld Mother Kloalk\nLittle Jumping Blarrn\nPat-a-Cake\nFrerkic and the Mar", + "vacatedSha256": "e80589ef836f343dbfb95e4e4f89b8d94c5ce2c374c9557bbec9ff45b2f72114", + "vacatedChars": 88629, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 440, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 434, + "stemsTotal": 1680, + "stemsVacated": 384, + "tokensTotal": 16000, + "tokensVacated": 1879, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.2899375, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3209952065373761, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0435625, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.093875, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.8625625, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "stemForms": { + "little": "little", + "pretty": "mienle", + "run": "run", + "eat": "eat", + "jump": "jump", + "away": "unpryng", + "squirrel": "squirrel", + "funny": "funny", + "today": "rekurd", + "gum": "gum", + "hang": "smeeg", + "crown": "trif", + "candlestick": "candlestick", + "crooked": "crooked", + "diddle": "gnoozzleel", + "moon": "glet", + "pussy": "pussy", + "goose": "kloalk", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "06921a8029f1b69a9fd1f52a9e2b66c93f2573396c7ebeea5a11e91f67a5befd" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed7-p0.35", + "map": "seed7", + "params": { + "p": 0.35, + "seed": 7, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE KRARRD\n MOTHER KLOALK\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA SHIRK OF THE SORFES\n\nLittle Bo-Peep\nLittle Boy Blue\nYurnk\nThe Clock\nZeller\nFingers and Toes\nA Seasonable Strieft\nSaun Shrurll and Her Florll\nThree Children on the Ice\nKroots Patch\nThe Kir Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Wousk!\nKir Mother Kloalk\nLittle Jumping Blarrn\nPat-a-Cake\nFrerkic and the ", + "vacatedSha256": "56dd28fe1ad3c925fdca2e7aca94de205a7992aba902a1cb19d7ad743f817960", + "vacatedChars": 89404, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 662, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 655, + "stemsTotal": 1680, + "stemsVacated": 577, + "tokensTotal": 16000, + "tokensVacated": 2746, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.2899375, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.320770368963141, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0433125, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.1355625, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.821125, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "stemForms": { + "little": "little", + "pretty": "mienle", + "run": "maik", + "eat": "eat", + "jump": "jump", + "away": "unpryng", + "squirrel": "squirrel", + "funny": "funny", + "today": "rekurd", + "gum": "thrern", + "hang": "smeeg", + "crown": "trif", + "candlestick": "candlestick", + "crooked": "crooked", + "diddle": "gnoozzleel", + "moon": "glet", + "pussy": "pussy", + "goose": "kloalk", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "b60ddf4590bdca13aa13eb65ee4f1ce8dd840c4eeb6dfa4cf8b49d27e7bb5ee1" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed7-p0.5", + "map": "seed7", + "params": { + "p": 0.5, + "seed": 7, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE KRARRD\n SORISH KLOALK\n\n _Illustrated by_\nBlanche Fisher Chouth\n\n1916\n\n\n\nA SHIRK OF THE SORFES\n\nShooventy Bo-Peep\nShooventy Boy Brermp\nYurnk\nThe Desp\nZeller\nYeebys and Spoults\nA Smeapicumen Strieft\nSaun Shrurll and Her Florll\nThree Children on the Ice\nKroots Patch\nThe Kir Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Wousk!\nKir Sorish Kloalk\nShooventy Scerrking Blarrn\nPat-a-Cake\nF", + "vacatedSha256": "3166e1c857398c7a3bbb94e77c8b5accda4874e0c4ef53462664cab7c2f531b1", + "vacatedChars": 91826, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 985, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 975, + "stemsTotal": 1680, + "stemsVacated": 849, + "tokensTotal": 16000, + "tokensVacated": 4289, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.290125, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.32048243825352346, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.025125, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.208625, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.76625, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "stemForms": { + "little": "shooventy", + "pretty": "mienle", + "run": "maik", + "eat": "drult", + "jump": "scerrk", + "away": "unpryng", + "squirrel": "squirrel", + "funny": "funny", + "today": "rekurd", + "gum": "thrern", + "hang": "smeeg", + "crown": "trif", + "candlestick": "candlestick", + "crooked": "crooked", + "diddle": "gnoozzleel", + "moon": "glet", + "pussy": "frarsish", + "goose": "kloalk", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "91913e4d037886983b6e9d7c67ce2b0d68ef87fb74f061fb21093cc9928d0888" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed7-p0.7", + "map": "seed7", + "params": { + "p": 0.7, + "seed": 7, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE KRARRD\n SORISH KLOALK\n\n _Wermishened by_\nBlanche Fisher Chouth\n\n1916\n\n\n\nA SHIRK OF THE SORFES\n\nShooventy Bo-Peep\nShooventy Boy Brermp\nYurnk\nThe Desp\nZeller\nYeebys and Spoults\nA Smeapicumen Strieft\nSaun Shrurll and Her Florll\nThree Broarmidy on the Ice\nKroots Patch\nThe Kir Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Wousk!\nKir Sorish Kloalk\nShooventy Scerrking Blarrn\nPat-a-Cake\n", + "vacatedSha256": "88e3d28102bccbeab9195d107d1b22dac7b535025334ae354b0586d26b0c2271", + "vacatedChars": 93436, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1349, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1336, + "stemsTotal": 1680, + "stemsVacated": 1164, + "tokensTotal": 16000, + "tokensVacated": 5841, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.2909375, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3207724075193964, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0195625, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.286, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.6944375, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "stemForms": { + "little": "shooventy", + "pretty": "mienle", + "run": "maik", + "eat": "drult", + "jump": "scerrk", + "away": "unpryng", + "squirrel": "squirrel", + "funny": "vezzleum", + "today": "rekurd", + "gum": "thrern", + "hang": "smeeg", + "crown": "trif", + "candlestick": "strooftishish", + "crooked": "prufed", + "diddle": "gnoozzleel", + "moon": "glet", + "pussy": "frarsish", + "goose": "kloalk", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "fafd1902158b2c9108240295d7046d42557bf4be3eee894a517791f06beb33d4" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed7-p0.75", + "map": "seed7", + "params": { + "p": 0.75, + "seed": 7, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE KRARRD\n SORISH KLOALK\n\n _Wermishened by_\nPirv Fisher Chouth\n\n1916\n\n\n\nA SHIRK OF THE SORFES\n\nShooventy Bo-Peep\nShooventy Boy Brermp\nYurnk\nThe Desp\nZeller\nYeebys and Spoults\nA Smeapicumen Strieft\nSaun Shrurll and Her Florll\nThree Broarmidy on the Ice\nKroots Patch\nThe Kir Fliespous Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Wousk!\nKir Sorish Kloalk\nShooventy Scerrking Blarrn\nPat-a-Cake", + "vacatedSha256": "82434ac3737bd7cfa810bba9cb22b23811887d9e0ca9dfa2071a377420b43753", + "vacatedChars": 94023, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1455, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1440, + "stemsTotal": 1680, + "stemsVacated": 1255, + "tokensTotal": 16000, + "tokensVacated": 6241, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.29125, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3215548164343356, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.018875, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.3063125, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.6748125, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "stemForms": { + "little": "shooventy", + "pretty": "mienle", + "run": "maik", + "eat": "drult", + "jump": "scerrk", + "away": "unpryng", + "squirrel": "squirrel", + "funny": "vezzleum", + "today": "rekurd", + "gum": "thrern", + "hang": "smeeg", + "crown": "trif", + "candlestick": "strooftishish", + "crooked": "prufed", + "diddle": "gnoozzleel", + "moon": "glet", + "pussy": "frarsish", + "goose": "kloalk", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "5367d0740cb247060a718707f6987be9362b2138a245b138b5a454af2b682124" + }, + "mapVocabWordsRejects": false + }, + { + "label": "seed7-p1.0", + "map": "seed7", + "params": { + "p": 1.0, + "seed": 7, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE KRARRD\n SORISH KLOALK\n\n _Wermishened by_\nPirv Smausker Chouth\n\n1916\n\n\n\nA SHIRK OF THE SORFES\n\nShooventy Bo-Peep\nShooventy Skurth Brermp\nYurnk\nThe Desp\nZeller\nYeebys and Spoults\nA Smeapicumen Strieft\nSaun Shrurll and Her Florll\nThree Broarmidy on the Skach\nKroots Ploank\nThe Kir Fliespous Under a Drier\nTweedle-Dum and Tweedle-Dee\nOh Wousk!\nKir Sorish Kloalk\nShooventy Scerrking Blarrn\nP", + "vacatedSha256": "263b8dd3733dbf4802a2bc4c74fb905388062952e45931e4ebcd561cbd4e76df", + "vacatedChars": 96334, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1944, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1922, + "stemsTotal": 1680, + "stemsVacated": 1680, + "tokensTotal": 16000, + "tokensVacated": 8202, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.291125, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.32142168255740294, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.012, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.4040625, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.5839375, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "stemForms": { + "little": "shooventy", + "pretty": "mienle", + "run": "maik", + "eat": "drult", + "jump": "scerrk", + "away": "unpryng", + "squirrel": "tielk", + "funny": "vezzleum", + "today": "rekurd", + "gum": "thrern", + "hang": "smeeg", + "crown": "trif", + "candlestick": "strooftishish", + "crooked": "prufed", + "diddle": "gnoozzleel", + "moon": "glet", + "pussy": "frarsish", + "goose": "kloalk", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "8ada5171ddcbeeee2b16e1e6a2f0f87fbfc0891985d04d863f469c81ff0b453f" + }, + "mapVocabWordsRejects": false + }, + { + "label": "noprosody-p0.7", + "map": "seed0-noprosody", + "params": { + "p": 0.7, + "seed": 0, + "consistent": true, + "matchProsody": false, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE PRIRLK\n WIELK BRARSP\n\n _Trirmed by_\nBlanche Neker Trurrk\n\n1916\n\n\n\nA SORP OF THE SHRENGES\n\nStrek Bo-Peep\nStrek Boy Gea\nRain\nThe Snuch\nWinter\nTits and Smurlks\nA Squrv Shyf\nStruv Vard and Her Cat\nThree Klies on the Ice\nDroalls Brem\nThe Garr Plaisp Under a Prornt\nTweedle-Dum and Tweedle-Dee\nOh Grirn!\nGarr Wielk Brarsp\nStrek Bylling Boub\nPat-a-Cake\nNied and the Meez\nSqorl Buff\nA Melanchol", + "vacatedSha256": "4e19fffc82f8ca95bbab20ecc1c8a900d5339c01a98017d644a848d7290a6720", + "vacatedChars": 89070, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1354, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1337, + "stemsTotal": 1680, + "stemsVacated": 1167, + "tokensTotal": 16000, + "tokensVacated": 6094, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.1546875, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.2863732246262396, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.017375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.30325, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.679375, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "stemForms": { + "little": "strek", + "pretty": "streesh", + "run": "run", + "eat": "ben", + "jump": "byll", + "away": "squz", + "squirrel": "straull", + "funny": "krun", + "today": "krurrd", + "gum": "fles", + "hang": "hang", + "crown": "strearm", + "candlestick": "vorm", + "crooked": "rieged", + "diddle": "diddle", + "moon": "smorft", + "pussy": "flyb", + "goose": "brarsp", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "a6cc85fd6a6701ef705037bb140c1ae7f84361fbfea5bebbc290d79635713a3b" + }, + "mapVocabWordsRejects": false + }, + { + "label": "noprosody-p1.0", + "map": "seed0-noprosody", + "params": { + "p": 1.0, + "seed": 0, + "consistent": true, + "matchProsody": false, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE PRIRLK\n WIELK BRARSP\n\n _Trirmed by_\nZoot Neker Trurrk\n\n1916\n\n\n\nA SORP OF THE SHRENGES\n\nStrek Bo-Peep\nStrek Sim Gea\nSkarf\nThe Snuch\nSwaper\nTits and Smurlks\nA Squrv Shyf\nStruv Vard and Her Chooz\nThree Klies on the Karn\nDroalls Brem\nThe Garr Plaisp Under a Prornt\nTweedle-Dum and Tweedle-Dee\nOh Grirn!\nGarr Wielk Brarsp\nStrek Bylling Boub\nPat-a-Cake\nNied and the Meez\nSqorl Buff\nA Torsly S", + "vacatedSha256": "512e8f77a256ef49f83bbf662b62f551e00079bb8b826f590ed8f8b21acbfe4a", + "vacatedChars": 89713, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1944, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1922, + "stemsTotal": 1680, + "stemsVacated": 1680, + "tokensTotal": 16000, + "tokensVacated": 8202, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.1075625, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.274221010365592, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.012, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.4040625, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.5839375, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "stemForms": { + "little": "strek", + "pretty": "streesh", + "run": "horlk", + "eat": "ben", + "jump": "byll", + "away": "squz", + "squirrel": "straull", + "funny": "krun", + "today": "krurrd", + "gum": "fles", + "hang": "pruch", + "crown": "strearm", + "candlestick": "vorm", + "crooked": "rieged", + "diddle": "norrn", + "moon": "smorft", + "pussy": "flyb", + "goose": "brarsp", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "15e3942f859b92940a17647051408a9acf8cab91d50f14a7dab2c324349d5c3f" + }, + "mapVocabWordsRejects": false + }, + { + "label": "swap-seed0-p0.0", + "map": "swap-seed0", + "params": { + "p": 0.0, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "swap" + }, + "head400": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\n", + "vacatedSha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c", + "vacatedChars": 86408, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 0, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 0, + "stemsTotal": 1680, + "stemsVacated": 0, + "tokensTotal": 16000, + "tokensVacated": 0, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.29, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3209324983421379, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0514375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.0, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.9485625, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "little", + "pretty": "pretty", + "run": "run", + "eat": "eat", + "jump": "jump", + "away": "away", + "squirrel": "squirrel", + "funny": "funny", + "today": "today", + "gum": "gum", + "hang": "hang", + "crown": "crown", + "candlestick": "candlestick", + "crooked": "crooked", + "diddle": "diddle", + "moon": "moon", + "pussy": "pussy", + "goose": "goose", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "2ba43046604884d431a9b8a503ae906a703235c578f7d588b877302c95b74983" + }, + "mapVocabWordsRejects": false + }, + { + "label": "swap-seed0-p0.7", + "map": "swap-seed0", + "params": { + "p": 0.7, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "swap" + }, + "head400": " THE SHOEING\n ROBIN SNOW\n\n _Grandmothered by_\nBlanche Doorer Comfits\n\n1916\n\n\n\nA OAK OF THE SAMES\n\nBirthday Bo-Peep\nBirthday Boy King\nRain\nThe Swan\nWinter\nNeedlers and Eyers\nA Elizabeth Worse\nFlew Tobago and Her Cat\nThree Visited on the Ice\nTakes North\nThe Boy Chicken Under a Bell\nTweedle-Dum and Tweedle-Dee\nOh Poor!\nBoy Robin Snow\nBirthday Tailing Pairs\nPat-a-Cake\nDerby and the Clap\nRiddl", + "vacatedSha256": "875901bb858cfd24bc94d7260628d406f7098f632de229e82d44b52fc447e2cd", + "vacatedChars": 88453, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1354, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1337, + "stemsTotal": 1680, + "stemsVacated": 1167, + "tokensTotal": 16000, + "tokensVacated": 6094, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.32025, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3294038374477789, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0503125, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.0, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.9496875, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "birthday", + "pretty": "mary", + "run": "run", + "eat": "say", + "jump": "tail", + "away": "upon", + "squirrel": "works", + "funny": "picture", + "today": "pricked", + "gum": "fish", + "hang": "hang", + "crown": "harm", + "candlestick": "clergyman", + "crooked": "welled", + "diddle": "diddle", + "moon": "bird", + "pussy": "crooked", + "goose": "snow", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": null, + "mapVocabWordsRejects": true + }, + { + "label": "swap-seed0-p1.0", + "map": "swap-seed0", + "params": { + "p": 1.0, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "swap" + }, + "head400": " THE SHOEING\n ROBIN SNOW\n\n _Grandmothered by_\nBrings Doorer Comfits\n\n1916\n\n\n\nA OAK OF THE SAMES\n\nBirthday Bo-Peep\nBirthday Watch King\nHen\nThe Swan\nSeasonablner\nNeedlers and Eyers\nA Elizabeth Worse\nFlew Tobago and Her Went\nThree Visited on the Pick\nTakes North\nThe Boy Chicken Under a Bell\nTweedle-Dum and Tweedle-Dee\nOh Poor!\nBoy Robin Snow\nBirthday Tailing Pairs\nPat-a-Cake\nDerby and the Cl", + "vacatedSha256": "68edf0cd793e55581dc54b8c64ca1b5adb8ae260269693f3f3f85541f480824b", + "vacatedChars": 89304, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1944, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1922, + "stemsTotal": 1680, + "stemsVacated": 1680, + "tokensTotal": 16000, + "tokensVacated": 8202, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.3296875, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3303667914227805, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0490625, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.0, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.9509375, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "birthday", + "pretty": "mary", + "run": "sneeze", + "eat": "say", + "jump": "tail", + "away": "upon", + "squirrel": "works", + "funny": "picture", + "today": "pricked", + "gum": "fish", + "hang": "mouth", + "crown": "harm", + "candlestick": "clergyman", + "crooked": "welled", + "diddle": "mother", + "moon": "bird", + "pussy": "crooked", + "goose": "snow", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "a5b600a49ada15f838c96f08e9a5515de07a4c84f1e09e372efa07d9a5a5482c" + }, + "mapVocabWordsRejects": false + }, + { + "label": "swap-seed7-p0.0", + "map": "swap-seed7", + "params": { + "p": 0.0, + "seed": 7, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "swap" + }, + "head400": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nLittle Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbreast\n", + "vacatedSha256": "03769632905e2b4c78f9a57b3ce16eec1cc33014eb7edefdbc3aae07e64f3f7c", + "vacatedChars": 86408, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 0, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 0, + "stemsTotal": 1680, + "stemsVacated": 0, + "tokensTotal": 16000, + "tokensVacated": 0, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.29, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3209324983421379, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0514375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.0, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.9485625, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "little", + "pretty": "pretty", + "run": "run", + "eat": "eat", + "jump": "jump", + "away": "away", + "squirrel": "squirrel", + "funny": "funny", + "today": "today", + "gum": "gum", + "hang": "hang", + "crown": "crown", + "candlestick": "candlestick", + "crooked": "crooked", + "diddle": "diddle", + "moon": "moon", + "pussy": "pussy", + "goose": "goose", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "2ba43046604884d431a9b8a503ae906a703235c578f7d588b877302c95b74983" + }, + "mapVocabWordsRejects": false + }, + { + "label": "swap-seed7-p0.7", + "map": "swap-seed7", + "params": { + "p": 0.7, + "seed": 7, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "swap" + }, + "head400": " THE RAW\n DIDDLE DOVE\n\n _Higgledyed by_\nBlanche Fisher Kitty\n\n1916\n\n\n\nA NEAT OF THE ROBES\n\nBirthday Bo-Peep\nBirthday Boy Sat\nTail\nThe Leg\nBiggerer\nMerrys and Mindls\nA Piccadilly Woods\nGet Ann's and Her Sing\nThree Another on the Ice\nCakes Patch\nThe Floor Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Ding!\nFloor Diddle Dove\nBirthday Mening Flour\nPat-a-Cake\nPenny and the Mare\nRobin Witho", + "vacatedSha256": "ab5776ee99817b0b8615ca77a67479dd919fac4b102986932f2713657f7fc939", + "vacatedChars": 88915, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1349, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1336, + "stemsTotal": 1680, + "stemsVacated": 1164, + "tokensTotal": 16000, + "tokensVacated": 5841, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.322625, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3294169279109046, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.0503125, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.0, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.9496875, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "birthday", + "pretty": "woman", + "run": "more", + "eat": "play", + "jump": "men", + "away": "upon", + "squirrel": "squirrel", + "funny": "wished", + "today": "away", + "gum": "kill", + "hang": "fetch", + "crown": "dish", + "candlestick": "clergyman", + "crooked": "nowed", + "diddle": "jenny", + "moon": "dame", + "pussy": "taffy", + "goose": "dove", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": null, + "mapVocabWordsRejects": true + }, + { + "label": "swap-seed7-p1.0", + "map": "swap-seed7", + "params": { + "p": 1.0, + "seed": 7, + "consistent": true, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "swap" + }, + "head400": " THE RAW\n DIDDLE DOVE\n\n _Higgledyed by_\nBlithe Buncher Kitty\n\n1916\n\n\n\nA NEAT OF THE ROBES\n\nBirthday Bo-Peep\nBirthday Home Sat\nTail\nThe Leg\nBiggerer\nMerrys and Mindls\nA Piccadilly Woods\nGet Ann's and Her Sing\nThree Another on the Fred\nCakes Mark\nThe Floor Funny Under a Bell\nTweedle-Dum and Tweedle-Dee\nOh Ding!\nFloor Diddle Dove\nBirthday Mening Flour\nPat-a-Cake\nPenny and the Till\nFlower Wit", + "vacatedSha256": "08287932cff42c0b0c6fd658a53e96e85833b099d935547318625d6d1fa2b5b6", + "vacatedChars": 89807, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1944, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1922, + "stemsTotal": 1680, + "stemsVacated": 1680, + "tokensTotal": 16000, + "tokensVacated": 8202, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.3290625, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3296101720623341, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.052625, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.0, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.947375, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": { + "little": "birthday", + "pretty": "woman", + "run": "more", + "eat": "play", + "jump": "men", + "away": "upon", + "squirrel": "wool", + "funny": "wished", + "today": "away", + "gum": "kill", + "hang": "fetch", + "crown": "dish", + "candlestick": "clergyman", + "crooked": "nowed", + "diddle": "jenny", + "moon": "dame", + "pussy": "taffy", + "goose": "dove", + "the": "the", + "and": "and", + "you": "you", + "not": "not", + "ox": "ox", + "good-bye": "good-bye" + }, + "idStream": { + "digest": "c0567be6d5df1a254ed64c72e7fb2fba313b5221a591e1f1e557d6ddc5d436af", + "length": 19071, + "first16": [ + 35, + 0, + 2, + 283, + 0, + 2, + 0, + 102, + 2, + 0, + 0, + 0, + 2, + 4, + 0, + 120 + ], + "last16": [ + 0, + 138, + 127, + 0, + 120, + 236, + 2, + 5, + 132, + 0, + 118, + 0, + 300, + 20, + 0, + 2 + ], + "mappedWordsSha256": "bf19c502e64042b5048ae4ef10278b6ab58381489c3bdbaaf7f36521924803f0" + }, + "mapVocabWordsRejects": false + }, + { + "label": "control-inconsistent", + "map": "control-inconsistent", + "params": { + "p": 0.7, + "seed": 0, + "consistent": false, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE GLAIRN\n SCIRRDER PLERNT\n\n _Wroormowyed by_\nBlanche Thursper Styn\n\n1916\n\n\n\nA HURF OF THE YORCKES\n\nWrerkenle Bo-Peep\nFlorlkishle Boy Poun\nRain\nThe Sqyft\nWinter\nNietens and Klisks\nA Gnuntingenle Jai\nSculk Mursh and Her Cat\nThree Thoochousish on the Ice\nNefs Gask\nThe Soamp Therltle Under a Gloang\nTweedle-Dum and Tweedle-Dee\nOh Steb!\nSwouck Felkid Rarp\nLainowid Yersking Snof\nPat-a-Cake\nSt", + "vacatedSha256": "610a0d8375890ec68a788457b957d248e5e8a279644fc5b339aa7849451a68cf", + "vacatedChars": 94748, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1354, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1337, + "stemsTotal": 1680, + "stemsVacated": 1167, + "tokensTotal": 16000, + "tokensVacated": 6094, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.28975, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.3215726757806852, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.017375, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.303375, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.67925, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": null, + "idStream": null, + "mapVocabWordsRejects": true + }, + { + "label": "control-inconsistent-seed7", + "map": "control-inconsistent-seed7", + "params": { + "p": 1.0, + "seed": 7, + "consistent": false, + "matchProsody": true, + "revealAfter": 0, + "keep": [], + "mint": "nonce" + }, + "head400": " THE BLORM\n SWERFFISH MORT\n\n _Plourerered by_\nTreesp Mearter Krad\n\n1916\n\n\n\nA YOORN OF THE GERLTES\n\nWreazzleerum Bo-Peep\nShilidy Throuf Squrm\nPook\nThe Jorrk\nFyler\nYichums and Virks\nA Scaitenticen Flirck\nPlerp Gnuz and Her Zersk\nThree Sqyrmouser on the Rars\nGeashs Horrd\nThe Naull Thistid Under a Nourn\nTweedle-Dum and Tweedle-Dee\nOh Skoa!\nBoaff Worbish Wrog\nStortelle Heasting Waud\nPat-a-Cake", + "vacatedSha256": "25df4453ac73c95e3a61db45fca3329b4b03bf08b3b9a74b161cebe618772292", + "vacatedChars": 97263, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1944, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 1922, + "stemsTotal": 1680, + "stemsVacated": 1680, + "tokensTotal": 16000, + "tokensVacated": 8202, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.2903125, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.32138700134802295, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.012, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.4041875, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.5838125, + "bijective": true, + "imageSize": 2233, + "remintRounds": 1 + }, + "stemForms": null, + "idStream": null, + "mapVocabWordsRejects": true + }, + { + "label": "control-reveal-after-2", + "map": "control-reveal-after-2", + "params": { + "p": 0.7, + "seed": 0, + "consistent": true, + "matchProsody": true, + "revealAfter": 2, + "keep": [], + "mint": "nonce" + }, + "head400": " THE REAL\n MOTHER GOOSE\n\n _Illustrated by_\nBlanche Fisher Wright\n\n1916\n\n\n\nA LIST OF THE RHYMES\n\nLittle Bo-Peep\nLittle Boy Blue\nRain\nThe Clock\nWinter\nFingers and Toes\nA Seasonable Song\nDame Trot and Her Cat\nThree Children on the Ice\nCross Patch\nThe Old Woman Under a Hill\nTweedle-Dum and Tweedle-Dee\nOh Dear!\nOld Mother Goose\nSkoufenty Jumping Joan\nPat-a-Cake\nMoney and the Mare\nRobin Redbrea", + "vacatedSha256": "c3eb24f1d280ec4855055f69fd2ff27883ff485d70e047e7f70dff4c4c99a543", + "vacatedChars": 92183, + "stats": { + "domainTypesTotal": 2233, + "domainTypesEligible": 1944, + "domainTypesVacated": 1354, + "corpusTypesTotal": 2211, + "corpusTypesEligible": 1922, + "corpusTypesVacated": 665, + "stemsTotal": 1680, + "stemsVacated": 1167, + "tokensTotal": 16000, + "tokensVacated": 4205, + "meanSyllablesBefore": 1.29, + "meanSyllablesAfter": 1.2898125, + "meanAnapestBefore": 0.3209324983421379, + "meanAnapestAfter": 0.32100662220258347, + "stressFromTableBefore": 0.0514375, + "stressFromTableAfter": 0.02025, + "stressFromMintedBefore": 0.0, + "stressFromMintedAfter": 0.2174375, + "stressFromRuleBefore": 0.9485625, + "stressFromRuleAfter": 0.7623125, + "bijective": true, + "imageSize": 2233, + "remintRounds": 0 + }, + "stemForms": null, + "idStream": null, + "mapVocabWordsRejects": true + } + ], + "nesting": { + "seed": 0, + "map": "seed0", + "levels": [ + { + "p": 0.0, + "stems": [] + }, + { + "p": 0.35, + "stems": [ + "adieu", + "admittance", + "advice", + "agree", + "alehouse", + "alley", + "alphabetical", + "appoint", + "around", + "arrow", + "art", + "ask", + "aunt", + "bak", + "bare", + "bark", + "bean", + "bear", + "bed", + "beetle", + "begin", + "beginn", + "begun", + "believe", + "belleisle", + "bes", + "bessy", + "bett", + "big", + "bil", + "bill", + "bird", + "bit", + "bite", + "bitt", + "bles", + "blind", + "blue", + "bobby", + "bolin", + "bombay", + "bone", + "bonnet", + "bonny", + "bough", + "bounc", + "bow", + "bowl", + "box", + "bramble", + "brandy", + "bras", + "bright", + "bristol", + "broke", + "broken", + "brook", + "brother", + "brought", + "brown", + "build", + "bull", + "bullet", + "butt", + "button", + "bye", + "cage", + "cake", + "calf", + "cap", + "captain", + "car", + "care", + "cast", + "catch", + "chair", + "chamb", + "chanc", + "charley", + "chid", + "chief", + "child", + "children", + "chimney", + "chirrup", + "choice", + "choose", + "christen", + "claw", + "clean", + "clergyman", + "clerk", + "clothe", + "cloudy", + "cobbler", + "colt", + "come", + "comin", + "compliment", + "contrary", + "contrive", + "cost", + "count", + "cover", + "cream", + "creep", + "crept", + "cri", + "croak", + "crook", + "cros", + "crumb", + "cutery", + "daddy", + "daff", + "daffodil", + "dainti", + "dame", + "dapple", + "darlington", + "daughter", + "day", + "dear", + "death", + "deck", + "dee", + "deep", + "delve", + "diamond", + "dick", + "dickery", + "dickory", + "dicky", + "die", + "died", + "difficult", + "ding", + "dish", + "dishy", + "dog", + "don", + "door", + "doubt", + "drak", + "dreary", + "dress", + "driv", + "drive", + "dropp", + "drum", + "duck", + "dumpl", + "dumpling", + "dumpty", + "dun", + "dwell", + "eat", + "egg", + "eith", + "else", + "elspeth", + "england", + "enough", + "etc", + "etticoat", + "evil", + "eye", + "fail", + "fair", + "fall", + "fan", + "fare", + "fast", + "fat", + "fear", + "feather", + "fed", + "feed", + "feet", + "fell", + "fifteen", + "fight", + "fill", + "find", + "fing", + "finger", + "fire", + "fit", + "flinder", + "floor", + "flow", + "flower", + "flute", + "fool", + "footman", + "forev", + "forgot", + "forth", + "fought", + "found", + "fourteen", + "fourth", + "france", + "friday", + "frighten", + "frump", + "full", + "fun", + "gallant", + "gallop", + "gamberal", + "gand", + "gang", + "gap", + "gather", + "gentleman", + "georgy", + "goe", + "goest", + "going", + "goose", + "gown", + "grace", + "gray", + "greenwood", + "grew", + "griev", + "grundy", + "half", + "hall", + "hand", + "handkerchief", + "handy", + "hard", + "hare", + "hart", + "hath", + "hatter", + "haystack", + "healthy", + "heel", + "hem", + "hero", + "herring", + "hickory", + "hide", + "higgledy", + "high", + "highnes", + "hill", + "hillock", + "home", + "honey", + "hood", + "horn", + "horrid", + "horseshoe", + "house", + "hubbard", + "humpty", + "hunt", + "hurry", + "jag", + "jam", + "jerry", + "jig", + "jog", + "john", + "joke", + "joyou", + "jump", + "ken", + "kept", + "kettle", + "kiss", + "kite", + "knave", + "knee", + "knife", + "knight", + "knock", + "laddie", + "lady", + "ladybird", + "lan", + "land", + "lark", + "late", + "laugh", + "lauk", + "least", + "led", + "leed", + "leg", + "lengthen", + "lent", + "let", + "lett", + "like", + "load", + "lol", + "london", + "longman", + "look", + "lov", + "love", + "low", + "mad", + "made", + "maid", + "main", + "mamma", + "man", + "many", + "margaret", + "market", + "marr", + "mary", + "match", + "matt", + "mayor", + "mean", + "meat", + "meet", + "mercy", + "merri", + "mice", + "mickle", + "middle", + "mis", + "mischievou", + "moisty", + "mol", + "more", + "morning", + "mortal", + "mother", + "motion", + "mouse", + "mow", + "mrs", + "multiplication", + "music", + "myself", + "nag", + "nancy", + "nasty", + "nibble", + "niggledy", + "nimble", + "nob", + "nor", + "north", + "noth", + "old", + "orange", + "oth", + "pail", + "pan", + "pandy", + "parson", + "patch", + "pear", + "penny", + "perhap", + "picture", + "pie", + "pip", + "plain", + "plast", + "poker", + "pony", + "pooh", + "poppety", + "porgy", + "porring", + "posses", + "potato", + "pound", + "practice", + "prayer", + "princes", + "prod", + "proper", + "pull", + "quarrel", + "quick", + "rabbit", + "rac", + "ran", + "rapp", + "rare", + "ready", + "real", + "reason", + "redbreast", + "rest", + "rhym", + "rhyme", + "ribbon", + "ride", + "rig", + "right", + "rise", + "riv", + "robin", + "robinson", + "rock", + "rode", + "rook", + "rough", + "round", + "rule", + "safe", + "said", + "sailor", + "sam", + "same", + "saw", + "seam", + "seasonable", + "see", + "seen", + "selfsame", + "sell", + "sempster", + "send", + "septemb", + "seventeen", + "shalt", + "sheep", + "shepherdes", + "shilling", + "shine", + "shiv", + "shod", + "shook", + "shop", + "shoreditch", + "shorn", + "short", + "show", + "siege", + "sigh", + "sill", + "silv", + "simon", + "simple", + "skipp", + "slatherum", + "slice", + "slipper", + "slitherum", + "smith", + "snipe", + "soft", + "someth", + "son", + "soon", + "sound", + "sow", + "sparrow", + "spent", + "spid", + "splish", + "spoil", + "sport", + "sprat", + "spry", + "squirrel", + "staff", + "steal", + "steel", + "stew", + "stiff", + "stile", + "stone", + "stood", + "storm", + "story", + "stout", + "strange", + "street", + "strife", + "strong", + "stump", + "stumpaty", + "sung", + "sup", + "surprise", + "sweep", + "swum", + "tack", + "tail", + "tale", + "talk", + "tarr", + "tarry", + "taste", + "tatter", + "taught", + "test", + "thank", + "thi", + "thief", + "thing", + "think", + "thirty", + "threescore", + "throat", + "throw", + "thrush", + "thu", + "thursday", + "thy", + "tick", + "til", + "till", + "tim", + "tinker", + "tir", + "tisha", + "tobago", + "today", + "told", + "tong", + "took", + "top", + "touch", + "tramp", + "trick", + "tripe", + "trot", + "trott", + "trow", + "true", + "try", + "tuck", + "twa", + "twelve", + "unto", + "upon", + "use", + "vast", + "venture", + "vinegar", + "visit", + "vow", + "waggl", + "wait", + "wak", + "wake", + "walnut", + "want", + "wash", + "wealthy", + "wear", + "weath", + "wedd", + "wee", + "weep", + "welshman", + "went", + "whatev", + "wherev", + "whipp", + "whitechapel", + "whith", + "whoop", + "willie", + "wing", + "winkle", + "winter", + "wire", + "wise", + "wiv", + "woman", + "women", + "wool", + "word", + "worm", + "wrap", + "wren", + "wright", + "yard", + "yes", + "young" + ] + }, + { + "p": 0.7, + "stems": [ + "abc", + "adieu", + "admittance", + "advice", + "afraid", + "aft", + "age", + "agree", + "ain", + "alabone", + "alack", + "alehouse", + "alive", + "alley", + "alone", + "along", + "alphabet", + "alphabetical", + "ampersand", + "ann", + "answer", + "apple", + "appoint", + "april", + "around", + "arrow", + "art", + "ask", + "asleep", + "ate", + "aught", + "aunt", + "away", + "awhile", + "aye", + "babylon", + "bachelor", + "back", + "backward", + "bacon", + "bad", + "bade", + "bag", + "bailey", + "bak", + "bake", + "baker", + "balloon", + "bare", + "bark", + "barm", + "barn", + "barrel", + "battle", + "bean", + "bear", + "beat", + "beaten", + "bed", + "bedtime", + "bee", + "beef", + "beetle", + "began", + "beggar", + "begin", + "beginn", + "begun", + "behind", + "believe", + "bell", + "belleisle", + "bent", + "bes", + "bessy", + "best", + "bett", + "bid", + "big", + "bigg", + "bil", + "bill", + "bird", + "bit", + "bite", + "bitt", + "bitten", + "black", + "bleak", + "bleat", + "bles", + "blind", + "blis", + "blithe", + "blow", + "blue", + "boat", + "bobby", + "boggen", + "boil", + "bolin", + "bombay", + "bone", + "bonnet", + "bonnie", + "bonny", + "born", + "bottle", + "bough", + "bounc", + "bow", + "bowl", + "box", + "bramble", + "brandy", + "bras", + "bray", + "brickbat", + "bride", + "bright", + "bristol", + "broke", + "broken", + "brook", + "brother", + "brought", + "brown", + "buckl", + "buckle", + "build", + "built", + "bull", + "bullet", + "bullfinch", + "bump", + "bun", + "bunt", + "burial", + "bush", + "butt", + "button", + "bye", + "cage", + "cake", + "calf", + "came", + "candle", + "candlestick", + "cap", + "captain", + "car", + "care", + "carr", + "carrion", + "cast", + "catch", + "caught", + "ceas", + "ceil", + "cellar", + "chain", + "chair", + "chamb", + "champ", + "chanc", + "charley", + "cherry", + "chid", + "chief", + "child", + "children", + "chimney", + "chin", + "chirrup", + "choice", + "choose", + "chopp", + "christen", + "cinder", + "city", + "claw", + "clay", + "clean", + "clergyman", + "clerk", + "cloak", + "clock", + "cloth", + "clothe", + "cloudy", + "coachman", + "coal", + "coat", + "cobbler", + "cobweb", + "codlin", + "coffee", + "cold", + "cole", + "colt", + "come", + "comin", + "compliment", + "consid", + "consider", + "contrary", + "contrive", + "copp", + "coral", + "corn", + "cost", + "count", + "cov", + "cover", + "cow", + "cream", + "creep", + "crept", + "cri", + "croak", + "crook", + "cros", + "crow", + "crown", + "crumb", + "crusoe", + "cry", + "cup", + "cupboard", + "cur", + "currant", + "cushy", + "custard", + "cutery", + "daddy", + "daff", + "daffodil", + "dainti", + "dainty", + "dam", + "dame", + "danc", + "dang", + "dapple", + "dare", + "darlington", + "daught", + "daughter", + "day", + "dear", + "death", + "deceit", + "decide", + "deck", + "declar", + "dee", + "deed", + "deep", + "delight", + "delve", + "derby", + "determin", + "diamond", + "dick", + "dickery", + "dickory", + "dicky", + "die", + "died", + "difficult", + "ding", + "dinkety", + "dish", + "dishy", + "division", + "dock", + "doe", + "doff", + "dog", + "dol", + "doll", + "dollar", + "dolly", + "don", + "doo", + "doodle", + "door", + "dost", + "doubt", + "dov", + "dove", + "dozen", + "drak", + "drake", + "dreamt", + "dreary", + "dress", + "drink", + "driv", + "drive", + "dropp", + "drove", + "drum", + "drumm", + "duck", + "dumpl", + "dumpling", + "dumpty", + "dun", + "durst", + "dusty", + "dwell", + "dwelt", + "ear", + "earth", + "eat", + "egg", + "eighteen", + "eith", + "else", + "elspeth", + "empty", + "england", + "enough", + "equal", + "espi", + "etc", + "etticoat", + "everyone", + "evil", + "eye", + "face", + "fail", + "fair", + "fall", + "fan", + "fare", + "farmer", + "fast", + "fat", + "fear", + "feast", + "feath", + "feather", + "fed", + "feed", + "feet", + "fell", + "fetch", + "fiddl", + "fiddle", + "fiddler", + "field", + "fifteen", + "fight", + "fill", + "fin", + "find", + "fing", + "finger", + "fire", + "first", + "fish", + "fishy", + "fit", + "flame", + "flapp", + "fleece", + "flew", + "flinder", + "floor", + "flow", + "flower", + "flute", + "fly", + "fol", + "folk", + "fool", + "footman", + "forc", + "forev", + "forgot", + "forth", + "fost", + "fought", + "found", + "fourteen", + "fourth", + "france", + "friday", + "frighten", + "fruit", + "fruiterer", + "frump", + "full", + "fun", + "funny", + "gai", + "gall", + "gallant", + "gallop", + "gamberal", + "game", + "gand", + "gang", + "gap", + "garden", + "gather", + "geese", + "gent", + "gentleman", + "gentlemen", + "georgy", + "gett", + "gil", + "girl", + "giving", + "gloucest", + "goat", + "gobbl", + "god", + "goe", + "goest", + "going", + "gold", + "gone", + "good", + "goose", + "got", + "gown", + "grace", + "grandmoth", + "gras", + "gray", + "greedy", + "green", + "greenwood", + "grew", + "griev", + "grim", + "grocer", + "grow", + "gruel", + "grundy", + "gum", + "hair", + "half", + "halfpenny", + "hall", + "hame", + "hand", + "handkerchief", + "handsome", + "handy", + "hard", + "hare", + "hark", + "harrow", + "hart", + "hath", + "hatter", + "hay", + "haystack", + "healthy", + "hear", + "heart", + "hector", + "heel", + "heighty", + "helen", + "help", + "hem", + "hen", + "hero", + "herring", + "hey", + "hickety", + "hickory", + "hid", + "hide", + "higgledy", + "high", + "highnes", + "highway", + "hill", + "hillock", + "himself", + "hire", + "hobble", + "hog", + "hold", + "holiday", + "home", + "hon", + "honey", + "honor", + "hood", + "hopp", + "horn", + "horrid", + "hors", + "horse", + "horseshoe", + "hosier", + "hot", + "hound", + "house", + "hubbard", + "humpty", + "hunt", + "hurry", + "hurt", + "hush", + "huzza", + "icicle", + "ill", + "illustrat", + "instead", + "ive", + "jacky", + "jag", + "jam", + "jerry", + "jig", + "joan", + "jog", + "john", + "joke", + "jol", + "joyou", + "jump", + "june", + "ken", + "kept", + "kettl", + "kettle", + "key", + "kill", + "kind", + "king", + "kiss", + "kite", + "kitten", + "kitty", + "knave", + "knee", + "knife", + "knight", + "knock", + "kyloe", + "laddie", + "lady", + "ladybird", + "laid", + "lal", + "lam", + "lan", + "land", + "lane", + "lark", + "lass", + "latch", + "late", + "laugh", + "lauk", + "lay", + "lea", + "lead", + "lean", + "leap", + "least", + "leath", + "leave", + "led", + "lee", + "leed", + "left", + "leg", + "lend", + "lengthen", + "lent", + "let", + "lett", + "lick", + "life", + "like", + "lin", + "lion", + "list", + "little", + "littleman", + "liv", + "live", + "load", + "lol", + "london", + "long", + "longman", + "look", + "lord", + "lost", + "loud", + "lov", + "love", + "low", + "lump", + "lumpety", + "mad", + "made", + "maid", + "main", + "make", + "mamma", + "mammy", + "man", + "many", + "mare", + "margaret", + "margery", + "mark", + "market", + "marr", + "marry", + "mary", + "match", + "matt", + "mayor", + "meadow", + "mean", + "meat", + "meet", + "mercy", + "merri", + "merrymen", + "met", + "mice", + "mickle", + "middle", + "mil", + "mild", + "milk", + "mintery", + "minute", + "mire", + "mis", + "mischievou", + "mist", + "mistaken", + "mistres", + "misty", + "moisty", + "mol", + "money", + "monkey", + "monstrou", + "moon", + "more", + "morning", + "mortal", + "mother", + "motion", + "mourn", + "mouse", + "mouth", + "move", + "mow", + "mrs", + "much", + "multiplication", + "music", + "muskidun", + "mutton", + "myself", + "nag", + "nancy", + "narrow", + "nasty", + "naughty", + "nay", + "needle", + "neith", + "new", + "next", + "nibble", + "nice", + "niggledy", + "night", + "nightgown", + "nimble", + "nob", + "nobleman", + "nodd", + "noise", + "noon", + "nor", + "north", + "notch", + "noth", + "often", + "old", + "ope", + "orange", + "oth", + "oven", + "owl", + "pail", + "pair", + "pan", + "pancake", + "pandy", + "pap", + "papa", + "parent", + "parson", + "pas", + "pat", + "patch", + "pay", + "peace", + "peaceable", + "peacock", + "pear", + "pease", + "peep", + "penny", + "people", + "perhap", + "peter", + "petticoat", + "piccadil", + "pick", + "pickety", + "pickl", + "picture", + "pie", + "piece", + "pig", + "pigeon", + "piggledy", + "pin", + "pinch", + "pint", + "piou", + "pip", + "piper", + "pippen", + "plac", + "plain", + "plast", + "plate", + "playfellow", + "playmat", + "plenty", + "point", + "poker", + "pony", + "pooh", + "poor", + "poppety", + "porgy", + "porridge", + "porring", + "pos", + "posses", + "pot", + "potato", + "pound", + "practice", + "pray", + "prayer", + "pretti", + "pretty", + "prick", + "prince", + "princes", + "prod", + "promis", + "proper", + "pudding", + "pull", + "pussy", + "put", + "quack", + "quarrel", + "queen", + "quick", + "rabbit", + "rac", + "ran", + "rapp", + "rare", + "rat", + "rattle", + "raw", + "reach", + "ready", + "real", + "reason", + "red", + "redbreast", + "reigate", + "request", + "rest", + "return", + "rhym", + "rhyme", + "ribbon", + "rice", + "rich", + "rid", + "riddle", + "ride", + "rig", + "right", + "ring", + "rise", + "riv", + "rob", + "robber", + "robin", + "robinson", + "rock", + "rode", + "rook", + "room", + "rough", + "round", + "row", + "rule", + "rye", + "saddle", + "safe", + "said", + "sailor", + "sam", + "same", + "sang", + "sat", + "saw", + "say", + "scholar", + "school", + "schoolroom", + "sea", + "seal", + "seam", + "seasonable", + "see", + "seek", + "seen", + "seldom", + "selfsame", + "sell", + "sempster", + "send", + "septemb", + "serv", + "servant", + "serve", + "set", + "seventeen", + "shalt", + "shave", + "shaven", + "shed", + "sheep", + "shelf", + "shell", + "shepherdes", + "shilling", + "shine", + "ship", + "shiv", + "shod", + "shook", + "shoot", + "shop", + "shoreditch", + "shorn", + "short", + "show", + "shower", + "shroud", + "shut", + "sick", + "siege", + "sigh", + "silk", + "sill", + "silv", + "simon", + "simple", + "sing", + "sir", + "sit", + "skin", + "skipp", + "slash", + "slatherum", + "sleepy", + "slend", + "slice", + "slipper", + "slitherum", + "slow", + "sly", + "small", + "smile", + "smith", + "snapp", + "sneez", + "snipe", + "snook", + "snow", + "sobb", + "soft", + "solomon", + "someth", + "son", + "song", + "soon", + "sore", + "sound", + "sow", + "spain", + "sparrow", + "spent", + "spid", + "spilt", + "spin", + "spinn", + "splash", + "splish", + "spoil", + "spoke", + "spoon", + "sport", + "sprat", + "spry", + "squirrel", + "staff", + "stair", + "stand", + "start", + "stay", + "steal", + "steel", + "stepney", + "stepp", + "stew", + "stiff", + "stile", + "stone", + "stood", + "stopp", + "storm", + "story", + "stout", + "str", + "straight", + "strang", + "strange", + "straw", + "stray", + "street", + "strife", + "strong", + "strow", + "struck", + "stump", + "stumpaty", + "such", + "sue", + "sukey", + "summer", + "sun", + "sung", + "sunshine", + "sunshiny", + "sup", + "suppose", + "sure", + "surprise", + "swarm", + "sweep", + "sweet", + "swimm", + "swoon", + "swum", + "table", + "tack", + "tail", + "take", + "tale", + "talk", + "talkative", + "tarr", + "tarry", + "taste", + "tatter", + "taught", + "tavern", + "tea", + "tell", + "test", + "thank", + "thi", + "thick", + "thief", + "thing", + "think", + "third", + "thirteen", + "thirty", + "thistle", + "thou", + "thought", + "thousand", + "thread", + "threescore", + "thrive", + "throat", + "throw", + "thrush", + "thu", + "thumb", + "thursday", + "thy", + "tick", + "tickl", + "til", + "till", + "tim", + "time", + "tinker", + "tip", + "tipple", + "tir", + "tis", + "tisha", + "tittlemouse", + "tobago", + "today", + "toe", + "togeth", + "told", + "toll", + "tom", + "tommy", + "tong", + "tongue", + "took", + "top", + "torch", + "torn", + "tos", + "toss", + "touch", + "town", + "tramp", + "trap", + "tri", + "trick", + "trip", + "tripe", + "trot", + "trott", + "troubl", + "trow", + "true", + "try", + "tuck", + "turn", + "turnip", + "twa", + "tweedle", + "twelve", + "twenty", + "twill", + "twitchett", + "unicorn", + "unto", + "upon", + "upstair", + "use", + "vale", + "vast", + "velvet", + "venture", + "vexation", + "victual", + "vinegar", + "visit", + "visitor", + "vow", + "waggl", + "wait", + "wak", + "wake", + "wall", + "walnut", + "wand", + "want", + "ware", + "wash", + "watch", + "watt", + "way", + "wealthy", + "wear", + "weath", + "wedd", + "wee", + "weep", + "well", + "welshman", + "went", + "west", + "whatev", + "wheel", + "whenev", + "wherev", + "whey", + "whipp", + "whitechapel", + "whith", + "whoop", + "wife", + "wig", + "wild", + "wildernes", + "wilkin", + "willie", + "wilt", + "wind", + "window", + "wing", + "winkle", + "winter", + "wip", + "wipe", + "wire", + "wise", + "wish", + "within", + "without", + "wiv", + "woman", + "women", + "woo", + "woodbin", + "wool", + "word", + "world", + "worm", + "worri", + "worry", + "wrap", + "wren", + "wright", + "write", + "yard", + "yea", + "year", + "yellow", + "yes", + "young" + ] + }, + { + "p": 1.0, + "stems": [ + "abc", + "abroad", + "adieu", + "admittance", + "advice", + "afraid", + "aft", + "age", + "agree", + "ain", + "air", + "alabone", + "alack", + "ale", + "alehouse", + "alive", + "alley", + "almanac", + "alone", + "along", + "alphabet", + "alphabetical", + "alway", + "amble", + "ampersand", + "angry", + "ann", + "announce", + "anoth", + "answer", + "appl", + "apple", + "appoint", + "april", + "arm", + "around", + "arrow", + "art", + "ask", + "asleep", + "ate", + "aught", + "aunt", + "away", + "awhile", + "awoke", + "axe", + "aye", + "baa", + "babby", + "baby", + "babylon", + "bachelor", + "back", + "backward", + "bacon", + "bad", + "bade", + "bag", + "bailey", + "bak", + "bake", + "baker", + "ball", + "balloon", + "banbury", + "bandy", + "barb", + "barber", + "bare", + "bark", + "barley", + "barm", + "barn", + "barrel", + "basket", + "bat", + "battle", + "bead", + "bean", + "bear", + "beat", + "beaten", + "bed", + "bedtime", + "bee", + "beef", + "beehive", + "beer", + "beetle", + "began", + "beggar", + "begin", + "beginn", + "begun", + "behind", + "believe", + "bell", + "belleisle", + "bend", + "bent", + "berr", + "bes", + "beside", + "bessy", + "best", + "betsy", + "bett", + "betty", + "betwixt", + "bid", + "big", + "bigg", + "bil", + "bill", + "bind", + "bird", + "birthday", + "bit", + "bite", + "bitt", + "bitten", + "black", + "blackbird", + "blacksmith", + "blanche", + "ble", + "bleak", + "bleat", + "bles", + "blessing", + "blind", + "blis", + "blithe", + "blood", + "blow", + "blue", + "blush", + "boat", + "bob", + "bobby", + "body", + "boggen", + "boil", + "boldero", + "bolin", + "bombay", + "bone", + "bonn", + "bonnet", + "bonnie", + "bonny", + "book", + "born", + "bottle", + "bough", + "bought", + "bounc", + "bow", + "bowl", + "box", + "boy", + "bramble", + "brandy", + "bras", + "bray", + "bread", + "break", + "breech", + "brew", + "bri", + "brickbat", + "bride", + "bridge", + "bridle", + "bright", + "bring", + "bristol", + "broke", + "broken", + "brook", + "broom", + "broth", + "brother", + "brought", + "brown", + "buckl", + "buckle", + "build", + "built", + "bull", + "bullet", + "bullfinch", + "bump", + "bumpety", + "bun", + "bunch", + "bunt", + "buri", + "burial", + "burnie", + "burnt", + "bush", + "butt", + "button", + "buy", + "bye", + "cabin", + "caesar", + "cage", + "cake", + "calf", + "call", + "came", + "canary", + "candl", + "candle", + "candlestick", + "candy", + "cannot", + "cant", + "cap", + "captain", + "car", + "care", + "carr", + "carri", + "carrion", + "carry", + "carv", + "cast", + "cat", + "catch", + "caught", + "ceas", + "ceil", + "cellar", + "chain", + "chair", + "chamb", + "champ", + "chanc", + "charley", + "che", + "cheek", + "cheese", + "cherry", + "chicken", + "chid", + "chief", + "child", + "children", + "chimney", + "chin", + "chirp", + "chirrup", + "choice", + "choose", + "chopp", + "christen", + "christma", + "cinder", + "city", + "clap", + "claw", + "clay", + "clean", + "clear", + "clergyman", + "clerk", + "clev", + "cloak", + "clock", + "cloth", + "clothe", + "cloudy", + "coachman", + "coal", + "coat", + "cobbler", + "cobweb", + "cock", + "codlin", + "coffee", + "coffin", + "cold", + "cole", + "collar", + "colt", + "com", + "comb", + "come", + "comfit", + "comical", + "comin", + "command", + "compare", + "compliment", + "consid", + "consider", + "contrary", + "contrive", + "coo", + "copp", + "coral", + "corn", + "corner", + "cost", + "cottage", + "count", + "court", + "cov", + "cover", + "cow", + "crackabone", + "cradle", + "cream", + "creep", + "crept", + "cri", + "croak", + "crook", + "cros", + "crow", + "crown", + "crumb", + "crumpl", + "crusoe", + "cry", + "cup", + "cupboard", + "cur", + "curd", + "curl", + "currant", + "curtsy", + "cushion", + "cushy", + "custard", + "cut", + "cutery", + "daddie", + "daddy", + "daff", + "daffodil", + "dainti", + "dainty", + "dairy", + "dam", + "dame", + "danc", + "dance", + "dang", + "dapple", + "dare", + "dark", + "darlington", + "dat", + "daught", + "daughter", + "daw", + "dawson", + "day", + "dead", + "dear", + "deary", + "death", + "deceit", + "decide", + "deck", + "declar", + "ded", + "dee", + "deed", + "deep", + "delight", + "delve", + "derby", + "determin", + "dew", + "diamond", + "dick", + "dickery", + "dickory", + "dicky", + "diddle", + "die", + "died", + "diet", + "difficult", + "dig", + "dill", + "ding", + "dinkety", + "dinn", + "dirty", + "dish", + "dishy", + "displeas", + "ditch", + "division", + "dob", + "dock", + "doctor", + "doe", + "doff", + "dog", + "dol", + "doll", + "dollar", + "dolly", + "don", + "dong", + "donkey", + "doo", + "doodle", + "door", + "dost", + "doth", + "doubt", + "dov", + "dove", + "downstair", + "dozen", + "drak", + "drake", + "draw", + "dream", + "dreamt", + "dreary", + "dres", + "dress", + "drink", + "driv", + "drive", + "dropp", + "drove", + "drown", + "drum", + "drumm", + "dry", + "duck", + "dumpl", + "dumpling", + "dumpty", + "dun", + "durst", + "dusty", + "dwell", + "dwelt", + "ear", + "earth", + "east", + "eat", + "egg", + "eighteen", + "eith", + "eleven", + "elizabeth", + "ell", + "else", + "elspeth", + "empty", + "end", + "england", + "enough", + "equal", + "espi", + "etc", + "etticoat", + "evermore", + "everyone", + "evil", + "except", + "exet", + "eye", + "face", + "fail", + "fair", + "fall", + "fan", + "fare", + "farm", + "farmer", + "farth", + "farthing", + "fast", + "fat", + "father", + "fear", + "feast", + "feath", + "feather", + "february", + "fed", + "feed", + "feet", + "fell", + "fellow", + "fetch", + "fiddl", + "fiddle", + "fiddler", + "fie", + "field", + "fife", + "fifteen", + "fight", + "fill", + "fin", + "find", + "fine", + "fing", + "finger", + "fir", + "fire", + "first", + "fish", + "fishy", + "fit", + "flame", + "flapp", + "fleece", + "fleet", + "flew", + "flinder", + "flock", + "floor", + "flour", + "flow", + "flower", + "flung", + "flute", + "fly", + "fol", + "folk", + "fond", + "fool", + "foot", + "footman", + "forc", + "forehead", + "foreman", + "forev", + "forgot", + "forlorn", + "forth", + "fortune", + "forward", + "fost", + "fought", + "found", + "fourpence", + "fourteen", + "fourth", + "france", + "fred", + "freeze", + "fret", + "friday", + "frighten", + "frosty", + "fruit", + "fruiterer", + "frump", + "frumpaty", + "full", + "fun", + "funny", + "gai", + "gall", + "gallant", + "gallop", + "gamberal", + "game", + "gand", + "gang", + "gap", + "garden", + "garter", + "gate", + "gather", + "gave", + "gay", + "geese", + "gent", + "gentle", + "gentleman", + "gentlemen", + "georgy", + "get", + "gett", + "giblet", + "gil", + "girl", + "give", + "giving", + "gloucest", + "goat", + "gobbl", + "gobble", + "god", + "goe", + "goest", + "going", + "gold", + "goldfinch", + "gone", + "good", + "goose", + "goosey", + "got", + "gotham", + "gown", + "grace", + "grandmoth", + "gras", + "grave", + "gravel", + "gray", + "great", + "greedy", + "green", + "greenwood", + "grew", + "griev", + "grim", + "grin", + "groat", + "grocer", + "ground", + "grow", + "gruel", + "grundy", + "guinea", + "gum", + "gun", + "hair", + "half", + "halfpence", + "halfpenny", + "hall", + "hame", + "hand", + "handkerchief", + "handsome", + "handy", + "hang", + "happen", + "hard", + "hare", + "hark", + "harm", + "harrow", + "hart", + "hat", + "hath", + "hatter", + "hawk", + "hay", + "haystack", + "hea", + "head", + "healthy", + "hear", + "heard", + "heart", + "hearty", + "heav", + "hector", + "heel", + "heighty", + "helen", + "help", + "hem", + "hen", + "hero", + "herring", + "hey", + "hickery", + "hickety", + "hickory", + "hid", + "hide", + "higgledy", + "high", + "highnes", + "highway", + "hill", + "hillock", + "himself", + "hire", + "hobble", + "hog", + "hold", + "hole", + "holiday", + "home", + "hon", + "honey", + "honor", + "hood", + "hop", + "hope", + "hopp", + "horn", + "horrid", + "hors", + "horse", + "horseshoe", + "hose", + "hosier", + "hot", + "hound", + "hour", + "house", + "housetop", + "hubbard", + "huff", + "humpty", + "hundr", + "hung", + "hunt", + "hurry", + "hurt", + "husband", + "hush", + "huzza", + "ice", + "icicle", + "ifs", + "ill", + "illustrat", + "inde", + "ink", + "instead", + "intery", + "iron", + "ive", + "jack", + "jacky", + "jag", + "jam", + "jelf", + "jen", + "jenny", + "jerry", + "jig", + "jiggety", + "jill", + "jingle", + "joan", + "jog", + "john", + "johnny", + "joke", + "jol", + "joy", + "joyou", + "july", + "jump", + "june", + "keep", + "ken", + "kept", + "kettl", + "kettle", + "key", + "kilkenny", + "kill", + "kind", + "king", + "kingdom", + "kirk", + "kis", + "kiss", + "kit", + "kitchen", + "kite", + "kitten", + "kitty", + "knave", + "kne", + "knee", + "knife", + "knight", + "knock", + "know", + "kyloe", + "lad", + "ladd", + "laddie", + "laden", + "lady", + "ladybird", + "lag", + "laid", + "lal", + "lam", + "lamb", + "lan", + "land", + "lane", + "lard", + "lark", + "lass", + "last", + "latch", + "late", + "laugh", + "lauk", + "lay", + "lea", + "lead", + "lean", + "leap", + "least", + "leath", + "leav", + "leave", + "led", + "lee", + "leed", + "left", + "leg", + "lend", + "lengthen", + "lent", + "les", + "let", + "lett", + "lick", + "lie", + "life", + "lift", + "light", + "like", + "lin", + "linen", + "linnet", + "lion", + "list", + "little", + "littleman", + "liv", + "live", + "load", + "lock", + "locket", + "lol", + "london", + "long", + "longman", + "look", + "lord", + "lost", + "loud", + "lov", + "love", + "low", + "luck", + "lucy", + "lump", + "lumpety", + "mad", + "made", + "maid", + "maiden", + "main", + "maintain", + "mak", + "make", + "malt", + "mamma", + "mammie", + "mammy", + "man", + "many", + "march", + "mare", + "margaret", + "margery", + "mark", + "market", + "marr", + "marri", + "marry", + "martin", + "mary", + "mast", + "master", + "match", + "matt", + "maybe", + "mayor", + "meadow", + "meal", + "mean", + "meat", + "meet", + "melancho", + "men", + "mend", + "merchant", + "mercy", + "merri", + "merry", + "merrymen", + "met", + "mew", + "mice", + "mickle", + "middle", + "mil", + "mild", + "mile", + "milk", + "mill", + "mind", + "mine", + "mintery", + "minute", + "mire", + "mis", + "mischievou", + "miss", + "mist", + "mistaken", + "mistres", + "misty", + "moisty", + "mol", + "monday", + "money", + "monkey", + "monstrou", + "moon", + "moppet", + "more", + "morn", + "morning", + "mortal", + "mother", + "motion", + "mourn", + "mouse", + "mouth", + "mov", + "move", + "mow", + "mrs", + "much", + "muffet", + "mulberry", + "multiplication", + "music", + "muskidun", + "mutton", + "myself", + "nag", + "nail", + "nam", + "name", + "nan", + "nancy", + "nanny", + "narrow", + "nasty", + "naught", + "naughty", + "nay", + "neary", + "neat", + "neck", + "needl", + "needle", + "neighbor", + "neith", + "nest", + "new", + "next", + "nibble", + "nice", + "niggledy", + "night", + "nightgown", + "nimble", + "nineteen", + "nob", + "nobleman", + "nobody", + "nodd", + "noise", + "noon", + "nor", + "north", + "norwich", + "nose", + "notch", + "note", + "noth", + "novemb", + "now", + "oak", + "often", + "old", + "ope", + "open", + "orange", + "organ", + "oth", + "oven", + "owe", + "owl", + "own", + "packet", + "pail", + "pair", + "pan", + "pancake", + "pandy", + "pantry", + "pap", + "papa", + "parent", + "parlor", + "parrot", + "parson", + "party", + "pas", + "pat", + "patch", + "pay", + "peace", + "peaceable", + "peacock", + "pear", + "pease", + "peck", + "pedlar", + "peep", + "pen", + "penny", + "people", + "pepper", + "perhap", + "pet", + "peter", + "petticoat", + "physician", + "piccadil", + "pick", + "pickety", + "pickl", + "picture", + "pie", + "piece", + "pieman", + "pig", + "pigeon", + "piggledy", + "pin", + "pinch", + "pint", + "piou", + "pip", + "pipe", + "piper", + "pippen", + "pitch", + "plac", + "plain", + "plast", + "plat", + "plate", + "platt", + "play", + "playfellow", + "playmat", + "please", + "plenty", + "plum", + "pocket", + "pocketful", + "point", + "poker", + "pol", + "poll", + "pony", + "pooh", + "poor", + "poppety", + "porgy", + "porridge", + "porring", + "pos", + "posses", + "pot", + "potato", + "pound", + "powd", + "practice", + "pray", + "prayer", + "pretti", + "pretty", + "pri", + "prick", + "prince", + "princes", + "prithee", + "prod", + "promis", + "proper", + "protector", + "proud", + "psalm", + "pudd", + "pudding", + "puddle", + "pull", + "pumpkin", + "pussy", + "put", + "puzzle", + "quack", + "quarrel", + "queen", + "quick", + "quiet", + "quite", + "rabbit", + "rac", + "rag", + "rage", + "rain", + "ram", + "ran", + "rapp", + "rare", + "rat", + "rattle", + "raven", + "raw", + "reach", + "read", + "ready", + "real", + "reason", + "receive", + "red", + "redbreast", + "reel", + "reigate", + "remedy", + "repartee", + "repli", + "request", + "resolv", + "rest", + "return", + "rhym", + "rhyme", + "ribbon", + "rice", + "rich", + "richard", + "rid", + "riddle", + "ride", + "rig", + "right", + "ring", + "ringman", + "rise", + "riv", + "roast", + "rob", + "robber", + "robert", + "robin", + "robinson", + "rock", + "rode", + "rog", + "roll", + "rook", + "room", + "root", + "ros", + "rosy", + "rough", + "round", + "row", + "rule", + "run", + "runn", + "rush", + "rye", + "sabbath", + "sack", + "saddle", + "safe", + "sage", + "sago", + "said", + "sail", + "sailor", + "salt", + "sam", + "same", + "sang", + "sat", + "saturday", + "saw", + "say", + "scar", + "scarce", + "scarlet", + "scholar", + "school", + "schoolroom", + "scotch", + "scratch", + "scuttle", + "sea", + "seal", + "seam", + "seasonable", + "second", + "see", + "seed", + "seek", + "seen", + "seldom", + "selfsame", + "sell", + "sempster", + "send", + "sent", + "septemb", + "serv", + "servant", + "serve", + "set", + "seventeen", + "sew", + "shaftoe", + "shake", + "shalt", + "shape", + "shave", + "shaven", + "shed", + "sheep", + "shelf", + "shell", + "shepherdes", + "shilling", + "shin", + "shine", + "ship", + "shiv", + "sho", + "shod", + "shoe", + "shook", + "shoot", + "shop", + "shoreditch", + "shorn", + "short", + "shot", + "show", + "shower", + "shroud", + "shut", + "sick", + "side", + "siege", + "sieve", + "sigh", + "silk", + "sill", + "silv", + "simon", + "simple", + "sing", + "single", + "sir", + "sister", + "sit", + "sitt", + "sixpence", + "sixteen", + "skin", + "skipp", + "sky", + "slash", + "slat", + "slatherum", + "sleep", + "sleepy", + "slend", + "slice", + "slid", + "slipper", + "slitherum", + "slow", + "sly", + "small", + "smile", + "smith", + "smok", + "snail", + "snap", + "snapp", + "sneez", + "sneeze", + "sniff", + "snipe", + "snook", + "snow", + "snuff", + "sobb", + "soft", + "sold", + "solomon", + "someth", + "son", + "song", + "soon", + "sore", + "sorrow", + "sorrowful", + "soul", + "sound", + "south", + "sow", + "spade", + "spain", + "sparrow", + "speak", + "spell", + "spend", + "spent", + "spice", + "spid", + "spilt", + "spin", + "spinn", + "spit", + "splash", + "splish", + "spoil", + "spoke", + "spoon", + "sport", + "spr", + "sprat", + "spright", + "spry", + "spun", + "squirrel", + "staff", + "stair", + "stand", + "star", + "start", + "stay", + "steal", + "steel", + "stepney", + "stepp", + "stew", + "stick", + "stiff", + "stile", + "stingy", + "stock", + "stole", + "stone", + "stood", + "stop", + "stopp", + "storm", + "story", + "stout", + "str", + "straight", + "straightway", + "strang", + "strange", + "straw", + "strawberr", + "stray", + "street", + "strife", + "strong", + "strow", + "struck", + "stump", + "stumpaty", + "such", + "sue", + "sugar", + "sukey", + "sulky", + "summ", + "summer", + "sun", + "sunday", + "sung", + "sunshine", + "sunshiny", + "sup", + "supp", + "suppose", + "sure", + "surprise", + "surrey", + "swan", + "swarm", + "sweep", + "sweet", + "swim", + "swimm", + "swine", + "swoon", + "sword", + "swore", + "swum", + "table", + "tack", + "taffy", + "tail", + "tailor", + "tak", + "take", + "tale", + "talk", + "talkative", + "tapp", + "tar", + "tarr", + "tarry", + "tart", + "taste", + "tatter", + "taught", + "tavern", + "tea", + "tear", + "tee", + "teeth", + "tell", + "test", + "thank", + "thee", + "thi", + "thick", + "thief", + "thigh", + "thing", + "think", + "third", + "thirteen", + "thirty", + "thistle", + "thorn", + "thou", + "thought", + "thousand", + "thread", + "threescore", + "threw", + "thrive", + "throat", + "throw", + "thrush", + "thu", + "thumb", + "thumbkin", + "thump", + "thumpaty", + "thursday", + "thy", + "thyself", + "tick", + "tickl", + "tie", + "tied", + "til", + "till", + "tim", + "time", + "tinker", + "tip", + "tipple", + "tir", + "tis", + "tisha", + "tittlemouse", + "tobago", + "today", + "toe", + "togeth", + "told", + "toll", + "tom", + "tommy", + "tong", + "tongu", + "tongue", + "took", + "top", + "torch", + "torn", + "tos", + "toss", + "touch", + "town", + "toy", + "tramp", + "trap", + "tre", + "tree", + "trencher", + "tri", + "trick", + "trip", + "tripe", + "tripp", + "trot", + "trott", + "troubl", + "trouble", + "trow", + "trowel", + "true", + "try", + "tuck", + "tuesday", + "tuffet", + "tumbl", + "tune", + "turn", + "turnip", + "twa", + "twaddle", + "twee", + "tweedle", + "twelve", + "twenty", + "twiddle", + "twig", + "twill", + "twitchett", + "twopence", + "undertaker", + "unicorn", + "unto", + "upon", + "upstair", + "upward", + "use", + "used", + "ush", + "vale", + "vast", + "velvet", + "venture", + "vex", + "vexation", + "victual", + "vinegar", + "visit", + "visitor", + "vow", + "wag", + "waggl", + "wail", + "wainscot", + "wait", + "wak", + "wake", + "walk", + "wall", + "walnut", + "wand", + "want", + "ware", + "warm", + "wash", + "wat", + "watch", + "watt", + "way", + "wealthy", + "wear", + "weath", + "wed", + "wedd", + "wednesday", + "wee", + "weed", + "week", + "weep", + "welcome", + "well", + "welshman", + "went", + "west", + "whale", + "whatev", + "wheel", + "wheelbarrow", + "whenev", + "wherev", + "whey", + "whip", + "whipp", + "whistle", + "white", + "whitechapel", + "whith", + "whoop", + "wife", + "wig", + "wil", + "wild", + "wildernes", + "wilkin", + "willie", + "wilt", + "wind", + "window", + "wine", + "wing", + "winkie", + "winkle", + "wint", + "winter", + "wip", + "wipe", + "wire", + "wise", + "wish", + "within", + "without", + "wiv", + "woe", + "woman", + "women", + "wond", + "wondrou", + "woo", + "wood", + "woodbin", + "wool", + "word", + "work", + "world", + "worm", + "worri", + "worry", + "worse", + "worth", + "wrap", + "wren", + "wright", + "write", + "wrote", + "yard", + "yea", + "year", + "yellow", + "yes", + "yon", + "young" + ] + } + ] + } +} diff --git a/code/frontend/tests/unit/archVacancy.test.ts b/code/frontend/tests/unit/archVacancy.test.ts new file mode 100644 index 0000000..e4a044e --- /dev/null +++ b/code/frontend/tests/unit/archVacancy.test.ts @@ -0,0 +1,212 @@ +/** + * The pretrained arm, browser side (contract §8; FR-717…720a). + * + * Real tokenizers and the real transform — no fabricated pieces anywhere. What this file + * does NOT do is download 280 MB of ONNX weights: the arithmetic on real logits is + * asserted by the backend's suite (which runs the same algorithm at float32) and by the + * static e2e, which drives the built site with the real quantized model. What is left + * here is exactly what can be wrong silently: + * + * - byte-level alignment, which mis-attributes rather than failing; + * - the passage cut, which must be the SAME six excerpts the backend scores; + * - the static build's reporting policy, which is what keeps a quantized number from + * being presented as a measurement. + */ + +import { describe, expect, it } from "vitest"; +import { AutoTokenizer } from "@huggingface/transformers"; + +import { WORD_RE } from "../../src/lib/lexEngine"; +import { + defaultVacancyPassages, + pairedDifference, + pooledStats, + preservedWordIndices, + staticVacancyDifferences, + vacancyVariantTexts, + VACANCY_ABSOLUTE_REFUSAL, + VACANCY_PER_PASSAGE_REFUSAL, +} from "../../src/lib/staticClient/arch"; +import { + preservedTokenIndices, + tokenByteSpans, + wordSpans, +} from "../../src/lib/staticClient/byteSpans"; +import { readStaticJson } from "./staticTestUtils"; +import golden from "../fixtures/arch-vacancy-passages.json"; + +const TEXTS = [ + "The cow jumped over the moon, and the little dog laughed.", + "café naïve — “owl” ≈ ç√ 東京 end", + " leading and\ttabbed\nnewlines ", + "don't good-bye o'clock", +]; + +async function pieces(modelId: string, text: string): Promise { + const tok = await AutoTokenizer.from_pretrained(modelId); + const inner = (tok as unknown as { _tokenizer: { encode(t: string): { tokens: string[] } } }) + ._tokenizer; + return inner.encode(text).tokens; +} + +async function sha256Hex(text: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)); + return Array.from(new Uint8Array(digest)) + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); +} + +describe("byte-level alignment (§8.2, FR-718)", () => { + for (const modelId of ["gpt2", "Qwen/Qwen2.5-0.5B-Instruct"]) { + for (const raw of TEXTS) { + it(`tiles ${JSON.stringify(raw.slice(0, 24))} exactly on ${modelId}`, async () => { + const text = raw.normalize("NFC"); + const spans = tokenByteSpans(await pieces(modelId, text), text); + const bytes = new TextEncoder().encode(text); + // A true partition: contiguous, covering, ordered — so a per-token quantity can + // be summed over a word without double-counting. + expect(spans[0][0]).toBe(0); + expect(spans[spans.length - 1][1]).toBe(bytes.length); + for (let i = 1; i < spans.length; i++) expect(spans[i][0]).toBe(spans[i - 1][1]); + const joined = spans.flatMap(([a, b]) => Array.from(bytes.slice(a, b))); + expect(joined).toEqual(Array.from(bytes)); + }, 60_000); + } + } + + it("raises rather than mis-attributing when the pieces do not rebuild the text", async () => { + const text = TEXTS[0]; + const p = await pieces("gpt2", text); + expect(() => tokenByteSpans(p.slice(0, -1), text)).toThrowError(/alignment failed/); + }, 60_000); + + it("attributes a leading-space token to its word (overlap, not 'starts inside')", async () => { + const text = "the cow and the moon"; + const spans = tokenByteSpans(await pieces("gpt2", text), text); + const words = wordSpans(text, WORD_RE); + expect(words.map((w) => w.word)).toEqual(["the", "cow", "and", "the", "moon"]); + const got = preservedTokenIndices(spans, words, new Set([0, 2, 3])); + // gpt2 gives one token per word here, so the three closed-class words are three + // tokens — each of which starts one byte BEFORE its word, on the space. + expect(got.length).toBe(3); + expect(spans[got[0]][0]).toBeLessThanOrEqual(words[0].start); + }, 60_000); + + it("refuses a token that spans a preserved and a vacated word", () => { + const words = wordSpans("the cow", WORD_RE); + expect(() => preservedTokenIndices([[0, 7]], words, new Set([0]))).toThrowError( + /spans both/, + ); + }); +}); + +describe("the three variants (§8.3)", () => { + it("preserves the scaffolding byte for byte and moves everything else", () => { + const passage = "Hey diddle diddle, the cat and the fiddle,\nThe cow jumped over the moon."; + const texts = vacancyVariantTexts(passage, { p: 1, seed: 0, matchProsody: true, keep: [] }); + const { words, preserved } = preservedWordIndices(texts); + expect(preserved.size).toBeGreaterThan(0); + for (const name of ["swap", "nonce"] as const) { + const variant = wordSpans(texts[name], WORD_RE); + expect(variant.length).toBe(words.length); + for (const i of preserved) expect(variant[i].word).toBe(words[i].word); + } + // …and the two vacated variants really differ from each other: a real English word + // where the nonce variant invented one. Without that the decomposition is vacuous. + const swap = wordSpans(texts.swap, WORD_RE); + const nonce = wordSpans(texts.nonce, WORD_RE); + const moved = words.filter((w) => !preserved.has(w.index)); + expect(moved.length).toBeGreaterThan(0); + expect(moved.some((w) => swap[w.index].word !== w.word)).toBe(true); + expect(moved.some((w) => nonce[w.index].word !== swap[w.index].word)).toBe(true); + }); + + it("is the identity at p = 0, so every variant is the same text", () => { + const passage = "The cow jumped over the moon and the little dog laughed."; + const texts = vacancyVariantTexts(passage, { p: 0, seed: 0, matchProsody: true, keep: [] }); + expect(new Set(Object.values(texts)).size).toBe(1); + }); +}); + +describe("the default passage set is the one the backend scores", () => { + it("cuts byte-identical excerpts from the shipped corpus", async () => { + const corpus = await readStaticJson<{ text: string }>("lex/corpus.json"); + const cut = defaultVacancyPassages(corpus.text.normalize("NFC")); + expect(cut.length).toBe(golden.count); + for (const row of golden.passages) { + expect(await sha256Hex(cut[row.index])).toBe(row.sha256); + expect((cut[row.index].match(new RegExp(WORD_RE.source, "g")) ?? []).length).toBe( + row.n_words, + ); + } + }); +}); + +describe("what the quantized static build may say (§8.3a, FR-720a)", () => { + const swap = { nats: 0.83, se: 0.09, nPairs: 780 }; + const nonce = { nats: 1.01, se: 0.1, nPairs: 780 }; + const diffs = staticVacancyDifferences(swap, nonce); + const byId = Object.fromEntries(diffs.map((d) => [d.id, d])); + + it("refuses nonce − swap with a typed error that names the full stack", () => { + const d = byId.unknown_form; + expect(d.nats).toBeNull(); + expect(d.se).toBeNull(); + expect(d.refused?.type).toBe("StaticModeError"); + expect(d.refused?.message).toMatch(/full stack/); + expect(d.refused?.message).toMatch(/uvicorn/); + // It must say WHY, in measured terms — not "unavailable in this demo". + expect(d.refused?.message).toMatch(/0\.16–0\.27/); + expect(d.refused?.message).toMatch(/sign flip/); + }); + + it("reports the two pooled differences it has a measured bound for", () => { + for (const id of ["wrong_content", "total"]) { + expect(byId[id].nats).toBeTypeOf("number"); + // The stated ± was MEASURED for q8; nothing here invents one. + expect(byId[id].quantizationUncertaintyNats).toBe(0.2); + expect(byId[id].refused).toBeUndefined(); + } + }); + + it("never headlines the conflated difference", () => { + expect(byId.wrong_content.headline).toBe(true); + expect(byId.unknown_form.headline).toBe(true); + expect(byId.total.headline).toBe(false); + expect(byId.total.note).toMatch(/conflates/); + expect(byId.unknown_form.upperBound).toBe(true); + }); + + it("refuses absolute NLLs and per-passage deltas by name", () => { + expect(VACANCY_ABSOLUTE_REFUSAL.type).toBe("StaticModeError"); + expect(VACANCY_ABSOLUTE_REFUSAL.message).toMatch(/−0\.19|\+0\.40/); + expect(VACANCY_PER_PASSAGE_REFUSAL.message).toMatch(/115 %/); + expect(VACANCY_PER_PASSAGE_REFUSAL.message).toMatch(/full stack/); + }); + + it("withholds the absolute numbers but not the token counts", () => { + const stats = pooledStats( + [{ pieces: [], nll: [NaN, 1, 2, 3], nChars: 40 }], + [[1, 3]], + ); + expect(stats.nllPreserved).toBeNull(); + expect(stats.nllAll).toBeNull(); + expect(stats.bitsPerChar).toBeNull(); + // Counts are exact at any dtype, and they are what show a nonce variant fragmenting. + expect(stats.nTokens).toBe(3); + expect(stats.nPreservedTokens).toBe(2); + expect(stats.nChars).toBe(40); + }); +}); + +describe("paired differences", () => { + it("pairs preserved tokens one-for-one and refuses a mismatch", () => { + const a = [{ pieces: [], nll: [NaN, 1, 2, 3], nChars: 10 }]; + const b = [{ pieces: [], nll: [NaN, 1.5, 2.5, 9], nChars: 10 }]; + const d = pairedDifference(a, [[1, 2]], b, [[1, 2]]); + expect(d.nats).toBeCloseTo(0.5, 12); + expect(d.nPairs).toBe(2); + expect(d.se).toBeCloseTo(0, 12); + expect(() => pairedDifference(a, [[1, 2]], b, [[1]])).toThrowError(/cannot be paired/); + }); +}); diff --git a/code/frontend/tests/unit/geoEngine.test.ts b/code/frontend/tests/unit/geoEngine.test.ts index 3466da9..d40db76 100644 --- a/code/frontend/tests/unit/geoEngine.test.ts +++ b/code/frontend/tests/unit/geoEngine.test.ts @@ -19,6 +19,7 @@ import { beforeAll, describe, expect, it } from "vitest"; import { GeoEngine, GeoEngineError } from "../../src/lib/geoEngine"; import { clipPrompt } from "../../src/lib/geoEngine/fields"; +import { sha256Hex, utf8Bytes } from "../../src/lib/geoEngine/hash"; import { GeoModel } from "../../src/lib/geoEngine/model"; import type { GeoVectorFieldData, @@ -422,4 +423,61 @@ describe("minted-set persistence hooks (static reload survival) [fixtures]", () const engine3 = GeoEngine.fromAssets(fixtureSrc.checkpoint, fixtureSrc.vocab); expect(engine3.importWeightSet(minted.weights_token, bad)).toBe(false); }); + + /** + * A model trained from scratch (or loaded from a file) has a vocabulary of its OWN: + * its token id 17 is not the shipped model's token id 17. The static build persists + * minted sets to sessionStorage and restores them after a reload — and it used to + * persist the weights WITHOUT the word list, so the restored set fell back to the + * shipped tokenizer. The damage was not a wrong label on screen: `exportBundle` then + * wrote a `.llmgeo.json` pairing those weights with the shipped word list and hashed + * THAT list into `vocab_sha256`, producing a file no integrity check can reject — the + * exact corruption the three digests exist to prevent, committed by the writer. So + * "save → reload → save" silently changed which words the model file described. + */ + it("carries a loaded model's OWN vocabulary across the persistence hop", () => { + const engine = GeoEngine.fromAssets(fixtureSrc.checkpoint, fixtureSrc.vocab); + + // A real model file whose word list is NOT the shipped one — the situation every + // from-scratch run and every `.llmgeo.json` load produces. + // Distinct weights (so the token is not the canonical one) AND distinct words. + const minted = engine.postWeights({ + base: "learned", + edits: [{ layer: 1, matrix: "W_K", preset: "identity" }], + }); + const shipped = engine.exportBundle(minted.weights_token); + const words = (JSON.parse(shipped.vocab) as { words: string[] }).words.map((w, i) => + i === 0 ? `${w}zz` : w, + ); + const vocabJson = JSON.stringify({ + format: "geo-tokenizer-v1", + specials: { "": 0, "": 1, "": 2 }, + words, + }); + const file = { + ...shipped, + vocab: vocabJson, + vocab_sha256: sha256Hex(utf8Bytes(vocabJson)), + }; + const { weights_token: token } = engine.importBundle(file); + expect(JSON.parse(engine.exportBundle(token).vocab).words).toEqual(words); + + // The reload hop: persist, restore into a fresh engine, save again. + const saved = engine.exportWeightSet(token); + expect(saved.vocabWords).toEqual(words); + const reloaded = GeoEngine.fromAssets(fixtureSrc.checkpoint, fixtureSrc.vocab); + expect(reloaded.importWeightSet(token, saved)).toBe(true); + const after = engine.exportBundle(token); + expect(JSON.parse(reloaded.exportBundle(token).vocab).words).toEqual(words); + // Byte-for-byte the same file, which is the user-visible claim. + expect(reloaded.exportBundle(token)).toEqual(after); + + // A payload that LOST the word list is dropped, not restored half-right: the token + // simply is not there afterwards, so the caller deletes it and the evicted-token + // self-heal resets visibly instead of quietly relabelling the model. + const { vocabWords: _dropped, ...withoutVocab } = saved; + const stale = GeoEngine.fromAssets(fixtureSrc.checkpoint, fixtureSrc.vocab); + expect(stale.importWeightSet(token, withoutVocab)).toBe(false); + expect(() => stale.exportBundle(token)).toThrow(/unknown/); + }); }); diff --git a/code/frontend/tests/unit/geoNodeEnv.d.ts b/code/frontend/tests/unit/geoNodeEnv.d.ts index ab10090..0f73dfc 100644 --- a/code/frontend/tests/unit/geoNodeEnv.d.ts +++ b/code/frontend/tests/unit/geoNodeEnv.d.ts @@ -17,6 +17,11 @@ declare module "node:path" { const path: { resolve(...parts: string[]): string; join(...parts: string[]): string; + // Consumed by tests/unit/staticTestUtils.ts and tests/e2e/static.spec.ts, both of + // which resolve a directory from `import.meta.url`. Declared here because this file + // is the project's only description of node:path — the tsconfig pins + // `types: ["vitest/globals"]`, so @types/node is deliberately absent. + dirname(p: string): string; }; export default path; } diff --git a/code/frontend/tests/unit/logitsSanity.test.ts b/code/frontend/tests/unit/logitsSanity.test.ts new file mode 100644 index 0000000..3a8f255 --- /dev/null +++ b/code/frontend/tests/unit/logitsSanity.test.ts @@ -0,0 +1,128 @@ +// The load-time non-degeneracy invariant (src/lib/staticClient/logitsSanity.ts). +// +// These exercise the predicate itself over the exact shapes the real failure produced, +// measured in a real browser on a real GPU (Chrome 150 / Apple Metal-3, ORT-web +// 1.26.0-dev, transformers.js 4.2.0): +// onnx-community/gpt2-ONNX webgpu/q4f16 → row separation 0 (identical rows) +// onnx-community/SmolLM2-135M-…-ONNX webgpu/q4f16 → row separation 0, every logit 0 +// onnx-community/SmolLM2-360M-…-ONNX webgpu/q4f16 → row separation 0, every logit 0 +// versus the healthy sessions on the same machine: +// gpt2 webgpu/q8 91.99 · SmolLM2-135M webgpu/q8 36.97 · Qwen2.5-0.5B webgpu/q8 20.27 +// The end-to-end version of this, against the real model in a WebGPU browser, is +// tests/e2e/webgpu.spec.ts. +import { describe, expect, it } from "vitest"; + +import { + MIN_ROW_SEPARATION, + assertNonDegenerateLogits, + rowSeparation, +} from "../../src/lib/staticClient/logitsSanity"; +import { FP16_ACTIVATION_DTYPES, RUNTIME_LADDER } from "../../src/lib/staticClient/runtimeTypes"; + +const VOCAB = 64; +const SEQ = 6; + +/** A [1, SEQ, VOCAB] buffer whose rows differ, like any working causal LM. */ +function healthy(): Float32Array { + const out = new Float32Array(SEQ * VOCAB); + for (let t = 0; t < SEQ; t++) { + for (let v = 0; v < VOCAB; v++) out[t * VOCAB + v] = Math.sin(v * 0.37) * 8 + t * 1.5; + } + return out; +} + +/** One row, repeated — the gpt2 webgpu/q4f16 failure. */ +function repeatedRows(): Float32Array { + const out = new Float32Array(SEQ * VOCAB); + for (let t = 0; t < SEQ; t++) { + for (let v = 0; v < VOCAB; v++) out[t * VOCAB + v] = Math.cos(v * 0.11) * 6; + } + return out; +} + +describe("rowSeparation", () => { + it("measures the L∞ gap between the first and last next-token distribution", () => { + // Constructed so the true answer is known: rows differ by exactly 1.5*(SEQ-1). + expect(rowSeparation(healthy(), SEQ, VOCAB)).toBeCloseTo(1.5 * (SEQ - 1), 5); + }); + + it("is exactly 0 for repeated rows and for an all-zero tensor", () => { + expect(rowSeparation(repeatedRows(), SEQ, VOCAB)).toBe(0); + expect(rowSeparation(new Float32Array(SEQ * VOCAB), SEQ, VOCAB)).toBe(0); + }); + + it("reports NaN when any compared entry is not finite", () => { + const nan = healthy(); + nan[(SEQ - 1) * VOCAB + 3] = Number.NaN; + expect(rowSeparation(nan, SEQ, VOCAB)).toBeNaN(); + const inf = healthy(); + inf[7] = Number.POSITIVE_INFINITY; + expect(rowSeparation(inf, SEQ, VOCAB)).toBeNaN(); + }); + + it("refuses a single-position pass, where the invariant is not defined", () => { + expect(() => rowSeparation(new Float32Array(VOCAB), 1, VOCAB)).toThrow(/at least 2 positions/); + }); +}); + +// The dtype-preference half of the fix. tests/e2e/webgpu.spec.ts checks it end to end +// on a real GPU, which SKIPS wherever there is no adapter (all GitHub-hosted runners); +// this runs everywhere, so CI always verifies at least that no fp16-activation dtype +// has crept back into the ladder. +describe("the runtime's dtype ladder", () => { + it("asks only for dtypes verified correct in a real browser", () => { + expect(RUNTIME_LADDER.length).toBeGreaterThan(0); + for (const rung of RUNTIME_LADDER) { + expect(["q8"]).toContain(rung.dtype); + expect(["webgpu", "wasm"]).toContain(rung.device); + } + }); + + it("never requests a dtype with fp16 ACTIVATIONS, the path that returns garbage", () => { + const asked = new Set(RUNTIME_LADDER.map((r) => r.dtype)); + for (const bad of FP16_ACTIVATION_DTYPES) expect(asked.has(bad)).toBe(false); + }); + + it("tries the GPU first and keeps a non-GPU rung to fall back to", () => { + expect(RUNTIME_LADDER[0].device).toBe("webgpu"); + expect(RUNTIME_LADDER.some((r) => r.device === "wasm")).toBe(true); + // Both rungs read the same model_quantized.onnx, so a rejection costs no + // second download — the property that makes the fallback cheap. + expect(new Set(RUNTIME_LADDER.map((r) => r.dtype)).size).toBe(1); + }); +}); + +describe("assertNonDegenerateLogits", () => { + it("accepts a session whose output depends on its input, returning the separation", () => { + expect(assertNonDegenerateLogits(healthy(), SEQ, VOCAB, "wasm/q8")).toBeCloseTo(7.5, 5); + }); + + it("rejects identical rows, naming the configuration under test", () => { + expect(() => assertNonDegenerateLogits(repeatedRows(), SEQ, VOCAB, "webgpu/q4f16")).toThrow( + /webgpu\/q4f16 produced degenerate logits/, + ); + }); + + it("rejects an all-zero tensor as the same single failure, not a special case", () => { + expect(() => + assertNonDegenerateLogits(new Float32Array(SEQ * VOCAB), SEQ, VOCAB, "webgpu/q4f16"), + ).toThrow(/does not depend on its input/); + }); + + it("rejects NaN output", () => { + const nan = new Float32Array(SEQ * VOCAB).fill(Number.NaN); + expect(() => assertNonDegenerateLogits(nan, SEQ, VOCAB, "webgpu/fp16")).toThrow(/NaN/); + }); + + it("accepts separations just above the threshold and rejects just below", () => { + const near = new Float32Array(SEQ * VOCAB); + near[(SEQ - 1) * VOCAB] = MIN_ROW_SEPARATION * 2; + expect(assertNonDegenerateLogits(near, SEQ, VOCAB, "wasm/q8")).toBeCloseTo( + MIN_ROW_SEPARATION * 2, + 9, + ); + const under = new Float32Array(SEQ * VOCAB); + under[(SEQ - 1) * VOCAB] = MIN_ROW_SEPARATION / 2; + expect(() => assertNonDegenerateLogits(under, SEQ, VOCAB, "wasm/q8")).toThrow(/degenerate/); + }); +}); diff --git a/code/frontend/tests/unit/staticClient.test.ts b/code/frontend/tests/unit/staticClient.test.ts index b3f0dfc..baea31a 100644 --- a/code/frontend/tests/unit/staticClient.test.ts +++ b/code/frontend/tests/unit/staticClient.test.ts @@ -156,11 +156,21 @@ describe("live runtime seam (no model downloads)", () => { model_id: "gpt2", onnx_repo: "onnx-community/gpt2-ONNX", error: null, + // No fallback rungs were rejected on the way to this one — the badge's "you are on + // a fallback path" signal must be explicitly empty, not absent. + rejected: [], }), tokenize: async (modelId, revision, text): Promise => { calls.push(["tokenize", modelId, revision, text]); return { model_id: modelId, tokens: [{ token: 1, token_str: "x" }] }; }, + // The vacancy measurement is not the seam under test here; it has its own suite + // (archVacancy.test.ts) and its own e2e. Recording the call and refusing keeps a + // stray route from silently scoring nothing. + scoreTexts: async (onnxRepo, texts) => { + calls.push(["scoreTexts", onnxRepo, texts.length]); + throw new Error("this runtime seam test does not exercise scoring"); + }, generate: async (body, onnxRepo): Promise => { calls.push(["generate", body.model_id, onnxRepo]); return { diff --git a/code/frontend/tests/unit/staticVacancy.test.ts b/code/frontend/tests/unit/staticVacancy.test.ts new file mode 100644 index 0000000..64f8981 --- /dev/null +++ b/code/frontend/tests/unit/staticVacancy.test.ts @@ -0,0 +1,336 @@ +/** + * `POST /api/lex/vacancy` in the STATIC build — and the parity that makes it worth having. + * + * The Lexicon Lab computes in the browser in both modes, so the vacancy transform is not + * something the Pages build refuses or approximates: it runs the real + * `lexEngine/vacancy.ts` over the real committed corpus and answers the same request the + * real FastAPI route answers. FR-722 says the static build "serves the same capability"; + * this file is the measurement of that sentence rather than a restatement of it. + * + * THE CENTRAL TEST is `matches the live backend field for field`. Its fixture, + * `tests/fixtures/vacancy-api-golden.json`, was produced by + * + * python scripts/export_vacancy_api_golden.py + * + * running the REAL app through FastAPI's TestClient on the REAL corpus — the responses in + * it are transcripts, not expectations someone typed. `test_api_lex.py:: + * test_vacancy_matches_the_static_client_fixture` asserts the live route still returns + * exactly the same file, so the two stacks are pinned to ONE document: if either drifts, + * one of the two tests fails and names the field it drifted on. + * + * What that comparison actually covers, per case: every §10 statistic, the resolved + * vocabulary word for word, the budget's measured coverage of the vacated corpus, the + * preview text character for character, and `vacated_sha256` — the sha256 of the WHOLE + * 86 kB rewrite, which pins every byte the preview does not show. Floats need no tolerance + * because both stacks round to 6 significant digits before serving (`jsonable_6sig` and + * `sig6`), so the wire values are identical, not merely close. + * + * The remaining tests cover what a single-request fixture cannot: the properties that only + * exist ACROSS requests (nesting, stability, the invariance theorem's effect on coverage) + * and the parameter errors, which have no fixture because they have no body. + */ + +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; + +import { createStaticClient, type StaticClient } from "../../src/lib/staticClient"; +import type { FetchLike } from "../../src/lib/staticClient/assets"; +import { + VACANCY_PREVIEW_CHARS, + VACANCY_PREVIEW_MAX, + type LexVacancyBody, + type LexVacancyResult, +} from "../../src/lib/staticClient/lex"; +import { fsStaticFetch } from "./staticTestUtils"; + +function client(fetchImpl: FetchLike = fsStaticFetch()): StaticClient { + return createStaticClient({ baseUrl: "/", fetchImpl }); +} + +// --- the parity fixture --------------------------------------------------------------- + +interface ApiGoldenCase { + label: string; + request: LexVacancyBody; + response: LexVacancyResult; +} + +interface ApiGolden { + format: string; + git_sha: string; + command: string; + contract: string; + tolerance: number; + endpoint: string; + defaults: { preview_chars: number; preview_max: number }; + corpus: { sha256: string; chars: number }; + cases: ApiGoldenCase[]; +} + +const GOLDEN: ApiGolden = JSON.parse( + fs.readFileSync( + path.resolve(__dirname, "../fixtures/vacancy-api-golden.json"), + "utf-8", + ), +) as ApiGolden; + +describe("the API-parity fixture is the real route's own output", () => { + it("declares its format, its generator and the contract it pins", () => { + expect(GOLDEN.format).toBe("vacancy-api-golden-v1"); + expect(GOLDEN.command).toBe("python scripts/export_vacancy_api_golden.py"); + expect(GOLDEN.endpoint).toBe("/api/lex/vacancy"); + expect(GOLDEN.contract).toBe("specs/002-interactive-model-explorer/contracts/api.md"); + expect(GOLDEN.git_sha).toMatch(/^[0-9a-f]{7,40}$/); + expect(GOLDEN.cases.length).toBeGreaterThanOrEqual(6); + }); + + /** + * The two stacks each hold their own copy of the preview defaults. A fixture generated + * with one and consumed by the other would paper over a mismatch precisely where it + * matters — the excerpt length — so the numbers are compared directly as well. + */ + it("was generated with the preview defaults this build uses", () => { + expect(GOLDEN.defaults.preview_chars).toBe(VACANCY_PREVIEW_CHARS); + expect(GOLDEN.defaults.preview_max).toBe(VACANCY_PREVIEW_MAX); + const usingDefault = GOLDEN.cases.filter((c) => c.request.preview_chars === undefined); + expect(usingDefault.length).toBeGreaterThan(0); + for (const c of usingDefault) { + expect(c.response.preview_chars).toBe(VACANCY_PREVIEW_CHARS); + } + }); +}); + +describe("the static build answers /api/lex/vacancy exactly as the backend does", () => { + for (const testCase of GOLDEN.cases) { + it(`matches the live backend field for field — ${testCase.label}`, async () => { + const got = await client().lexVacancy(testCase.request); + const want = testCase.response; + + // The digest first and by name: it is the one assertion that covers all 86 kB, and + // a failure here means the two stacks vacated the corpus DIFFERENTLY, which is a + // different (and worse) bug than a statistic being reported differently. + expect(got.vacated_sha256).toBe(want.vacated_sha256); + expect(got.original_sha256).toBe(want.original_sha256); + expect(got.vacated_chars).toBe(want.vacated_chars); + + expect(got.vacancy_stats).toEqual(want.vacancy_stats); + expect(got.vocabulary_rule).toBe(want.vocabulary_rule); + expect(got.words).toEqual(want.words); + expect(got.budget).toEqual(want.budget); + expect(got.corpus).toEqual(want.corpus); + expect(got.preview).toBe(want.preview); + expect(got.original_preview).toBe(want.original_preview); + + // …and then the whole object, so a field added on one side and not the other is a + // failure rather than something the assertions above happen not to look at. + expect(got).toEqual(want); + }); + } +}); + +describe("what the response says about itself", () => { + it("returns an excerpt plus a digest, never the whole corpus", async () => { + const res = await client().lexVacancy({ p: 1, seed: 0 }); + expect(res.preview_chars).toBe(VACANCY_PREVIEW_CHARS); + expect(res.preview.length).toBe(VACANCY_PREVIEW_CHARS); + expect(res.truncated).toBe(true); + // The corpus is ~86 kB; the excerpt is a fortieth of it and the digest covers the rest. + expect(res.vacated_chars).toBeGreaterThan(VACANCY_PREVIEW_CHARS * 10); + expect(res.vacated_sha256).toMatch(/^[0-9a-f]{64}$/); + }); + + it("serves a longer excerpt on request, up to the ceiling", async () => { + const res = await client().lexVacancy({ p: 1, seed: 0, preview_chars: 10 }); + expect(res.preview.length).toBe(10); + expect(res.truncated).toBe(true); + await expect( + client().lexVacancy({ p: 1, preview_chars: VACANCY_PREVIEW_MAX + 1 }), + ).rejects.toMatchObject({ type: "InvalidParamError" }); + }); + + it("is the identity at p = 0, digest included", async () => { + const res = await client().lexVacancy({ p: 0, seed: 0 }); + // `u ∈ [0, 1)`, so `u < 0` is never true and nothing can vacate. + expect(res.vacated_sha256).toBe(res.original_sha256); + expect(res.preview).toBe(res.original_preview); + expect(res.vacancy_stats.corpusTypesVacated).toBe(0); + expect(res.vacancy_stats.tokensVacated).toBe(0); + expect(res.vacancy_stats.stemsVacated).toBe(0); + }); +}); + +describe("the properties a single request cannot show", () => { + /** + * SC-703 through the API surface. Under the mapped condition the transform is a pure + * relabelling of the vocabulary, so a budget's measured coverage of the VACATED corpus + * must equal its coverage of the English one — same tokens in budget, same `` in + * the same places. This is the invariance theorem stated in the units the panel shows. + */ + it("leaves coverage bit-identical under the mapped vocabulary, at every p", async () => { + const c = client(); + const english = await c.lexCoverage({ source: "dolch", budget: "primer" }); + for (const p of [0, 0.25, 0.5, 0.75, 1]) { + for (const seed of [0, 7]) { + const res = await c.lexVacancy({ p, seed, source: "dolch", budget: "primer" }); + expect(res.vocabulary_rule).toBe("mapped"); + expect(res.budget.coverage).toEqual(english.coverage); + expect(res.budget.rows).toBe(english.rows); + expect(res.corpus.n_tokens).toBe(english.corpus.n_tokens); + expect(res.corpus.n_distinct).toBe(english.corpus.n_distinct); + expect(res.corpus.n_lines).toBe(english.corpus.n_lines); + } + } + }); + + /** SC-705: the controls BREAK it, and the break is what the panel measures. */ + it("collapses coverage under both control conditions, at the same p", async () => { + const c = client(); + const mapped = await c.lexVacancy({ p: 0.5, seed: 0, source: "dolch", budget: "primer" }); + const inconsistent = await c.lexVacancy({ + p: 0.5, + seed: 0, + consistent: false, + source: "dolch", + budget: "primer", + }); + const revealed = await c.lexVacancy({ + p: 0.5, + seed: 0, + reveal_after: 2, + source: "dolch", + budget: "primer", + }); + expect(mapped.vocabulary_rule).toBe("mapped"); + expect(inconsistent.vocabulary_rule).toBe("rebuilt"); + expect(revealed.vocabulary_rule).toBe("rebuilt"); + expect(inconsistent.budget.coverage.unk_rate).toBeGreaterThan(mapped.budget.coverage.unk_rate); + expect(revealed.budget.coverage.unk_rate).toBeGreaterThan(mapped.budget.coverage.unk_rate); + }); + + /** + * SC-701 / SC-702 as the API reports them: `stemsVacated` never falls as `p` rises, and + * a stem minted at a low `p` is byte-identical at every higher one. The second half is + * checked through the mapped word list, which is the map restricted to the budget. + */ + it("nests and stays stable as p rises", async () => { + const c = client(); + const seen: { p: number; stems: number; words: string[] }[] = []; + for (const p of [0, 0.25, 0.5, 0.75, 1]) { + const res = await c.lexVacancy({ p, seed: 0, source: "dolch", budget: "primer" }); + seen.push({ p, stems: res.vacancy_stats.stemsVacated, words: res.words }); + } + for (let i = 1; i < seen.length; i++) { + expect(seen[i].stems).toBeGreaterThanOrEqual(seen[i - 1].stems); + // Stability: any word already rewritten at the lower `p` is unchanged at the higher. + for (let w = 0; w < seen[i].words.length; w++) { + const lower = seen[i - 1].words[w]; + const higher = seen[i].words[w]; + if (lower !== seen[0].words[w]) expect(higher).toBe(lower); + } + } + // The two ends are identities, not observations: `u ∈ [0, 1)`, so nothing vacates at + // `p = 0` and EVERY eligible stem vacates at `p = 1` (§10). + const last = await client().lexVacancy({ p: 1, seed: 0, source: "dolch", budget: "primer" }); + expect(seen[0].stems).toBe(0); + expect(last.vacancy_stats.stemsVacated).toBe(last.vacancy_stats.stemsTotal); + expect(last.vacancy_stats.corpusTypesVacated).toBe(last.vacancy_stats.corpusTypesEligible); + }); + + /** FR-716: vacancy composes with a user's own text, not only with the shipped corpus. */ + it("transforms pasted text as readily as the shipped corpus", async () => { + const text = "The little brown squirrel ate the pretty acorn.\nThe squirrel ran away.\n"; + const res = await client().lexVacancy({ text, p: 1, seed: 0, preview_chars: 200 }); + expect(res.original_preview).toBe(text); + expect(res.preview).not.toBe(text); + // §1: only WORD_RE matches are replaced — punctuation and line breaks pass through. + expect(res.preview.split("\n").length).toBe(text.split("\n").length); + expect((res.preview.match(/\./g) ?? []).length).toBe(2); + expect(res.vacancy_stats.tokensTotal).toBe(12); + // Three of the twelve are `the`/`The`, which §2.1 preserves. The other nine — + // little, brown, squirrel, ate, pretty, acorn, squirrel, ran, away — are open class, + // and at `p = 1` every one of them moves. + expect(res.vacancy_stats.tokensVacated).toBe(9); + expect(res.vacancy_stats.corpusTypesVacated).toBe(8); // `squirrel` occurs twice + }); +}); + +describe("bad vacancy parameters are refused in the shared envelope", () => { + const bad: [string, LexVacancyBody][] = [ + ["p above 1", { p: 1.5 }], + ["p below 0", { p: -0.1 }], + ["a negative reveal_after", { reveal_after: -1 }], + ["preview_chars above the ceiling", { preview_chars: VACANCY_PREVIEW_MAX + 1 }], + ["preview_chars below zero", { preview_chars: -1 }], + ["a bare string for keep", { keep: "little" as unknown as string[] }], + ["size on a Dolch budget", { source: "dolch", size: 50 }], + ["an unknown budget", { budget: "not-a-budget" }], + ]; + for (const [what, body] of bad) { + it(`rejects ${what}`, async () => { + await expect(client().lexVacancy(body)).rejects.toMatchObject({ + type: "InvalidParamError", + }); + }); + } +}); + +describe("training on a vacated corpus", () => { + /** + * FR-713 / SC-703's corollary, run for real in the browser engine: with the mapped + * vocabulary the token id stream is unchanged, so a real training run at `p = 0.5` must + * produce BIT-IDENTICAL losses to the same run on the English corpus. Not "close" — + * identical, because `runTraining` sees the same integers either way. + */ + it("is bit-identical to training on the English corpus under the mapped vocabulary", async () => { + const c = client(); + const shape = { + source: "dolch", + budget: "pre_primer", + steps: 12, + d_model: 16, + n_layers: 1, + n_heads: 1, + ctx: 32, + batch_size: 8, + seed: 3, + } as const; + const english = await trainToCompletion(c, { ...shape }); + const vacated = await trainToCompletion(c, { + ...shape, + vacancy: { p: 0.5, seed: 0 }, + }); + expect(vacated.final_loss).toBe(english.final_loss); + expect(vacated.first_loss).toBe(english.first_loss); + expect(vacated.val_loss).toBe(english.val_loss); + expect(vacated.n_tokens).toBe(english.n_tokens); + expect(vacated.vocab_rows).toBe(english.vocab_rows); + // Same numbers, DIFFERENT words: the model is blind to the relabelling, which is the + // finding rather than a caveat about it. + expect(vacated.model_token).not.toBe(english.model_token); + }, 120_000); + + it("rejects a vacancy block that is not an object", async () => { + await expect( + client().lexTrain({ vacancy: 0.5 as unknown as Record }), + ).rejects.toMatchObject({ type: "InvalidParamError" }); + }); +}); + +/** Start a training run and wait for its result, exactly as a view would. */ +async function trainToCompletion( + c: StaticClient, + body: Parameters[0], +): Promise & { model_token: string }> { + const started = await c.lexTrain(body); + if (started.ready) { + return started as unknown as Record & { model_token: string }; + } + const done = await new Promise>((resolve, reject) => { + c.subscribeProgress(started.job_id, { + onDone: (data) => resolve(data ?? {}), + onError: (type, message) => reject(new Error(`${type}: ${message}`)), + }); + }); + return done as Record & { model_token: string }; +} diff --git a/code/frontend/tests/unit/vacancy.test.ts b/code/frontend/tests/unit/vacancy.test.ts new file mode 100644 index 0000000..9769060 --- /dev/null +++ b/code/frontend/tests/unit/vacancy.test.ts @@ -0,0 +1,1090 @@ +/** + * The vacancy transform, measured on the REAL committed corpus. + * + * No mocks and no toy strings standing in for the thing: every assertion below runs + * against `public/static-data/lex/corpus.json` — *The Real Mother Goose*, the same bytes + * the Pages build ships and the same bytes the Python backend trains on. The properties + * being checked are the ones `specs/007-vacancy-transform-field/architecture.md` says the + * instrument must have, and they are the reason a `p`-sweep means anything: + * + * SC-701 NESTING `{stems vacated at p} ⊆ {stems vacated at p'}` for `p < p'`, + * because `u` is a function of `(seed, stem)` alone. + * SC-702 STABILITY a stem's nonce is identical at every `p` where it is vacated, and + * unchanged if the input type order is shuffled — the map is built + * once in canonical order, which is the correction §5.2 makes to the + * source's order-dependent `used` set. + * SC-704 INJECTIVITY the image of the real type set is the same size as the type set, + * verified rather than assumed, with `remintRounds` reported. + * FR-706 no minted form is ever a real corpus type. + * + * Plus the segmentation guarantee of §1 — every output is a single complete `WORD_RE` + * match, so `tokenize(vacated)` aligns with `tokenize(original)` element for element and + * line structure survives — which is what the invariance theorem of §7.3 rests on. + * + * The final `it` prints the measured numbers (types, tokens, prosody, stress-table + * coverage) rather than hiding them: §10 forbids transcribing the source's numbers from + * ITS corpus, so ours have to come from a run like this one. + */ + +import { describe, expect, it } from "vitest"; + +import { DOLCH_ORDER, LexVocab, WORD_RE, dolchBudget, splitLines, tokenize } from "../../src/lib/lexEngine"; +import { + CODAS, + DEFAULT_VACANCY_PARAMS, + FUNCTION_WORDS, + NUCLEI, + ONSETS, + SPLIT_EXCEPTIONS, + STRESS_TABLE, + SUFFIXES, + UNSTRESSED_TAILS, + buildVacancyMap, + effectiveKeepSet, + isEligible, + mapVocabWords, + meterScore, + stemAndSuffix, + stress, + syllables, + transformWord, + typeCounts, + vacancyDomain, + vacancyParams, + vacancyStats, + vacancyU, + vacateText, + type VacancyMap, + type VacancyParams, +} from "../../src/lib/lexEngine/vacancy"; +import type { LexCorpusAsset } from "../../src/lib/staticClient/lex"; +import { readStaticJson } from "./staticTestUtils"; + +// --- the real corpus, once ------------------------------------------------------------ + +const corpusAsset = await readStaticJson("lex/corpus.json"); +const CORPUS = corpusAsset.text; +const CORPUS_TYPES = new Set(tokenize(CORPUS)); +const BUDGET = dolchBudget("full"); +/** §5.2: corpus types ∪ the FULL Dolch list, via the helper — never built by hand here, + * since building it by hand at one call site and not another is the failure the helper + * exists to prevent. */ +const DOMAIN = vacancyDomain(CORPUS_TYPES); + +const P_GRID = [0, 0.25, 0.5, 0.75, 1] as const; +const SEEDS = [0, 7] as const; + +function params(partial: Partial): VacancyParams { + return vacancyParams(partial); +} + +/** One map per seed — it is `p`-independent by construction, which is the point. */ +const MAPS = new Map( + SEEDS.map((seed) => [seed, buildVacancyMap(DOMAIN, params({ seed }))]), +); + +function mapFor(seed: number): VacancyMap { + const m = MAPS.get(seed); + if (m === undefined) throw new Error(`no map for seed ${seed}`); + return m; +} + +/** The map as a sorted array of pairs, so two maps compare by value. */ +function mappingEntries(vmap: VacancyMap): [string, string][] { + return [...vmap.mapping].sort((a, b) => (a[0] < b[0] ? -1 : 1)); +} + +/** The source types whose surface actually changed, measured from the two texts. */ +function changedTypes(original: string, rewritten: string): Set { + const before = original.match(new RegExp(WORD_RE.source, "g")) ?? []; + const after = rewritten.match(new RegExp(WORD_RE.source, "g")) ?? []; + expect(before.length).toBe(after.length); + const out = new Set(); + for (let i = 0; i < before.length; i++) { + if (before[i] !== after[i]) out.add(before[i].toLowerCase()); + } + return out; +} + +/** The stems the map would vacate at this `p`, as a set. */ +function vacatedStems(seed: number, p: number): Set { + const out = new Set(); + for (const stem of mapFor(seed).mapping.keys()) { + if (vacancyU(stem, seed) < p) out.add(stem); + } + return out; +} + +/** Cache the vacated corpus per (seed, p) — each pass rewrites 86 kB. */ +const vacatedCache = new Map(); +function vacated(seed: number, p: number): string { + const key = `${seed}:${p}`; + const hit = vacatedCache.get(key); + if (hit !== undefined) return hit; + const out = vacateText(CORPUS, mapFor(seed), params({ seed, p })); + vacatedCache.set(key, out); + return out; +} + +// --- the verbatim tables -------------------------------------------------------------- + +describe("the ported tables are the source's, unchanged", () => { + it("keeps the curated closed class and nothing else", () => { + expect(FUNCTION_WORDS.size).toBe(137); + // The warning the source carries: the closed class is the curated list ONLY. Union it + // with Dolch service words and these content verbs are silently protected. + for (const verb of ["run", "eat", "see", "get", "let", "put"]) { + expect(FUNCTION_WORDS.has(verb)).toBe(false); + } + for (const w of ["the", "not", "under", "ten"]) expect(FUNCTION_WORDS.has(w)).toBe(true); + }); + + it("keeps the phonotactic tables in their source order", () => { + expect(ONSETS.length).toBe(47); + expect(NUCLEI.length).toBe(19); + // architecture.md §5.4 says 49; the source list it is copied verbatim from has 46. + expect(CODAS.length).toBe(46); + expect(CODAS[0]).toBe(""); + expect(ONSETS[0]).toBe("b"); + expect(ONSETS[ONSETS.length - 1]).toBe("sq"); + expect(NUCLEI[0]).toBe("a"); + expect(NUCLEI[NUCLEI.length - 1]).toBe("er"); + expect(UNSTRESSED_TAILS.length).toBe(13); + expect(UNSTRESSED_TAILS[0]).toBe("y"); + expect(UNSTRESSED_TAILS[UNSTRESSED_TAILS.length - 1]).toBe("ing"); + }); + + it("keeps the suffix list in its trial order and the audited exceptions", () => { + expect([...SUFFIXES]).toEqual(["ing", "edly", "est", "ies", "'s", "n't", "ed", "es", "er", "ly", "s"]); + expect(SPLIT_EXCEPTIONS.size).toBe(10); + // Departure 9: without these, `brother -> broth+er` and `morning -> morn+ing`. + expect(stemAndSuffix("brother")).toEqual(["brother", ""]); + expect(stemAndSuffix("morning")).toEqual(["morning", ""]); + // ... and the artifact that remains, honestly: this is a spelling heuristic. + expect(stemAndSuffix("ladder")).toEqual(["ladd", "er"]); + }); + + it("keeps the 61-entry hand stress table", () => { + expect(STRESS_TABLE.size).toBe(61); + expect(STRESS_TABLE.get("together")).toBe("0100"); + expect(STRESS_TABLE.get("Christmas")).toBe("10"); + }); +}); + +// --- §6 prosody ----------------------------------------------------------------------- + +describe("prosody", () => { + it("uses the hand table before the rule, case-sensitively", () => { + expect(stress("Christmas")).toBe("10"); + expect(stress("little")).toBe("100"); + expect(stress("away")).toBe("01"); + // Not in the table -> the spelling rule. + expect(stress("cat")).toBe("1"); + expect(stress("candle")).toBe("10"); + }); + + it("prefers a minted pattern over both", () => { + const minted = new Map([["zorble", "010"]]); + expect(stress("zorble", minted)).toBe("010"); + expect(syllables("zorble", minted)).toBe(3); + expect(stress("zorble")).toBe("10"); + }); + + it("scores a foot as the fraction of matching syllable positions", () => { + expect(meterScore("", "anapest")).toBe(0); + // "the little cat" scans "1" + "100" + "1" = "11001": monosyllables are stressed by + // the rule, so the scan is not the metrist's reading — it is the table's and the + // rule's, which is exactly what `stressTableCoverage` exists to qualify. + expect(meterScore("the little cat", "trochee")).toBeCloseTo(3 / 5, 12); + expect(meterScore("the little cat", "anapest")).toBeCloseTo(1 / 5, 12); + // "away away" scans "0101": a perfect iamb, and the hand table is why. + expect(meterScore("away away", "iamb")).toBe(1); + expect(() => meterScore("x", "spondee")).toThrow(/unknown foot/); + }); +}); + +// --- §2.2 eligibility ----------------------------------------------------------------- + +describe("eligibility (architecture.md §2.2)", () => { + const keep = effectiveKeepSet(); + + it("never vacates good-bye: no suffix matches and the stem carries a hyphen", () => { + expect(stemAndSuffix("good-bye")).toEqual(["good-bye", ""]); + expect(isEligible("good-bye", keep)).toBe(false); + // ...and it survives the real transform at p = 1, in the corpus's own spelling. + const out = vacateText("good-bye", mapFor(0), params({ seed: 0, p: 1 })); + expect(out).toBe("good-bye"); + }); + + it("never vacates don't", () => { + // architecture.md §2.2 says the `n't` suffix splits this to stem `do`, which then + // fails tests 1 and 3. It does not: §3's own rule requires `len(word) - len(s) >= 3` + // and `len("don't") - len("n't")` is 2, so NO suffix matches and the stem is the + // whole word — which then fails test 2 on the apostrophe. Same verdict, different + // clause; §3 is the operative rule and the source behaves this way too. + expect(stemAndSuffix("don't")).toEqual(["don't", ""]); + expect(isEligible("don't", keep)).toBe(false); + expect(isEligible("do", keep)).toBe(false); // and would fail tests 1 and 3 anyway + expect(vacateText("don't", mapFor(0), params({ seed: 0, p: 1 }))).toBe("don't"); + // A longer contraction DOES split, and its stem is what §2.2 describes. + expect(stemAndSuffix("couldn't")).toEqual(["could", "n't"]); + }); + + it("vacates dog's as 's", () => { + expect(stemAndSuffix("dog's")).toEqual(["dog", "'s"]); + expect(isEligible("dog", keep)).toBe(true); + const out = vacateText("dog's", mapFor(0), params({ seed: 0, p: 1 })); + expect(out).not.toBe("dog's"); + expect(out.endsWith("'s")).toBe(true); + expect(out.slice(0, -2)).toBe(mapFor(0).mapping.get("dog")); + }); + + it("rejects non-ASCII letters where Python's isalpha() would accept them", () => { + // Test 2 is `^[A-Za-z]+$`, not `str.isalpha()` — the difference is invisible in the + // shipped corpus and visible the moment someone pastes one. + expect(isEligible("café", keep)).toBe(false); + expect(isEligible("naïve", keep)).toBe(false); + }); + + it("respects an extra keep set", () => { + const extended = effectiveKeepSet(["Dog", "cat"]); + expect(isEligible("dog", extended)).toBe(false); + expect(isEligible("cat", extended)).toBe(false); + const vmap = buildVacancyMap(["dog", "cat", "hill"], params({ keep: ["dog"] })); + expect(vmap.mapping.has("dog")).toBe(false); + expect(vmap.mapping.has("hill")).toBe(true); + }); +}); + +// --- §4 nesting (SC-701) -------------------------------------------------------------- + +describe("SC-701 nesting: the vacated sets grow monotonically with p", () => { + it("nests across the p grid, for both seeds, on the real corpus", () => { + for (const seed of SEEDS) { + const sets = P_GRID.map((p) => vacatedStems(seed, p)); + for (let i = 0; i + 1 < sets.length; i++) { + for (const stem of sets[i]) { + expect(sets[i + 1].has(stem)).toBe(true); + } + expect(sets[i].size).toBeLessThan(sets[i + 1].size); + } + expect(sets[0].size).toBe(0); + expect(sets[sets.length - 1].size).toBe(mapFor(seed).mapping.size); + } + }); + + it("nests at the level of the rewritten text, not just the decision", () => { + // Whatever changed at p = 0.25 must still be changed, identically, at p = 0.5. + const low = tokenize(vacated(0, 0.25)); + const mid = tokenize(vacated(0, 0.5)); + const base = tokenize(CORPUS); + for (let i = 0; i < base.length; i++) { + if (low[i] !== base[i]) expect(mid[i]).toBe(low[i]); + } + }); + + it("u depends on (seed, stem) alone, so the two seeds disagree", () => { + expect(vacancyU("hill", 0)).toBe(vacancyU("HILL", 0)); + expect(vacancyU("hill", 0)).not.toBe(vacancyU("hill", 7)); + for (const seed of SEEDS) { + for (const stem of ["hill", "jack", "candle"]) { + const u = vacancyU(stem, seed); + expect(u).toBeGreaterThanOrEqual(0); + expect(u).toBeLessThan(1); + // 53-bit numerator over 2**53 — exactly representable, which is departure 2. + expect(Number.isInteger(u * 2 ** 53)).toBe(true); + } + } + }); +}); + +// --- §5 stability (SC-702) ------------------------------------------------------------ + +describe("SC-702 stability: a stem's nonce does not depend on p or on input order", () => { + it("mints the same nonce at every p where the stem is vacated", () => { + const seed = 0; + // Real corpus types whose stem is already vacated at the bottom of the grid, so every + // higher p must reproduce the identical surface form. + const low = vacatedStems(seed, 0.25); + const early = [...CORPUS_TYPES].filter((t) => low.has(stemAndSuffix(t)[0])).sort().slice(0, 300); + expect(early.length).toBeGreaterThan(100); + const reference = new Map(); + for (const p of [0.25, 0.5, 0.75, 1]) { + const surface = mapVocabWords(early, mapFor(seed), params({ seed, p })); + early.forEach((type, i) => { + const seen = reference.get(type); + if (seen === undefined) reference.set(type, surface[i]); + else expect(surface[i]).toBe(seen); + expect(surface[i]).not.toBe(type); + }); + } + }); + + it("is unchanged when the input type order is shuffled", () => { + // A deterministic shuffle: the assertion is about the map, not about randomness. + const shuffled = [...DOMAIN]; + let state = 123456789; + for (let i = shuffled.length - 1; i > 0; i--) { + state = (state * 1103515245 + 12345) >>> 0; + const j = state % (i + 1); + const tmp = shuffled[i]; + shuffled[i] = shuffled[j]; + shuffled[j] = tmp; + } + expect(shuffled).not.toEqual(DOMAIN); + const rebuilt = buildVacancyMap(shuffled, params({ seed: 0 })); + const original = mapFor(0); + expect(rebuilt.mapping.size).toBe(original.mapping.size); + for (const [stem, nonce] of original.mapping) expect(rebuilt.mapping.get(stem)).toBe(nonce); + expect(rebuilt.remintRounds).toBe(original.remintRounds); + }); + + it("is unchanged when the corpus is rewritten a second time", () => { + expect(vacateText(CORPUS, mapFor(0), params({ seed: 0, p: 0.5 }))).toBe(vacated(0, 0.5)); + }); +}); + +// --- §7.3 injectivity (SC-704) -------------------------------------------------------- + +describe("§5.2 the map is a pure function of (domain, seed, matchProsody)", () => { + it("builds the domain by the union rule, and refuses a text", () => { + expect(DOMAIN.length).toBe(CORPUS_TYPES.size + 22); + for (const w of BUDGET) expect(DOMAIN).toContain(w.toLowerCase()); + for (const t of CORPUS_TYPES) expect(DOMAIN).toContain(t); + // A string is itself an iterable of characters, so this would silently yield a domain + // of single letters — every one failing §2.2's length test, giving an empty map and a + // transform that does nothing, with no error anywhere. + expect(() => vacancyDomain(CORPUS)).toThrow(/expected an iterable of TYPES/); + expect(() => vacancyDomain("hello")).toThrow(/tokenize\(text\)/); + // Idempotent, and insensitive to case and duplicates in the input. + expect(vacancyDomain(DOMAIN)).toEqual(DOMAIN); + expect(vacancyDomain([...CORPUS_TYPES].map((t) => t.toUpperCase()))).toEqual(DOMAIN); + }); + + it("is byte-identical through two different call paths", () => { + // The whole point of removing `avoid`: the map can no longer depend on what a caller + // remembered to pass, so two call sites that build the domain differently — but to the + // same set — must produce the same map, key for key. + for (const seed of SEEDS) { + const viaHelper = buildVacancyMap(vacancyDomain(CORPUS_TYPES), params({ seed })); + // A different path to the same set: reversed, duplicated, upper-cased, budget first. + const scrambled = [ + ...BUDGET.map((w) => w.toUpperCase()), + ...[...CORPUS_TYPES].reverse(), + ...BUDGET, + ...[...CORPUS_TYPES].map((t) => t.toUpperCase()), + ]; + const viaScrambled = buildVacancyMap(vacancyDomain(scrambled), params({ seed })); + expect(viaScrambled.mapping.size).toBe(viaHelper.mapping.size); + for (const [stem, nonce] of viaHelper.mapping) expect(viaScrambled.mapping.get(stem)).toBe(nonce); + expect(viaScrambled.remintRounds).toBe(viaHelper.remintRounds); + expect(viaScrambled.imageSize).toBe(viaHelper.imageSize); + expect([...viaScrambled.mintedStress].sort()).toEqual([...viaHelper.mintedStress].sort()); + } + }); + + it("is identical across all five Dolch domains", () => { + // §5.2 measured this and says to assert it rather than rely on it: the domain rule is + // "always the FULL list" precisely so switching budgets cannot re-mint the corpus in + // front of the reader. A future change to the canonical order could break it silently. + const reference = mapFor(7); + for (const name of DOLCH_ORDER) { + const domain = vacancyDomain([...CORPUS_TYPES, ...dolchBudget(name)]); + const vmap = buildVacancyMap(domain, params({ seed: 7 })); + expect(vmap.mapping.get("gum")).toBe(reference.mapping.get("gum")); + expect(vmap.mapping.get("hang")).toBe(reference.mapping.get("hang")); + } + }); + + it("depends on seed and on matchProsody, and on nothing else", () => { + const a = buildVacancyMap(DOMAIN, params({ seed: 0 })); + expect(a.mapping.get("gum")).not.toBe(mapFor(7).mapping.get("gum")); + const flat = buildVacancyMap(DOMAIN, params({ seed: 0, matchProsody: false })); + let differ = 0; + for (const [stem, nonce] of flat.mapping) if (a.mapping.get(stem) !== nonce) differ++; + expect(differ).toBeGreaterThan(0); + // p and the other knobs are NOT inputs to the map — it is built once, for all p. + for (const p of P_GRID) { + const atP = buildVacancyMap(DOMAIN, params({ seed: 0, p, revealAfter: 3, consistent: false })); + for (const [stem, nonce] of a.mapping) expect(atP.mapping.get(stem)).toBe(nonce); + } + }); +}); + +describe("SC-704 injectivity on the real corpus", () => { + it("maps the type set one-to-one, verified rather than assumed", () => { + for (const seed of SEEDS) { + const vmap = mapFor(seed); + expect(vmap.bijective).toBe(true); + expect(vmap.imageSize).toBe(DOMAIN.length); + // Seed 0 needs no re-mint; seed 7 needs exactly one, for `hang` (see the regression + // test below). Both are measured facts about this corpus, not aspirations. + expect(vmap.remintRounds).toBe(seed === 7 ? 1 : 0); + // The nonces themselves are distinct — a weaker statement than the above, but the + // one the source's `used` set was supposed to guarantee. + expect(new Set(vmap.mapping.values()).size).toBe(vmap.mapping.size); + } + }); + + it("stays injective at EVERY p, where vacated and English types mix", () => { + // Conditions A and B of §5.2 are p-independent, so injectivity has to hold at every p + // and not only at full vacancy. This is the assertion the contract's §7.3 now makes. + for (const seed of SEEDS) { + const types = [...DOMAIN]; + for (const p of P_GRID) { + const image = mapVocabWords(types, mapFor(seed), params({ seed, p })); + expect(new Set(image).size).toBe(types.length); + } + } + }); + + it("re-mints the seed-7 hanged/waked collision away (regression)", () => { + // The named case that forced conditions A/B into the contract. At seed 7 the stem + // `hang` originally minted `wak`; no corpus type equals `wak`, so a bare-nonce check + // passed, and at p = 1 `waked` is itself vacated, so a full-vacancy check passed too. + // At p ∈ {0.25, 0.5} `hanged` was vacated and `waked` was not, so both became `waked`. + const vmap = mapFor(7); + expect(vmap.remintRounds).toBe(1); + expect(vmap.mapping.get("hang")).not.toBe("wak"); + // The re-mint is still prosody-matched — `smeeg` is monosyllabic like `hang` — which + // is the observable proving §5.5's thresholds are on the ATTEMPT counter and not on + // the absolute salt. On the absolute reading a re-mint from base salt 1001 would start + // with every check relaxed and the replacement could carry any syllable count. + expect(vmap.mapping.get("hang")).toBe("smeeg"); + expect(syllables("smeeg")).toBe(1); + expect(transformWord("hanged", vmap, params({ seed: 7, p: 1 }))).toBe("smeeged"); + // Condition B, stated directly: no assembled surface form is a domain type. + for (const t of DOMAIN) { + const [stem, suffix] = stemAndSuffix(t); + const nonce = vmap.mapping.get(stem); + if (nonce === undefined) continue; + const surface = mapVocabWords([t], vmap, params({ seed: 7, p: 1 }))[0]; + expect(DOMAIN.includes(surface)).toBe(false); + expect(surface.endsWith(suffix)).toBe(true); + } + // And the specific pair no longer meets, at the p where it used to. + for (const p of [0.25, 0.5]) { + const [hanged, waked] = mapVocabWords(["hanged", "waked"], vmap, params({ seed: 7, p })); + expect(hanged).not.toBe(waked); + expect(waked).toBe("waked"); // still English at this p — u(wak) = 0.571179 + } + }); + + it("maps a budget onto ids the pre-images had, order preserved (§7.2)", () => { + const seed = 0; + for (const p of P_GRID) { + const mapped = mapVocabWords(BUDGET, mapFor(seed), params({ seed, p })); + expect(mapped.length).toBe(BUDGET.length); + expect(new Set(mapped).size).toBe(BUDGET.length); + if (p === 0) expect(mapped).toEqual([...BUDGET]); + // Order is the contract: word i of the budget becomes word i of the mapped budget. + BUDGET.forEach((w, i) => { + const [stem] = stemAndSuffix(w); + const eligible = isEligible(stem, effectiveKeepSet()); + if (!eligible || !(vacancyU(stem, seed) < p)) expect(mapped[i]).toBe(w); + else expect(mapped[i]).not.toBe(w); + }); + } + }); +}); + +// --- FR-706 no minted form is a real word -------------------------------------------- + +describe("FR-706: a nonce never collides with a real corpus type", () => { + it("holds for every minted form at both seeds", () => { + for (const seed of SEEDS) { + for (const nonce of mapFor(seed).mapping.values()) { + expect(CORPUS_TYPES.has(nonce)).toBe(false); + } + } + }); + + it("holds for the rewritten corpus: no new type is an old type in disguise", () => { + // Every type of the vacated corpus that is NOT a type of the original must be minted, + // and every minted surface form must be absent from the original. + const after = new Set(tokenize(vacated(0, 1))); + const survivors = new Set(); + for (const t of after) if (CORPUS_TYPES.has(t)) survivors.add(t); + for (const t of survivors) { + const [stem] = stemAndSuffix(t); + // A surviving English type must be one the transform was never allowed to touch. + const untouchable = !isEligible(stem, effectiveKeepSet()); + expect(untouchable).toBe(true); + } + }); +}); + +// --- §1 segmentation ------------------------------------------------------------------ + +describe("§1 segmentation: the transform is a word-for-word bijection", () => { + it("emits a single complete WORD_RE match for every corpus type", () => { + const whole = new RegExp(`^(?:${WORD_RE.source})$`); + const surface = mapVocabWords([...CORPUS_TYPES], mapFor(0), params({ seed: 0, p: 1 })); + expect(surface.length).toBe(CORPUS_TYPES.size); + for (const w of surface) { + expect(whole.test(w)).toBe(true); + const re = new RegExp(WORD_RE.source, "g"); + expect(w.match(re)).toEqual([w]); + } + }); + + it("preserves the token count and ordering on the real corpus", () => { + const base = tokenize(CORPUS); + expect(base.length).toBe(corpusAsset.n_tokens); + for (const seed of SEEDS) { + for (const p of P_GRID) { + expect(tokenize(vacated(seed, p)).length).toBe(base.length); + } + } + }); + + it("preserves line structure, so the -per-line rule fires in the same places", () => { + const baseLines = splitLines(CORPUS); + for (const p of P_GRID) { + const lines = splitLines(vacated(0, p)); + expect(lines.length).toBe(baseLines.length); + lines.forEach((line, i) => { + expect(tokenize(line).length).toBe(tokenize(baseLines[i]).length); + }); + } + }); + + it("§5.7 commutes with lowercasing, over every type in three casings", () => { + // `lower(transformWord(w)) === transformWord(lower(w))`, normative. The first + // implementation sliced the suffix case-preserved and ran the seam test against it, so + // `gums -> flels` while `GUMS -> FLESS`: one source type, two surface forms, and since + // the tokenizer lowercases those are two different types — §7.3 false. + for (const seed of SEEDS) { + const vmap = mapFor(seed); + const cfg = params({ seed, p: 1 }); + for (const t of CORPUS_TYPES) { + const lowerImage = transformWord(t.toLowerCase(), vmap, cfg); + for (const cased of [t.toLowerCase(), t.toUpperCase(), t[0].toUpperCase() + t.slice(1)]) { + expect(transformWord(cased, vmap, cfg).toLowerCase()).toBe(lowerImage); + } + } + } + // The `gums` case by name, since it is the one that was wrong. + const vmap = mapFor(0); + const cfg = params({ seed: 0, p: 1 }); + expect(transformWord("GUMS", vmap, cfg).toLowerCase()).toBe(transformWord("gums", vmap, cfg)); + // ...and case is still carried, not discarded. + expect(transformWord("GUMS", vmap, cfg)).toBe(transformWord("gums", vmap, cfg).toUpperCase()); + const capital = transformWord("Gums", vmap, cfg); + expect(capital[0]).toBe(capital[0].toUpperCase()); + expect(capital.slice(1)).toBe(transformWord("gums", vmap, cfg).slice(1)); + }); + + it("commutes with lowercasing over the whole real corpus", () => { + expect(vacated(0, 1).toLowerCase()).toBe( + vacateText(CORPUS.toLowerCase(), mapFor(0), params({ seed: 0, p: 1 })), + ); + }); + + it("passes everything that is not a word through byte for byte", () => { + const out = vacated(0, 1); + const strip = (s: string): string => s.replace(new RegExp(WORD_RE.source, "g"), ""); + expect(strip(out)).toBe(strip(CORPUS)); + }); +}); + +// --- the endpoints of the sweep ------------------------------------------------------- + +describe("the endpoints of the p sweep", () => { + it("p = 0 is the identity, for both seeds", () => { + for (const seed of SEEDS) { + expect(vacated(seed, 0)).toBe(CORPUS); + } + }); + + it("p = 1 vacates every eligible type", () => { + const keep = effectiveKeepSet(); + const out = vacated(0, 1); + const before = tokenize(CORPUS); + const after = tokenize(out); + let eligibleTokens = 0; + for (let i = 0; i < before.length; i++) { + const [stem] = stemAndSuffix(before[i]); + if (!isEligible(stem, keep)) { + expect(after[i]).toBe(before[i]); + continue; + } + eligibleTokens++; + expect(after[i]).not.toBe(before[i]); + } + expect(eligibleTokens).toBeGreaterThan(0); + }); +}); + +// --- §6/§7.1 the control conditions --------------------------------------------------- + +describe("the control conditions really are different conditions", () => { + const seed = 0; + + it("consistent = false destroys type identity while holding the vacancy rate", () => { + const vmapA = buildVacancyMap(DOMAIN, params({ seed })); + const inconsistent = vacateText(CORPUS, vmapA, params({ seed, p: 1, consistent: false })); + const consistent = vacated(seed, 1); + expect(inconsistent).not.toBe(consistent); + // Same number of tokens, same tokens changed, far more distinct types. + expect(tokenize(inconsistent).length).toBe(tokenize(consistent).length); + const changed = (text: string): number => { + const base = tokenize(CORPUS); + const t = tokenize(text); + let n = 0; + for (let i = 0; i < base.length; i++) if (t[i] !== base[i]) n++; + return n; + }; + expect(changed(inconsistent)).toBe(changed(consistent)); + expect(new Set(tokenize(inconsistent)).size).toBeGreaterThan(new Set(tokenize(consistent)).size); + }); + + it("condition B applies to the per-occurrence path too — the seed-7 `tak` case", () => { + // §5.8. Condition B — no minted form may equal a domain type — was enforced when + // building the map and NOT on the `consistent = false` minting path. Observable at + // seed 7, `p = 1`: the stem `tak` (of `taking`) minted the nonce `tak`, so + // `Taking -> Taking` and one token silently failed to vacate — `corpusTypesVacated` + // 1921 against the consistent path's 1922, `tokensVacated` 8201 against 8202. + // + // §7.1 denies this control a STABILITY property, which is about a nonce being reused + // across occurrences; it does not license a word surviving the transform. A control + // whose vacancy rate is not the stated rate is not a control, so a per-occurrence nonce + // must equal neither a domain type nor the stem it replaces. + // + // `tak` is a stem, not a type, which is why the domain did not already forbid it: the + // domain holds the corpus's TYPES (`taking`, `takes`, …) plus the Dolch list. + expect(stemAndSuffix("taking")).toEqual(["tak", "ing"]); + expect(DOMAIN.includes("tak")).toBe(false); + expect(DOMAIN.includes("taking")).toBe(true); + + for (const s of SEEDS) { + // A fresh map per condition: `consistent = false` writes to `mintedStress`. + const cfg = params({ seed: s, p: 1, consistent: false }); + const vmap = buildVacancyMap(DOMAIN, cfg); + const text = vacateText(CORPUS, vmap, cfg); + const stats = vacancyStats(CORPUS, text, vmap, cfg); + // At `p = 1` every eligible type vacates (§10) — in this control exactly as in the + // mapped condition, which is the whole claim. + expect(stats.corpusTypesVacated, `seed ${s}`).toBe(1922); + expect(stats.corpusTypesEligible, `seed ${s}`).toBe(1922); + expect(stats.tokensVacated, `seed ${s}`).toBe(8202); + + // No eligible token survives the transform as itself. + const before = tokenize(CORPUS); + const after = tokenize(text); + const survivors: string[] = []; + for (let i = 0; i < before.length; i++) { + if (!isEligible(stemAndSuffix(before[i])[0], effectiveKeepSet([]))) continue; + if (after[i] === before[i]) survivors.push(before[i]); + } + expect(survivors, `seed ${s}`).toEqual([]); + } + }); + + it("revealAfter > 0 leaves the first occurrences in English", () => { + const revealed = vacateText(CORPUS, mapFor(seed), params({ seed, p: 1, revealAfter: 2 })); + const pure = vacated(seed, 1); + expect(revealed).not.toBe(pure); + const base = tokenize(CORPUS); + const r = tokenize(revealed); + const q = tokenize(pure); + let revealedUnchanged = 0; + let pureUnchanged = 0; + for (let i = 0; i < base.length; i++) { + if (r[i] === base[i]) revealedUnchanged++; + if (q[i] === base[i]) pureUnchanged++; + } + expect(revealedUnchanged).toBeGreaterThan(pureUnchanged); + }); + + it("matchProsody = false drops the syllable/stress match", () => { + const flat = buildVacancyMap(DOMAIN, params({ seed, matchProsody: false })); + const prosodic = mapFor(seed); + expect(flat.mapping.size).toBe(prosodic.mapping.size); + let differ = 0; + for (const [stem, nonce] of flat.mapping) if (prosodic.mapping.get(stem) !== nonce) differ++; + expect(differ).toBeGreaterThan(0); + // Every flat nonce is monosyllabic by construction; the prosodic ones are not. + for (const pattern of flat.mintedStress.values()) expect(pattern).toBe("1"); + const polysyllabic = [...prosodic.mintedStress.values()].filter((s) => s.length > 1).length; + expect(polysyllabic).toBeGreaterThan(0); + }); + + it("the two seeds are different assignments of the same instrument", () => { + expect(vacated(0, 1)).not.toBe(vacated(7, 1)); + expect(tokenize(vacated(0, 1)).length).toBe(tokenize(vacated(7, 1)).length); + }); +}); + +// --- §10 statistics ------------------------------------------------------------------- + +describe("§10 statistics", () => { + it("returns exactly the contracted field names", () => { + const stats = vacancyStats(CORPUS, vacated(0, 0.5), mapFor(0), params({ seed: 0, p: 0.5 })); + expect(Object.keys(stats).sort()).toEqual( + [ + "bijective", + "corpusTypesEligible", + "corpusTypesTotal", + "corpusTypesVacated", + "domainTypesEligible", + "domainTypesTotal", + "domainTypesVacated", + "imageSize", + "meanAnapestAfter", + "meanAnapestBefore", + "meanSyllablesAfter", + "meanSyllablesBefore", + "remintRounds", + "stemsTotal", + "stemsVacated", + "stressFromMintedAfter", + "stressFromMintedBefore", + "stressFromRuleAfter", + "stressFromRuleBefore", + "stressFromTableAfter", + "stressFromTableBefore", + "tokensTotal", + "tokensVacated", + ].sort(), + ); + }); + + it("forbids an unprefixed types* name, which is what let the two stacks diverge", () => { + // §10 now requires every type count to declare its scope. An unprefixed `typesTotal` + // read as "domain" in one stack and "corpus" in the other, and both were defensible. + const stats = vacancyStats(CORPUS, vacated(0, 0.5), mapFor(0), params({ seed: 0, p: 0.5 })); + for (const key of Object.keys(stats)) { + if (!key.startsWith("types")) continue; + throw new Error(`unprefixed type count ${JSON.stringify(key)} — §10 forbids it`); + } + expect(stats.domainTypesTotal).not.toBe(stats.corpusTypesTotal); + }); + + it("counts stems ACTUALLY vacated, not the size of the prebuilt map", () => { + // Departure 11: the zip copy reports `len(self.mapping)`, which is p-independent. + const full = mapFor(0).mapping.size; + const half = vacancyStats(CORPUS, vacated(0, 0.5), mapFor(0), params({ seed: 0, p: 0.5 })); + const none = vacancyStats(CORPUS, vacated(0, 0), mapFor(0), params({ seed: 0, p: 0 })); + expect(none.stemsVacated).toBe(0); + expect(none.domainTypesVacated).toBe(0); + expect(none.corpusTypesVacated).toBe(0); + expect(none.tokensVacated).toBe(0); + expect(half.stemsVacated).toBeGreaterThan(0); + expect(half.stemsVacated).toBeLessThan(full); + expect(half.stemsTotal).toBe(full); + // The vacancy rate tracks p over the eligible stems, which is what p means. + expect(half.stemsVacated / half.stemsTotal).toBeGreaterThan(0.4); + expect(half.stemsVacated / half.stemsTotal).toBeLessThan(0.6); + }); + + it("satisfies the p = 1 identities that exposed the counting gap", () => { + for (const seed of SEEDS) { + const s = vacancyStats(CORPUS, vacated(seed, 1), mapFor(seed), params({ seed, p: 1 })); + // u ∈ [0, 1) by construction, so at p = 1 everything eligible vacates — in BOTH + // scopes. These three identities are what caught the domain/corpus ambiguity. + expect(s.stemsVacated).toBe(s.stemsTotal); + expect(s.domainTypesVacated).toBe(s.domainTypesEligible); + expect(s.corpusTypesVacated).toBe(s.corpusTypesEligible); + // Inflected forms share a stem, so there are always at least as many types as stems. + expect(s.domainTypesEligible).toBeGreaterThanOrEqual(s.stemsTotal); + expect(s.domainTypesTotal).toBe(DOMAIN.length); + expect(s.corpusTypesTotal).toBe(CORPUS_TYPES.size); + } + }); + + it("separates the two scopes by exactly the 22 domain-only words", () => { + // The domain-only words are budget entries the reader never meets in the text, which + // is why the panel shows the CORPUS scope: counting them inflates the vacancy rate + // being reported to someone looking at that text. + const domainOnly = [...DOMAIN].filter((w) => !CORPUS_TYPES.has(w)); + expect(domainOnly.length).toBe(22); + expect(domainOnly).toContain("funny"); + expect(domainOnly).toContain("squirrel"); + expect(domainOnly).toContain("today"); + // All 22 happen to be eligible, so the two scopes differ by exactly 22. + const keep = effectiveKeepSet(); + expect(domainOnly.every((w) => isEligible(stemAndSuffix(w)[0], keep))).toBe(true); + for (const seed of SEEDS) { + const s = vacancyStats(CORPUS, vacated(seed, 1), mapFor(seed), params({ seed, p: 1 })); + expect(s.domainTypesEligible).toBe(s.corpusTypesEligible + 22); + expect(s.domainTypesTotal).toBe(s.corpusTypesTotal + 22); + } + }); + + it("measures corpusTypesVacated from the texts, not from map membership, under revealAfter", () => { + // The defect the golden fixture caught. A type whose every occurrence falls inside the + // reveal window is STILL LISTED IN THE MAP but has changed nowhere in the text, so map + // membership over-reports it. The two readings coincide at revealAfter = 0, which is + // why only a control condition exposed it. + const seed = 0; + const cfg = params({ seed, p: 1, revealAfter: 1 }); + const revealed = vacateText(CORPUS, mapFor(seed), cfg); + const s = vacancyStats(CORPUS, revealed, mapFor(seed), cfg); + + // The text-measured count, computed here independently of the implementation. + const beforeToks = tokenize(CORPUS); + const afterToks = tokenize(revealed); + const changed = new Set(); + for (let i = 0; i < beforeToks.length; i++) { + if (beforeToks[i] !== afterToks[i]) changed.add(beforeToks[i]); + } + expect(s.corpusTypesVacated).toBe(changed.size); + + // ...and it is strictly smaller than what map membership would have said, which is the + // number the domain scope still reports (deliberately — see below). + const byMap = [...CORPUS_TYPES].filter((t) => { + const [stem] = stemAndSuffix(t); + return isEligible(stem, effectiveKeepSet()) && vacancyU(stem, seed) < 1; + }).length; + expect(s.corpusTypesVacated).toBeLessThan(byMap); + expect(byMap).toBe(s.corpusTypesEligible); + }); + + it("pins the exact golden-fixture case the defect was found at: p = 0.7, revealAfter = 2", () => { + // The coordinate the two stacks split on, reproduced to the type. Map membership says + // 1337 and the texts say 665 — the 2x over-report, at the golden fixture's own p. + const seed = 0; + const cfg = params({ seed, p: 0.7, revealAfter: 2 }); + const s = vacancyStats(CORPUS, vacateText(CORPUS, mapFor(seed), cfg), mapFor(seed), cfg); + const byMap = [...CORPUS_TYPES].filter((t) => { + const [stem] = stemAndSuffix(t); + return isEligible(stem, effectiveKeepSet()) && vacancyU(stem, seed) < 0.7; + }).length; + expect(byMap).toBe(1337); + expect(s.corpusTypesVacated).toBe(665); + // The domain scope keeps the map reading, and so is unmoved by revealAfter. + expect(s.domainTypesVacated).toBe(1354); + }); + + it("agrees at revealAfter = 0 and diverges at revealAfter > 0", () => { + // The property that would have caught the defect, asserted directly. + for (const seed of SEEDS) { + const pure = params({ seed, p: 1, revealAfter: 0 }); + const held = params({ seed, p: 1, revealAfter: 1 }); + const sPure = vacancyStats(CORPUS, vacated(seed, 1), mapFor(seed), pure); + const sHeld = vacancyStats(CORPUS, vacateText(CORPUS, mapFor(seed), held), mapFor(seed), held); + + // At revealAfter = 0 the text reading and the map reading are the same number. + expect(sPure.corpusTypesVacated).toBe(sPure.corpusTypesEligible); + // At revealAfter > 0 the text-measured count is STRICTLY smaller... + expect(sHeld.corpusTypesVacated).toBeLessThan(sPure.corpusTypesVacated); + // ...while the domain scope, which reads map membership, does not move at all. + expect(sHeld.domainTypesVacated).toBe(sPure.domainTypesVacated); + // Tokens are text-measured in both, so they drop too — by one per revealed type. + expect(sHeld.tokensVacated).toBeLessThan(sPure.tokensVacated); + } + }); + + it("splits stress three ways, summing to 1 on each side", () => { + for (const p of P_GRID) { + const s = vacancyStats(CORPUS, vacated(0, p), mapFor(0), params({ seed: 0, p })); + expect(s.stressFromTableBefore + s.stressFromMintedBefore + s.stressFromRuleBefore).toBeCloseTo(1, 12); + expect(s.stressFromTableAfter + s.stressFromMintedAfter + s.stressFromRuleAfter).toBeCloseTo(1, 12); + // The original corpus contains no minted form — `avoid` and condition B guarantee it. + expect(s.stressFromMintedBefore).toBe(0); + if (p > 0) expect(s.stressFromMintedAfter).toBeGreaterThan(0); + } + }); + + it("agrees with the corpus manifest the backend wrote", () => { + const stats = vacancyStats(CORPUS, CORPUS, mapFor(0), DEFAULT_VACANCY_PARAMS); + expect(stats.tokensTotal).toBe(corpusAsset.n_tokens); + // The manifest counts the CORPUS, so that is the scope that has to match it. + expect(stats.corpusTypesTotal).toBe(corpusAsset.n_distinct); + expect(stats.domainTypesTotal).toBe(DOMAIN.length); + }); + + it("reports the measured numbers on the real corpus", () => { + const rows: string[] = []; + for (const seed of SEEDS) { + for (const p of P_GRID) { + const s = vacancyStats(CORPUS, vacated(seed, p), mapFor(seed), params({ seed, p })); + rows.push( + [ + `seed=${seed}`, + `p=${p.toFixed(2)}`, + `domainTypes=${s.domainTypesTotal}/${s.domainTypesEligible}/${s.domainTypesVacated}`, + `corpusTypes=${s.corpusTypesTotal}/${s.corpusTypesEligible}/${s.corpusTypesVacated}`, + `stemsTotal=${s.stemsTotal}`, + `stemsVacated=${s.stemsVacated}`, + `tokensTotal=${s.tokensTotal}`, + `tokensVacated=${s.tokensVacated}`, + `syl=${s.meanSyllablesBefore.toFixed(4)}->${s.meanSyllablesAfter.toFixed(4)}`, + `anapest=${s.meanAnapestBefore.toFixed(4)}->${s.meanAnapestAfter.toFixed(4)}`, + `table=${s.stressFromTableBefore.toFixed(4)}->${s.stressFromTableAfter.toFixed(4)}`, + `minted=${s.stressFromMintedBefore.toFixed(4)}->${s.stressFromMintedAfter.toFixed(4)}`, + `rule=${s.stressFromRuleBefore.toFixed(4)}->${s.stressFromRuleAfter.toFixed(4)}`, + `bijective=${s.bijective}`, + `imageSize=${s.imageSize}`, + `remintRounds=${s.remintRounds}`, + ].join(" "), + ); + expect(s.bijective).toBe(true); + } + } + // eslint-disable-next-line no-console + console.log(["", "vacancy transform, measured on The Real Mother Goose:", ...rows, ""].join("\n")); + expect(rows.length).toBe(SEEDS.length * P_GRID.length); + }); +}); + +// --- `forbidden`, §5.8 ---------------------------------------------------------------- + +describe("`forbidden` is stored, and keeps superseded re-mint nonces (§5.8)", () => { + it("carries `wak`, the nonce seed 7's re-mint of `hang` replaced", () => { + // THE CASE THAT DISTINGUISHES a stored set from `domain ∪ mapping.values()`, and the + // only one the shipped corpus produces: at seed 7 the stem `hang` first minted `wak`, + // whose surface `wak` + `ed` is the real English word `waked`; condition B rejected it + // and the re-mint returned `smeeg`. `wak` is now no stem's nonce, so a reconstruction + // drops it — but it must stay forbidden, because it was rejected for cause and the + // `consistent = false` control draws against this very set. + const vmap = mapFor(7); + expect(vmap.mapping.get("hang")).toBe("smeeg"); + expect(vmap.remintRounds).toBe(1); + expect(vmap.forbidden.has("wak")).toBe(true); + expect([...vmap.mapping.values()]).not.toContain("wak"); // what a rebuild would lose + expect(vmap.domain.has("waked")).toBe(true); // ... and why it was superseded + }); + + it("contains the whole domain and every nonce, at both seeds", () => { + for (const seed of SEEDS) { + const vmap = mapFor(seed); + for (const t of vmap.domain) expect(vmap.forbidden.has(t)).toBe(true); + for (const n of vmap.mapping.values()) expect(vmap.forbidden.has(n)).toBe(true); + expect(vmap.forbidden.size).toBe( + vmap.domain.size + new Set(vmap.mapping.values()).size + (seed === 7 ? 1 : 0), + ); + } + }); + + it("keeps the superseded nonce out of the inconsistent control's output", () => { + const p = params({ p: 1, seed: 7, consistent: false }); + const text = vacateText(CORPUS, buildVacancyMap(DOMAIN, p), p); + expect(new Set(tokenize(text)).has("wak")).toBe(false); + }); +}); + +// --- the swap control, §8.3 / §5.2a --------------------------------------------------- + +const COUNTS = typeCounts(tokenize(CORPUS)); +const SWAP_MAPS = new Map( + SEEDS.map((seed) => [seed, buildVacancyMap(DOMAIN, params({ seed, mint: "swap" }), COUNTS)]), +); +function swapMapFor(seed: number): VacancyMap { + const m = SWAP_MAPS.get(seed); + if (m === undefined) throw new Error(`no swap map for seed ${seed}`); + return m; +} + +describe("mint = 'swap' draws a real English word (§8.3)", () => { + it("replaces every stem with a word the corpus or the budget already had", () => { + const real = new Set([...CORPUS_TYPES, ...BUDGET.map((w) => w.toLowerCase())]); + for (const seed of SEEDS) { + const vmap = swapMapFor(seed); + expect(vmap.mapping.size).toBeGreaterThan(0); + for (const [stem, word] of vmap.mapping) { + expect(real.has(word), `${seed}: ${stem} -> ${word}`).toBe(true); + expect(word).not.toBe(stem); // a stem keeping its form is a word that failed to vacate + } + } + }); + + it("needs the frequency counts, and says so rather than ranking alphabetically", () => { + expect(() => buildVacancyMap(DOMAIN, params({ mint: "swap" }))).toThrow(/type counts/); + }); + + it("refuses the inconsistent control — there is no supply of fresh real words", () => { + expect(() => + buildVacancyMap(DOMAIN, params({ mint: "swap", consistent: false }), COUNTS), + ).toThrow(/consistent/); + }); + + it("leaves the nonce map a pure function of (domain, seed, matchProsody)", () => { + for (const seed of SEEDS) { + const withCounts = buildVacancyMap(DOMAIN, params({ seed }), COUNTS); + expect([...withCounts.mapping]).toEqual([...mapFor(seed).mapping]); + expect(withCounts.remintRounds).toBe(mapFor(seed).remintRounds); + } + }); + + it("registers no minted stress — the replacements are real English words", () => { + for (const seed of SEEDS) expect(swapMapFor(seed).mintedStress.size).toBe(0); + }); + + it("is stable in (seed, stem): rebuilding gives the same map (SC-702)", () => { + for (const seed of SEEDS) { + const again = buildVacancyMap(DOMAIN, params({ seed, p: 1, mint: "swap" }), COUNTS); + expect(mappingEntries(again)).toEqual(mappingEntries(swapMapFor(seed))); + } + }); + + it("is nested in `p` — the `u(stem) < p` decision is untouched (SC-701)", () => { + let previous = new Set(); + for (const p of P_GRID) { + const text = vacateText(CORPUS, swapMapFor(0), params({ p, seed: 0, mint: "swap" })); + const changed = changedTypes(CORPUS, text); + for (const t of previous) expect(changed.has(t), `p=${p}: ${t}`).toBe(true); + previous = changed; + } + }); + + it("is a bijection of the domain at full vacancy (A + B₁ of §5.2a)", () => { + for (const seed of SEEDS) { + const vmap = swapMapFor(seed); + expect(vmap.bijective).toBe(true); + expect(vmap.imageSize).toBe(vmap.domain.size); + expect(vmap.remintRounds).toBe(0); + expect(vmap.injectiveAtEveryP).toBe(false); + } + }); +}); + +describe("the invariance theorem under mint = 'swap' (SC-703 / §5.2a)", () => { + it("holds at p ∈ {0, 1}, exactly as it does for mint = 'nonce'", () => { + const base = new LexVocab(BUDGET, "dolch", "full"); + const reference = base.encode(tokenize(CORPUS)); + for (const seed of SEEDS) { + for (const p of [0, 1]) { + const ps = params({ p, seed, mint: "swap" }); + const vmap = swapMapFor(seed); + const text = vacateText(CORPUS, vmap, ps); + const words = mapVocabWords(BUDGET, vmap, ps); + expect(new Set(words).size).toBe(words.length); + const mapped = new LexVocab(words, "dolch", "full"); + expect(mapped.rows).toBe(base.rows); + expect(mapped.encode(tokenize(text))).toEqual(reference); + } + } + }); + + it("refuses the mapped vocabulary at intermediate p rather than duplicating a row", () => { + // §5.2a: no `p`-stable map whose images are domain types is injective at 0 < p < 1 + // unless it is the identity. That is a theorem, not a defect to be re-drawn away, so + // the mapped vocabulary is refused exactly as it is for the two controls. + for (const p of [0.25, 0.5, 0.75]) { + expect(() => + mapVocabWords(BUDGET, swapMapFor(0), params({ p, seed: 0, mint: "swap" })), + ).toThrow(/full vacancy/); + } + }); + + it("measures WHY: a vacated type lands on a word that has not moved", () => { + // Pinned so the refusal above can never be mistaken for over-caution. If a future change + // makes swap injective at p = 0.5, this fails and the contract is wrong. + const vmap = swapMapFor(0); + const ps = params({ p: 0.5, seed: 0, mint: "swap" }); + const images = new Map(); + let collisions = 0; + for (const t of [...vmap.domain].sort()) { + const image = transformWord(t, vmap, ps).toLowerCase(); + if (images.has(image)) collisions++; + images.set(image, t); + } + expect(collisions).toBeGreaterThan(0); + + const full = params({ p: 1, seed: 0, mint: "swap" }); + const atOne = new Set([...vmap.domain].map((t) => transformWord(t, vmap, full).toLowerCase())); + expect(atOne.size).toBe(vmap.domain.size); + }); +}); diff --git a/code/frontend/tests/unit/vacancyGolden.test.ts b/code/frontend/tests/unit/vacancyGolden.test.ts new file mode 100644 index 0000000..c8b978b --- /dev/null +++ b/code/frontend/tests/unit/vacancyGolden.test.ts @@ -0,0 +1,666 @@ +/** + * Golden-vector parity for the VACANCY TRANSFORM: the TypeScript engine vs the real + * Python one, on the real committed corpus. + * + * `specs/007-vacancy-transform-field/architecture.md` §11 requires this file and names + * exactly what it must pin. SC-706 requires the two implementations to agree — strings and + * id streams EXACTLY, floats within `tolerance`. This test is that proof, and it follows + * `lexGolden.test.ts` (feature 006) and `geoEngine.test.ts` (feature 003): a Python script + * runs the REAL backend and writes what it measured; this file runs the REAL browser + * engine against the same inputs and asserts. + * + * The fixture is `tests/fixtures/vacancy-golden.json`, regenerated by + * + * python scripts/export_vacancy_golden.py + * + * and it records the git sha, the command, the corpus digest and each case's parameters. + * `TOLERANCE` is read FROM the file rather than declared here, so the exporter and this + * test cannot drift apart — the same discipline `lexGolden.test.ts` keeps. + * + * WHAT IS PINNED, per §11: + * + * 1. `vacancyU` for 24 stems spanning eligible/ineligible and both budgets, compared as + * the EXACT double. That exactness is the point of departure 2 (§4): `u` shifts a + * 64-bit digest right by 11 before dividing by 2^53, so the numerator is exactly + * representable and Python and JavaScript get THE SAME double rather than two + * neighbours that straddle a `p` boundary. + * 2. The FULL stem -> nonce map at seed 0 and seed 7 — every pair, both directions of + * the size check, plus `remintRounds` / `bijective` / `imageSize`. + * 3. The first 400 characters of the vacated corpus at each `p`, exactly; plus the + * sha256 of the WHOLE 86 kB rewrite, which is the same assertion at 64 bytes. + * 4. Every §10 statistic, at `p in {0, 0.25, 0.35, 0.5, 0.7, 0.75, 1}` and both seeds. + * 5. Nesting as explicit SETS: `vacated(0.35) ⊆ vacated(0.7) ⊆ vacated(1)`, checked + * against the literal stem lists rather than against an assertion this language makes + * about itself (SC-701). + * 6. Stability: each pinned stem's surface form at every `p`, and — derived across the + * cases — the fact that it is byte-identical at every `p` where the stem is vacated + * (SC-702). + * 7. The token-id-stream digest under the mapped vocabulary at each `p`. All of them are + * EQUAL, across both seeds and both `matchProsody` settings: the invariance theorem + * of §7.3 / SC-703 pinned as DATA rather than as an assertion written twice. + * 8. Both control conditions and both `matchProsody` settings. + * + * COMPARISON RULE. Strings, digests, counts, booleans and `u` are compared with `toBe` — + * exact. Only the six prosody means (`meanSyllables*`, `meanAnapest*`, `stressFrom*`) use + * `tolerance`, and the measured worst deviation is printed at the end of the run so a + * silent creep toward the bound is visible. + * + * NO KNOWN DIVERGENCES REMAIN. Every case here — both seeds, both `matchProsody` settings, + * every `p`, and all three controls — agrees field for field, string for string, digest for + * digest. Three earlier disagreements were pinned in this file rather than hidden, and each + * was retired by fixing the stack the contract said was wrong, never by loosening a + * comparison: + * + * 1. `control-reveal-after-2`: `corpusTypesVacated` read 665 in Python and 1337 here, + * because Python MEASURED the count from the two texts while this engine computed it + * through the map. §10 defines `corpusTypes*` as what the panel shows a READER — i.e. + * what the text does — so the measured reading was the contract's. `vacancyStats` now + * measures the texts; `domainTypes*` keeps map membership, which §10 also requires. + * 2. `control-inconsistent`: the two stacks minted DIFFERENT per-occurrence nonces. §5.8 + * pins the key as `${stem}#${idx}` and §5.5's mint read its stress pattern off the + * string it was handed, so Python got `stress("little#0") = "10"` where this engine + * gets `stress("little") = "100"` — `Little` minted as `Wrerken` there and `Wrerkenle` + * here. §7.1 says the nonce carries THE STEM's syllable count and stress, so the key + * must not reach the prosody lookup; Python's `_mint` now takes `stem` separately. + * 3. Condition B on the per-occurrence path (§5.8). It was enforced when building the map + * and not in the `consistent = false` control, so at seed 7, `p = 1`, the stem `tak` + * minted the nonce `tak` and `Taking -> Taking`: one token silently failed to vacate. + * Both stacks now forbid a per-occurrence nonce from equalling the stem it replaces as + * well as any domain type, and `control-inconsistent-seed7` pins the result — 1922 + * corpus types and 8202 tokens vacated, exactly what `consistent = true` reads. + * + * The `knownDivergence` mechanism itself stays: it carries both readings per field, this + * test asserts each side against its own, and `attach_divergence` in the exporter refuses to + * write a fixture at all once Python starts agreeing with a recorded TypeScript value. That + * guard is what retired all three entries above, and it is what an outstanding cross-stack + * defect would use again. + */ + +import { afterAll, describe, expect, it } from "vitest"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { LexVocab, dolchBudget, tokenize } from "../../src/lib/lexEngine"; +import { tokenStream } from "../../src/lib/lexEngine/train"; +import { + buildVacancyMap, + effectiveKeepSet, + isEligible, + mapVocabWords, + stemAndSuffix, + transformWord, + typeCounts, + vacancyDomain, + vacancyParams, + vacancyStats, + vacancyU, + vacateText, + type VacancyMap, + type VacancyParams, + type VacancyStats, +} from "../../src/lib/lexEngine/vacancy"; +import type { LexCorpusAsset } from "../../src/lib/staticClient/lex"; + +// --- the fixture ---------------------------------------------------------------------- + +interface GoldenStem { + stem: string; + eligible: boolean; + stemOf: string; + suffixOf: string; + inDomain: boolean; + inCorpus: boolean; + inDolchFull: boolean; + u: Record; +} + +interface GoldenMap { + label: string; + seed: number; + matchProsody: boolean; + mint: "nonce" | "swap"; + injectiveAtEveryP: boolean; + remintRounds: number; + bijective: boolean; + imageSize: number; + domainSize: number; + mappingSize: number; + mappingSha256: string; + mapping: Record | null; + sampleNonces?: Record; +} + +interface GoldenIdStream { + digest: string; + length: number; + first16: number[]; + last16: number[]; + mappedWordsSha256: string; +} + +/** One field the two stacks disagree on, with BOTH readings — see the header. */ +interface GoldenDivergentField { + /** Either a case key (`head400`, `vacatedSha256`, `vacatedChars`) or `stats.`. */ + field: string; + python: string | number; + typescript: string | number; +} + +interface GoldenDivergence { + cause: string; + status: string; + fields: GoldenDivergentField[]; +} + +interface GoldenCase { + label: string; + map: string; + params: { + p: number; + seed: number; + consistent: boolean; + matchProsody: boolean; + revealAfter: number; + keep: string[]; + mint: "nonce" | "swap"; + }; + head400: string; + vacatedSha256: string; + vacatedChars: number; + stats: Record; + stemForms: Record | null; + idStream: GoldenIdStream | null; + mapVocabWordsRejects: boolean; + knownDivergence?: GoldenDivergence; +} + +interface Golden { + format: string; + generated: string; + git_sha: string; + command: string; + contract: string; + tolerance: number; + corpus: { + sha256: string; + chars: number; + tokens: number; + corpusTypes: number; + domainSize: number; + budget: string; + budgetSize: number; + }; + stems: GoldenStem[]; + maps: GoldenMap[]; + cases: GoldenCase[]; + nesting: { seed: number; map: string; levels: { p: number; stems: string[] }[] }; +} + +const FIXTURE = path.resolve(__dirname, "../fixtures/vacancy-golden.json"); +const golden = JSON.parse(fs.readFileSync(FIXTURE, "utf-8")) as Golden; + +/** The contract's tolerance, read from the file so the two cannot drift (see the header). */ +const TOLERANCE = golden.tolerance; + +// --- the real corpus, the same bytes both stacks use ---------------------------------- + +const CORPUS_ASSET = path.resolve(__dirname, "../../public/static-data/lex/corpus.json"); +const CORPUS = (JSON.parse(fs.readFileSync(CORPUS_ASSET, "utf-8")) as LexCorpusAsset).text; +const CORPUS_TYPES = new Set(tokenize(CORPUS)); +const DOMAIN = vacancyDomain(CORPUS_TYPES); +/** The corpus's per-type occurrence counts — the frequency source `mint = "swap"` ranks by. */ +const COUNTS = typeCounts(tokenize(CORPUS)); +const BUDGET_WORDS = dolchBudget("full"); + +const sha256 = (s: string): string => createHash("sha256").update(s, "utf8").digest("hex"); + +/** The exporter's canonical form for a map digest: `stem\tnonce\n`, stems ASCII-ascending. */ +function mappingSha256(mapping: ReadonlyMap): string { + const stems = [...mapping.keys()].sort(); + return sha256(stems.map((s) => `${s}\t${mapping.get(s)}\n`).join("")); +} + +function paramsOf(c: GoldenCase): VacancyParams { + return vacancyParams({ + p: c.params.p, + seed: c.params.seed, + consistent: c.params.consistent, + matchProsody: c.params.matchProsody, + revealAfter: c.params.revealAfter, + keep: c.params.keep, + mint: c.params.mint, + }); +} + +/** + * A FRESH map per case, always. Under `consistent = false` the rewrite writes the stress + * pattern of every form it mints onto `vmap.mintedStress`, so a map shared between cases + * would score one case's text against another case's minted patterns. The exporter builds + * fresh maps for the controls for the same reason; doing it for every case here also + * proves that reuse is irrelevant in the mapped condition. + */ +function mapFor(params: VacancyParams): VacancyMap { + // `typeCounts` is passed unconditionally. `mint = "nonce"` ignores it — the nonce map is a + // pure function of `(domain, seed, matchProsody)`, and every pre-existing case in this + // fixture proves it by still matching — while `mint = "swap"` needs it and throws without + // it. Passing it always is what keeps the two call sites from differing by what one of + // them remembered, which is §5.2's whole objection to an optional `avoid`. + return buildVacancyMap(DOMAIN, params, COUNTS); +} + +/** The fixture's own reading of a field a `knownDivergence` block names. */ +function goldenField(gc: GoldenCase, field: string): string | number | boolean { + if (field.startsWith("stats.")) { + const name = field.slice("stats.".length); + const value = gc.stats[name]; + if (value === undefined) throw new Error(`vacancy-golden.json: no stat ${name}`); + return value; + } + if (field === "head400") return gc.head400; + if (field === "vacatedSha256") return gc.vacatedSha256; + if (field === "vacatedChars") return gc.vacatedChars; + throw new Error(`vacancy-golden.json: unknown divergence field ${field}`); +} + +/** Every float in this fixture is order-1 or smaller, so absolute deviation is the honest read. */ +const deviations: { what: string; dev: number }[] = []; +function expectClose(actual: number, expected: number, what: string): void { + const dev = Math.abs(actual - expected); + deviations.push({ what, dev }); + expect(dev, `${what}: ${actual} vs golden ${expected}`).toBeLessThanOrEqual(TOLERANCE); +} + +/** The §10 fields that are exact integers or booleans, and must compare with `toBe`. */ +const EXACT_STATS = [ + "domainTypesTotal", + "domainTypesEligible", + "domainTypesVacated", + "corpusTypesTotal", + "corpusTypesEligible", + "corpusTypesVacated", + "stemsTotal", + "stemsVacated", + "tokensTotal", + "tokensVacated", + "bijective", + "imageSize", + "remintRounds", +] as const; + +/** The §10 fields that are means, and are the ONLY things `tolerance` applies to. */ +const FLOAT_STATS = [ + "meanSyllablesBefore", + "meanSyllablesAfter", + "meanAnapestBefore", + "meanAnapestAfter", + "stressFromTableBefore", + "stressFromTableAfter", + "stressFromMintedBefore", + "stressFromMintedAfter", + "stressFromRuleBefore", + "stressFromRuleAfter", +] as const; + +// --- provenance ----------------------------------------------------------------------- + +describe("the fixture is the one this test was written against", () => { + it("declares its format, contract, tolerance and provenance", () => { + expect(golden.format).toBe("vacancy-golden-v2"); + expect(golden.contract).toBe("specs/007-vacancy-transform-field/architecture.md"); + expect(golden.command).toBe("python scripts/export_vacancy_golden.py"); + expect(golden.git_sha).toMatch(/^[0-9a-f]{40}$/); + expect(golden.generated).toMatch(/^\d{4}-\d{2}-\d{2}$/); + expect(TOLERANCE).toBeGreaterThan(0); + expect(TOLERANCE).toBeLessThanOrEqual(1e-5); + }); + + it("was generated from the same corpus bytes this engine is reading", () => { + // Not a formality: every nonce, every count and every digest below is a function of + // these bytes, so a fixture built from a different corpus would fail everywhere at + // once with no indication of why. + expect(sha256(CORPUS)).toBe(golden.corpus.sha256); + expect(CORPUS.length).toBe(golden.corpus.chars); + expect(tokenize(CORPUS).length).toBe(golden.corpus.tokens); + expect(CORPUS_TYPES.size).toBe(golden.corpus.corpusTypes); + expect(DOMAIN.length).toBe(golden.corpus.domainSize); + expect(golden.corpus.budget).toBe("dolch/full"); + expect(BUDGET_WORDS.length).toBe(golden.corpus.budgetSize); + }); +}); + +// --- 1. u(stem), exactly --------------------------------------------------------------- + +describe("u(stem) is THE SAME double in both languages (§4, departure 2)", () => { + const keep = effectiveKeepSet([]); + + it("pins 24 stems spanning eligible/ineligible and both budgets", () => { + expect(golden.stems).toHaveLength(24); + // The span §11 asks for is a property of the list, so it is asserted rather than + // assumed: a fixture of 24 ordinary words would pin nothing about the eligibility test. + expect(golden.stems.some((s) => s.eligible)).toBe(true); + expect(golden.stems.some((s) => !s.eligible)).toBe(true); + expect(golden.stems.some((s) => s.inDolchFull)).toBe(true); + expect(golden.stems.some((s) => s.inCorpus && !s.inDolchFull)).toBe(true); + expect(golden.stems.some((s) => !s.inCorpus && s.inDolchFull)).toBe(true); + }); + + for (const entry of golden.stems) { + it(`${entry.stem}: exact u at both seeds, and the same eligibility verdict`, () => { + // `toBe` on a double is deliberate. The `>> 11` in `vacancyU` exists precisely so + // that this is bit-exact; a tolerance here would hide the bug it was added to fix. + expect(vacancyU(entry.stem, 0)).toBe(entry.u["0"]); + expect(vacancyU(entry.stem, 7)).toBe(entry.u["7"]); + const [stem, suffix] = stemAndSuffix(entry.stem); + expect(stem).toBe(entry.stemOf); + expect(suffix).toBe(entry.suffixOf); + expect(isEligible(stem, keep)).toBe(entry.eligible); + expect(DOMAIN.includes(entry.stem)).toBe(entry.inDomain); + expect(CORPUS_TYPES.has(entry.stem)).toBe(entry.inCorpus); + expect(BUDGET_WORDS.includes(entry.stem)).toBe(entry.inDolchFull); + }); + } +}); + +// --- 2. the maps ------------------------------------------------------------------------ + +describe("the stem -> nonce map (§5.2)", () => { + for (const gm of golden.maps) { + it(`${gm.label}: reproduces Python's assignment`, () => { + const params = vacancyParams({ + seed: gm.seed, + matchProsody: gm.matchProsody, + mint: gm.mint, + // The control labels differ from the mapped condition only in knobs that do not + // reach minting, so the map is a pure function of (domain, seed, matchProsody); + // building it from the seed and the flag alone is itself part of the claim. + }); + const vmap = mapFor(params); + expect(vmap.remintRounds).toBe(gm.remintRounds); + expect(vmap.bijective).toBe(gm.bijective); + expect(vmap.injectiveAtEveryP).toBe(gm.injectiveAtEveryP); + expect(vmap.imageSize).toBe(gm.imageSize); + expect(vmap.domain.size).toBe(gm.domainSize); + expect(vmap.mapping.size).toBe(gm.mappingSize); + expect(mappingSha256(vmap.mapping)).toBe(gm.mappingSha256); + + if (gm.mapping !== null) { + // The full map, pair by pair — §11 asks for every one at seeds 0 and 7. Both + // directions, so neither a missing stem nor an extra one can pass. + const expectedStems = Object.keys(gm.mapping); + expect(expectedStems).toHaveLength(vmap.mapping.size); + const mismatched: string[] = []; + for (const [stem, nonce] of Object.entries(gm.mapping)) { + if (vmap.mapping.get(stem) !== nonce) { + mismatched.push(`${stem}: ${vmap.mapping.get(stem)} != ${nonce}`); + } + } + expect(mismatched).toEqual([]); + for (const stem of vmap.mapping.keys()) expect(gm.mapping[stem]).toBeDefined(); + } + if (gm.sampleNonces !== undefined) { + for (const [stem, nonce] of Object.entries(gm.sampleNonces)) { + expect(vmap.mapping.get(stem) ?? null).toBe(nonce); + } + } + }); + } +}); + +// --- 3-4, 6-8. the measured cases ------------------------------------------------------- + +describe("the vacated corpus, its statistics and its id stream", () => { + for (const gc of golden.cases) { + it(`${gc.label}: matches Python`, () => { + const params = paramsOf(gc); + const vmap = mapFor(params); + // Order matters and it is Python's: the rewrite runs first, because under + // `consistent = false` it registers the minted stress patterns that the statistics + // then score the vacated side with. + const vacated = vacateText(CORPUS, vmap, params); + const stats: VacancyStats = vacancyStats(CORPUS, vacated, vmap, params); + + // A field the fixture flags as divergent is asserted against the MEASURED TypeScript + // reading, AND its Python reading is asserted to still differ. Two exact assertions, + // not a widened tolerance and not a skip: either stack changing fails one of them. + const diverging = new Map((gc.knownDivergence?.fields ?? []).map((f) => [f.field, f])); + const consumed = new Set(); + function reference(field: string): string | number | boolean { + const d = diverging.get(field); + if (d === undefined) return goldenField(gc, field); + consumed.add(field); + expect(goldenField(gc, field), `${gc.label}.${field} (python reading)`).toBe(d.python); + expect(d.python).not.toBe(d.typescript); + return d.typescript; + } + + // §11: strings compare EXACTLY. + expect(vacated.slice(0, 400), `${gc.label}.head400`).toBe(reference("head400")); + expect(vacated.length, `${gc.label}.vacatedChars`).toBe(reference("vacatedChars")); + expect(sha256(vacated), `${gc.label}.vacatedSha256`).toBe(reference("vacatedSha256")); + + for (const field of EXACT_STATS) { + expect(stats[field], `${gc.label}.${field}`).toBe(reference(`stats.${field}`)); + } + for (const field of FLOAT_STATS) { + expectClose(stats[field], reference(`stats.${field}`) as number, `${gc.label}.${field}`); + } + // No stale exemption may sit in the fixture unread. + expect([...consumed].sort()).toEqual([...diverging.keys()].sort()); + // The three-way stress split is token-weighted and sums to 1 on each side (§10). + expectClose( + stats.stressFromTableAfter + stats.stressFromMintedAfter + stats.stressFromRuleAfter, + 1, + `${gc.label}.stressSplitAfter sums to 1`, + ); + + // §11 stability: the surface form of each pinned stem at this `p`. + if (gc.stemForms !== null) { + for (const [stem, form] of Object.entries(gc.stemForms)) { + expect(transformWord(stem, vmap, params), `${gc.label}.stemForms.${stem}`).toBe(form); + } + } + + // §7.2 / §7.3: the id stream under the MAPPED vocabulary. + if (gc.idStream === null) { + expect(gc.mapVocabWordsRejects).toBe(true); + // Both stacks refuse: a mapped vocabulary in a control condition would be a word + // list matching no corpus, built without error. + expect(() => mapVocabWords(BUDGET_WORDS, vmap, params)).toThrow(); + } else { + expect(gc.mapVocabWordsRejects).toBe(false); + const mapped = mapVocabWords(BUDGET_WORDS, vmap, params); + expect(mapped).toHaveLength(BUDGET_WORDS.length); + expect(sha256(mapped.join("\n"))).toBe(gc.idStream.mappedWordsSha256); + const ids = tokenStream(vacated, new LexVocab(mapped, "dolch", "full")); + expect(ids.length).toBe(gc.idStream.length); + expect(ids.slice(0, 16)).toEqual(gc.idStream.first16); + expect(ids.slice(-16)).toEqual(gc.idStream.last16); + expect(sha256(ids.join(","))).toBe(gc.idStream.digest); + } + }); + } +}); + +// --- 5. nesting, as explicit sets (SC-701) --------------------------------------------- + +describe("nesting is a structural fact, checked against explicit sets (SC-701)", () => { + const gn = golden.nesting; + const vmap = mapFor(vacancyParams({ seed: gn.seed })); + + function vacatedStems(p: number): string[] { + const out: string[] = []; + for (const stem of vmap.mapping.keys()) if (vacancyU(stem, gn.seed) < p) out.push(stem); + return out.sort(); + } + + for (const level of gn.levels) { + it(`p = ${level.p}: the vacated stem set is exactly Python's ${level.stems.length}`, () => { + expect(vacatedStems(level.p)).toEqual(level.stems); + }); + } + + it("each level contains the one below it", () => { + for (let i = 1; i < gn.levels.length; i++) { + const smaller = new Set(gn.levels[i - 1].stems); + const larger = new Set(gn.levels[i].stems); + expect(larger.size).toBeGreaterThan(smaller.size); + for (const stem of smaller) { + expect(larger.has(stem), `${stem} vacated at ${gn.levels[i - 1].p} but not later`).toBe( + true, + ); + } + } + // At p = 1 every eligible stem vacates, since u ∈ [0, 1) by construction (§10). + const top = gn.levels[gn.levels.length - 1]; + expect(top.p).toBe(1); + expect(top.stems).toHaveLength(vmap.mapping.size); + }); +}); + +// --- 6. stability across p (SC-702) ----------------------------------------------------- + +describe("a stem's nonce is identical at every p where it is vacated (SC-702)", () => { + for (const gm of golden.maps.filter((m) => m.mapping !== null)) { + it(`${gm.label}: the pinned stems never change form as p grows`, () => { + const cases = golden.cases.filter((c) => c.map === gm.label && c.stemForms !== null); + expect(cases.length).toBeGreaterThan(1); + const seen = new Map(); + for (const gc of cases) { + const params = paramsOf(gc); + const vmap = mapFor(params); + for (const stem of Object.keys(gc.stemForms as Record)) { + const form = transformWord(stem, vmap, params); + if (form === stem) continue; // not vacated at this p — nothing to be stable about + const previous = seen.get(stem); + if (previous !== undefined) expect(form).toBe(previous); + seen.set(stem, form); + } + } + // The pinned list must actually exercise the property, or this describe is vacuous. + expect(seen.size).toBeGreaterThan(0); + }); + } +}); + +// --- 7. the invariance theorem, as data (SC-703) --------------------------------------- + +describe("the id stream is unchanged by vacancy at every p (§7.3, SC-703)", () => { + it("every mapped-condition case has the SAME token-id digest", () => { + const withStream = golden.cases.filter((c) => c.idStream !== null); + // Both seeds, both matchProsody settings, every p — one digest. + expect(withStream.length).toBeGreaterThanOrEqual(16); + const digests = new Set(withStream.map((c) => (c.idStream as GoldenIdStream).digest)); + const lengths = new Set(withStream.map((c) => (c.idStream as GoldenIdStream).length)); + expect(digests.size).toBe(1); + expect(lengths.size).toBe(1); + expect(withStream.some((c) => c.params.seed === 7)).toBe(true); + expect(withStream.some((c) => !c.params.matchProsody)).toBe(true); + expect(withStream.some((c) => c.params.p === 1)).toBe(true); + }); + + it("and that digest is the UNVACATED corpus's own stream", () => { + // The theorem is that relabelling the vocabulary alongside the corpus leaves training + // bit-identical, so the p = 0 stream — plain English, plain Dolch — must be it. + const ids = tokenStream(CORPUS, new LexVocab(BUDGET_WORDS, "dolch", "full")); + const golden0 = golden.cases.find((c) => c.idStream !== null)?.idStream as GoldenIdStream; + expect(sha256(ids.join(","))).toBe(golden0.digest); + expect(ids.length).toBe(golden0.length); + }); +}); + +// --- 8. the controls really are controls ----------------------------------------------- + +describe("the control conditions are present and behave as controls (§7.1, SC-705)", () => { + it("both controls and both matchProsody settings are pinned", () => { + const labels = golden.cases.map((c) => c.label); + expect(labels).toContain("control-inconsistent"); + expect(labels).toContain("control-reveal-after-2"); + expect(golden.cases.some((c) => !c.params.consistent)).toBe(true); + expect(golden.cases.some((c) => c.params.revealAfter > 0)).toBe(true); + expect(golden.cases.some((c) => c.params.matchProsody)).toBe(true); + expect(golden.cases.some((c) => !c.params.matchProsody)).toBe(true); + }); + + it("matchProsody = false really mints a different map", () => { + const withProsody = golden.maps.find((m) => m.label === "seed0"); + const without = golden.maps.find((m) => m.label === "seed0-noprosody"); + expect(withProsody?.mappingSha256).not.toBe(without?.mappingSha256); + }); + + it("reveal_after leaves fewer tokens vacated than the pure case at the same p", () => { + const pure = golden.cases.find((c) => c.label === "seed0-p0.7"); + const reveal = golden.cases.find((c) => c.label === "control-reveal-after-2"); + expect(reveal?.params.p).toBe(pure?.params.p); + expect(reveal?.stats.tokensVacated as number).toBeLessThan(pure?.stats.tokensVacated as number); + }); + + it("no divergence remains, and any that appeared would still be constrained", () => { + // The two stacks now agree on every case, so this list is EMPTY — asserted, because a + // fixture that quietly reacquired an exemption would look exactly like a green run. + const flagged = golden.cases.filter((c) => c.knownDivergence !== undefined); + expect(flagged.map((c) => c.label)).toEqual([]); + + // The rest of this test is the constraint the mechanism carries, kept live for the day + // it is needed again: a divergence may appear only on a CONTROL case, it must carry the + // explanation and the status, and it may never cover a count the control exists to + // produce. A mapped-condition case acquiring one would mean the invariance theorem's + // own arm had stopped agreeing. + for (const gc of flagged) { + const d = gc.knownDivergence as GoldenDivergence; + expect(gc.params.consistent && gc.params.revealAfter === 0).toBe(false); + expect(d.cause.length).toBeGreaterThan(80); + expect(d.status).toContain("neither implementation was modified"); + expect(d.fields.length).toBeGreaterThan(0); + for (const f of d.fields) expect(f.python).not.toBe(f.typescript); + } + // The counts a control exists to produce must agree in BOTH stacks — a divergence there + // would mean SC-705's measurement itself had moved, which no exemption may cover. + for (const gc of flagged) { + const names = new Set((gc.knownDivergence as GoldenDivergence).fields.map((f) => f.field)); + expect(names.has("stats.tokensVacated")).toBe(false); + expect(names.has("stats.stemsVacated")).toBe(false); + expect(names.has("stats.domainTypesVacated")).toBe(false); + } + }); + + it("consistent = false holds the vacancy rate but destroys the identity", () => { + const pure = golden.cases.find((c) => c.label === "seed0-p0.7"); + const control = golden.cases.find((c) => c.label === "control-inconsistent"); + // Same rate — that is what makes it a control and not just a different transform. + expect(control?.stats.tokensVacated).toBe(pure?.stats.tokensVacated); + // Different text, because each occurrence gets its own type. + expect(control?.vacatedSha256).not.toBe(pure?.vacatedSha256); + }); + + it("consistent = false vacates EVERYTHING at p = 1, seed 7 — the `tak` regression", () => { + // §5.8, condition B on the per-occurrence path. `tak` (the stem of `taking`) once minted + // the nonce `tak` at seed 7, so `Taking -> Taking` and one token silently survived the + // transform: 1921 corpus types and 8201 tokens against the consistent path's 1922/8202. + // A control whose vacancy rate is not the stated rate is not a control, so the identity + // §10 states at `p = 1` — every eligible type vacates — has to hold here too. + const control = golden.cases.find((c) => c.label === "control-inconsistent-seed7"); + const pure = golden.cases.find((c) => c.label === "seed7-p1.0"); + expect(control).toBeDefined(); + expect(control?.params.consistent).toBe(false); + expect(control?.params.p).toBe(1); + expect(control?.stats.corpusTypesVacated).toBe(1922); + expect(control?.stats.tokensVacated).toBe(8202); + expect(control?.stats.corpusTypesVacated).toBe(control?.stats.corpusTypesEligible); + // The consistent path at the same seed and `p` reads the same counts; only the text + // differs, which is exactly what this control is for. + expect(control?.stats.corpusTypesVacated).toBe(pure?.stats.corpusTypesVacated); + expect(control?.stats.tokensVacated).toBe(pure?.stats.tokensVacated); + expect(control?.vacatedSha256).not.toBe(pure?.vacatedSha256); + }); +}); + +afterAll(() => { + if (deviations.length === 0) return; + const worst = deviations.reduce((a, b) => (b.dev > a.dev ? b : a)); + const nonZero = deviations.filter((d) => d.dev > 0).length; + console.log( + `vacancy golden: ${deviations.length} float comparisons, ${nonZero} non-zero, ` + + `worst |Δ| = ${worst.dev.toExponential(3)} (${worst.what}), tolerance ${TOLERANCE}`, + ); +}); diff --git a/docs/screenshots/feature-007/arch-vacancy-score.png b/docs/screenshots/feature-007/arch-vacancy-score.png new file mode 100644 index 0000000..f8ca64f Binary files /dev/null and b/docs/screenshots/feature-007/arch-vacancy-score.png differ diff --git a/docs/screenshots/feature-007/arch-vacancy-static-refusals.png b/docs/screenshots/feature-007/arch-vacancy-static-refusals.png new file mode 100644 index 0000000..ad69c58 Binary files /dev/null and b/docs/screenshots/feature-007/arch-vacancy-static-refusals.png differ diff --git a/docs/screenshots/feature-007/info-vacancy-section.png b/docs/screenshots/feature-007/info-vacancy-section.png new file mode 100644 index 0000000..9c0e204 Binary files /dev/null and b/docs/screenshots/feature-007/info-vacancy-section.png differ diff --git a/docs/screenshots/feature-007/vacancy-panel.png b/docs/screenshots/feature-007/vacancy-panel.png new file mode 100644 index 0000000..3a9c0a1 Binary files /dev/null and b/docs/screenshots/feature-007/vacancy-panel.png differ diff --git a/docs/screenshots/feature-007/vacancy-swap-full.png b/docs/screenshots/feature-007/vacancy-swap-full.png new file mode 100644 index 0000000..a868b86 Binary files /dev/null and b/docs/screenshots/feature-007/vacancy-swap-full.png differ diff --git a/docs/screenshots/feature-007/vacancy-swap-refusal.png b/docs/screenshots/feature-007/vacancy-swap-refusal.png new file mode 100644 index 0000000..7331091 Binary files /dev/null and b/docs/screenshots/feature-007/vacancy-swap-refusal.png differ diff --git a/notes/2026-08-04-feature-007-vacancy-transform.md b/notes/2026-08-04-feature-007-vacancy-transform.md new file mode 100644 index 0000000..245766d --- /dev/null +++ b/notes/2026-08-04-feature-007-vacancy-transform.md @@ -0,0 +1,262 @@ +# Feature 007 — the vacancy transform (session notes) + +Started 2026-08-04. Branch `007-vacancy-transform`. Follows feature 006 (Lexicon Lab). + +## What the user asked for + +> take a look at ~/Desktop/TinyModelsDoc for what the tiny model is *trying* to build. the +> limited vocabulary and word-level tokenizers are one part, but the vacancy transform is +> important too. Can you add this to the demo? This needs to be done carefully! + +Scope decision (asked, answered): **both arms** — the tiny arm in the Lexicon Lab AND the +pretrained arm in the Architecture Explorer, completing the doc's T4 2×2. + +## Source material + +`~/Desktop/TinyModelsDoc/` — `tiny_models.tex` (the proposal, 1055 lines) and +`tiny-seuss.zip`. The zip is the ORIGINAL bundle; `~/Desktop/tiny-models/tiny-seuss/` is the +AUDITED copy from the previous session and differs in three files. Where they disagree, the +audited copy is right: +- `split_suffix` gains an `exceptions` set (`brother` etc.) — without it `brother → broth+er` +- `main` reports `len(j.vacated)` not `len(j.map)` — the zip prints the wrong count +- `README.md` retracts the unverified TTR 0.098 baseline (measured 0.121) and withdraws the + "exact prosody" claim + +## Key artifacts + +- `specs/007-vacancy-transform-field/spec.md` — FR-701…726, SC-701…710 +- `specs/007-vacancy-transform-field/architecture.md` — **the normative TS↔Python contract**. + Written before any code, deliberately: feature 006's one CRITICAL bug was a contract gap. + +## The load-bearing idea + +For a word-level model trained from scratch, a word's "location" is a row index — the model +never sees the letters. So with `consistent = true`, `revealAfter = 0`, and the vocabulary +*mapped* through the transform (order preserved), the vacancy transform is a **pure +relabelling** and training is **bit-identical**. That is contract §7.3, spec SC-703. + +I checked the proof by hand against `token_stream`'s actual behaviour: +- `t ∈ V` → `map(t) ∈ V_p` at the same index ✓ +- `t ∉ V` → `map(t) ∉ V_p`, since `map(t) = map(w)` with `w ∈ V` would force `t = w` ✓ +- both depend on **injectivity over `corpus types ∪ V`** — which is why the contract makes + injectivity a verified property with a re-mint loop, not an assumption +- case is safe because `tokenize` lowercases and `avoid`/`used` are lowercase +- all types sharing a stem are vacated together, so a budget word can never be vacated while a + corpus type sharing its stem is not + +**Framing risk.** "The model is exactly invariant" can read as trivial — *of course, you +relabelled the vocabulary*. That IS the point, and the docs must say so: the doc asks whether an +embedding is an independent carrier of content or a summary of contextual support; in the tiny +regime it is provably the latter. The tiny arm's job is to establish the **baseline of zero**, +so that the pretrained arm's delta has a scale. Do not dress the null up as a curve. + +## Eleven departures from the source + +Contract §9 lists them all. Four are corrections to bugs that break properties the source +*claims* for itself: +- the map is built lazily while rewriting, so `used` makes a nonce depend on `p` — breaks stability +- the give-up path is `syllable + str(len(used))` — order-dependent +- the seam fix draws from a shared RNG — order-dependent +- injectivity is assumed; `avoid` is accepted and never passed, so a nonce can merge with a real word + +Plus one that only bites across our two stacks: `top64 / 2**64` is not exactly representable, so +Python and JS can disagree at the boundary. Contract §4 uses `(top64 >> 11) / 2**53`, which is. + +## RESOLVED — the swap control is in (contract §8.3, FR-719a, SC-707a) + +The pretrained arm's `ΔnllPreserved` has a confound the contract states (§8.3): the vacated +passage genuinely has higher entropy, so every prediction degrades, scaffolding included. + +A **swap control** would make the number interpretable rather than merely caveated: replace each +vacated stem with a *real English word* (drawn from the corpus, frequency-matched) instead of a +nonce form. Same machinery, different minting strategy — `mint: "nonce" | "swap"`. The context +is then equally wrong semantically but the forms are all known, so +`nonce − swap` isolates *unknown form* from *wrong content*. + +Added to the contract as §8.3 while the two modules were still being written, since it does not +change §§1-7 and the modules are implementing those. It only adds a minting strategy. + +The decomposition the UI must report: +- `nll(swap) − nll(english)` — the cost of **wrong content** +- `nll(nonce) − nll(swap)` — the cost of **unknown form** + +and never `nll(nonce) − nll(english)` alone, which conflates them. The residual — that nonce +forms fragment into more subword tokens — is not separable without a tokenizer-level control, +and the UI says so rather than pretending the remainder is pure location. + +The correctness check for the control is elegant: `swap` must satisfy the invariance theorem +exactly as `nonce` does, because the tiny model is equally blind to both. + +## Two defects found in SHIPPED code while building this + +Neither is a feature-007 bug; both were found because 007 forced us to drive the real app. + +**1. Silent vocabulary substitution in the Geometry Lab** (fixed, `d6e9d5d`). Static build only, +across a reload, for a model with its own vocabulary (scratch-trained or file-loaded). Train → +Save (your word list) → reload → Save → same weights under *Alice in Wonderland*'s word list. +No error, and unrejectable: `vocab_sha256` is computed over the list that was written, so the +file verifies on both sides. Exactly the corruption the three digests exist to prevent, +committed by the writer. The persisted `ExportedWeightSet` carried weights but not the +vocabulary, so `tokenizerFor()` fell back to the canonical tokenizer — right for edited and +fine-tuned sets, catastrophic for scratch and imported ones. + +**2. q4f16 produces garbage on WebGPU** (FIXED — see below). The dtype the app tries FIRST. The +session builds successfully and then returns degenerate output: every row of the `[1,T,V]` +logits bit-identical, gpt2 greedy-generating `,,,,,,,`, SmolLM2's logits all exactly 0 so every +NLL is `ln(49152) = 10.80267`. The fallback in `transformersRuntime.ts` only fires on a thrown +exception and nothing throws, so **the deployed Architecture Explorer is already showing wrong +probabilities on any `shader-f16` machine** — reproduced in real Chrome 150, not just Playwright. +q8, fp32 and q4 are correct; fp16 fails identically, so the fp16 *activation* path is the cause. + +Why it survived: **plain headless Chromium exposes no WebGPU adapter** (`requestAdapter()` → +null), so the entire e2e suite has only ever exercised wasm/q8. That coverage gap is part of the +fix. + +**The fix** (its own commit, independent of 007). Re-verified first in a real browser on the +real Apple Metal-3 adapter, per repo, `maxAbsRowDiff` between the first and last logit row of one +teacher-forced pass + a greedy continuation: + +| repo | webgpu/q4f16 | webgpu/q8 | +|-|-|-| +| gpt2-ONNX | 0.000, `,,,,,,,,,,` | 91.99, ` Berlin. The capital of the United States is Washington` | +| SmolLM2-135M-Instruct-ONNX | 0.000, all logits 0, empty | 36.97, ` Berlin.\n\nThe capital of Italy is Rome` | +| SmolLM2-360M-Instruct-ONNX | 0.000, all logits 0, empty | (correct) | +| Qwen2.5-0.5B-Instruct | 16.47 — NOT degenerate, but worse text | 20.27, ` Berlin. What is…` | + +Three of the four curated models are destroyed by q4f16; Qwen survives it. `q4` is correct +(SmolLM2-135M: 35.66, ` Berlin. The capital of the United States is Washington`) but is **not** +the smaller download the earlier note assumed — in every curated repo `model_q4.onnx` is LARGER +than `model_quantized.onnx` (gpt2 498 vs 280 MB, SmolLM2-135M 181 vs 136, SmolLM2-360M 386 vs +363, Qwen2.5-0.5B 786 vs 512). So the ladder is now **webgpu/q8 → wasm/q8**: both rungs read the +same file, so a rejected rung costs no second download. + +1. `staticClient/logitsSanity.ts` — ONE invariant, in the spirit of the Geometry Lab's training + gates: a causal LM's output must depend on its input, so the L∞ gap between the first and last + next-token distribution of a fixed 12-token probe must exceed 1e-3. All-identical rows and + all-zero logits are the same failure, not two rules. Asserted on every session at load, before + any number is shown; a rejected rung falls through and is NAMED in the badge (`· fallback`). +2. `RUNTIME_LADDER` + `FP16_ACTIVATION_DTYPES` in `runtimeTypes.ts`, unit-tested + (`tests/unit/logitsSanity.test.ts`) so CI always checks that no fp16-activation dtype creeps + back, even where it cannot run a GPU. +3. `tests/e2e/webgpu.spec.ts` + a `webgpu` Playwright project. **The flag that matters on macOS + is `--use-angle=metal`**: `--enable-unsafe-webgpu` alone still hands back google/swiftshader + (no `shader-f16`); with it, headless Chromium gets the real apple/metal-3 adapter. Verified to + FAIL on the pre-fix code (badge `webgpu · q4f16`; 2 distinct top-5 lists across 64 generated + positions) and PASS after (64/64 distinct). + +**Named residual gap:** GitHub-hosted runners have no GPU, so this test SKIPS in CI with a loud +reason. The WebGPU path is verified only on a developer machine with a real GPU; CI verifies the +invariant, the ladder, and one real session through the same gate on the WASM rung. + +Lesson worth keeping: both defects are invisible to unit tests and to any test that checks for +thrown errors. They produce *plausible* wrong answers. The only thing that caught them was +running the real thing and comparing against a known-good reference. + +## The swap control in the Lexicon Lab, and how its constraint is surfaced + +`mint = "swap"` is now live in `VacancyPanel` (it had been rendered disabled). Three decisions +worth keeping, because each replaces a tempting shortcut: + +1. **The constraint is shown, never enforced by the UI.** `p` is not clamped, `swap` is not + silently downgraded to `nonce`, and the typed error is not caught-and-replaced with a + fallback. `LexiconLab` asks the engine (`buildVacancyMap` with the real `consistent`, + `mapVocabWords` at the real `p`) and CARRIES the refusal up as a string; the panel prints it + verbatim in a refusal card, with buttons for the two exits (`p = 1`, `p = 0`, switch to + nonce, use the consistent condition). With no vocabulary, the budget counters, the trainer + and the invariance check simply have nothing to report — which is the honest state. +2. **The theorem is COUNTED, not asserted.** Beside the mint control, every domain type is + pushed through the real transform at the current `p` and the distinct images are counted: + `|domain| − |images|` lost image slots. Measured on the shipped corpus, seed 0: + **244 / 322 / 233 at p = .25/.5/.75, and 0 at both endpoints** (reproduced independently in + Python while writing the docs). Under `nonce` it is 0 everywhere. So §5.2a happens in front + of the reader. +3. **The `bijective` chip branches on `injectiveAtEveryP`.** Under swap it reads "injective at + p = 0, 1" rather than a bare tick — the map property is real, but claiming it at every `p` + would be false. + +Two shipped-code defects fixed in passing: `LexiconLab` never passed `mint` into `vacParams` at +all (so the control could not have worked), and `staticClient/arch.ts` still documented a +"±0.1 nats" quantization uncertainty that its own constant had superseded with 0.2. + +## Documentation (FR-724/725/726, ui.md §3) + +Info tab gains `

    `: the T4 2×2 with the vacancy cell marked, the transform's +definition (with `u` as an equation), nesting + stability and the four properties the source +implementation claims and breaks, the invariance theorem with §7.4's framing (the exact zero IS +the finding), the swap decomposition with "cost of unknown form" stated as an UPPER BOUND, the +stress table's real status, and a by-name list of what the static build refuses plus the WebGPU +/ CI coverage gap. `#real`, `#limits` and `#refs` updated (Gutenberg #12 — *Through the +Looking-Glass* — added; link checked). + +**Every number is pinned** in `tests/e2e/docs.spec.ts` (5 new tests): the counts come from a live +`POST /api/lex/vacancy` at p = 1 (2,233 / 2,211 / 1,944 / 1,680 / 8,202 / 16,000 and the 5.1% +stress-table coverage), the swap collisions are read off the running panel, the stress-table size +is read off the panel's own honesty line, and the static-mode ±0.2 nats / 700-token floor are +regex'd out of `staticClient/arch.ts`. Nothing in the section is a number a human retyped. + +Note for anyone extending this: the pretrained arm's measured deltas are deliberately NOT quoted +in the Info tab. They depend on model, passage set and dtype, and the only honest pin would be a +real Qwen run per CI job. The panel reports them from the run the reader triggers. + +## CI: HuggingFace rate limiting, third occurrence in two sessions + +Run 30929570326 failed in `Export static assets`. **Not our code.** The Hub returned 429 through +all five retries on `get_safetensors_metadata` and `model_info`, so the export refused: + +> `gpt2: revision did not resolve to a commit sha (got 'main'). The HuggingFace API is probably +> rate-limiting or down — rerun the export rather than publishing unpinned weight URLs.` + +That refusal is correct — it is the guard behind issue #5, and publishing a build whose weight +URLs point at `main` rather than a pinned sha is exactly what it exists to prevent. Re-running +succeeds, but this is now the third occurrence, so the two durable fixes are worth doing: + +1. **An `HF_TOKEN` repository secret.** Needs the repo owner; CI is unauthenticated today and is + rate-limited hard. This is the robust fix. +2. **Pin the curated models to explicit commit shas** (issue #5). Removes the `main → sha` + resolution from the build path entirely, so a `model_info` 429 cannot block a build. Does not + need the owner, and shrinks the API surface even once a token exists. + +## Status + +### Contract defects found by BUILDING it (13, not one of which I caught by re-reading) + +The two stacks were written independently from the contract so they could disagree. Every +disagreement turned out to be a defect in the document, never in one implementation: + +1. case — suffix sliced case-preserved, so `gums→flels` but `GUMS→FLESS` (one type, two surfaces) +2. injectivity checked only at `p=1` and over bare nonces — `hanged→waked` collides at `p=0.25` +3. `CODAS` documented as 49 entries; it has 46 +4. `typesVacated` undefined between stems and types (1922 vs 1665) +5. salt thresholds ambiguous between attempt counter and absolute salt +6. domain readable as the *active* budget — would re-mint on every budget switch +7. `avoid` optional, so the map depended on caller memory (0 vs 1 re-mint rounds, different nonces) +8. `vacancyDomain` helper existed in Python only +9. `VacancyMap.map` vs `.mapping` +10. `consistent=false` prosody drawn from the mint key, not the stem +11. `corpusTypesVacated` from map membership vs measured from the texts (1337 vs 665) +12. condition B not applied to the per-occurrence path — `tak→tak`, a word silently surviving +13. `forbidden` stored in one stack, reconstructed in the other (drops superseded nonces) + +Four of these (1, 2, 10, 12) would have broken the invariance theorem. Numbers 4, 11 and 13 were +found only because a stack was told to STOP and report rather than reconcile to the other — a +golden fixture built over a silent reconciliation would have cemented both stacks being +consistently wrong. + +- [x] Spec + contract written and committed (`36b9f3d`) +- [ ] Python `lex/vacancy.py` + tests +- [ ] TS `lexEngine/vacancy.ts` + tests +- [ ] transformers.js alignment probe (contract §8.2 requires this be settled empirically first) +- [ ] Golden fixture + parity +- [ ] Lexicon Lab vacancy panel +- [ ] API routes + static client +- [ ] Pretrained arm +- [x] Docs (Info tab + in-tab prose) — `#vacancy` section, `#real`/`#limits`/`#refs` updated, + the swap control enabled in the Lexicon Lab +- [ ] Full suite, deploy, live verification + +## Standing constraints (from CLAUDE.md) + +No mocks, ever. Real corpus, real models, real browsers. Re-run **all** checks after any fix. +Never transcribe a number from the source document as if it were ours — the doc's +`0.351 → 0.345` anapest and `1.224 → 1.211` syllables are its numbers on a corpus we do not +have. Measure ours. diff --git a/scripts/export_arch_vacancy_golden.py b/scripts/export_arch_vacancy_golden.py new file mode 100644 index 0000000..a25ab52 --- /dev/null +++ b/scripts/export_arch_vacancy_golden.py @@ -0,0 +1,69 @@ +"""Pin the pretrained arm's default passage set across both stacks (contract §8.3a). + +The six excerpts the measurement was made on are cut from the shipped corpus by +`llm_geometry.arch.vacancy_score.default_passages`, and the browser cuts its own from +`static-data/lex/corpus.json` with `defaultVacancyPassages` in +`src/lib/staticClient/arch.ts`. Those two are the same measurement or they are not the +same measurement; this writes the digests that decide it. + +Only digests and counts are written — the corpus text itself is already committed, and +duplicating 9 kB of it into a fixture would just be a second copy to keep in sync. + + python scripts/export_arch_vacancy_golden.py + +Regenerate whenever `default_passages` or the corpus changes; `tests/unit/archVacancy. +test.ts` fails until you do, which is the point. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "code" / "backend" / "src")) + +from llm_geometry.arch.vacancy_score import ( # noqa: E402 + DEFAULT_PASSAGE_COUNT, + DEFAULT_PASSAGE_WORDS, + default_passages, +) +from llm_geometry.lex.corpus import corpus_sha256 # noqa: E402 +from llm_geometry.lex.vocab import WORD_RE # noqa: E402 + +OUT = ROOT / "code" / "frontend" / "tests" / "fixtures" / "arch-vacancy-passages.json" + + +def main() -> None: + passages = default_passages() + payload = { + "note": ( + "Digests of the default passage set of contract §8.3a. Written by " + "scripts/export_arch_vacancy_golden.py from the real corpus; asserted " + "against the browser's own cut in tests/unit/archVacancy.test.ts." + ), + "corpus_sha256": corpus_sha256(), + "count": DEFAULT_PASSAGE_COUNT, + "words_per_passage": DEFAULT_PASSAGE_WORDS, + "passages": [ + { + "index": i, + "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), + "n_words": len(WORD_RE.findall(text)), + "n_chars": len(text), + "head": re.sub(r"\s+", " ", text[:60]).strip(), + } + for i, text in enumerate(passages) + ], + } + OUT.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n") + print(f"wrote {OUT.relative_to(ROOT)}") + for row in payload["passages"]: + print(f" #{row['index']} {row['n_words']} words {row['sha256'][:12]}… {row['head']!r}") + + +if __name__ == "__main__": + main() diff --git a/scripts/export_vacancy_api_golden.py b/scripts/export_vacancy_api_golden.py new file mode 100644 index 0000000..9c15699 --- /dev/null +++ b/scripts/export_vacancy_api_golden.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +"""Emit the vacancy **API** golden vectors — the endpoint/static-client parity fixture. + +`scripts/export_vacancy_golden.py` pins the TRANSFORM across the two stacks (§11 of +`specs/007-vacancy-transform-field/architecture.md`). This script pins the layer above it: +what `POST /api/lex/vacancy` actually puts on the wire, so that + +* `code/backend/tests/contract/test_api_lex.py` can assert the live FastAPI route still + returns exactly this, and +* `code/frontend/tests/unit/staticVacancy.test.ts` can assert the browser's + `staticClient.lexVacancy()` returns exactly this too. + +Two tests, one file, and therefore one claim: **the full stack and the static build answer +the same request with the same numbers, the same vocabulary, and the same sha256 of the +whole vacated corpus.** FR-722 is that sentence; this fixture is what makes it checkable +rather than asserted. If either side drifts, one of the two tests fails and names the +field. + +The route is exercised through FastAPI's `TestClient` against the REAL app with the REAL +committed corpus — no mocks, no stubbed transform, no hand-written expectations. Every +value in the output was produced by running the endpoint. + +Usage (from the backend venv, at the repo root): + + python scripts/export_vacancy_api_golden.py + python scripts/export_vacancy_api_golden.py --out /tmp/vacancy-api-golden.json + +Determinism: the transform is a pure function of `(corpus, params)` and the route adds no +clock or randomness, so regenerating this twice must produce byte-identical bytes. +`--generated` is a date rather than a timestamp for exactly that reason. + +WHAT THE CASES COVER, and why each is here rather than being one more of the same: + +* `p1-seed7-pre_primer` — full vacancy at the seed that needs a re-mint (`remintRounds` 1, + where `hang` first minted `wak` and `hanged` surfaced as the real word `waked`). Pins the + DEFAULT `preview_chars`, so a drift in either stack's default fails here. +* `p035-seed0-full` — the source's own figure `p`, against the largest Dolch budget, so the + mapped word list being pinned is 314 words long. +* `p0-seed0-pre_primer` — the identity boundary. `u ∈ [0, 1)`, so nothing vacates and the + vacated digest must equal the original's. A transform that quietly did something at + `p = 0` would show up nowhere else. +* `control-inconsistent` — `consistent = false`. The vocabulary is REBUILT rather than + mapped (§7.2) and coverage collapses; that collapse is the measurement (FR-715), and it + is the case where the two stacks previously minted different per-occurrence forms. +* `control-reveal-after-2` — `revealAfter = 2`. The other rebuilt condition, and the one + where `corpusTypesVacated` is measured from the two TEXTS rather than from map + membership (§10) — the definition the stacks split on. +* `p07-seed0-frequency100` — a frequency budget, whose word list is drawn from the corpus + rather than from a fixed list, plus a non-default `preview_chars`. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from datetime import date +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "code" / "backend" / "src")) + +from fastapi.testclient import TestClient # noqa: E402 + +from llm_geometry.api.app import app # noqa: E402 +from llm_geometry.api.routes_lex import ( # noqa: E402 + VACANCY_PREVIEW_CHARS, + VACANCY_PREVIEW_MAX, +) +from llm_geometry.lex.corpus import load_corpus_text # noqa: E402 + +# Reuse the transform exporter's git-sha helper rather than writing a second one. +sys.path.insert(0, str(REPO_ROOT / "scripts")) +from export_vacancy_golden import git_sha, sha256_text # noqa: E402 + +DEFAULT_OUT = REPO_ROOT / "code" / "frontend" / "tests" / "fixtures" / "vacancy-api-golden.json" + +FORMAT = "vacancy-api-golden-v1" + +#: Both stacks round every float in a response to 6 significant digits before it reaches a +#: caller (`api/encoding.py::jsonable_6sig` and `staticClient/lex.ts::sig6`), so the wire +#: values are IDENTICAL, not merely close. The tests compare with `toBe` / `==`; this bound +#: exists only so a future float that escapes the rounding has a documented allowance +#: instead of a silently loosened assertion. +TOLERANCE = 0.0 + +#: `(label, request body)`. The body is sent verbatim; the corpus is the shipped one in +#: every case, because the point of the fixture is parity on the text both stacks ship. +CASES: tuple[tuple[str, dict[str, Any]], ...] = ( + ( + "p1-seed7-pre_primer", + {"p": 1.0, "seed": 7, "source": "dolch", "budget": "pre_primer"}, + ), + ( + "p035-seed0-full", + {"p": 0.35, "seed": 0, "source": "dolch", "budget": "full", "preview_chars": 400}, + ), + ( + "p0-seed0-pre_primer", + {"p": 0.0, "seed": 0, "source": "dolch", "budget": "pre_primer", "preview_chars": 400}, + ), + ( + "control-inconsistent", + { + "p": 0.5, + "seed": 0, + "consistent": False, + "source": "dolch", + "budget": "primer", + }, + ), + ( + "control-reveal-after-2", + { + "p": 0.5, + "seed": 0, + "reveal_after": 2, + "source": "dolch", + "budget": "primer", + "preview_chars": 400, + }, + ), + ( + "p07-seed0-frequency100", + { + "p": 0.7, + "seed": 0, + "match_prosody": False, + "source": "frequency", + "budget": "full", + "size": 100, + "preview_chars": 400, + }, + ), +) + + +def build_document(generated: str) -> dict[str, Any]: + client = TestClient(app) + corpus = load_corpus_text() + + cases: list[dict[str, Any]] = [] + for label, body in CASES: + response = client.post("/api/lex/vacancy", json=body) + if response.status_code != 200: + raise SystemExit( + f"case {label!r}: the route returned {response.status_code}: {response.text}" + ) + cases.append({"label": label, "request": body, "response": response.json()}) + + return { + "format": FORMAT, + "generated": generated, + "git_sha": git_sha(), + "command": "python scripts/export_vacancy_api_golden.py", + "source": ( + "POST /api/lex/vacancy on the real FastAPI app with the real committed " + "corpus, through fastapi.testclient — real route, real transform, no mocks" + ), + "contract": "specs/002-interactive-model-explorer/contracts/api.md", + "tolerance": TOLERANCE, + "endpoint": "/api/lex/vacancy", + "defaults": { + "preview_chars": VACANCY_PREVIEW_CHARS, + "preview_max": VACANCY_PREVIEW_MAX, + }, + "encoding": ( + "exactly what the route serves: every float already rounded to 6 significant " + "digits by api/encoding.py::jsonable_6sig, which staticClient/lex.ts::sig6 " + "reproduces, so every field compares EXACTLY. `vacated_sha256` is over the " + "UTF-8 bytes of the WHOLE vacated corpus — 86 kB pinned in 64 characters." + ), + "corpus": { + "path": "code/backend/src/llm_geometry/lex/data/real-mother-goose.txt", + "sha256": sha256_text(corpus), + "chars": len(corpus), + }, + "cases": cases, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--generated", default=date.today().isoformat()) + args = parser.parse_args() + + document = build_document(args.generated) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(document, indent=1) + "\n", encoding="utf-8") + print( + f"wrote {args.out} ({args.out.stat().st_size / 1024:.0f} KB, {len(document['cases'])} cases)" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/export_vacancy_golden.py b/scripts/export_vacancy_golden.py new file mode 100644 index 0000000..15e9fdc --- /dev/null +++ b/scripts/export_vacancy_golden.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python +"""Emit the vacancy-transform golden vectors (spec 007, §11 of architecture.md). + +Both stacks implement ``specs/007-vacancy-transform-field/architecture.md``: + +* Python — ``code/backend/src/llm_geometry/lex/vacancy.py`` +* browser — ``code/frontend/src/lib/lexEngine/vacancy.ts`` + +This script runs the **real Python side** on the **real committed corpus** (*The Real +Mother Goose*, verified against its recorded digest) and writes what it measured to +``code/frontend/tests/fixtures/vacancy-golden.json``. ``tests/unit/vacancyGolden.test.ts`` +then runs the real TypeScript side against that file and asserts agreement. Nothing here +re-derives the maths; every number is read out of the shipped implementation. + +Same pattern and the same discipline as ``scripts/export_lex_golden.py`` -> +``tests/fixtures/lex-golden.json`` -> ``tests/unit/lexGolden.test.ts``: the file carries +``format``, ``tolerance``, ``git_sha`` and the generator versions, and the test reads the +tolerance FROM the file so the two cannot drift apart. + +Usage (from the backend venv): + + python scripts/export_vacancy_golden.py + python scripts/export_vacancy_golden.py --out /tmp/vacancy-golden.json + +The output is a pure function of the committed corpus plus this file, so regenerating it +twice must produce byte-identical bytes; ``--generated`` is pinned to a date rather than a +timestamp for that reason. + +What §11 requires pinned, and where it lives in the document: + +1. ``u(stem)`` for 24 stems spanning eligible/ineligible and both budgets, as the EXACT + float64 — ``stems[].u``. Plain JSON numbers are exact here: ``repr`` and JavaScript's + number formatting are both shortest-round-trip, so the double survives the trip. That + exactness is the whole point of departure 2 (``>> 11`` before the divide, §4). +2. The FULL stem -> nonce map at ``seed in {0, 7}`` — ``maps[].mapping``, every pair. +3. The first 400 characters of the vacated corpus at ``p in {0, 0.35, 0.7, 1}``, seed 0 — + ``cases[].head400``. The corpus is ASCII, so "character" means the same thing in both + languages. Each case also carries the sha256 of the WHOLE vacated corpus, which pins + all 86 kB for 64 bytes. +4. ``vacancyStats`` with every §10 field — ``cases[].stats``. +5. Nesting as explicit sets — ``nesting.levels[].stems``, the literal vacated-stem sets at + ``p in {0, 0.35, 0.7, 1}``, so the test checks containment on data rather than on an + assertion one language makes about itself. +6. Stability — ``cases[].stemForms``, the surface form of each of the 24 stems at every + ``p``. A stem's nonce must be byte-identical at every ``p`` where it is vacated. +7. The token-id-stream digest under the mapped vocabulary at each ``p`` — + ``cases[].idStream``. These are all EQUAL, which is §7.3 (the invariance theorem) pinned + as DATA rather than as an assertion written twice, once per language. +8. Both control conditions (``consistent = false``, ``revealAfter > 0``) and both + ``matchProsody`` settings — the ``control-*`` and ``noprosody-*`` cases. +9. The swap control of §8.3 — the ``swap-*`` maps and cases. Four maps (both seeds × both + ``matchProsody`` settings) pinned by digest and 24 samples, and three cases per seed at + ``p in {0, 0.7, 1}``. The endpoints carry a real ``idStream``: SC-703 holds for + ``mint="swap"`` exactly as for ``mint="nonce"`` wherever a swap map can be injective. + ``swap-*-p0.7`` carries ``idStream: null`` and ``mapVocabWordsRejects: true``, which is + §5.2a's theorem pinned as data — a map whose images are domain types and which does not + depend on `p` cannot be injective at intermediate `p` unless it is the identity, so the + mapped vocabulary does not exist there and BOTH stacks refuse it. + +MEASURED, and recorded here because the fixture's shape depends on it: the mapped +vocabulary of §7.2 is defined ONLY for ``consistent = true, revealAfter = 0``; both stacks +raise otherwise. So a control case has ``idStream: null`` and +``mapVocabWordsRejects: true``, and the test asserts the TypeScript side refuses it too — +a silently-accepted control would manufacture a vocabulary matching no corpus. + +Also measured: ``consistent = false`` mints a fresh form per OCCURRENCE and registers its +stress pattern on the map's ``minted_stress``, i.e. it MUTATES the map. Every control case +therefore builds its own map, and the test must do the same or its statistics will be +scored against patterns left behind by a previous case. + +NO DISAGREEMENTS REMAIN BETWEEN THE STACKS. Every case in this fixture — both seeds, both +``matchProsody`` settings, every `p`, and both control conditions — agrees field for field, +string for string, digest for digest. That was not true when the fixture was first written, +and the history is worth keeping because it is what the mechanism below exists for. + +Three defects were pinned here as ``knownDivergence`` blocks and have since been fixed in +the stack the contract said was wrong. Each was fixed in the IMPLEMENTATION; none was fixed +by loosening this file: + +1. **``corpusTypesVacated`` under ``revealAfter > 0``** — Python 665, TypeScript 1337. + Python MEASURED the count from the two texts; TypeScript computed it through the map, so + a type whose every occurrence fell inside the reveal window was still counted. §10 says + ``corpusTypes*`` is "what the panel shows a reader", and the reader is looking at the + text, so the measured reading was the contract's. TypeScript now measures the texts too + (``domainTypes*`` keeps map membership, which §10 also requires — the 22 Dolch-only words + have no occurrences to measure). +2. **Prosody of the per-occurrence mint under ``consistent = false``.** §5.8 pins the key as + ``f"{stem}#{idx}"`` and §5.5's mint read its stress pattern off the string it was handed, + so Python got ``stress("little#0") == "10"`` instead of ``stress("little") == "100"`` and + ``Little`` minted as ``Wrerken`` rather than ``Wrerkenle``. §7.1 says the nonce carries + THE STEM's syllable count and stress, so the key must not reach the prosody lookup. + Python's ``_mint`` now takes ``stem`` separately from ``key``. +3. **Condition B on the per-occurrence path** (§5.8, added to the contract with this fix). + It was enforced when building the map and not in the ``consistent = false`` control, so + at seed 7, `p = 1`, the stem ``tak`` minted the nonce ``tak`` and ``Taking -> Taking``: + ``corpusTypesVacated`` 1921 against the consistent path's 1922, ``tokensVacated`` 8201 + against 8202. Both stacks now forbid a per-occurrence nonce from equalling the stem it + replaces as well as any domain type, and the control vacates 1922 types / 8202 tokens at + both seeds, exactly as ``consistent = true`` does. + +``KNOWN_DIVERGENCES`` is therefore EMPTY, and it is kept — with ``attach_divergence`` and +``DIVERGENCE_STATUS`` — rather than deleted, because it is the honest way to ship a fixture +while a real cross-stack defect is outstanding: record BOTH readings, assert each side +against its own, and refuse to write the fixture at all the moment Python starts agreeing +with the recorded TypeScript value. That guard is what retired all three entries above; it +fired on ``control-inconsistent.vacatedSha256`` and would not let the stale exemption be +regenerated. An entry added here must be MEASURED on both sides, never predicted. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import subprocess +import sys +from datetime import date +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "code" / "backend" / "src")) + +from llm_geometry.lex.corpus import load_corpus_text # noqa: E402 +from llm_geometry.lex.dolch import dolch_budget # noqa: E402 +from llm_geometry.lex.train import token_stream # noqa: E402 +from llm_geometry.lex.vocab import LexVocab, tokenize # noqa: E402 +from llm_geometry.lex.vacancy import ( # noqa: E402 + VacancyMap, + VacancyParams, + build_vacancy_map, + is_eligible, + map_vocab_words, + stem_and_suffix, + type_counts, + vacancy_domain, + vacancy_stats, + vacancy_u, + vacate_text, +) + +DEFAULT_OUT = ( + REPO_ROOT / "code" / "frontend" / "tests" / "fixtures" / "vacancy-golden.json" +) + +FORMAT = "vacancy-golden-v2" + +#: Both stacks compute the vacancy transform in float64 and in the same order, so the only +#: floats that can differ at all are the prosody means (a sum over the same tokens in the +#: same sequence). Everything else in this fixture — `u`, every count, every string, every +#: digest — is compared EXACTLY by the test; this bound applies to the means alone, and it +#: is deliberately far tighter than the 1e-5 of `lex-golden.json`, where torch's float32 +#: was on the other side. +TOLERANCE = 1e-12 + +#: The `p` grid. §11 names {0, 0.35, 0.7, 1}; {0.25, 0.5, 0.75} are added because §10 +#: quotes measured `corpusTypesVacated` / `domainTypesVacated` at exactly those values, and +#: a number quoted in a contract that no test reads is a number free to rot. +P_GRID: tuple[float, ...] = (0.0, 0.25, 0.35, 0.5, 0.7, 0.75, 1.0) + +#: §11's "24 stems spanning eligible/ineligible and both budgets". +#: +#: * eligible, in the Dolch list AND in the corpus — the ordinary case +#: * eligible, Dolch-only — one of the 22 domain-only words that +#: have images but never appear in the +#: text (§10) +#: * eligible, corpus-only — `crown`, `candlestick`, `diddle`, … +#: * eligible, in the MAP but in neither budget — `gum` and `hang`, which reach the map +#: only as the stems of `gums`/`hanged` +#: and are the pair §5.7's case-commuting +#: bug (`GUMS` -> `FLESS`) was found on +#: * ineligible, closed class — `is_eligible` test 1 +#: * ineligible, too short — test 3, `len(stem) > 2` +#: * ineligible, non-alphabetic stem — test 2; `good-bye` matches no suffix, +#: so its stem keeps the hyphen +PINNED_STEMS: tuple[str, ...] = ( + "little", + "pretty", + "run", + "eat", + "jump", + "away", + "squirrel", + "funny", + "today", + "gum", + "hang", + "crown", + "candlestick", + "crooked", + "diddle", + "moon", + "pussy", + "goose", + "the", + "and", + "you", + "not", + "ox", + "good-bye", +) + +#: §11's excerpt length. The corpus is ASCII (asserted below), so Python code points and +#: JavaScript UTF-16 code units count the same characters. +HEAD_CHARS = 400 + +#: The nesting sets §11 asks for, as explicit data. +NESTING_SEED = 0 +NESTING_PS: tuple[float, ...] = (0.0, 0.35, 0.7, 1.0) + +#: Measured cross-stack disagreements, recorded instead of reconciled — see the module +#: docstring. **Currently EMPTY: the two stacks agree on every case in this fixture.** The +#: three entries this dict used to carry are listed there, with what each one was and which +#: stack was fixed; none of them was retired by editing this file. +#: +#: The mechanism stays because it is the honest way to ship a fixture over an outstanding +#: defect. To add an entry, every ``typescript`` value must be MEASURED, by running the real +#: browser engine on the real corpus at that case's parameters: +#: +#: const vmap = buildVacancyMap(vacancyDomain(new Set(tokenize(CORPUS))), params); +#: const vacated = vacateText(CORPUS, vmap, params); +#: const stats = vacancyStats(CORPUS, vacated, vmap, params); +#: +#: Never a prediction, and never derived here. ``attach_divergence`` then asserts that the +#: Python side still disagrees with each recorded value, so the moment the defect is fixed +#: this exporter refuses to write a fixture at all rather than shipping a stale exemption. +#: Shape: ``{case label: {"cause": str, "fields": {field path: typescript value}}}``, where +#: a field path is a case key (``head400``) or ``stats.``. +KNOWN_DIVERGENCES: dict[str, dict[str, Any]] = {} + +#: Shared by every entry, so the wording cannot drift between them. +DIVERGENCE_STATUS = ( + "reported, unfixed — neither implementation was modified to build this fixture. Both " + "readings are pinned exactly, so fixing either stack turns vacancyGolden.test.ts red at " + "this block and the exemption cannot outlive the defect." +) + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def mapping_sha256(mapping: dict[str, str]) -> str: + """A digest over the whole assignment, so a map can be pinned without shipping it. + + Canonical form: `stem\\tnonce\\n` per pair, stems in ASCII-ascending order — the same + order §5.2 mints in, and reproducible in one line of TypeScript. + """ + body = "".join(f"{stem}\t{mapping[stem]}\n" for stem in sorted(mapping)) + return sha256_text(body) + + +def params_json(params: VacancyParams) -> dict[str, Any]: + """The parameter block, in the TypeScript field names both stacks' UIs use (§5.8).""" + return { + "p": params.p, + "seed": params.seed, + "consistent": params.consistent, + "matchProsody": params.match_prosody, + "revealAfter": params.reveal_after, + "keep": sorted(params.keep), + "mint": params.mint, + } + + +def stem_block( + domain: set[str], corpus_types: set[str], budget: set[str] +) -> list[dict[str, Any]]: + """`u` for the 24 pinned stems at both seeds, plus why each one is (in)eligible.""" + out: list[dict[str, Any]] = [] + for stem in PINNED_STEMS: + out.append( + { + "stem": stem, + "eligible": is_eligible(stem, VacancyParams().keep_set), + "stemOf": stem_and_suffix(stem)[0], + "suffixOf": stem_and_suffix(stem)[1], + "inDomain": stem in domain, + "inCorpus": stem in corpus_types, + "inDolchFull": stem in budget, + # EXACT float64 — see the module docstring on round-tripping. + "u": {"0": vacancy_u(stem, 0), "7": vacancy_u(stem, 7)}, + } + ) + return out + + +def vacated_stems(vmap: VacancyMap, seed: int, p: float) -> list[str]: + """The stems the map vacates at `p`: `{stem : u(stem) < p}`, in canonical order.""" + return sorted(stem for stem in vmap.mapping if vacancy_u(stem, seed) < p) + + +def _mapped_condition(vmap: VacancyMap, params: VacancyParams) -> bool: + """Is the MAPPED vocabulary of §7.2 defined for this (map, params) pair? + + Three ways it is not, and both stacks raise on each: the inconsistent-assignment control + and ``reveal_after > 0`` (§7.2 — a source type no longer has a single image), and + ``mint="swap"`` at intermediate `p` (§5.2a — swap's replacements are domain types, so a + vacated type can land on an un-vacated one and no `p`-stable swap avoids it). + """ + if not params.consistent or params.reveal_after: + return False + return vmap.injective_at_every_p or params.p in (0.0, 1.0) + + +def id_stream_block( + vacated: str, budget_words: list[str], vmap: VacancyMap, params: VacancyParams +) -> dict[str, Any] | None: + """The §7.3 measurement: the id stream under the MAPPED vocabulary. + + `map_vocab_words` preserves order, so `itos_p = SPECIALS ++ mapped` gives every word the + id its pre-image had; the stream of ids is then unchanged by vacancy and training is + bit-identical. Returning the digest at every `p` turns that theorem into data. + + ``None`` for the control conditions, where the mapped vocabulary is undefined and both + stacks raise — see the module docstring. + """ + if not _mapped_condition(vmap, params): + return None + mapped = map_vocab_words(budget_words, vmap, params) + vocab_p = LexVocab(tuple(mapped), source="dolch", budget_name="full") + ids = [int(i) for i in token_stream(vacated, vocab_p)] + return { + "digest": sha256_text(",".join(str(i) for i in ids)), + "length": len(ids), + "first16": ids[:16], + "last16": ids[-16:], + "mappedWordsSha256": sha256_text("\n".join(mapped)), + } + + +def build_case( + label: str, + map_label: str, + corpus: str, + vmap: VacancyMap, + params: VacancyParams, + budget_words: list[str], +) -> dict[str, Any]: + """One measured condition: the rewritten corpus, its statistics, and the id stream. + + Order matters and the test must repeat it: `vacate_text` runs FIRST, because under + `consistent = false` it registers the stress pattern of every form it mints on the map, + and `vacancy_stats` scores the vacated side with exactly those patterns. + """ + vacated = vacate_text(corpus, vmap, params) + stats = vacancy_stats(corpus, vacated, vmap, params) + mapped_condition = _mapped_condition(vmap, params) + return { + "label": label, + "map": map_label, + "params": params_json(params), + "head400": vacated[:HEAD_CHARS], + "vacatedSha256": sha256_text(vacated), + "vacatedChars": len(vacated), + "stats": stats, + # §11's stability assertion: the surface form of each pinned stem at this `p`. It + # is stated for — and only meaningful in — the MAPPED condition; §7.1 says the two + # controls have DELIBERATELY no stability property, so there is nothing to pin + # there and both stacks say so in their own way (TypeScript's single-word + # `transformWord` refuses an order-dependent condition outright). + # Stability (§11) is stated for the ORDER-INDEPENDENT conditions, which is a weaker + # requirement than the mapped vocabulary's: `mint="swap"` at intermediate `p` has no + # mapped vocabulary (§5.2a) but its map is still built once and still stable, so its + # surface forms are pinned here exactly as the nonce strategy's are. + "stemForms": ( + {stem: vmap.apply_word(stem, params) for stem in PINNED_STEMS} + if params.consistent and not params.reveal_after + else None + ), + "idStream": id_stream_block(vacated, budget_words, vmap, params), + "mapVocabWordsRejects": not mapped_condition, + } + + +def _read_field(case: dict[str, Any], field: str) -> Any: + """Read a `knownDivergence` field path — either a case key or ``stats.``.""" + if field.startswith("stats."): + return case["stats"][field.split(".", 1)[1]] + return case[field] + + +def attach_divergence(case: dict[str, Any]) -> None: + """Record BOTH readings of a field the two stacks disagree on (see the docstring). + + Nothing is reconciled and nothing is loosened: the Python value is what this run + measured, the TypeScript value is what a real browser-engine run measured, and the + golden test asserts each side against its own. The guard below is what stops the + exemption outliving the defect — the moment Python agrees with the recorded TypeScript + reading, this exporter refuses to write a fixture at all. + """ + entry = KNOWN_DIVERGENCES.get(case["label"]) + if entry is None: + return + fields = [] + for field, ts_value in entry["fields"].items(): + py_value = _read_field(case, field) + if py_value == ts_value: + raise SystemExit( + f"{case['label']}.{field} now reads the TypeScript value on the Python " + "side — the divergence this fixture pins has been resolved. Drop it from " + "KNOWN_DIVERGENCES, drop the exemption in vacancyGolden.test.ts, and say " + "so in architecture.md." + ) + fields.append({"field": field, "python": py_value, "typescript": ts_value}) + case["knownDivergence"] = { + "cause": entry["cause"], + "status": DIVERGENCE_STATUS, + "fields": fields, + } + + +def git_sha() -> str: + try: + return subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except ( + OSError, + subprocess.CalledProcessError, + ) as err: # pragma: no cover - dev only + raise SystemExit(f"cannot read the git sha for provenance: {err}") from err + + +def build_document(generated: str) -> dict[str, Any]: + corpus = load_corpus_text() + if not corpus.isascii(): + raise SystemExit( + "the corpus is no longer ASCII, so `head400` no longer means the same 400 " + "characters in Python and JavaScript — re-derive the excerpt before shipping" + ) + corpus_types = set(tokenize(corpus)) + counts = type_counts(tokenize(corpus)) + budget_words = list(dolch_budget("full")) + domain = vacancy_domain(corpus_types) + domain_set = set(domain) + + maps: list[dict[str, Any]] = [] + cases: list[dict[str, Any]] = [] + + # --- the two pinned maps, in full (§11) ------------------------------------------ + prosody_maps: dict[int, VacancyMap] = {} + for seed in (0, 7): + vmap = build_vacancy_map(domain, VacancyParams(seed=seed)) + prosody_maps[seed] = vmap + maps.append( + { + "label": f"seed{seed}", + "seed": seed, + "matchProsody": True, + "mint": "nonce", + "injectiveAtEveryP": vmap.injective_at_every_p, + "remintRounds": vmap.remint_rounds, + "bijective": vmap.bijective, + "imageSize": vmap.image_size, + "domainSize": len(vmap.domain), + "mappingSize": len(vmap.mapping), + "mappingSha256": mapping_sha256(dict(vmap.mapping)), + "mapping": {stem: vmap.mapping[stem] for stem in sorted(vmap.mapping)}, + } + ) + for p in P_GRID: + cases.append( + build_case( + f"seed{seed}-p{p}", + f"seed{seed}", + corpus, + vmap, + VacancyParams(seed=seed, p=p), + budget_words, + ) + ) + + # --- matchProsody = false: a DIFFERENT map, since minting reads the flag ---------- + # Pinned by digest plus the 24 sample nonces rather than in full: §11 asks for the + # complete map at the two seeds above, and a sha256 over the canonical form is exactly + # as strong a check for this one at 1/1000th the bytes. + noprosody = build_vacancy_map(domain, VacancyParams(seed=0, match_prosody=False)) + maps.append( + { + "label": "seed0-noprosody", + "seed": 0, + "matchProsody": False, + "mint": "nonce", + "injectiveAtEveryP": noprosody.injective_at_every_p, + "remintRounds": noprosody.remint_rounds, + "bijective": noprosody.bijective, + "imageSize": noprosody.image_size, + "domainSize": len(noprosody.domain), + "mappingSize": len(noprosody.mapping), + "mappingSha256": mapping_sha256(dict(noprosody.mapping)), + "mapping": None, + "sampleNonces": { + stem: noprosody.mapping.get(stem) for stem in PINNED_STEMS + }, + } + ) + for p in (0.7, 1.0): + cases.append( + build_case( + f"noprosody-p{p}", + "seed0-noprosody", + corpus, + noprosody, + VacancyParams(seed=0, p=p, match_prosody=False), + budget_words, + ) + ) + + # --- the swap control (§8.3), on its own maps ------------------------------------ + # `mint="swap"` draws a REAL English word from the domain's open-class types by + # frequency rank, so it needs the corpus's counts and it produces a different map at + # every (seed, matchProsody). Pinned by digest plus the 24 sample replacements, like the + # noprosody map above: a sha256 over the canonical form is exactly as strong a check. + # + # The `p` grid here is deliberately {0, 0.7, 1}. §5.2a proves that a map whose images are + # domain types and which does not depend on `p` CANNOT be injective at intermediate `p` + # unless it is the identity, so `swap-p0.7` carries `idStream: null` and + # `mapVocabWordsRejects: true` — measured, and the same refusal both stacks make. The two + # endpoints carry a real id stream, which is SC-703 holding for swap exactly as it does + # for nonce wherever a swap map can be injective at all. + for seed in (0, 7): + for prosody in (True, False): + label = f"swap-seed{seed}" + ("" if prosody else "-noprosody") + base = VacancyParams(seed=seed, mint="swap", match_prosody=prosody) + swap_map = build_vacancy_map(domain, base, counts) + maps.append( + { + "label": label, + "seed": seed, + "matchProsody": prosody, + "mint": "swap", + "injectiveAtEveryP": swap_map.injective_at_every_p, + "remintRounds": swap_map.remint_rounds, + "bijective": swap_map.bijective, + "imageSize": swap_map.image_size, + "domainSize": len(swap_map.domain), + "mappingSize": len(swap_map.mapping), + "mappingSha256": mapping_sha256(dict(swap_map.mapping)), + "mapping": None, + "sampleNonces": { + stem: swap_map.mapping.get(stem) for stem in PINNED_STEMS + }, + } + ) + if not prosody: + continue + for p in (0.0, 0.7, 1.0): + cases.append( + build_case( + f"swap-seed{seed}-p{p}", + label, + corpus, + swap_map, + VacancyParams(seed=seed, p=p, mint="swap"), + budget_words, + ) + ) + + # --- the two control conditions (§7.1), each on its OWN map ----------------------- + # `consistent = false` writes to `minted_stress`; sharing a map across cases would let + # one case's minted patterns score another case's text. + # + # `control-inconsistent-seed7` is the seed-7 CONDITION-B regression, pinned as data. + # It is at `p = 1` and not `0.7` deliberately: §10's identity says every eligible type + # vacates at full vacancy, so this case must read `corpusTypesVacated == 1922` and + # `tokensVacated == 8202` — exactly what `consistent = true` reads. Before the fix it + # read 1921 / 8201, because the stem `tak` minted the nonce `tak` and `Taking` survived + # the transform. Seed 0 shows nothing here (no stem mints itself), which is why the + # defect lived in the one control the fixture already had. + for label, params in ( + ("control-inconsistent", VacancyParams(seed=0, p=0.7, consistent=False)), + ("control-inconsistent-seed7", VacancyParams(seed=7, p=1.0, consistent=False)), + ("control-reveal-after-2", VacancyParams(seed=0, p=0.7, reveal_after=2)), + ): + fresh = build_vacancy_map(domain, params) + maps.append( + { + "label": label, + "seed": params.seed, + "matchProsody": params.match_prosody, + "mint": params.mint, + "injectiveAtEveryP": fresh.injective_at_every_p, + "remintRounds": fresh.remint_rounds, + "bijective": fresh.bijective, + "imageSize": fresh.image_size, + "domainSize": len(fresh.domain), + "mappingSize": len(fresh.mapping), + "mappingSha256": mapping_sha256(dict(fresh.mapping)), + "mapping": None, + "sampleNonces": { + stem: fresh.mapping.get(stem) for stem in PINNED_STEMS + }, + } + ) + case = build_case(label, label, corpus, fresh, params, budget_words) + attach_divergence(case) + cases.append(case) + + nesting = { + "seed": NESTING_SEED, + "map": f"seed{NESTING_SEED}", + "levels": [ + { + "p": p, + "stems": vacated_stems(prosody_maps[NESTING_SEED], NESTING_SEED, p), + } + for p in NESTING_PS + ], + } + + return { + "format": FORMAT, + "generated": generated, + "git_sha": git_sha(), + "command": "python scripts/export_vacancy_golden.py", + "source": ( + "llm_geometry.lex.vacancy run directly on the committed corpus — real code, " + "real text, no mocks" + ), + "contract": "specs/007-vacancy-transform-field/architecture.md", + "tolerance": TOLERANCE, + "python_version": platform.python_version(), + "encoding": ( + "every value is plain JSON; floats are shortest-round-trip in both languages, " + "so `u` and the digests compare EXACTLY. Only the prosody means " + "(meanSyllables*, meanAnapest*, stressFrom*) use `tolerance`. Digests are " + "sha256 hex: `vacatedSha256` over the vacated corpus's UTF-8 bytes, " + "`mappingSha256` over `stem\\tnonce\\n` in ASCII-ascending stem order, " + "`idStream.digest` over the ids joined by ','." + ), + "corpus": { + "path": "code/backend/src/llm_geometry/lex/data/real-mother-goose.txt", + "note": "the Gutenberg body, trimmed exactly as lex/corpus.py trims it", + "sha256": sha256_text(corpus), + "chars": len(corpus), + "tokens": len(tokenize(corpus)), + "corpusTypes": len(corpus_types), + "domainSize": len(domain), + "budget": "dolch/full", + "budgetSize": len(budget_words), + }, + "stems": stem_block(domain_set, corpus_types, set(budget_words)), + "maps": maps, + "cases": cases, + "nesting": nesting, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--generated", default=date.today().isoformat()) + args = parser.parse_args() + + document = build_document(args.generated) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(document, indent=1) + "\n", encoding="utf-8") + size_kb = args.out.stat().st_size / 1024 + print( + f"wrote {args.out} ({size_kb:.0f} KB, {len(document['maps'])} maps, " + f"{len(document['cases'])} cases)" + ) + + +if __name__ == "__main__": + main() diff --git a/specs/002-interactive-model-explorer/contracts/api.md b/specs/002-interactive-model-explorer/contracts/api.md index a6e5df8..869bf41 100644 --- a/specs/002-interactive-model-explorer/contracts/api.md +++ b/specs/002-interactive-model-explorer/contracts/api.md @@ -15,6 +15,28 @@ SSE progress events MAY now carry an optional `"phase"` field Array encoding: all tensors are nested JSON lists of finite floats, row-major, rounded to 6 significant digits. Shapes documented as `(rows, cols)`. +## Additive namespaces (this file's endpoints are unchanged) + +Later features add whole namespaces beside `/api/geo/*` and `/api/arch/*`. Nothing +below is altered by them — no path, parameter, field, status code, or error type in +this document changes meaning — and each addition is recorded here so that "frozen" +means *frozen*, not *undocumented*: + +| Added by | Namespace | Contract | +|-|-|-| +| 006 | `/api/lex/*` (Lexicon Lab) | `specs/006-lexicon-lab-tiny/contracts/api-lex.md` | +| 007 | `POST /api/lex/vacancy`, `vacancy` on `POST /api/lex/train` | same file, "Feature 007" section | + +**Why 007 needed an addition rather than a parameter on something existing.** The +vacancy transform rewrites a *corpus*, and every existing endpoint here takes a model +or a prompt. Folding it into `/api/lex/coverage` would have made that endpoint's +response mean two different things depending on a flag — the exact failure this file +is frozen to prevent. The one existing endpoint that did change, `/api/lex/train`, +gained an **optional** object whose absence is byte-for-byte the previous behaviour, +because a vacated corpus has to be tokenized under the vocabulary the transform +assigns it and shipping the ~86 kB rewritten text back and forth to achieve that +would have been a worse contract than a parameter. + --- ## Geometry Lab — `/api/geo/*` diff --git a/specs/003-static-pages-site/plan.md b/specs/003-static-pages-site/plan.md index 5b57240..e9df99f 100644 --- a/specs/003-static-pages-site/plan.md +++ b/specs/003-static-pages-site/plan.md @@ -28,6 +28,9 @@ Python backend — per issue #1's "ideally it could run entirely in a browser". - Generation/chat: LIVE via transformers.js v4 pipeline, device webgpu dtype q4f16 (117 MB from HF CDN) with wasm+q8 fallback; per-token probs from logits (direct forward calls). + AMENDED (2026-08-04, defect fix): q4f16 returns input-independent logits on + WebGPU and is never requested; the ladder is webgpu/q8 → wasm/q8 with a + load-time non-degeneracy check on every session. - Trace panel (per-layer attentions/hidden norms): NOT available from ONNX → precomputed traces for ~6 example prompts (dropdown), clearly labeled; the playback/diagram animation runs on the precomputed node_activations. Arbitrary diff --git a/specs/003-static-pages-site/spec.md b/specs/003-static-pages-site/spec.md index f7efc63..3ba72ae 100644 --- a/specs/003-static-pages-site/spec.md +++ b/specs/003-static-pages-site/spec.md @@ -31,6 +31,11 @@ Python backend's trace/field/finetune outputs to ≤1e-5. - Generation/chat: LIVE via transformers.js v4 (`device: "webgpu"`, `dtype: "q4f16"`; WASM+q8 fallback), per-token probabilities from real logits. + **AMENDED (2026-08-04, defect fix): `q4f16` is never requested. It builds a session + on WebGPU and returns logits identical at every position — measured on three of the + four curated models. The ladder is `webgpu/q8 → wasm/q8`, and every session must pass + a load-time non-degeneracy check (`staticClient/logitsSanity.ts`) before its numbers + are shown. Do not reinstate an fp16-activation dtype from this sentence.** - Tokenization strip: LIVE via transformers.js AutoTokenizer (vendored tokenizer.json, no model download). - Weight inspector: LIVE exact windows via safetensors HTTP **Range** reads from diff --git a/specs/006-lexicon-lab-tiny/contracts/api-lex.md b/specs/006-lexicon-lab-tiny/contracts/api-lex.md index 10f9a0a..cf47a67 100644 --- a/specs/006-lexicon-lab-tiny/contracts/api-lex.md +++ b/specs/006-lexicon-lab-tiny/contracts/api-lex.md @@ -1,4 +1,4 @@ -# API Contract: Lexicon Lab — `/api/lex/*` (feature 006) +# API Contract: Lexicon Lab — `/api/lex/*` (features 006, 007) **Additive.** This file adds a new namespace; it does **not** change anything in the frozen feature-002 contract (`specs/002-interactive-model-explorer/contracts/api.md`). @@ -25,8 +25,17 @@ byte-identical weights but different word lists are different models and get dif tokens: the vocabulary is this tab's independent variable, and a shared token would let a cache hit serve the wrong labels. +**Feature 007 additions**, marked as such where they appear below: +`POST /api/lex/vacancy` (new path) and an **optional** `vacancy` object on +`POST /api/lex/train` (absent ⇒ the endpoint is byte for byte what it was). Their +semantics are fixed by `specs/007-vacancy-transform-field/architecture.md`, which both +stacks implement; this file specifies only what goes on the wire. + Implementation: `code/backend/src/llm_geometry/api/routes_lex.py`. Contract tests: `code/backend/tests/contract/test_api_lex.py`. +The static build serves the same surface from +`code/frontend/src/lib/staticClient/lex.ts` — the Lexicon Lab computes in the browser +in **both** modes, so nothing in this namespace is refused or approximated there. --- @@ -141,6 +150,105 @@ a corpus with no word tokens, or a corpus shorter than one full context window. `404 NotFoundError` — unknown `base`. `500 TrainingFailedError` (via the job error event) — a real training failure, surfaced verbatim, never replaced by a partial result. +### Feature 007: the optional `vacancy` object + +← `"vacancy": { "p", "seed", "consistent", "match_prosody", "reveal_after", "keep" }` + +**Optional and additive: absent, everything above is unchanged, byte for byte.** +Present, the resolved corpus is vacated (`POST /api/lex/vacancy` below defines the +parameters) *before* training, and the model is trained under the vocabulary +`specs/007-vacancy-transform-field/architecture.md` §7.2 assigns it — **mapped** when +`consistent` and `reveal_after = 0`, **rebuilt** from the vacated corpus otherwise. + +The transform runs server-side rather than in the client because `/api/lex/vacancy` +deliberately returns an excerpt: sending the whole rewritten corpus back just to train +on it would move ~86 kB per request in each direction. + +With `base` set the base model's vocabulary is used unchanged, as it always is; only +the text is vacated. + +Under the mapped condition this is a **pure relabelling**: the token id stream is +element-for-element identical, so `first_loss`, `final_loss` and `val_loss` are +*bit-identical* to the same run on the English corpus. That is the tiny arm's result +(architecture.md §7.3), not a caveat about it. The `model_token` still differs, +because the vocabulary is part of it. + +The transform's parameters are in the cache key even though `(corpus, vocabulary)` +already determines the run — so that a knob added to the transform later cannot land +on an entry made before it existed. + +Errors: `400 InvalidParamError` — `vacancy` not an object, or any parameter outside +the ranges given for `/api/lex/vacancy`. + +## POST /api/lex/vacancy + +**Feature 007.** The vacancy transform applied to a corpus, with the statistics of +`specs/007-vacancy-transform-field/architecture.md` §10. Additive: it adds a path and +changes nothing that existed. Same corpus-source and budget rules as +`/api/lex/coverage`, because the interesting question about a vacated corpus is always +"under which vocabulary?". + +← `{ "source", "budget", "size", // as /coverage + "text" | "hf_dataset" (+ "hf_split", "max_samples"), // as /coverage + "p": , "seed": , + "consistent": , "match_prosody": , + "reveal_after": , "keep": [, …], + "preview_chars": }` + +All fields optional. The five transform knobs are architecture.md §7.1's, in this +API's `snake_case`; `keep` must be a **list**, since a bare string would be read +letter by letter and quietly protect six single letters. + +→ `200 { "p", "seed", "consistent", "match_prosody", "reveal_after", "keep": [, …], + "vocabulary_rule": "mapped" | "rebuilt", + "words": [, …], + "budget": { "source", "budget", "size", "rows", "coverage": {…} }, + "corpus": { "n_tokens", "n_distinct", "n_lines", "n_chars" }, + "vacancy_stats": { …§10's 23 fields, camelCase… }, + "bijective": , "remint_rounds": , + "preview": , "original_preview": , + "preview_chars": , "truncated": , + "vacated_chars": , "vacated_sha256": "<64 hex>", + "original_chars": , "original_sha256": "<64 hex>" }` + +**An excerpt and a digest, never the whole vacated corpus.** The shipped corpus is +~86 kB of body text and the panel re-runs this on every tick of the `p` slider, so +returning it whole would put megabytes on the wire across one sweep to show a reader a +screenful. Nothing needs it whole: the panel shows an excerpt (the source's own figure +is its first 400 characters), and a caller that wants to *train* on the vacated corpus +sends the same parameters to `/api/lex/train`, which vacates in place. What an excerpt +cannot do by itself is prove which text it came from, so `vacated_sha256` covers all of +it in 64 characters — and that digest is the single value the static build's +in-browser transform is checked against. + +`vacancy_stats` carries §10's field names **verbatim**, camelCase inside this API's +snake_case envelope on purpose: they are a cross-language contract between +`llm_geometry/lex/vacancy.py` and `lexEngine/vacancy.ts`, not this API's naming. An +unprefixed `types*` is forbidden there; every count names its scope (`domainTypes*` vs +`corpusTypes*`). `bijective` and `remint_rounds` also appear at the top level, because +injectivity is the guarantee the mapped vocabulary rests on and a caller checking it +should not have to reach into a statistics block. + +`vocabulary_rule` says which of §7.2's two rules produced `words`, and a client must +not have to infer it from the parameters: `"mapped"` is the only condition under which +the ids are the English ids, and that is the difference between an invariance result +and a coverage collapse. + +Every number returned is measured on the corpus in the request. The source document's +own prosody figures are its numbers on a corpus we do not have and are transcribed +nowhere. + +Errors: `400 InvalidParamError` — `p` outside `[0, 1]` or not a number, +`reveal_after < 0`, `preview_chars` outside `0..20000`, `keep` not a list of strings, +`size` with `source="dolch"`, an unknown budget, a corpus with no word tokens, or both +`text` and `hf_dataset`. + +Parity: `code/frontend/tests/fixtures/vacancy-api-golden.json` is a transcript of this +route (`python scripts/export_vacancy_api_golden.py`, real app, real corpus, no mocks). +`test_api_lex.py` asserts the live route still returns it and +`tests/unit/staticVacancy.test.ts` asserts the browser's in-page implementation +reproduces it field for field, so neither stack can drift alone. + ## GET /api/lex/spectrum The geometry of a trained model's embedding (FR-620..FR-623). diff --git a/specs/007-vacancy-transform-field/architecture.md b/specs/007-vacancy-transform-field/architecture.md new file mode 100644 index 0000000..c9f3067 --- /dev/null +++ b/specs/007-vacancy-transform-field/architecture.md @@ -0,0 +1,925 @@ +# Feature 007 — the vacancy transform: TS ↔ Python contract + +**Status:** normative. Both stacks implement *this document*, not each other. + +This is the file that feature 006 taught us to write first. In 006 the contract omitted one +sentence — how a corpus becomes a token stream — and the two stacks silently trained on +different data for a day. Everything here that reads like pedantry is load-bearing; if a +sentence looks obvious, it is because someone would otherwise have guessed differently. + +Source material: `~/Desktop/TinyModelsDoc/tiny_models.tex` §"The vacancy transform" +(`\label{sec:how-vacancy}`) and `tiny-seuss/synth/jabberwockify.py`. We port the *design* and +correct the *implementation*; §9 lists every deliberate departure with its reason. As in 006, +we never port a claim we have not measured ourselves. + +--- + +## 0. What the instrument is for + +The doc's T4 is a 2×2 over *location* (a token has a prior embedding) and *field* (a token's +distributional neighbourhood is fully specified by context): + +| | no field | field supplied | +|-|-|-| +| **no location** | (i) random init, no data | (iii) **vacancy** — nonce form, full syntactic support | +| **location** | (ii) minting at a hub centroid | (iv) normal word learning | + +The vacancy transform manufactures condition (iii) at a controlled rate `p` on any corpus: +closed-class words, inflectional suffixes, syntax and line structure are preserved exactly; +open-class stems are replaced by phonotactically legal nonce forms carrying the same syllable +count and stress pattern. + +**The result the tiny arm can prove.** For a *word-level model trained from scratch*, a word's +"location" is nothing but a row index — the model never sees the letters. Under the conditions +of §7 the transform is therefore a **pure relabelling of the vocabulary**, and the model is +*exactly* invariant to it: same token id stream, same loss, bit for bit. That is stronger than +the doc's prediction (it predicts (iii) ≫ (ii); in the tiny regime (iii) ≡ (iv) identically), +and §7 states it as a theorem with a test that would catch it becoming false. + +An invariance is only worth showing against something that breaks it. §6 defines the three +control conditions the doc itself calls for, and §8 defines the pretrained arm — a model that +*does* have locations — which is where the number that says what location was worth comes from. + +--- + +## 1. Word segmentation + +The transform rewrites a raw text in place. It finds words with **exactly the tokenizer's +regex** and passes everything else — whitespace, punctuation, digits, line breaks — through +unchanged, byte for byte. + +``` +WORD_RE = /[A-Za-z]+(?:['\-][A-Za-z]+)*/g # TS: lexEngine/vocab.ts +WORD_RE = re.compile(r"[A-Za-z]+(?:['\-][A-Za-z]+)*") # Py: lex/vocab.py +``` + +This is **not** the regex the source used (`[A-Za-z][A-Za-z']*`, which splits `good-bye` into +two words). Using the tokenizer's own regex is a hard requirement: the relabelling theorem of +§7 is false the moment the transform's idea of a word differs from the trainer's. + +Each match is replaced by the output of `transformWord` (§5). **Every output is itself a +single, complete `WORD_RE` match** — checked, not assumed (§7.3). Therefore +`tokenize(vacate(text))` has exactly the same length and ordering as `tokenize(text)`, and +because line breaks are untouched, the ``-per-line rule produces the same number of +`` in the same places. + +TS must construct a fresh `RegExp(WORD_RE.source, "g")` per call — the shared literal carries +`lastIndex`. (This bit `vocab.ts` already; do not re-learn it.) + +--- + +## 2. Eligibility: what may be vacated + +### 2.1 The closed class + +`FUNCTION_WORDS` is the source's curated list, ported verbatim, whitespace-split and +lowercased: + +``` +a an the this that these those my your his her its our their some any all both each +every no none i me you he she it we they him them us who whom whose which what where +when why how is am are was were be been being do does did done have has had having +will would shall should can could may might must not and or but so if then than as of +to in on at by for with from into onto up down out off over under again once here there +very too also only just even still yet ever never always about after before while +because though although unless until since during between among against through above +below near far one two three four five six seven eight nine ten +``` + +The source carries a warning we keep: an earlier version of it added short Dolch service words +to the closed class, which silently protected content verbs (`run`, `eat`, `see`, `get`, `let`, +`put`) and understated the vacancy rate. The closed class is **this curated list only**. Do not +union it with a Dolch budget. + +Callers may extend it with a `keep` set; the effective set is `FUNCTION_WORDS ∪ lower(keep)`. + +### 2.2 The eligibility test + +A word is eligible for vacancy iff, after suffix splitting (§3), its **stem** satisfies all of: + +1. `lower(stem) ∉ keepSet` +2. `stem` matches `^[A-Za-z]+$` — ASCII letters only +3. `len(stem) > 2` + +Test 2 is why hyphenated and apostrophised words behave as they do, and both stacks must agree +on it exactly: + +- `good-bye` — no suffix matches, stem is `good-bye`, which contains a hyphen, so **test 2 + fails and the word is never vacated**. +- `don't` — `n't` does *not* split it, because §3's length rule requires + `len(word) - len(suffix) >= 3` and `5 - 3 = 2`. The stem is therefore `don't`, which contains + an apostrophe and fails test 2. Never vacated. (An earlier draft of this document said the + suffix splits it to `do`; that was wrong about the mechanism, though right about the outcome.) +- `dog's` — `'s` splits it (`5 - 2 = 3`) to stem `dog`, which passes; output is `'s`. + +Python must use `re.fullmatch(r"[A-Za-z]+", stem)`, **not** `str.isalpha()`. `isalpha()` is +Unicode-aware and would accept letters JS's `^[A-Za-z]+$` rejects. Nothing in the shipped +corpus exercises the difference; a pasted corpus would. + +--- + +## 3. Suffix splitting + +Inflectional morphology is preserved: the stem is vacated, the suffix is re-attached, so the +nonce still looks inflected and the syntax still parses. + +``` +SUFFIXES = ["ing", "edly", "est", "ies", "'s", "n't", "ed", "es", "er", "ly", "s"] +``` + +Tried **in this order**; the first match wins. A suffix `s` matches iff +`lower(word).endswith(s)` **and** `len(word) - len(s) >= 3`. The split slices the **original** +word, so case is preserved. + +``` +EXCEPTIONS = {brother, father, mother, sister, never, over, under, morning, giving, thing} +``` + +A word whose lowercase form is in `EXCEPTIONS` is **never split** (stem = word, suffix = `""`). +This list comes from the audited copy of the source, not the copy in the zip: without it +`brother → broth+er` and `morning → morn+ing`, which the source itself flags as a known +artifact. It is a spelling heuristic, not a morphological analyser, and it is wrong on words +outside the list (`ladder → ladd+er`). That is acceptable — the nonce still carries a +consistent identity and an inflected-looking surface — but it must be *documented in the UI*, +not quietly tolerated. + +--- + +## 4. The vacancy decision — nesting + +A stem is vacated iff `u(stem) < p`, where `u` depends only on the stem and the seed: + +``` +digest = sha256(utf8(f"{seed}:{lower(stem)}")) # 32 bytes +top64 = int(digest[0:8], big-endian) # first 8 bytes +u = (top64 >> 11) / 2**53 +vacate iff u < p +``` + +**The `>> 11` is mandatory and is a departure from the source** (which used +`top64 / 2**64`). + +The reason is *not* that the source's expression diverges across the two languages. It was +measured, and it does not: `int / 2**64` in CPython is a single correctly-rounded division, +while `Number(bigint) / 2**64` in JS rounds to float64 and then divides by a power of two — +which is exact — so the two agree. This was checked on 200 006 values, including random 64-bit +integers and hand-picked ties at the rounding boundary (`(1<<63)|((1<<11)-1)`, `(1<<64)-1`, and +neighbours). Every one matched bit for bit. + +The reason is that `(top64 >> 11) / 2**53` needs **no such argument**. A 53-bit integer over +2⁵³ is exactly representable, so `u` is self-evidently the same double in both languages, and +the property survives a reimplementation that assembles the value differently — from two 32-bit +halves, say, where the rounding argument above stops holding. A cross-language equality that +depends on a subtle proof is one refactor away from being false; this one does not. + +- TS: `Number(BigInt("0x" + hex.slice(0, 16)) >> 11n) / 2 ** 53` +- Py: `(int.from_bytes(digest[:8], "big") >> 11) / 2 ** 53` + +`p` is compared as given. Callers must pass the identical double; the UI emits `p` at two +decimal places and both stacks parse it as float64. + +**Nesting.** `u` is a function of `(seed, stem)` alone — not of `p`, not of traversal order, not +of which other words exist. So `{stems vacated at p} ⊆ {stems vacated at p'}` for `p < p'`, +which is the first of the two properties that make a `p`-sweep interpretable. + +--- + +## 5. Minting — stability + +### 5.1 Why the source's minter cannot be ported as written + +The source mints with `random.Random(f"{seed}:{k}")` and then guards uniqueness with a +`used` set on a long-lived `Minter`. Two consequences, both of which break the stability +property the source claims for itself: + +1. **`used` is order-dependent.** If stem *A* mints `flim` and stem *B* would too, *B* retries — + but only if *A* was minted first. At `p = 0.5` only *B* may be vacated, so *B* gets `flim`; + at `p = 1.0` both are, so *B* gets something else. The nonce for a word then depends on `p`, + which is exactly what stability forbids. +2. **The give-up path is order-dependent.** After 400 failed attempts the source returns + `syllable + str(len(self.used))` — a counter of how many words happened to be minted before. + +There is also the practical problem that Python's Mersenne Twister seeded from a string is not +reproducible in TypeScript without reimplementing MT19937 and `Random.choice`'s masking. + +### 5.2 What we do instead + +**The map is built once over the whole type set, in a canonical order, independent of `p`.** +The map at any `p` is then the restriction of that single map to `{stem : u(stem) < p}`. +Nesting and stability become structural facts rather than properties to be hoped for. + +``` +buildVacancyMap(types, params) -> Map + domain := { lower(t) for t in types } + stems := sorted({ stemOf(t) for t in domain if eligible(stemOf(t)) }) # ASCII sort, ascending + used := {} + for stem in stems: # canonical order — never p, never document order + nonce := mint(stem, seed, matchProsody, forbidden = used ∪ domain) + used.add(nonce); map[stem] = nonce +``` + +**There is no caller-supplied `avoid` parameter.** The domain is always avoided, implicitly. + +The first implementations both gave `avoid` a default of empty and left it to the caller to pass +the type set. Both stacks agreed with each other, so no parity test caught it — but the map is +then a function of *what the caller remembered to pass*. Measured: at seed 0 the same corpus and +seed produce `remintRounds = 0` with the domain passed and `1` without, and **different nonces** +either way. Both maps are valid; that is the problem. One caller passing it and another not — +the panel and the golden fixture, say — is a silent divergence with no failing test. + +Since condition B below already requires that no surface form equal any domain type, avoiding +the domain at mint time is not an extra policy, only the cheaper way to reach the same fixed +point. Making it implicit costs nothing and makes the map a pure function of +`(domain, seed, matchProsody)`. + +**The domain is `corpus types ∪ the full Dolch list`** — the *full* list, always, never the +active budget. §7.2 explains why budget words must be in the domain at all; the reason it is the +full list is that the domain must not depend on which budget the reader has selected, or +switching budgets would re-mint the corpus in front of them and the stability the panel is +demonstrating would look false. A frequency budget needs no special case, since its words are +corpus types by construction. + +Making the domain the forbidden set (below) turns this from a convenience into a requirement, +and the earlier draft of this paragraph is now wrong in an instructive way. It said the map was +"identical across all five Dolch domains" — true when `avoid` was a caller-passed corpus type +set independent of the domain, false now. A **smaller domain forbids less**, so it mints +differently: building over `corpus ∪ dolch_budget(name)` for any name below `full` moves exactly +one stem, `jam → floor` instead of `scirmp`, because `floor` is a full-list Dolch word that +never occurs in *Mother Goose* and so is only forbidden when the full list is in the domain. + +That is a *reason* the domain must be the full list rather than a coincidence that it may be. +`vacancyDomain` makes the smaller domains unreachable, and the test asserts the property that +actually matters — **the map does not move when the active budget changes** — rather than the +stronger claim that happened to hold before. + +The source accepts an `avoid` parameter and then never passes one, which lets a minted form +silently merge with an English type. We do not repeat that by making it optional — see above. + +**Both stacks expose a `vacancyDomain(types)` / `vacancy_domain(types)` helper** that applies the +union rule, and every call site uses it. Python had one and TypeScript did not, which is the kind +of asymmetry that ends with two call sites building the domain two different ways. + +Both take an **iterable of types, not a text**. Python must reject a bare `str` explicitly: +`Iterable[str]` happily accepts one and iterates it character by character, so +`vacancy_domain(corpus_text)` silently yields a domain of single letters. Raise a `TypeError` +naming `tokenize()`, rather than returning a subtly wrong answer. + +**The check is over surface forms, not bare nonces, and it must hold at every `p`.** This is +the second defect the first implementation exposed. A bare-nonce check is not enough: + +> At `seed = 7`, the stem `hang` minted the nonce `wak`. No corpus type equals `wak`, so a +> bare-nonce `avoid` check passes. But the corpus contains `hanged`, whose surface form is +> `wak` + `ed` = `waked` — and the corpus *also* contains the English word `waked`. At `p = 1` +> both are vacated and nothing collides, which is why a check performed only at full vacancy +> sees nothing. At `p = 0.25` and `p = 0.5`, `hanged` is vacated and `waked` is not, so two +> distinct source types both map to `waked`. Injectivity fails, and with it §7.3. + +So define, over the domain: + +- `pairs` := `{(stem, suffix)}` for every domain type whose stem is eligible +- `surface(stem, suffix)` := the assembled, lowercased output of §5.7 + +and require **both**: + +- **A.** the surface forms are pairwise distinct +- **B.** no surface form equals any lowercased type in the domain — whether or not that type is + itself eligible + +B is deliberately conservative: it forbids a minted form from equalling a word that would always +have been vacated alongside it. That costs a re-mint and buys a condition that is independent of +`p`, which is what the theorem needs. Because un-vacated words map to themselves, minted words +map into the surface set, and A and B keep those two sets internally distinct and mutually +disjoint, injectivity holds **simultaneously for every `p`** rather than at `p = 1` only. + +On violation, re-mint the offending stems at a higher salt in canonical order and re-check; +raise after 8 rounds. + +### 5.2a What A and B are standing in for, and what `mint = "swap"` can therefore satisfy + +B is **sufficient, not necessary**. Writing the necessary condition out is what makes the swap +control of §8.3 statable at all, because swap draws its replacements *from* the domain and so +violates B by construction. + +Fix `p` and let `T_p` be the type map: `T_p(t) = surface(t)` when `t`'s stem is eligible and +`u(stem(t)) < p`, and `T_p(t) = t` otherwise. `T_p` is injective for **every** `p` iff both: + +- **A.** the surface forms are pairwise distinct — unchanged; and +- **B′.** for domain types `t₁` (eligible stem) and `t₃`, if `surface(t₁) = t₃` then `t₃`'s stem + is eligible **and** `u(stem(t₃)) < u(stem(t₁))`. + +*Why.* The only way two types can merge is that one moved onto another that had not: an image +`surface(t₁)` colliding with an un-vacated `t₃`. That is possible at some `p` exactly when +`u(stem(t₁)) < p ≤ u(stem(t₃))`, i.e. exactly when `u(stem(t₃)) ≥ u(stem(t₁))` — which B′ forbids. +The case `t₃ = t₁` is included, so B′ also rules out a type that silently fails to vacate (the +`tak → tak` defect of §5.8). Image-on-image collisions are A. + +**B ⟹ B′ vacuously** (B makes B′'s premise unsatisfiable), so nothing about the nonce strategy +changes and nothing in this document about it is weakened. B stays the *enforced* rule for +`mint = "nonce"`: it is cheaper to check, it costs one re-mint on the shipped corpus, and it is +the reason §7.3 holds simultaneously at every `p`. + +**Theorem (why `swap` cannot have that).** Suppose the map is stable in `p` (§5.6) and every +image is a domain type — both true of swap by construction. `T_p` injective for every `p` forces +`T_p` to be a bijection of the domain onto itself, hence to map the vacated set `V_p` onto `V_p`. +The `V_p` are nested and grow one *stem family* at a time, so a bijection mapping every `V_p` +onto itself maps each stem family onto itself: `u(stem(σ(s))) = u(stem(s))`, hence `σ = id`. +**So no non-trivial swap is injective at intermediate `p`.** Measured, to make it concrete rather +than merely proved: the frequency-rank swap below produces 191 / 246 / 190 colliding types at +`p = 0.25 / 0.5 / 0.75` on the shipped corpus, and 0 at `p ∈ {0, 1}`. + +What swap *can* satisfy, and does, is the condition at **full vacancy**, where the un-vacated set +is exactly the ineligible types: + +- **B₁.** no surface form equals an **ineligible** domain type, and no surface form equals its own + source type. + +A + B₁ make `T_1` a bijection of the domain, so the invariance theorem of §7.3 holds for +`mint = "swap"` at `p ∈ {0, 1}` — and *provably cannot* hold at `0 < p < 1`. The engine therefore +**refuses** the mapped vocabulary of §7.2 for `swap` at intermediate `p`, with a typed error +naming this theorem, rather than shipping a vocabulary with two words on one row. `vacateText` +itself is unrestricted: the pretrained arm of §8.3 measures a passage, and a passage does not +need an injective map. `VacancyMap.injectiveAtEveryP` reports which of the two regimes a map is +in — `true` for nonce, `false` for swap — so no caller has to infer it. + +### 5.3 The deterministic byte stream + +`random.Random` is replaced by a sha256 counter stream, which is trivially identical in both +languages: + +``` +bytesFor(seed, stem, salt, counter) = sha256(utf8(f"{seed}:mint:{stem}:{salt}:{counter}")) +``` + +Consume the stream 4 bytes at a time, big-endian, as an unsigned 32-bit integer; refill from +the next `counter` when exhausted. A choice from a list is `list[nextU32() % len(list)]`. +(All lists here are shorter than 256, so the modulo bias is aesthetic, not statistical — but +both stacks must bias *identically*, which they do.) + +TS: `((b0 << 24) | (b1 << 16) | (b2 << 8) | b3) >>> 0`. The `>>> 0` is required; without it JS +produces a negative number and `%` returns a negative index. + +### 5.4 The phonotactic tables + +Ported verbatim from the source, order significant (the index into each list is what the byte +stream selects, so reordering silently changes every nonce): + +- `ONSETS` — 47 entries, `b … sq` +- `NUCLEI` — 19 entries, `a … er` +- `CODAS` — **46** entries, `"" … zzle` +- `UNSTRESSED_TAILS` — 13 entries, `y … ing` + +An earlier draft said `CODAS` had 49. It has 46 — I miscounted. Both implementations copied the +source verbatim, flagged the discrepancy, and correctly took "verbatim" over the tally, so +nothing diverged. **The lists are normative; the counts here are commentary.** If they ever +disagree again, the source lists win, because a nonce is a function of the strings and their +indices and not of a number in a document. +- unstressed-onset prefixes — `["a", "be", "re", "de", "un", "en"]` +- reduced coda set for unstressed syllables — `["", "", "l", "n", "r", "s"]` (the duplicated + empty string doubles its weight; keep it) + +### 5.5 The mint loop + +A mint call carries a **base salt** `S` (0 for the first build; §5.8 sets it for re-mints) and +runs an **attempt counter** `a = 0, 1, 2, …`. The byte stream of §5.3 is keyed on +`salt = S + a`; **the quality thresholds below are on `a`, not on `salt`.** + +That distinction is load-bearing. Read the other way — thresholds on the absolute salt — a +re-mint at `S = 1001` would begin with every quality check already relaxed, so the replacement +nonce would not be prosody-matched, and a second round at `S = 2001` would exceed the give-up +bound and raise, contradicting §5.2's "raise after 8 rounds". Counting attempts per call means a +re-mint is held to exactly the same standard as an original mint, which is what makes the +seed-7 replacement (`hang → smeeg`) monosyllabic like the word it replaces. + +For `a = 0, 1, 2, …`: + +1. `pattern` := the stem's stress pattern (§6) when `matchProsody`, else `"1"`. + `nSyl` := `len(pattern)`. +2. Build a candidate: for syllable `i`, + - if `pattern[i] == "1"` emit `choice(ONSETS) + choice(NUCLEI) + choice(CODAS)` + - else if `i == 0` emit `choice(prefixes)` + - else emit `choice(UNSTRESSED_TAILS)` + + Exactly three branches, in that order. An earlier draft added a sentence about "stressed + syllables in a non-initial unstressed position", which is self-contradictory — a syllable is + stressed or it is not. **The reduced coda set of §5.4 is therefore unreachable**, exactly as + it is in the source, where `_syl(stressed=False)` is never called. It is retained in §5.4 for + fidelity to the source's tables and because the byte stream's list indices must not shift. + **Do not "fix" this in either stack** — doing so would change every multi-syllable nonce. +3. Collapse runs: `re.sub(r"([bcdfghjklmnpqrstvwxz])\1{2,}", r"\1\1", w)`. +4. Accept iff `len(w) >= 3` **and** `w ∉ forbidden` **and** `syllables(w) == nSyl`. +5. On `a >= 400`, drop the syllable-count check. On `a >= 800`, drop the length check. + These relaxations are deterministic and order-independent, unlike the source's counter. + Reaching `a >= 1200` raises — it has never happened and if it does we want to know. + +### 5.6 Stability + +`mint` depends only on `(seed, stem, matchProsody, forbidden)`, and `forbidden` depends only on +the canonically-ordered prefix of stems before it. Nothing depends on `p`, on the document, or +on the order words are encountered while rewriting. Therefore **a stem's nonce is the same at +every `p`** — the second property the `p`-sweep needs. + +### 5.7 Seams, and the case-commuting invariant + +Re-attaching a suffix can produce a seam (`wee` + `er` → `weeer`). When +`nonce[-1] == suffix[0]`, replace the nonce's last character with +`"lnrtk"[u32(sha256(f"{seed}:seam:{stem}:{suffix}")) % 5]`. Deterministic, order-independent; +the source used a shared RNG here, which is order-dependent. + +**Everything in the transform is computed on the lowercased word.** The stem and the suffix are +lowercased before the seam test, before the seam hash, and before the surface form is assembled; +`matchCase` is then applied **to the whole assembled surface form**, with the *original whole +word* as the case source. + +This is not a stylistic preference. The first implementation of this contract followed the +source and sliced the suffix case-preserved, then ran the seam test against it. So `gums` → +`flels` while `GUMS` → `FLESS`: `suffix[0]` was `s` in one and `S` in the other, the seam test +fired in one and not the other, and one source **type** acquired two distinct surface forms. +The tokenizer lowercases, so those are two different types — and §7.3 is false. + +> **Invariant (normative, and a test).** For every word `w`: +> `lower(transformWord(w)) == transformWord(lower(w))`. +> +> The transform must **commute with lowercasing**, because the tokenizer lowercases. Any step +> that branches on a character's case — seam tests, hash inputs, table lookups other than the +> deliberate case-sensitive `STRESS_TABLE` probe of §6.3 — violates it. Assert it over the whole +> real corpus, and over each type upper-cased, capitalised, and lower-cased. + +--- + +### 5.8 Details the first implementation had to invent — now pinned + +These were gaps, not choices. Both stacks do it this way or the golden fixture fails. + +- **Re-mint selection.** When conditions A/B of §5.2 fail, re-mint **only the losing stem** — + the one later in canonical (ASCII-ascending) order among those involved in the collision — at + **base salt** `1000 * round + previousBaseSalt + 1`. Round counts from 1. Winners keep their + nonce, so a re-mint never cascades. The attempt counter restarts at `a = 0` inside the new + call, so §5.5's quality checks apply in full (see the note there). +- **`consistent = false` key, and where its prosody comes from.** The per-occurrence nonce is + minted for the key `f"{stem}#{idx}"`, `idx` being the 0-based occurrence index of that **stem** + in document order. The `#` is not a legal `WORD_RE` character, so the key can never collide + with a stem. + + **Condition B applies to the per-occurrence path too.** It was enforced for the map and not + for this control, and the gap is observable: at seed 7, `p = 1`, the stem `tak` minted the + nonce `tak`, so `Taking → Taking` — a token that silently failed to vacate, leaving + `corpusTypesVacated` at 1921 against the consistent path's 1922. §7.1 says this control has no + *stability* property, which is about a nonce being reused across occurrences; it does not + license a word quietly surviving the transform. A control whose vacancy rate is not actually + the stated rate is not a control. So a per-occurrence nonce must equal neither any domain type + **nor the stem it replaces**, and the same re-mint loop applies. + + **The key feeds the byte stream and the uniqueness check only. The stress pattern comes from + `stress(stem)`, never from `stress(key)`.** This was under-specified and the two stacks split + on it: one passed the key into the minter, so the pattern became `stress("little#0") = "10"` + instead of `stress("little") = "100"`, and `Little` minted as `Wrerken` rather than + `Wrerkenle`. §7.1 says the nonce carries *the stem's* syllable count and stress, so the key + must not reach the prosody lookup. Caught by the golden fixture, not by either test suite. +- **Minted stress is passed, never global.** `stress(word, mintedStress)` takes the map as an + argument. The source used a module-level `MINTED_STRESS` dict mutated by `register_minted`, + which makes two concurrently-live maps corrupt each other — and the Lexicon Lab holds several + at once (one per condition being compared). +- **`forbidden` is STORED, not reconstructed, and includes superseded re-mint nonces.** One + stack stored it; the other rebuilt it as `domain ∪ mapping.values()`, which silently drops + every nonce that a re-mint round replaced (`wak` at seed 7). Nothing observable diverges from + that today — all digests agree — but the two sets are genuinely different, and the + per-occurrence path of the `consistent = false` control now *draws against* `forbidden`, so it + is one unlucky hash away from mattering. Superseded nonces stay forbidden because they were + rejected for a reason: reusing one can recreate the very collision the re-mint resolved. + **Both stacks now store it**, and both assert `"wak" ∈ forbidden` at seed 7 — the superseded + nonce of `hang`, which re-minted to `smeeg`. It is the one case the shipped corpus produces, so + it is the one the test names; a reconstruction from `mapping.values()` fails that assertion. + The re-mint loop draws against the same accumulated set, so a re-mint can never hand a stem a + form some other stem has already given up. +- **`VacancyMap`'s stem→nonce field is `mapping` in both stacks.** TypeScript called it `map` + and Python `mapping`, which is the third naming asymmetry this feature produced (after the + missing `vacancyDomain` helper and the `avoid` default) and cost a debugging round for + anything driving both. `mapping` wins because a field called `map` sitting next to Python's + builtin reads badly. The remaining fields — `mintedStress`, `remintRounds`, `bijective`, + `imageSize`, `forbidden`, `domain` — already agree and are normative. +- **Statistic field names are camelCase in both stacks**, exactly as §10 spells them, including + in the Python JSON. This deviates from the rest of the Python API, which is snake_case; the + vacancy block is a nested object, so it is self-consistent and it lets the golden fixture + compare the two stacks key-for-key without a translation table. + +## 6. Prosody + +### 6.1 Provenance — read this before quoting a number + +`STRESS_TABLE` is ported verbatim from `tiny-seuss/synth/lexicon.py` (61 polysyllables of the +Dolch list). The source describes it as **"seeded by rule and then overridden by a hand table"** +and its own status table lists the stress table under *not yet exercised*: *"seeded by rule; +wants roughly an hour of human checking."* + +So: **we do not claim exact prosody, and no UI string may.** The doc's argument that a closed +lexicon *buys* exact prosody is sound in principle and false of this table today. Two +consequences, both mandatory: + +1. The table covers Dolch words. The shipped corpus is *The Real Mother Goose* with ~2 200 + types, most of which are not in it, so most words fall through to the spelling rule. +2. Every prosody statistic the UI shows must be accompanied by `stressTableCoverage` — the + fraction of tokens whose stress came from the hand table rather than the rule. That number + is the honesty of every other prosody number on the panel. + +### 6.2 Syllable rule (fallback) + +``` +w := lower(word), strip leading/trailing "'" and "-", then delete every non [a-z] +if w is empty: return 1 +n := count of matches of /[aeiouy]+/ in w +if w ends with "e" and n > 1 and w does not end with "le" | "ee" | "ye": n -= 1 +return max(1, n) +``` + +The source has a further `if w.endswith("le") …: pass` branch. It is dead code — a `pass` — and +is *not* ported; the behaviour above is byte-identical to the source's. + +### 6.3 `stress(word)` + +Lookup order, exactly: + +1. `mintedStress[lower(word)]` — the intended pattern of a form we minted ourselves, so + prosody scoring on a vacated corpus reflects what we built *for the minted forms*. Passed in + as an argument, never a module global (§5.8). +2. `STRESS_TABLE[word]` — **case-sensitive**, for `Christmas` +3. `STRESS_TABLE[lower(word)]` +4. the rule: `"1"` if `n == 1` else `"1" + "0" * (n - 1)` + +`syllables(word) := len(stress(word))`. + +### 6.4 Meter + +`meterScore(line, foot)` = the fraction of syllable positions in the line's concatenated stress +string that match the repeating foot, `0.0` for a line with no syllables. +`anapest = "001"`, `iamb = "01"`, `trochee = "10"`, `dactyl = "100"`. Reported as the mean over +lines that produce at least one token. + +--- + +## 7. Conditions, vocabulary, and the invariance theorem + +### 7.1 Parameters + +| name | type | default | meaning | +|-|-|-|-| +| `p` | float ∈ [0,1] | `0` | fraction of eligible **types** vacated | +| `seed` | int | `0` | selects both `u` and the nonce assignment | +| `consistent` | bool | `true` | one nonce per source type, corpus-wide | +| `matchProsody` | bool | `true` | nonce carries the stem's syllable count and stress | +| `revealAfter` | int | `0` | first N occurrences of a vacated stem keep the English form | +| `keep` | set | `{}` | extra words added to the closed class | +| `mint` | `"nonce"` \| `"swap"` | `"nonce"` | invent the replacement, or draw a real word (§8.3) | + +`consistent = false` derives the nonce from `(stem, occurrenceIndex)` in document order, so +every occurrence is a fresh type. This condition deliberately has **no** stability property; +it is the source's "inconsistent assignment" control and its purpose is to destroy the field +while holding the vacancy rate fixed. + +### 7.2 Vocabulary under vacancy + +Two rules, and which applies depends on the condition: + +**Mapped vocabulary** (`consistent = true`, `revealAfter = 0`) — the budget's word list is +pushed through the *same* `transformWord`, **preserving order**. Since the map is injective, +`itos_p = SPECIALS ++ [transformWord(w) for w in words]` assigns every word the id its +pre-image had. This is why the map's domain must include the budget's words as well as the +corpus's types (§5.2): a budget word absent from the corpus still needs an image. + +**Rebuilt vocabulary** (every other condition) — the budget is rebuilt from `C_p` by the tab's +normal rule (the Dolch list as-is, or `frequencyBudget(C_p, N)`). Coverage then collapses, and +the collapse is the measurement. + +### 7.3 The invariance theorem + +> **Theorem.** With `consistent = true` and `revealAfter = 0`, and the vocabulary mapped as +> above, for every `p`, `seed`, budget, and value of `matchProsody`: +> +> `tokenStream(vacate(C, p), V_p)` equals `tokenStream(C, V)` element for element. +> +> **Corollary.** Training is bit-identical: `runTraining` is a function of +> `(cfg, tokens, seed, hyperparameters)` and `cfg` depends on the vocabulary only through +> `vocabRows`, which is unchanged. + +Why it holds: §1 gives a bijection between word occurrences that preserves order and line +structure; §5.2 makes the type map injective on the union domain; §7.2 makes `stoi_p(map(w))` += `stoi(w)` for budget words and sends every non-budget type to a non-budget type, so +`` lands in exactly the same places. + +**The theorem is a test, not a comment** (SC-703). It is asserted on the real corpus across all +five Dolch budgets and a frequency budget, `p ∈ {0, 0.25, 0.5, 0.75, 1}`, `seed ∈ {0, 7}`, and +both settings of `matchProsody`. If a future change to the tokenizer, the suffix list, or the +minter breaks it, that test fails. + +**Injectivity is verified, not assumed** — by conditions A and B of §5.2, which are checked at +map-build time and reported as `bijective` and `remintRounds` in the statistics. + +Two traps, both found by implementing this document rather than by reading it: + +1. Checking `|image| == |types|` **at `p = 1` only** is insufficient. At full vacancy every + eligible type has moved, so a minted form cannot collide with a surviving English word; the + collision only exists at intermediate `p`. Condition B is `p`-independent precisely so this + cannot recur. +2. Checking **bare nonces** rather than assembled surface forms is insufficient, for the same + reason: the collision arrives through the suffix. + +The test asserts injectivity at every `p` in the grid, not just at the endpoints. + +### 7.4 What this predicts, stated before we measure it + +Three of the knobs — `p`, `seed`, `matchProsody` — are **invisible** to a word-level model +trained from scratch. Only the knobs that break type identity (`consistent = false`, +`revealAfter > 0`) can change a loss. That is the honest tiny-arm result, and it is worth +saying plainly rather than dressing a null up as a curve: *for this model class, all of a +word's meaning is field and none of it is form.* The prosody control matters for the corpus as +an artifact and not at all for this model — which is itself a finding about what a word-level +model can see. + +--- + +## 8. The pretrained arm (Architecture Explorer) + +The tiny arm has no model with a location, so it cannot say what a location is worth. This arm +does: it runs a **real pretrained model** over a passage and its vacated twin. + +### 8.1 The measurement + +Mean negative log-likelihood, in nats per token, restricted to the tokens of **preserved** +words — the closed-class scaffolding, which is character-identical in both passages. Formally, +`the __ __ did __ and __`: does a model that knows English still predict the scaffolding when +the content is vacant? Carroll's claim is that a reader does; this puts a number on it. + +Reported per passage (English `E`, vacated `J`): + +- `nllPreserved` — the headline; mean NLL over tokens belonging to preserved words +- `nllAll` — every scored token, for context +- `bitsPerChar` — `nllAll * nTokens / (ln 2 * nChars)`, comparable across tokenizations +- `nTokens`, `nPreservedTokens`, `nChars` + +and the deltas `ΔnllPreserved = nllPreserved(J) - nllPreserved(E)`. + +### 8.2 Alignment + +Tokens must be attributed to words. **Determine empirically which mechanism the installed +transformers.js actually provides before writing the implementation** — offsets if the +tokenizer exposes them, otherwise incremental decode with a character cursor. Whichever is +used, it is verified by reconstructing the passage from the token spans and asserting equality +with the input; a mismatch raises rather than mis-attributes. + +Do not tokenize word-by-word to force alignment. It suppresses cross-word merges and changes +the NLL being reported. + +### 8.3 The swap control — what makes the number interpretable + +`ΔnllPreserved > 0` on its own is uninterpretable, because at least three things change at once +when content words are vacated: + +1. the forms are unknown, so the model has no lexical entry to condition on; +2. nonce forms fragment into many subword tokens, so the context is longer and stranger; +3. the passage says something nonsensical. + +Only (1) is "location". A caveat cannot separate them; a control can. + +**Swap.** Mint by drawing a *real English word* instead of a nonce form — same eligibility, same +`u(stem) < p` decision, same suffix handling, and the injectivity §5.2a shows is available to it. +The replacement is drawn deterministically from the domain's own open-class **types** by +**frequency rank**: rank the eligible types by `(corpus count descending, type ascending)` — the +tie rule `frequencyBudget` already uses — and let `r` be the stem's rank. Attempt `a` draws an +offset `δ ∈ [-w, -1] ∪ [1, w]` from the byte stream of §5.3 under the tag `swap` (never `mint`, so +the two streams can never alias), and proposes `pool[(r + δ) mod |pool|]`; `w` starts at 32 and +doubles every 64 attempts up to `|pool|`, which is the same deterministic relaxation §5.5 uses and +is needed because "anything already used" depletes a window. A candidate is accepted iff it is not +already used, is not the stem itself, is not a type of the stem's own family, and — while +`a < 1024` and `matchProsody` — carries the stem's stress pattern. So the swapped passage is +equally nonsensical, but every form is a known word with ordinary tokenization. + +Two consequences of drawing from a *finite* pool, stated rather than hidden. The pool is 1 944 +types against 1 680 stems on the shipped corpus, so the tail of the canonical order draws from +what is left and its frequency match degrades; and a source type that carries a suffix may receive +an already-inflected replacement, giving a doubly-inflected surface. 66 % of eligible types are +suffix-free and receive a bare real word. + +The pool must be the *types*, not the stems: the stem set is exactly the set of keys, so drawing +from it would consume the pool exactly and leave a collision with nowhere to move. + +Because the replacements are real English words, they are **not** registered in `mintedStress` — +their stress comes from the table or the rule like any other English word, so `stressFromMinted` +is 0 on both sides of a swap and `stressFromTable`/`stressFromRule` say what they always say. + +`consistent = false` is **refused** under `swap`: that control needs a fresh type per occurrence, +and the corpus has 1 680 open-class stems against 8 202 vacated tokens, so there is no supply. It +raises rather than quietly reusing words and reporting a rate it is not achieving. + +This makes the minting strategy a parameter: + +``` +mint: "nonce" | "swap" # default "nonce" +``` + +and decomposes the measurement: + +- `nll(swap) − nll(english)` — the cost of **wrong content** (3) +- `nll(nonce) − nll(swap)` — the cost of **unknown form**, i.e. (1) together with (2) + +The second difference is the closest this instrument gets to "what location was worth", and the +UI must report it as *that difference*, never `nll(nonce) − nll(english)` alone. Residual (2) is +not separable without a tokenizer-level control and the UI must say so rather than pretend the +remainder is pure location. + +`mint: "swap"` is still **nested** in `p` (the `u(stem) < p` decision is untouched) and still +**stable** in `(seed, stem)` (the map is built once, in canonical order, independently of `p`). +Injectivity is where it and `nonce` part company, and §5.2a proves why they must: an earlier draft +of this paragraph claimed swap "preserves every property of §7", and that claim is false — a map +whose images are domain types and which does not depend on `p` cannot be injective at intermediate +`p` unless it is the identity. So the invariance theorem of §7.3 holds for `swap` at `p ∈ {0, 1}`, +where it is asserted exactly as for `nonce`, and the mapped vocabulary is refused in between. At +full vacancy the tiny model is exactly as blind to `swap` as to `nonce`, which is the check that +the control is implemented correctly; the pretrained arm measures at full vacancy, so nothing the +control exists for is lost. + +### 8.3a What was measured, and what the static build may therefore say + +Measured before implementing: 6 × 250-word real-corpus passages × 4 conditions (english / +frequency-matched real-word swap / nonce at two seeds) × {gpt2, SmolLM2-135M}, ~700 preserved +closed-class tokens per condition. ONNX fp32 ≡ torch to 5.3e-4 nats, and the two stacks' +tokenizations were identical (0 id mismatches across 48 texts), so the alignment of §8.2 is +sound and fp32 is the reference. + +**The result (fp32).** `nonce − english` ≈ **0.92–1.03 nats**, of which `nonce − swap` is only +**0.06–0.21**. So roughly 80–90 % of the damage is *wrong content* and only 10–20 % is *unknown +form*. Taken with the tiny arm's exact zero, that is the 2×2: + +| | what a word's form is worth | +|-|-| +| tiny, trained from scratch (no locations) | exactly 0 | +| pretrained (has locations) | ~0.1 of ~1.0 nats, i.e. 10–20 % | + +Even for a model that *has* locations, losing the location costs far less than losing the +content — the doc's T4 prediction that field ≫ location, on a model it did not consider. + +**The quantization verdict.** The app ships quantized ONNX, and the effect above is small: + +- **Absolute NLL is unusable.** q8 shifts `nllPreserved` by −0.19 nats (gpt2) and **+0.40** + (SmolLM2) — the sign is not even stable across models. +- **Pooled differences do cancel**: `|Δ_q8 − Δ_fp32| ≤ 0.054` nats on every contrast, against a + sampling standard error of 0.12–0.22. +- **Per-passage differences do not**: worst case 0.65 nats, **115 %** of that passage's fp32 + delta. +- **`nonce − swap` is destroyed.** Its true value is 0.06–0.21; q8's error on it is 14–23 % + pooled, up to 0.28 per passage, with one sign flip in six passages per model. +- The error is **not** a constant offset that a baseline could remove: q8 compresses extreme + surprisal, and 2.7 % of gpt2's preserved tokens carry `|e| > 5` nats, all on 15–20-nat + line-initial function words — precisely the tokens this measurement is about. Median and + trimmed means do not rescue it. +- **q4f16 — the app's first-choice dtype — could not be measured outside a browser** (session + init fails on the onnxruntime-node CPU EP for both models). Until it is measured in a real + browser, the deployed default path has **no error bar at all**. + **RESOLVED (2026-08-04):** measured in a real browser on a real Apple Metal-3 adapter — on + gpt2 and both SmolLM2 exports q4f16 returns logits identical at every position (SmolLM2: + exactly 0, every NLL = ln V), so it has no error bar because it measures nothing. It is no + longer requested: the app's ladder is now `webgpu/q8 → wasm/q8`, both rungs gated by a + load-time non-degeneracy check, so **q8 is the dtype whose measured bounds apply here**. + +**Policy.** The full stack (torch/fp32) reports everything. The static build may report a +number only where there is a measured bound for the dtype it actually ran: + +1. pooled `nonce − english` and `swap − english`, with the quantization uncertainty stated; +2. **never** `nonce − swap`, and **never** a per-passage delta — these are refused with a typed + error naming the full stack, exactly as the static build already refuses elsewhere; +3. if the dtype in use has no measured bound, refuse rather than invent one. A stated ± + that was never measured is a fabricated error bar, which is worse than no number. + +### 8.4 Confound, stated in the UI + +The vacated passage has genuinely higher entropy, so *every* prediction in it gets worse — +including the scaffolding. `ΔnllPreserved > 0` is therefore expected; its **magnitude** is the +result, and it is only interpretable against the tiny arm's exact zero. The UI must show both +arms together, and must not present `ΔnllPreserved > 0` as a surprise. + +Both stacks (backend PyTorch, static transformers.js) must produce the same numbers for the +same model and passage, to the tolerance the existing arch parity tests use. + +--- + +## 9. Deliberate departures from the source + +| # | Source | Here | Why | +|-|-|-|-| +| 1 | `[A-Za-z][A-Za-z']*` | the tokenizer's `WORD_RE` | otherwise the transform and the trainer disagree about `good-bye` and §7.3 is false | +| 2 | `top64 / 2**64` | `(top64 >> 11) / 2**53` | not a bug in the source — the two languages were *measured* to agree on it (§4). Exact representability makes the agreement structural instead of a proof that a refactor could invalidate | +| 3 | `random.Random(str)` | sha256 counter stream | MT19937 seeded from a string is not reproducible in TS | +| 4 | map built lazily while rewriting | map built once over all types in canonical order | the source's `used` set and give-up counter make the nonce depend on `p`, breaking its own stability claim | +| 5 | `avoid` accepted, never passed | `avoid` = corpus type set | a minted form could otherwise merge with a real English type | +| 6 | give-up = `syllable + str(len(used))` | deterministic salt relaxation | order-independence | +| 7 | seam fix via shared RNG | seam fix via hash of `(stem, suffix)` | order-independence | +| 8 | injectivity assumed | injectivity verified, re-mint on collision | §7.3 depends on it | +| 9 | zip copy of `split_suffix` | audited copy's `EXCEPTIONS` | `brother → broth+er` is a known artifact the audited copy fixes | +| 10 | "exact prosody" | measured prosody + `stressTableCoverage` | the source's own status table calls the stress table unverified | +| 11 | `vacated` count reported as `len(self.map)` | count of stems actually vacated | the zip copy reports the wrong number; the audited copy fixes it | + +Departures 4, 6, 7 and 8 are corrections to bugs that break properties the source *claims*. +Departure 2 is **not** a bug fix — the source's expression was tested and is fine; the change +buys structural rather than argued cross-language equality. + +--- + +## 10. Statistics contract + +`vacancyStats(originalText, vacatedText, map, p, seed, …)` returns, with these exact names: + +``` +domainTypesTotal, domainTypesEligible, domainTypesVacated, +corpusTypesTotal, corpusTypesEligible, corpusTypesVacated, +stemsTotal, stemsVacated, +tokensTotal, tokensVacated, +meanSyllablesBefore, meanSyllablesAfter, +meanAnapestBefore, meanAnapestAfter, +stressFromTableBefore, stressFromTableAfter, +stressFromMintedBefore, stressFromMintedAfter, +stressFromRuleBefore, stressFromRuleAfter, +bijective, imageSize, remintRounds +``` + +Both stacks compute these from the same definitions; the golden fixture (§11) pins them. + +**Counting: the scope is in the name.** This section cost two round trips between the stacks, +both times because "types" is ambiguous between the **corpus** (2 211 types of *Mother Goose*) +and the **domain** (2 233 = corpus ∪ the full Dolch list). The two stacks agreed on +`tokensVacated` to the token (8 202 at `p = 1`) and disagreed only on the type counts — a +reporting gap, never a disagreement about the transform. + +An unprefixed `types*` is therefore **forbidden**. Every count names its scope: + +- `domainTypes{Total,Eligible,Vacated}` — over the domain of §5.2. This is what governs the map + and the vocabulary, so it is the diagnostic number. +- `corpusTypes{Total,Eligible,Vacated}` — over the corpus's own type set. This is what the panel + shows a reader, because the 22 domain-only words (`funny`, `squirrel`, `today`, …) are in the + budget but never appear in the text, and counting words the reader cannot see inflates the + vacancy rate they are being shown. + + **`corpusTypesVacated` is measured from the two texts, not from map membership.** A type + counts as vacated iff at least one of its occurrences actually changed. Under + `revealAfter > 0` the two are not the same number and the stacks split on it — one measured + the texts (665) and one asked whether the stem was in the vacated set (1337), over-reporting + by 2×, because a type whose every occurrence falls inside the reveal window is still listed + in the map. Under `revealAfter = 0` the readings coincide, which is why it took a control + condition to expose. Measuring the texts is the definition that matches what this number + claims to the reader. +- `stemsTotal` — distinct eligible stems, i.e. the size of the map; `stemsVacated` — stems with + `u(stem) < p` +- `tokensTotal` / `tokensVacated` — over the **corpus** token stream + +`domainTypesEligible ≥ stemsTotal` always, since inflected forms share a stem, and +`domainTypesEligible = corpusTypesEligible + 22` on the shipped corpus. At `p = 1` every +eligible stem vacates, because `u ∈ [0, 1)` by construction, so `stemsVacated == stemsTotal` and +`{domain,corpus}TypesVacated == {domain,corpus}TypesEligible` — identities worth asserting, +since the first of them is what exposed all of this. + +Measured on the shipped corpus (seed 0 / seed 7, `p = 0/.25/.5/.75/1`): +`corpusTypesVacated` 0/461/954/1430/1922 and 0/434/975/1440/1922; +`domainTypesVacated` 0/469/966/1448/1944 and 0/440/985/1455/1944. + +**Where each token's stress came from.** The first draft asked for a single +`stressTableCoverage`, which is ambiguous the moment minted forms exist: read literally it counts +only the hand table (measured 1.2 % after full vacancy), and read as "stress we actually know" it +also counts minted forms, whose pattern we chose ourselves (41.6 %). Rather than pick, report the +**three-way split**, which is unambiguous and strictly more informative: + +- `stressFromTable` — the hand table of §6.1. This is the honesty number for English words, and + it is the one FR-712/SC-708 require beside every prosody statistic. +- `stressFromMinted` — forms we minted, whose intended pattern we registered. Known by + construction, but *asserted* rather than verified: §5.5 accepts a candidate on syllable + **count**, so the count is checked and the pattern is not. +- `stressFromRule` — the spelling heuristic of §6.2, i.e. a guess. + +The three are token-weighted fractions and sum to 1 on each side. + +For reference, the source reports mean anapest `0.351 → 0.345` and mean syllables +`1.224 → 1.211` on *its* corpus. **Those are its numbers on a corpus we do not have. Do not +transcribe them into any UI string, test, or doc.** Compute ours on Mother Goose and quote +only what we measured — this is the same rule that caught the fabricated "+29.3 … +decelerating" in feature 006. + +--- + +## 11. Golden fixture + +`code/frontend/tests/fixtures/vacancy-golden.json`, generated by +`scripts/export_vacancy_golden.py`, consumed by `code/frontend/tests/unit/vacancyGolden.test.ts`. +Same shape and discipline as `lex-golden.json`: `format`, `tolerance`, `git_sha`, generator +versions, then cases. + +Pinned per case, on the **real committed corpus**: + +- `u(stem)` for a fixed list of 24 stems spanning eligible/ineligible and both budgets — the + exact float64, which is the whole point of departure 2 +- the full `map` at `seed ∈ {0, 7}` — every stem→nonce pair +- the first 400 characters of the vacated corpus at `p ∈ {0, 0.35, 0.7, 1}`, seed 0 — this is + the source's own figure, reproduced on our corpus +- `vacancyStats` for each of those, all fields +- the nesting assertion: `vacated(0.35) ⊆ vacated(0.7) ⊆ vacated(1.0)` as explicit id sets +- the stability assertion: the nonce for each of 24 stems is identical at every `p` where it + is vacated +- the token id stream digest under the mapped vocabulary at each `p` — all equal, which is + §7.3 pinned as data rather than as an assertion in one language + +Strings are compared exactly. Only the prosody means use `tolerance`. diff --git a/specs/007-vacancy-transform-field/spec.md b/specs/007-vacancy-transform-field/spec.md new file mode 100644 index 0000000..9c7faa4 --- /dev/null +++ b/specs/007-vacancy-transform-field/spec.md @@ -0,0 +1,191 @@ +# Feature 007 — The vacancy transform: the field-without-location instrument + +**Feature Branch:** `007-vacancy-transform` +**Created:** 2026-08-04 +**Status:** In progress + +**Contract:** `specs/007-vacancy-transform-field/architecture.md` (normative — read it first) +**Builds on:** `specs/006-lexicon-lab-tiny/spec.md` (the Lexicon Lab this extends) +**Frozen HTTP contract:** `specs/002-interactive-model-explorer/contracts/api.md` — additive +endpoints only; any change to an existing endpoint gets its own commit with a note. + +## Why + +Feature 006 shipped the closed lexicon — the graded word budgets and the word-level +tokenizer — which is one half of what `~/Desktop/TinyModelsDoc/tiny_models.tex` argues a tiny +model is *for*. The other half is the **vacancy transform** (the doc's §"The vacancy +transform"), the instrument that manufactures Carroll's condition — full syntactic scaffolding, +vacant lexical content — at a controlled rate on any corpus. 006 deferred it as FR-624 on the +grounds that it needed a parameter-matched control. It has one: under the conditions of +contract §7 the transform preserves the vocabulary *exactly*, so the control is the design. + +The transform is what turns "field" and "location" from vocabulary into numbers. Without it the +Lexicon Lab shows a budget; with it, the lab shows what a word's *identity* is worth to a model +that has never seen one — which is zero, exactly — and the Architecture Explorer shows what it +is worth to a model that has. + +## Scope + +Two arms, both real, no mocks anywhere. + +**Tiny arm — Lexicon Lab.** The transform, a `p`-sweep, the doc's three control conditions, +live retraining against the existing in-browser trainer, and the invariance result. + +**Pretrained arm — Architecture Explorer.** A real HF model over a passage and its vacated +twin, scored on the preserved closed-class scaffolding only. + +Out of scope, and stated so it is not mistaken for an omission: the doc's lattice/trie decode +mask (`eval/mask_decode.py`, its T1/T6 instrument), the concept battery (its resource F), the +minting staircase (T3), and corpus synthesis by rejection sampling (its resource A). Each is a +separate instrument; none is needed for T4. + +## Functional requirements + +### The transform (shared, both stacks) + +- **FR-701** A vacancy module exists in both stacks implementing contract §§1–6 exactly: + `llm_geometry/lex/vacancy.py` and `code/frontend/src/lib/lexEngine/vacancy.ts`. +- **FR-702** The transform rewrites raw text in place, replacing only `WORD_RE` matches and + passing all other bytes — punctuation, whitespace, line breaks — through unchanged. +- **FR-703** Vacancy is decided by `u(stem) < p` with `u` derived as contract §4, so vacated + sets **nest** in `p`. +- **FR-704** The nonce for a stem is fixed by `(seed, stem)` alone — **stable** across `p`, + across document order, and across corpora. +- **FR-705** The map is injective, verified per build with re-minting on collision, and + `bijective` is reported in the statistics. +- **FR-706** A nonce never collides with a real type of the corpus (`avoid`, contract §5.2). +- **FR-707** Inflectional suffixes and closed-class words are preserved; the eligibility rules + of contract §2.2 are implemented identically in both stacks. +- **FR-708** The three control conditions are supported: `consistent = false` (inconsistent + assignment), `matchProsody = false` (no prosody matching), `revealAfter > 0` (partial reveal). +- **FR-709** `vacancyStats` returns exactly the fields of contract §10, including + `stressTableCoverage`, and **no prosody number from the source document is transcribed + anywhere** — every number shown is measured on our corpus. + +### Lexicon Lab (tiny arm) + +- **FR-710** A vacancy panel exposes `p`, `seed`, and the three conditions, and shows the + transform acting on the live corpus with minted forms visually distinguished from preserved + and not-yet-vacated words — the doc's Figure 5, interactive. +- **FR-711** Raising `p` visibly demonstrates nesting and stability: a form minted at a lower + `p` is still present, unchanged, at every higher `p`. The UI states this and the state is + derived from the real map, not annotated by hand. +- **FR-712** Prosody preservation is reported as measured before/after statistics, always + alongside `stressTableCoverage`, and the UI states that the stress table is rule-seeded and + unverified (contract §6.1). +- **FR-713** The lab can train on the vacated corpus with the existing trainer, at any + condition, reporting loss, held-out loss, and generated samples exactly as for the + untransformed corpus. +- **FR-714** The invariance result is *demonstrated*, not asserted: the panel runs the mapped + vocabulary at two values of `p` and shows the resulting losses are identical, with the + identity checked in the UI rather than claimed in prose. +- **FR-715** The conditions that break invariance show what breaks: coverage collapse and + `` rate for `consistent = false`, type splitting for `revealAfter > 0`. +- **FR-716** Vacancy composes with everything the lab already does — any budget, any budget + source, pasted text, a HuggingFace dataset, fine-tuning, weight editing, save/load. + +### Architecture Explorer (pretrained arm) + +- **FR-717** A passage and its vacated twin are scored by a real curated HF model, reporting + the fields of contract §8.1. +- **FR-718** Token→word alignment is verified by reconstruction; a mismatch raises rather than + mis-attributing (contract §8.2). +- **FR-719** The entropy confound is stated in the UI, and the tiny arm's exact zero is shown + next to the pretrained delta so the number is interpretable (contract §8.4). +- **FR-719a** A **swap control** exists (`mint: "nonce" | "swap"`, contract §8.3): the same + transform drawing a real, frequency-rank-matched English word instead of a nonce form. The + pretrained arm reports the decomposition `nll(swap) − nll(english)` (wrong content) and + `nll(nonce) − nll(swap)` (unknown form), and never reports `nll(nonce) − nll(english)` alone + as if it measured location. The residual tokenization component is stated, not hidden. +- **FR-720** Both stacks produce the same numbers for the same model and passage **at the same + dtype**. They do not at the dtypes actually shipped, which is a measured fact, not an + assumption (contract §8.3a): ONNX fp32 ≡ torch to 5.3e-4 nats, but q8 shifts absolute + `nllPreserved` by −0.19 nats on gpt2 and +0.40 on SmolLM2-135M. +- **FR-720a** The static build reports a quantity **only** where a measured error bound exists + for the dtype it actually ran. Pooled `nonce − english` and `swap − english` qualify under q8 + (|Δ| ≤ 0.054 nats). `nonce − swap` and every per-passage delta do **not** and are refused with + a typed error naming the full stack. If the running dtype has no measured bound, the panel + refuses — a stated ± that was never measured is a fabricated error bar and is worse than no + number. + +### API and static build + +- **FR-721** New endpoints are **additive**: `POST /api/lex/vacancy`, `POST /api/lex/train` + gains optional vacancy parameters, `POST /api/arch/vacancy-score`. No existing response field + changes meaning. +- **FR-722** The static build serves the same capability: `staticClient` implements every new + endpoint in-browser, or refuses loudly with a typed error naming the command that would fix + it. Nothing is fabricated and nothing silently degrades. +- **FR-723** A golden fixture pins the transform across both stacks (contract §11). + +### Documentation + +- **FR-724** The Info tab gains a vacancy section: the 2×2, the definition of the transform, + the nesting and stability properties, the invariance theorem and what it does and does not + say, and the honest status of the stress table. +- **FR-725** Both tabs carry orientation prose and `Explain` deep-dives to feature 005's + standard, and every number in that prose is transcribed from a source constant. +- **FR-726** The source document's provenance is stated: what we ported, what we corrected + (contract §9), and that its reported prosody figures are its own, on a corpus we do not have. + +## Success criteria + +- **SC-701** Nesting holds: for `p < p'`, the set of vacated types at `p` is a subset of that + at `p'`. Asserted on the real corpus across a `p` grid and two seeds, in both stacks. +- **SC-702** Stability holds: a stem's nonce is byte-identical at every `p` at which it is + vacated, and independent of document order. Asserted on the real corpus. +- **SC-703** **The invariance theorem holds** (contract §7.3): with `consistent = true` and + `revealAfter = 0`, the mapped-vocabulary token id stream is element-for-element identical to + the untransformed stream, across all five Dolch budgets, a frequency budget, + `p ∈ {0, 0.25, 0.5, 0.75, 1}`, `seed ∈ {0, 7}`, and both `matchProsody` settings. A real + short training run at two values of `p` produces bit-identical losses. +- **SC-704** The map is injective on the real corpus at every `p` tested, with `remintRounds` + reported. +- **SC-705** The control conditions measurably break invariance: `consistent = false` raises + the `` rate and the held-out loss relative to `consistent = true` at the same `p`, by a + margin recorded from a real run rather than assumed. +- **SC-706** TS↔Python parity: the golden fixture matches within `tolerance` for floats and + exactly for strings and id streams. +- **SC-707** The pretrained arm produces a `ΔnllPreserved` whose sign and magnitude are + reported from a real model run, next to the tiny arm's exact zero. +- **SC-707a** ~~The swap control satisfies the invariance theorem exactly as the nonce strategy + does — the tiny model is equally blind to both.~~ **This claim was false and is retracted.** + It cannot hold, and the reason is a theorem rather than a bug (contract §5.2a): a map that is + stable in `p` and whose images are *domain types* is injective at every `p` only if it is the + identity. `swap` draws its replacements from the domain by construction, so at intermediate + `p` a swapped word can collide with a word not yet vacated — measured at 191 / 246 / 190 + colliding types for `p` = 0.25 / 0.5 / 0.75. + + The corrected criterion, and what is actually asserted: `swap` satisfies the invariance + theorem at **`p ∈ {0, 1}`** — 48 of the 120 SC-703 cases — and the remaining 72 are + **refused with a typed error citing §5.2a**, never silently computed. `nonce` remains + 120/120. The refusal is the deliverable: an instrument that declines the configurations it + cannot support is sound, one that quietly returns a non-injective map is not. + + This costs the pretrained arm nothing, which is the point worth checking rather than + assuming: it scores at full vacancy, where `swap` *is* a bijection of the domain. +- **SC-707c** The decomposition `nll(swap) − nll(english)` and `nll(nonce) − nll(swap)` is + reported from a real model run **in the full stack**, where fp32 makes it measurable. +- **SC-707b** The measured 2×2 is reported: a word's form is worth **exactly 0** to the tiny + model and **10–20 % of ~1.0 nats** to a pretrained one (contract §8.3a). The static build + either states a measured uncertainty for the dtype it ran, or refuses — verified by driving + the deployed static build and confirming it does one or the other, never a bare number. +- **SC-708** Every prosody statistic displayed is accompanied by `stressTableCoverage`, and no + number from the source document appears as if it were ours. +- **SC-709** The full suite is green locally and in CI: backend `pytest` + `ruff` + `black`, + frontend `vitest` + `svelte-check` (0 errors, 0 warnings) + e2e in both projects, and the + Pages deploy. +- **SC-710** The deployed site is verified by *using* it: the vacancy panel is exercised on + https://context-lab.com/llm-geometry/ with a real training run, and the console is clean. + +## Risks + +- **The theorem could be false in a way the tests do not reach.** Mitigated by asserting it as + data (the golden fixture pins id-stream digests) as well as an assertion, and by testing at + the boundaries — `p = 0`, `p = 1`, and a budget word absent from the corpus. +- **Token alignment in the pretrained arm** depends on what transformers.js actually exposes. + Contract §8.2 requires this be determined empirically before implementation, and verified by + reconstruction at run time. +- **A null result reads as a bug.** The tiny arm's headline is an exact zero. The UI must + present it as the finding it is (contract §7.4), not hide it behind a curve that looks like + it is measuring something. diff --git a/specs/007-vacancy-transform-field/ui.md b/specs/007-vacancy-transform-field/ui.md new file mode 100644 index 0000000..e9d50de --- /dev/null +++ b/specs/007-vacancy-transform-field/ui.md @@ -0,0 +1,141 @@ +# Feature 007 — UI specification + +Normative for the two panels. Read `architecture.md` first; every number named here comes from +`vacancyStats` or a source constant, never from prose. + +--- + +## 1. `VacancyPanel.svelte` (Lexicon Lab) + +Lives in `src/viz/lex/`, owned by `LexiconLab.svelte` like every other panel — plain props in, +callback props out, no store. It sits **after `BudgetPanel`/`ModelPanel` and before +`TrainPanel`**, because it changes the corpus the trainer will see. + +### 1.1 Controls + +| control | type | default | notes | +|-|-|-|-| +| `p` | slider 0→1, step 0.05 | `0` | the headline knob; show the value numerically | +| `seed` | integer input | `0` | changing it re-mints everything, by design | +| condition | radio | `consistent` | `consistent` \| `inconsistent` \| `partial reveal` | +| reveal N | integer, shown only for `partial reveal` | `1` | `revealAfter` | +| prosody | checkbox | on | `matchProsody` | +| mint | radio | `nonce` | `nonce` \| `swap` (§8.3 of the contract) | + +Changing any control re-derives the view synchronously. The transform runs in ~ms on 16 000 +tokens; do **not** put it behind a spinner or a worker, and do not debounce the slider so hard +that the nesting demonstration stops feeling continuous. + +### 1.2 The corpus view — the doc's Figure 5, live + +Render the first ~40 token-producing lines of the active corpus with every word carrying one of +three classes, colour-coded to match the source document's figure: + +- **closed class, preserved** — full-contrast text +- **open class, not yet vacated** — muted +- **minted** — accent + +A legend states exactly that. The classification comes from the real map (`isEligible` + +`u(stem) < p`), never from a hand-annotated list — FR-711. + +### 1.3 The nesting ribbon + +The single most important thing this panel has to make *visible*, because it is the property +that makes a `p`-sweep interpretable and the source's own implementation gets it wrong. + +Pick ~8 eligible stems spanning the `u` range. For each, a row of cells at +`p = 0, 0.25, 0.5, 0.75, 1` showing the surface form at that `p`. The reader must be able to see +at a glance that: + +- once a cell turns minted it **never reverts** as `p` grows (nesting), and +- the minted string is **the same string** in every later cell (stability). + +Caption states both properties by name. This is FR-711 and it is not satisfied by prose. + +### 1.4 Statistics readout + +Show, from `vacancyStats` only: + +- `corpusTypesVacated` / `corpusTypesEligible` and `tokensVacated` / `tokensTotal` + — **corpus scope, not domain**: the 22 domain-only Dolch words never appear in the text and + counting them inflates the rate the reader is being shown (contract §10). +- prosody: `meanSyllablesBefore → After`, `meanAnapestBefore → After` +- **immediately beside them**, the three-way stress split (`stressFromTable` / + `stressFromMinted` / `stressFromRule`), plus one sentence: the stress table is rule-seeded and + unverified, it covers ~5 % of this corpus's tokens, so these are indicative and not exact. + FR-712 / SC-708. No prosody number may appear without it. +- `bijective`, `remintRounds` + +### 1.5 The invariance demonstration — FR-714 + +Two tiers, because the theorem is free to check and the training run is not. + +**Instant (always shown).** Compute `tokenStream` under the mapped vocabulary at the current `p` +and at `p = 0`, compare element-for-element, and display the verdict with the count actually +compared — e.g. *"token id streams identical · 19 071 ids compared"*. Recompute on every control +change. In a condition that breaks the theorem (`inconsistent`, `revealAfter > 0`) this must +show the **real** result — the streams differ, with how many positions and the resulting +`` rate. It is a live check, never a hard-coded ✓. + +**On demand (button).** Train at `p = 0` and at the current `p` with the same seed and +hyperparameters, then show both loss curves and `max |Δloss|`. Under `consistent` this is +exactly `0`. Report it as `0`, not "≈0" — and if it is ever not 0, that is a bug and the UI +should say so rather than round it away. + +Default the demo to a step count that finishes quickly; the point is the comparison, not the +final loss. Two runs, not three. + +### 1.6 Framing — do not let the null read as a bug + +The headline result is an exact zero, and a panel that presents it as a flat line looks broken. +Contract §7.4: state plainly that for a word-level model trained from scratch the transform is a +pure relabelling and the model is provably blind to it, that this is *the finding* — all of a +word's meaning is field, none is form — and that the number which is **not** zero lives in the +Architecture Explorer, with a link to it. + +--- + +## 2. `VacancyScorePanel.svelte` (Architecture Explorer) + +### 2.1 Controls + +Model (from the curated list), passage (default a fixed excerpt of the shipped corpus, editable), +`p`, `seed`, and mint strategy. A single **Score** button — this runs real forward passes and +must not fire on every keystroke. + +### 2.2 Output + +A three-row table, English / swap / nonce, each with `nllPreserved`, `nllAll`, `bitsPerChar`, +`nTokens`, `nPreservedTokens`; then the two differences that matter, labelled in words: + +- `nll(swap) − nll(english)` — **the cost of wrong content** +- `nll(nonce) − nll(swap)` — **the cost of unknown form** + +Never display `nll(nonce) − nll(english)` as a headline; it conflates the two (contract §8.3). + +Beside it, the tiny arm's exact `0`, labelled as the same measurement on a model with no +locations. That juxtaposition is the whole 2×2 and is the reason this panel exists. + +### 2.3 Honesty requirements + +- State that the residual — nonce forms fragmenting into more subword tokens — is not separable + without a tokenizer-level control, so "cost of unknown form" is an upper bound on what + location was worth. +- State the alignment mechanism (byte-level pieces → UTF-8 byte spans) and that it is verified + by reconstruction at run time; a mismatch raises rather than mis-attributing (FR-718). +- **Quantization**: the static build runs quantized ONNX whose absolute per-token logprobs + differ from fp32 by up to several nats. Whatever the measurement of that error's effect on the + *difference* concludes, the panel states the stated uncertainty in static mode, or refuses and + names the full stack. Not decided here — it is decided by measurement. + +--- + +## 3. Info tab + +A new `

    ` after `#lex`, covering: the 2×2 with the vacancy cell marked; the +transform's definition; nesting and stability, and that the source's implementation breaks both; +the invariance theorem, what it proves and what it does not; the swap control and the +decomposition; and the stress table's real status. Update `#real`, `#limits` and `#refs`. + +Every number transcribed from a source constant, pinned by `tests/e2e/docs.spec.ts` — feature +005's rule, so changing a constant without changing the sentence fails CI.