diff --git a/.gitignore b/.gitignore
index f5eac0f..c13af83 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@ checkpoints/
 viz/
 input.txt
 input_shakespeare.txt
-input_gutenberg.txt
+input_gutenberg*.txt
+*.cache.pt
 donotcommit.txt
 runs/
diff --git a/README.md b/README.md
index 057453b..a76cf51 100644
--- a/README.md
+++ b/README.md
@@ -45,6 +45,26 @@ Downloads verify sha256 and skip files that are already present.
 
 Switch profiles by editing `ACTIVE_PROFILE` in `gpt.py`. Each checkpoint embeds the architecture and vocab so it's self-contained for inference.
 
+### Tokenizer
+
+The training script supports two tokenizers:
+
+```bash
+uv run python gpt.py --tokenizer char                          # default; one token per character
+uv run python gpt.py --tokenizer bpe --tokenizer-vocab-size 1024  # byte-level BPE (minbpe-style)
+```
+
+`char` keeps the original behavior: vocab = unique characters in the corpus (~85–250). `bpe` trains byte-pair merges on a 5 MB prefix of the corpus and gives you ~3–4× sequence compression on English, so `block_size=512` covers ~1.5–2k chars of real context. The encoded corpus is cached to disk (`{dataset_path}.{tokenizer}_v{vocab_size}.cache.pt`) keyed on the tokenizer state, since naive-Python BPE encoding of a 2 GB corpus takes minutes.
+
+Checkpoints store the tokenizer state, so inference reconstructs the exact same tokenizer:
+
+```bash
+uv run python gpt.py --infer checkpoints/ckpt_default_step_04999.pt
+#   tokenizer: bpe (vocab=1024)
+```
+
+Legacy checkpoints (only `chars` saved) load through a compatibility path as if they were `char` checkpoints — nothing breaks.
+
 ## Monitoring with TensorBoard
 
 Each training run writes scalar loss curves and a 200-character text sample at every eval step to `./runs/<timestamp>_<profile>_<dataset>/`. Disable with `--no-tensorboard`, or keep the curves but skip generation with `--no-sample` (worth it on the `large` profile).
@@ -109,6 +129,7 @@ The sidebar lets you scrub across training-step checkpoints and watch the embedd
 ## Layout
 
 - `gpt.py` — the transformer, training loop, inference, and lookahead sampling
+- `tokenizers/` — char and byte-level BPE tokenizer classes (local package; not HuggingFace's)
 - `bigram.py` — the tiny bigram baseline from earlier in the lecture
 - `viz_embeddings.py` — Streamlit embedding viewer
 - `checkpoint_io.py` — shared helpers for checkpoint files (sha256, git SHA tag, HF Hub upload, manifest)
diff --git a/datasets/gutenberg.py b/datasets/gutenberg.py
index 71d1304..e5d1935 100644
--- a/datasets/gutenberg.py
+++ b/datasets/gutenberg.py
@@ -8,8 +8,10 @@
 
 import os
 import re
+import time
 import urllib.request
 from collections import Counter
+from concurrent.futures import ThreadPoolExecutor
 
 from huggingface_hub import (
     hf_hub_download,  # pyright: ignore[reportUnknownVariableType]
@@ -29,15 +31,30 @@
 PG19_MANIFEST_FILE = "data/train_files.txt"
 
 
+def _fetch_book(rel: str) -> str:
+    """Download one book from GCS, strip the trailing license footer if
+    present, and normalize trailing whitespace. Network-bound; safe to
+    run in many threads concurrently."""
+    with urllib.request.urlopen(PG19_GCS_BASE + rel) as r:
+        raw = r.read().decode("utf-8", errors="replace")
+    m = _END_MARKER.search(raw)
+    if m:
+        raw = raw[: m.start()]
+    return raw.strip() + "\n\n"
+
+
 class Gutenberg(Dataset):
     name = "gutenberg"
     url = PG19_GCS_BASE
-    default_path = "input_gutenberg.txt"
+    default_path = "input_gutenberg_10k.txt"
     description = (
         "DeepMind PG-19 subset (Project Gutenberg books pre-1919). "
-        "First 3000 books, ~2 GB. Set max_books=None to download all 28,602."
+        "First 10000 books, ~7 GB. Set max_books=None to download all 28,602."
     )
-    max_books: int | None = 3000
+    max_books: int | None = 10000
+    # Network-bound; 32 concurrent connections is comfortable against GCS
+    # and gets us roughly an order-of-magnitude speedup over sequential.
+    download_workers: int = 32
 
     def prepare(self, path: str | None = None) -> str:
         path = path or self.default_path
@@ -55,25 +72,32 @@ def prepare(self, path: str | None = None) -> str:
         if self.max_books is not None:
             book_paths = book_paths[: self.max_books]
 
-        print(f"downloading {len(book_paths):,} PG-19 books to {path}")
+        print(
+            f"downloading {len(book_paths):,} PG-19 books to {path} "
+            f"({self.download_workers} workers in parallel)"
+        )
         tmp = f"{path}.partial"
         chars = 0
+        t0 = time.time()
         with open(tmp, "w", encoding="utf-8") as out:
-            for i, rel in enumerate(book_paths):
-                with urllib.request.urlopen(PG19_GCS_BASE + rel) as r:
-                    raw = r.read().decode("utf-8", errors="replace")
-                m = _END_MARKER.search(raw)
-                if m:
-                    raw = raw[: m.start()]
-                block = raw.strip() + "\n\n"
-                out.write(block)
-                chars += len(block)
-                if (i + 1) % 50 == 0 or i + 1 == len(book_paths):
-                    print(
-                        f"\r  {i + 1}/{len(book_paths)} books · {chars / 1e9:.2f} GB",
-                        end="",
-                        flush=True,
-                    )
+            with ThreadPoolExecutor(max_workers=self.download_workers) as ex:
+                # executor.map preserves submission order, so the file is
+                # written in book_paths order even though fetches finish
+                # out of order across worker threads.
+                for i, block in enumerate(ex.map(_fetch_book, book_paths)):
+                    out.write(block)
+                    chars += len(block)
+                    if (i + 1) % 50 == 0 or i + 1 == len(book_paths):
+                        done = (i + 1) / len(book_paths)
+                        elapsed = time.time() - t0
+                        eta = elapsed * (1 - done) / done if done > 0 else 0.0
+                        print(
+                            f"\r  {i + 1}/{len(book_paths)} books · "
+                            f"{chars / 1e9:.2f} GB · "
+                            f"{elapsed:4.0f}s elapsed · ETA {eta:4.0f}s",
+                            end="",
+                            flush=True,
+                        )
         print()
 
         with open(tmp, "r", encoding="utf-8") as f:
diff --git a/export_onnx.py b/export_onnx.py
index e32b28c..5dc3b93 100644
--- a/export_onnx.py
+++ b/export_onnx.py
@@ -42,7 +42,7 @@ def main() -> None:
     args = ap.parse_args()
 
     print(f"loading {args.ckpt}")
-    model, chars, hp = load_model_from_checkpoint(args.ckpt, device="cpu")
+    model, tokenizer, hp = load_model_from_checkpoint(args.ckpt, device="cpu")
     wrapped = InferenceWrapper(model)
     wrapped.eval()
 
@@ -63,15 +63,27 @@ def main() -> None:
     size_mb = os.path.getsize(args.out) / 1e6
     print(f"  wrote {args.out} ({size_mb:.1f} MB)")
 
+    # One decoded string per token ID. For char-level each entry is a single
+    # character; for BPE entries are byte-decoded substrings (possibly empty
+    # / multi-byte). The JS frontend just renders `tokens[id]` after each
+    # sample, so both tokenizers display correctly.
+    tokens = [tokenizer.decode([i]) for i in range(tokenizer.vocab_size)]
     meta: dict[str, object] = {
-        "chars": chars,
+        "tokens": tokens,
+        "tokenizer": tokenizer.name,
         "block_size": hp.block_size,
-        "vocab_size": len(chars),
+        "vocab_size": tokenizer.vocab_size,
         "n_layer": hp.n_layer,
         "n_embd": hp.n_embd,
         "n_head": hp.n_head,
         "checkpoint": os.path.basename(args.ckpt),
     }
+    # Char-level: keep the legacy `chars` field so the existing JS frontend
+    # (which builds its stoi map from it for prompt encoding) keeps working
+    # without changes. BPE prompt encoding would need a JS BPE encoder; out
+    # of scope for this change.
+    if tokenizer.name == "char":
+        meta["chars"] = tokens
     with open(args.vocab_out, "w", encoding="utf-8") as f:
         json.dump(meta, f, indent=2)
     print(f"  wrote {args.vocab_out}")
diff --git a/gpt.py b/gpt.py
index 0436123..5dc8a7d 100644
--- a/gpt.py
+++ b/gpt.py
@@ -1,9 +1,10 @@
 import argparse
 import json
+import multiprocessing as mp
 import os
-from collections import Counter
-from collections.abc import Callable, Iterable
+import time
 from datetime import datetime
+from typing import Any
 
 import torch
 import torch.nn as nn
@@ -12,7 +13,9 @@
 from torch.utils.tensorboard.writer import SummaryWriter
 
 import datasets as corpora
+import tokenizers as tok
 from checkpoint_io import git_sha, manifest_repo, sha256_file, upload_checkpoint
+from tokenizers.base import Tokenizer
 
 
 class Hyperparameters(BaseModel):
@@ -27,7 +30,16 @@ class Hyperparameters(BaseModel):
     max_iters: int = 5000
     eval_interval: int = 25
     eval_iters: int = 200
-    learning_rate: float = 3e-4
+    # LR schedule: linear warmup from 0 to learning_rate over warmup_iters,
+    # then ReduceLROnPlateau — multiply lr by lr_factor whenever val loss
+    # hasn't improved by lr_threshold for lr_patience eval intervals.
+    # Bottoms out at min_lr.
+    learning_rate: float = 6e-4
+    min_lr: float = 6e-5
+    warmup_iters: int = 100
+    lr_factor: float = 0.5
+    lr_patience: int = 3
+    lr_threshold: float = 1e-3
 
     @model_validator(mode="after")
     def _check(self) -> "Hyperparameters":
@@ -36,6 +48,13 @@ def _check(self) -> "Hyperparameters":
         )
         return self
 
+    def warmup_lr(self, iter_num: int) -> float | None:
+        """LR during warmup, or None once warmup is over and the
+        plateau scheduler takes over."""
+        if iter_num < self.warmup_iters:
+            return self.learning_rate * (iter_num + 1) / (self.warmup_iters + 1)
+        return None
+
     def architecture_dict(self) -> dict[str, int | float]:
         return {
             "n_embd": self.n_embd,
@@ -278,21 +297,91 @@ def generate_lookahead(
 
 def load_model_from_checkpoint(
     path: str, device: str = "cpu"
-) -> tuple[GPTLanguageModel, list[str], Hyperparameters]:
-    """Load a checkpoint and return (model, chars, hp). Used by both the
+) -> tuple[GPTLanguageModel, Tokenizer, Hyperparameters]:
+    """Load a checkpoint and return (model, tokenizer, hp). Used by both the
     inference CLI in __main__ and external tools like viz_embeddings.py."""
     ckpt = torch.load(path, map_location=device, weights_only=False)
-    if "chars" not in ckpt or "hparams" not in ckpt:
-        raise ValueError(f"checkpoint {path} missing 'chars' or 'hparams'")
-    chars = ckpt["chars"]
+    if "hparams" not in ckpt:
+        raise ValueError(f"checkpoint {path} missing 'hparams'")
+    # New format: tokenizer_type + tokenizer_state. Legacy format: just `chars`.
+    if "tokenizer_type" in ckpt and "tokenizer_state" in ckpt:
+        tokenizer = tok.get(ckpt["tokenizer_type"])
+        tokenizer.load_state_dict(ckpt["tokenizer_state"])
+    elif "chars" in ckpt:
+        tokenizer = tok.get("char")
+        tokenizer.load_state_dict({"chars": ckpt["chars"]})
+    else:
+        raise ValueError(f"checkpoint {path} missing tokenizer info ('chars' or 'tokenizer_state')")
+
     hp = Hyperparameters(**ckpt["hparams"])
-    model = GPTLanguageModel(hp, vocab_size=len(chars)).to(device)
+    model = GPTLanguageModel(hp, vocab_size=tokenizer.vocab_size).to(device)
     # Older checkpoints (pre-tril-removal) carry per-head 'tril' buffers we
     # no longer hold; drop them so load_state_dict doesn't complain.
     state = {k: v for k, v in ckpt["model"].items() if not k.endswith(".tril")}
     model.load_state_dict(state)
     model.eval()
-    return model, chars, hp
+    return model, tokenizer, hp
+
+
+# Module-level worker state. multiprocessing.Pool's initializer runs once per
+# worker process and stashes the tokenizer here so each task call doesn't
+# need to (re)pickle it.
+_worker_tokenizer: Tokenizer | None = None
+
+
+def _init_worker(tok_type: str, tok_state: dict[str, Any]) -> None:
+    global _worker_tokenizer
+    _worker_tokenizer = tok.get(tok_type)
+    _worker_tokenizer.load_state_dict(tok_state)
+
+
+def _encode_one_chunk(chunk: str) -> list[int]:
+    assert _worker_tokenizer is not None
+    return _worker_tokenizer.encode(chunk)
+
+
+def _encode_corpus_with_progress(
+    text: str, tokenizer: Tokenizer, n_chunks: int = 200, n_workers: int | None = None
+) -> torch.Tensor:
+    """Encode `text` through `tokenizer` in roughly-equal-sized chunks, in
+    parallel across processes. Functionally equivalent to
+    `tokenizer.encode(text)` except for at most ~n_chunks tokens of slop
+    at chunk boundaries — negligible against a corpus of millions of
+    tokens, and the alternative is staring at a frozen prompt for hours
+    on the BPE path.
+    """
+    chunk_size = max(1, len(text) // n_chunks)
+    chunks = [text[i : i + chunk_size] for i in range(0, len(text), chunk_size)]
+
+    if n_workers is None:
+        n_workers = min(len(chunks), mp.cpu_count())
+
+    encoded: list[int] = []
+    total_mb = len(text) / 1e6
+    t0 = time.time()
+    report_every = max(1, len(chunks) // 50)
+    print(f"  encoding {len(chunks)} chunks across {n_workers} workers")
+    with mp.Pool(
+        processes=n_workers,
+        initializer=_init_worker,
+        initargs=(tokenizer.name, tokenizer.state_dict()),
+    ) as pool:
+        # imap preserves input order, so concatenation gives us back the
+        # original text's token sequence (modulo cross-chunk slop).
+        for i, result in enumerate(pool.imap(_encode_one_chunk, chunks)):
+            encoded.extend(result)
+            if (i + 1) % report_every == 0 or i + 1 == len(chunks):
+                done = (i + 1) / len(chunks)
+                elapsed = time.time() - t0
+                eta = elapsed * (1 - done) / done if done > 0 else 0.0
+                print(
+                    f"\r  encoding: {done * 100:5.1f}% · "
+                    f"{done * total_mb:5.1f}/{total_mb:.1f} MB · "
+                    f"{elapsed:4.0f}s elapsed · ETA {eta:4.0f}s",
+                    end="", flush=True,
+                )
+    print()
+    return torch.tensor(encoded, dtype=torch.long)
 
 
 def _main() -> None:
@@ -347,19 +436,30 @@ def _main() -> None:
         action="store_true",
         help="Skip generating sample text at every eval step. Useful on the `large` profile where generation is slow.",
     )
+    parser.add_argument(
+        "--tokenizer",
+        default="char",
+        choices=tok.names(),
+        help="Tokenizer to use. 'char' = one token per character; 'bpe' = byte-level BPE trained on the corpus.",
+    )
+    parser.add_argument(
+        "--tokenizer-vocab-size",
+        type=int,
+        default=1024,
+        help="Target vocab size when --tokenizer is 'bpe'. Ignored otherwise.",
+    )
     args = parser.parse_args()
 
     torch.manual_seed(1337)
 
     if args.infer is not None:
         print(f"loading checkpoint from {args.infer}")
-        model, chars, hp = load_model_from_checkpoint(args.infer, device=device)
-        itos = {i: ch for i, ch in enumerate(chars)}
-        decode: Callable[[Iterable[int]], str] = lambda l: "".join([itos[i] for i in l])
+        model, tokenizer, hp = load_model_from_checkpoint(args.infer, device=device)
         print(
             f"  architecture: n_embd={hp.n_embd} n_head={hp.n_head} n_layer={hp.n_layer} "
             f"block_size={hp.block_size} dropout={hp.dropout}"
         )
+        print(f"  tokenizer: {tokenizer.name} (vocab={tokenizer.vocab_size})")
         print(sum(p.numel() for p in model.parameters()) / 1e6, "M parameters")
         context = torch.zeros((1, 1), dtype=torch.long, device=device)
         if args.lookahead_depth > 1:
@@ -374,7 +474,7 @@ def _main() -> None:
             )
         else:
             out = model.generate(context, max_new_tokens=args.max_new_tokens)
-        print(decode(out[0].tolist()))
+        print(tokenizer.decode(out[0].tolist()))
         return
 
     assert device == "cuda", "training requires CUDA"
@@ -386,14 +486,40 @@ def _main() -> None:
     text = dataset.prepare()
     print(f"text is in RAM ({len(text):,} chars)")
 
-    chars = sorted(list(set(text)))
-    vocab_size = len(chars)
-    stoi = {ch: i for i, ch in enumerate(chars)}
-    itos = {i: ch for i, ch in enumerate(chars)}
-    encode: Callable[[str], list[int]] = lambda s: [stoi[c] for c in s]
-    decode: Callable[[Iterable[int]], str] = lambda l: "".join([itos[i] for i in l])
-
-    data = torch.tensor(encode(text), dtype=torch.long)
+    tokenizer_kwargs = (
+        {"vocab_size": args.tokenizer_vocab_size} if args.tokenizer == "bpe" else {}
+    )
+    tokenizer = tok.get(args.tokenizer, **tokenizer_kwargs)
+    print(f"tokenizer: {tokenizer.name} (training …)")
+    tokenizer.train(text)
+    print(f"  vocab_size={tokenizer.vocab_size}")
+    vocab_size = tokenizer.vocab_size
+
+    # Encoding the full corpus through naive BPE is slow (tens of minutes
+    # on Gutenberg), so we cache the result to disk keyed on dataset +
+    # tokenizer config. The cached file also stores the tokenizer state
+    # to detect staleness.
+    cache_id = (
+        f"{tokenizer.name}_v{tokenizer.vocab_size}"
+        if tokenizer.name != "char"
+        else "char"
+    )
+    cache_path = f"{dataset.default_path}.{cache_id}.cache.pt"
+    cached_state = tokenizer.state_dict()
+    if os.path.exists(cache_path):
+        cache = torch.load(cache_path, weights_only=False)
+        if cache.get("tokenizer_state") == cached_state:
+            print(f"using cached encoded corpus at {cache_path}")
+            data = cache["data"]
+        else:
+            print(f"  cache at {cache_path} is stale, re-encoding")
+            data = _encode_corpus_with_progress(text, tokenizer)
+            torch.save({"tokenizer_state": cached_state, "data": data}, cache_path)
+    else:
+        print(f"encoding corpus ({len(text):,} chars) — this can take a while for bpe")
+        data = _encode_corpus_with_progress(text, tokenizer)
+        torch.save({"tokenizer_state": cached_state, "data": data}, cache_path)
+        print(f"  cached encoded corpus to {cache_path}")
     n = int(0.9 * len(data))
     train_data = data[:n]
     val_data = data[n:]
@@ -428,15 +554,24 @@ def estimate_loss() -> dict[str, torch.Tensor]:
     # model's zero-context prediction starts at the corpus unigram
     # distribution rather than uniform, dropping initial loss from
     # log(vocab_size) to the unigram entropy.
-    token_counts = Counter(text)
+    token_counts = torch.bincount(data, minlength=tokenizer.vocab_size)
     with torch.no_grad():
-        freqs = torch.tensor(
-            [token_counts[ch] for ch in chars], dtype=torch.float, device=device
-        )
+        freqs = token_counts.float().to(device).clamp(min=1.0)
         model.lm_head.bias.copy_((freqs / freqs.sum()).log())
 
     os.makedirs(checkpoint_dir, exist_ok=True)
     optimizer = torch.optim.AdamW(model.parameters(), lr=hp.learning_rate)
+    # Plateau scheduler: drop lr only when val loss stops improving. We
+    # also do a manual linear warmup over the first `warmup_iters` steps,
+    # before letting this scheduler take over.
+    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
+        optimizer,
+        mode="min",
+        factor=hp.lr_factor,
+        patience=hp.lr_patience,
+        threshold=hp.lr_threshold,
+        min_lr=hp.min_lr,
+    )
 
     writer: SummaryWriter | None = None
     if not args.no_tensorboard:
@@ -446,73 +581,124 @@ def estimate_loss() -> dict[str, torch.Tensor]:
         writer.add_text("hparams", f"```\n{json.dumps(hp.model_dump(), indent=2)}\n```")
         writer.add_text("profile", ACTIVE_PROFILE)
         writer.add_text("dataset", args.dataset)
-        n_total = len(text)
-        rows = ["| rank | id | char | count | freq |", "|---|---|---|---|---|"]
-        for rank, (ch, k) in enumerate(token_counts.most_common()):
+        writer.add_text("tokenizer", f"{tokenizer.name} (vocab={tokenizer.vocab_size})")
+        n_total = int(token_counts.sum().item())
+        # pyright loses the int type through argsort().tolist(); the cast is safe.
+        sorted_ids: list[int] = token_counts.argsort(descending=True).tolist()  # pyright: ignore[reportUnknownVariableType]
+        rows = ["| rank | id | token | count | freq |", "|---|---|---|---|---|"]
+        for rank, token_id in enumerate(sorted_ids):
+            count = int(token_counts[token_id].item())
+            if count == 0:
+                break
+            tok_str = tokenizer.decode([token_id])
             rows.append(
-                f"| {rank} | {stoi[ch]} | `{ch!r}` | {k:,} | {100 * k / n_total:.3f}% |"
+                f"| {rank} | {token_id} | `{tok_str!r}` | {count:,} | {100 * count / n_total:.3f}% |"
             )
         writer.add_text("tokens", "\n".join(rows))
         writer.add_histogram("token_distribution", data, 0)
         print(f"tensorboard: logging to {log_dir}")
 
-    for iter in range(hp.max_iters):
-        if iter % hp.eval_interval == 0 or iter == hp.max_iters - 1:
-            losses = estimate_loss()
-            print(
-                f"step {iter}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}"
-            )
-            if writer is not None:
-                writer.add_scalar("loss/train", losses["train"].item(), iter)
-                writer.add_scalar("loss/val", losses["val"].item(), iter)
-                if not args.no_sample:
-                    model.eval()
-                    with torch.no_grad():
-                        ctx = torch.zeros((1, 1), dtype=torch.long, device=device)
-                        sample = model.generate(ctx, max_new_tokens=200)
-                    model.train()
-                    writer.add_text(
-                        "sample", f"```\n{decode(sample[0].tolist())}\n```", iter
+    last_saved_step: int | None = None
+    interrupted = False
+    try:
+        for iter in range(hp.max_iters):
+            # During warmup, overwrite the optimizer's lr manually. After
+            # warmup, the scheduler owns it (it modifies param_groups in
+            # place when it decides to step down).
+            warmup = hp.warmup_lr(iter)
+            if warmup is not None:
+                for pg in optimizer.param_groups:
+                    pg["lr"] = warmup
+
+            if iter % hp.eval_interval == 0 or iter == hp.max_iters - 1:
+                losses = estimate_loss()
+                val_loss = losses["val"].item()
+                lr_before = optimizer.param_groups[0]["lr"]
+                # Let the plateau scheduler decide whether to drop lr,
+                # but not during warmup — its internal "best so far"
+                # tracking shouldn't see the early ramp-up noise.
+                if iter >= hp.warmup_iters:
+                    scheduler.step(val_loss)
+                lr_now = optimizer.param_groups[0]["lr"]
+                if lr_now < lr_before:
+                    print(
+                        f"  lr reduced: {lr_before:.2e} → {lr_now:.2e} (val loss plateau)"
                     )
-            ckpt_path = os.path.join(
-                checkpoint_dir,
-                f"ckpt_{ACTIVE_PROFILE}_step_{iter:05d}.pt",
-            )
-            torch.save(
-                {
+                print(
+                    f"step {iter}: train loss {losses['train']:.4f}, val loss {val_loss:.4f}, lr {lr_now:.2e}"
+                )
+                if writer is not None:
+                    writer.add_scalar("loss/train", losses["train"].item(), iter)
+                    writer.add_scalar("loss/val", val_loss, iter)
+                    writer.add_scalar("lr", lr_now, iter)
+                    if not args.no_sample:
+                        model.eval()
+                        with torch.no_grad():
+                            ctx = torch.zeros((1, 1), dtype=torch.long, device=device)
+                            sample = model.generate(ctx, max_new_tokens=200)
+                        model.train()
+                        writer.add_text(
+                            "sample",
+                            f"```\n{tokenizer.decode(sample[0].tolist())}\n```",
+                            iter,
+                        )
+                ckpt_path = os.path.join(
+                    checkpoint_dir,
+                    f"ckpt_{ACTIVE_PROFILE}_step_{iter:05d}.pt",
+                )
+                ckpt_payload: dict[str, object] = {
                     "iter": iter,
                     "profile": ACTIVE_PROFILE,
                     "model": model.state_dict(),
                     "train_loss": losses["train"].item(),
                     "val_loss": losses["val"].item(),
-                    "chars": chars,
+                    "tokenizer_type": tokenizer.name,
+                    "tokenizer_state": tokenizer.state_dict(),
                     "hparams": hp.architecture_dict(),
-                },
-                ckpt_path,
-            )
-            print(f"  saved checkpoint to {ckpt_path}")
-
-        xb, yb = get_batch("train")
-        logits, loss = model(xb, yb)
-        assert loss is not None
-        optimizer.zero_grad(set_to_none=True)
-        loss.backward()
-        optimizer.step()
+                }
+                # Keep `chars` for backwards compat with viz_embeddings /
+                # export_onnx / the published checkpoints — only meaningful for
+                # the char tokenizer.
+                if isinstance(tokenizer, tok.CharTokenizer):
+                    ckpt_payload["chars"] = tokenizer.chars
+                torch.save(ckpt_payload, ckpt_path)
+                last_saved_step = iter
+                print(f"  saved checkpoint to {ckpt_path}")
+
+            xb, yb = get_batch("train")
+            logits, loss = model(xb, yb)
+            assert loss is not None
+            optimizer.zero_grad(set_to_none=True)
+            loss.backward()
+            optimizer.step()
+    except KeyboardInterrupt:
+        interrupted = True
+        print()
+        print("training interrupted by user (Ctrl-C)")
+        if last_saved_step is None:
+            print("  no checkpoint saved yet; nothing to upload")
+        else:
+            print(f"  last saved checkpoint: step {last_saved_step}")
 
     if writer is not None:
         writer.close()
 
-    context = torch.zeros((1, 1), dtype=torch.long, device=device)
-    print(
-        decode(model.generate(context, max_new_tokens=args.max_new_tokens)[0].tolist())
-    )
+    # Only run the final-generation sample on clean completion; on Ctrl-C the
+    # user wants to get out, not wait for 500 tokens of inference.
+    if not interrupted:
+        context = torch.zeros((1, 1), dtype=torch.long, device=device)
+        print(
+            tokenizer.decode(
+                model.generate(context, max_new_tokens=args.max_new_tokens)[0].tolist()
+            )
+        )
 
-    if args.no_upload:
+    if args.no_upload or last_saved_step is None:
         return
 
     repo = args.upload_repo or manifest_repo()
     sha = git_sha()
-    final_step = hp.max_iters - 1
+    final_step = last_saved_step
     final_ckpt = os.path.join(
         checkpoint_dir,
         f"ckpt_{ACTIVE_PROFILE}_step_{final_step:05d}.pt",
diff --git a/pyproject.toml b/pyproject.toml
index 07fd650..88f4aad 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,6 +13,7 @@ dependencies = [
     "huggingface-hub>=1.14.0",
     "tensorboard>=2.20.0",
     "onnx>=1.21.0",
+    "numba>=0.65.1",
 ]
 
 [dependency-groups]
@@ -22,7 +23,7 @@ dev = [
 ]
 
 [tool.pyright]
-include = ["gpt.py", "datasets", "viz_embeddings.py", "download_checkpoints.py", "checkpoint_io.py", "export_onnx.py"]
+include = ["gpt.py", "datasets", "tokenizers", "viz_embeddings.py", "download_checkpoints.py", "checkpoint_io.py", "export_onnx.py"]
 exclude = ["bigram.py", ".venv", "**/__pycache__"]
 pythonVersion = "3.13"
 typeCheckingMode = "basic"
diff --git a/tokenizers/__init__.py b/tokenizers/__init__.py
new file mode 100644
index 0000000..faaca62
--- /dev/null
+++ b/tokenizers/__init__.py
@@ -0,0 +1,28 @@
+"""Tokenizer registry.
+
+Maps text ↔ integer token IDs. Each tokenizer is a small subclass of
+`Tokenizer`. Register a new one by importing its class in this module
+and adding it to the list below.
+
+Note: this is the *local* package — not the PyPI `tokenizers` library
+from Hugging Face. We don't use that here.
+"""
+
+from typing import Any
+
+from .base import Tokenizer
+from .bpe import BPETokenizer
+from .char import CharTokenizer
+
+_TOKENIZERS: list[type[Tokenizer]] = [CharTokenizer, BPETokenizer]
+_REGISTRY: dict[str, type[Tokenizer]] = {cls.name: cls for cls in _TOKENIZERS}
+
+
+def get(name: str, **kwargs: Any) -> Tokenizer:
+    if name not in _REGISTRY:
+        raise KeyError(f"unknown tokenizer {name!r}. Available: {names()}")
+    return _REGISTRY[name](**kwargs)
+
+
+def names() -> list[str]:
+    return sorted(_REGISTRY)
diff --git a/tokenizers/base.py b/tokenizers/base.py
new file mode 100644
index 0000000..e1cef98
--- /dev/null
+++ b/tokenizers/base.py
@@ -0,0 +1,35 @@
+"""Base tokenizer class.
+
+A `Tokenizer` maps text to a sequence of integer token IDs and back.
+It can optionally be `train`ed on a corpus to learn its vocabulary,
+and round-trips through a torch checkpoint via state_dict / load_state_dict.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class Tokenizer(ABC):
+    """Maps text ↔ integer token IDs."""
+
+    name: str = ""
+
+    @abstractmethod
+    def encode(self, text: str) -> list[int]: ...
+
+    @abstractmethod
+    def decode(self, ids: list[int]) -> str: ...
+
+    @property
+    @abstractmethod
+    def vocab_size(self) -> int: ...
+
+    def train(self, corpus: str) -> None:
+        """Fit the tokenizer to a text corpus. Default: no-op."""
+        return None
+
+    @abstractmethod
+    def state_dict(self) -> dict[str, Any]: ...
+
+    @abstractmethod
+    def load_state_dict(self, state: dict[str, Any]) -> None: ...
diff --git a/tokenizers/bpe.py b/tokenizers/bpe.py
new file mode 100644
index 0000000..94c7ae1
--- /dev/null
+++ b/tokenizers/bpe.py
@@ -0,0 +1,168 @@
+"""Byte-level BPE tokenizer.
+
+Starts with the 256 byte tokens and greedily learns merges of the
+most-frequent adjacent pair until the target vocab size is reached.
+Same algorithm as Karpathy's minbpe; inlined here so the repo stays
+self-contained.
+
+Pros vs char-level: ~3-4x sequence compression on English, so the
+same block_size covers more actual context.
+Cons: bigger embedding + lm_head. Training is still pure-Python (one-time
+cost on a small corpus sample). Encoding is JIT-compiled via numba —
+the hot path is a tight loop over int64 arrays with O(1) merge lookup
+against a precomputed (vocab_size, vocab_size) table.
+"""
+
+from collections import Counter
+from typing import Any
+
+import numpy as np
+import numpy.typing as npt
+from numba import njit  # pyright: ignore[reportUnknownVariableType]
+
+from .base import Tokenizer
+
+
+def _pair_counts(ids: list[int]) -> Counter[tuple[int, int]]:
+    return Counter(zip(ids, ids[1:]))
+
+
+def _merge_py(ids: list[int], pair: tuple[int, int], new_id: int) -> list[int]:
+    out: list[int] = []
+    i = 0
+    n = len(ids)
+    while i < n:
+        if i + 1 < n and ids[i] == pair[0] and ids[i + 1] == pair[1]:
+            out.append(new_id)
+            i += 2
+        else:
+            out.append(ids[i])
+            i += 1
+    return out
+
+
+@njit(cache=True)
+def _bpe_encode_jit(
+    ids: npt.NDArray[np.int64], merge_lookup: npt.NDArray[np.int64]
+) -> npt.NDArray[np.int64]:
+    """Greedy BPE encode. At each iteration finds the lowest-new_id (=
+    earliest-learned, highest priority) merge currently present in `ids`
+    and applies it to all of its occurrences. Loops until no applicable
+    merge remains. `merge_lookup[a, b]` is the new_id for pair (a, b) or
+    -1 if no merge exists.
+    """
+    n = len(ids)
+    if n < 2:
+        return ids
+
+    while True:
+        found = False
+        best_a = np.int64(0)
+        best_b = np.int64(0)
+        best_new_id = np.int64(0)
+        for i in range(n - 1):
+            a = ids[i]
+            b = ids[i + 1]
+            nid = merge_lookup[a, b]
+            if nid >= 0 and (not found or nid < best_new_id):
+                found = True
+                best_a = a
+                best_b = b
+                best_new_id = nid
+        if not found:
+            break
+
+        out = np.empty(n, dtype=np.int64)
+        out_idx = 0
+        i = 0
+        while i < n:
+            if i + 1 < n and ids[i] == best_a and ids[i + 1] == best_b:
+                out[out_idx] = best_new_id
+                out_idx += 1
+                i += 2
+            else:
+                out[out_idx] = ids[i]
+                out_idx += 1
+                i += 1
+        ids = out[:out_idx]
+        n = out_idx
+
+    return ids
+
+
+class BPETokenizer(Tokenizer):
+    name = "bpe"
+
+    def __init__(self, vocab_size: int = 1024) -> None:
+        # `vocab_size` is the *target*; train() runs at most
+        # (vocab_size - 256) merges.
+        self._target = vocab_size
+        self._merges: list[tuple[tuple[int, int], int]] = []
+        self._merge_index: dict[tuple[int, int], int] = {}
+        self._vocab: dict[int, bytes] = {i: bytes([i]) for i in range(256)}
+        # Lazy: a (vocab_size, vocab_size) int64 array of merge -> new_id
+        # (or -1), built once after train/load for O(1) lookup in the JIT
+        # encode loop.
+        self._merge_lookup: npt.NDArray[np.int64] | None = None
+
+    def _build_merge_lookup(self) -> None:
+        size = max(self._target, 256)
+        lookup = np.full((size, size), -1, dtype=np.int64)
+        for (a, b), n in self._merges:
+            lookup[a, b] = n
+        self._merge_lookup = lookup
+
+    def train(self, corpus: str, sample_chars: int = 5_000_000) -> None:
+        """Learn BPE merges from `corpus`.
+
+        For corpora longer than `sample_chars`, train merges on a prefix
+        of that length — full-corpus training under naive Python takes
+        prohibitively long, and the prefix typically converges to
+        nearly-identical merges.
+        """
+        text = corpus[:sample_chars] if sample_chars else corpus
+        ids = list(text.encode("utf-8"))
+        n_merges = max(0, self._target - 256)
+        for i in range(n_merges):
+            stats = _pair_counts(ids)
+            if not stats:
+                break
+            top_pair = max(stats.items(), key=lambda kv: kv[1])[0]
+            new_id = 256 + i
+            ids = _merge_py(ids, top_pair, new_id)
+            self._merges.append((top_pair, new_id))
+            self._merge_index[top_pair] = new_id
+            self._vocab[new_id] = self._vocab[top_pair[0]] + self._vocab[top_pair[1]]
+        self._build_merge_lookup()
+
+    def encode(self, text: str) -> list[int]:
+        if self._merge_lookup is None:
+            self._build_merge_lookup()
+        assert self._merge_lookup is not None
+        # bytes() → uint8 view → writable int64 array for numba
+        ids_arr = np.frombuffer(text.encode("utf-8"), dtype=np.uint8).astype(np.int64)
+        result = _bpe_encode_jit(ids_arr, self._merge_lookup)
+        return result.tolist()
+
+    def decode(self, ids: list[int]) -> str:
+        return b"".join(self._vocab[i] for i in ids).decode("utf-8", errors="replace")
+
+    @property
+    def vocab_size(self) -> int:
+        return len(self._vocab)
+
+    def state_dict(self) -> dict[str, Any]:
+        return {
+            "merges": [(list(p), n) for p, n in self._merges],
+            "target_vocab_size": self._target,
+        }
+
+    def load_state_dict(self, state: dict[str, Any]) -> None:
+        self._target = state["target_vocab_size"]
+        self._merges = [((p[0], p[1]), n) for p, n in state["merges"]]
+        self._merge_index = {p: n for p, n in self._merges}
+        self._vocab = {i: bytes([i]) for i in range(256)}
+        for (p1, p2), new_id in self._merges:
+            self._vocab[new_id] = self._vocab[p1] + self._vocab[p2]
+        # Invalidate; the next encode() call will rebuild it.
+        self._merge_lookup = None
diff --git a/tokenizers/char.py b/tokenizers/char.py
new file mode 100644
index 0000000..a895ff1
--- /dev/null
+++ b/tokenizers/char.py
@@ -0,0 +1,49 @@
+"""Character-level tokenizer.
+
+Each unique character in the training corpus becomes a token. Tiny
+vocabularies (~85-250 for English-ish text), no OOV concept, no
+sequence compression — every input character is one token.
+"""
+
+from typing import Any
+
+from .base import Tokenizer
+
+
+class CharTokenizer(Tokenizer):
+    name = "char"
+
+    def __init__(self) -> None:
+        self._chars: list[str] = []
+        self._stoi: dict[str, int] = {}
+        self._itos: dict[int, str] = {}
+
+    def train(self, corpus: str) -> None:
+        self._chars = sorted(set(corpus))
+        self._rebuild_maps()
+
+    def encode(self, text: str) -> list[int]:
+        # Silently skip OOV characters (matches the prior inline behavior).
+        return [self._stoi[c] for c in text if c in self._stoi]
+
+    def decode(self, ids: list[int]) -> str:
+        return "".join(self._itos[i] for i in ids)
+
+    @property
+    def vocab_size(self) -> int:
+        return len(self._chars)
+
+    @property
+    def chars(self) -> list[str]:
+        return self._chars
+
+    def state_dict(self) -> dict[str, Any]:
+        return {"chars": list(self._chars)}
+
+    def load_state_dict(self, state: dict[str, Any]) -> None:
+        self._chars = list(state["chars"])
+        self._rebuild_maps()
+
+    def _rebuild_maps(self) -> None:
+        self._stoi = {c: i for i, c in enumerate(self._chars)}
+        self._itos = {i: c for i, c in enumerate(self._chars)}
diff --git a/uv.lock b/uv.lock
index e3a4048..ff6fbc6 100644
--- a/uv.lock
+++ b/uv.lock
@@ -501,6 +501,26 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" },
 ]
 
+[[package]]
+name = "llvmlite"
+version = "0.47.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/01/88/a8952b6d5c21e74cbf158515b779666f692846502623e9e3c39d8e8ba25f/llvmlite-0.47.0.tar.gz", hash = "sha256:62031ce968ec74e95092184d4b0e857e444f8fdff0b8f9213707699570c33ccc", size = 193614, upload-time = "2026-03-31T18:29:53.497Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/77/6f/4615353e016799f80fa52ccb270a843c413b22361fadda2589b2922fb9b0/llvmlite-0.47.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:a3c6a735d4e1041808434f9d440faa3d78d9b4af2ee64d05a66f351883b6ceec", size = 37232771, upload-time = "2026-03-31T18:29:01.324Z" },
+    { url = "https://files.pythonhosted.org/packages/31/b8/69f5565f1a280d032525878a86511eebed0645818492feeb169dfb20ae8e/llvmlite-0.47.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2699a74321189e812d476a43d6d7f652f51811e7b5aad9d9bba842a1c7927acb", size = 56275178, upload-time = "2026-03-31T18:29:05.748Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/da/b32cafcb926fb0ce2aa25553bf32cb8764af31438f40e2481df08884c947/llvmlite-0.47.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c6951e2b29930227963e53ee152441f0e14be92e9d4231852102d986c761e40", size = 55128632, upload-time = "2026-03-31T18:29:11.235Z" },
+    { url = "https://files.pythonhosted.org/packages/46/9f/4898b44e4042c60fafcb1162dfb7014f6f15b1ec19bf29cfea6bf26df90d/llvmlite-0.47.0-cp313-cp313-win_amd64.whl", hash = "sha256:c2e9adf8698d813a9a5efb2d4370caf344dbc1e145019851fee6a6f319ba760e", size = 38138695, upload-time = "2026-03-31T18:29:15.43Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/d4/33c8af00f0bf6f552d74f3a054f648af2c5bc6bece97972f3bfadce4f5ec/llvmlite-0.47.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:de966c626c35c9dff5ae7bf12db25637738d0df83fc370cf793bc94d43d92d14", size = 37232773, upload-time = "2026-03-31T18:29:19.453Z" },
+    { url = "https://files.pythonhosted.org/packages/64/1d/a760e993e0c0ba6db38d46b9f48f6c7dceb8ac838824997fb9e25f97bc04/llvmlite-0.47.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ddbccff2aeaff8670368340a158abefc032fe9b3ccf7d9c496639263d00151aa", size = 56275176, upload-time = "2026-03-31T18:29:24.149Z" },
+    { url = "https://files.pythonhosted.org/packages/84/3b/e679bc3b29127182a7f4aa2d2e9e5bea42adb93fb840484147d59c236299/llvmlite-0.47.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a7b778a2e144fc64468fb9bf509ac1226c9813a00b4d7afea5d988c4e22fca", size = 55128631, upload-time = "2026-03-31T18:29:29.536Z" },
+    { url = "https://files.pythonhosted.org/packages/be/f7/19e2a09c62809c9e63bbd14ce71fb92c6ff7b7b3045741bb00c781efc3c9/llvmlite-0.47.0-cp314-cp314-win_amd64.whl", hash = "sha256:694e3c2cdc472ed2bd8bd4555ca002eec4310961dd58ef791d508f57b5cc4c94", size = 39153826, upload-time = "2026-03-31T18:29:33.681Z" },
+    { url = "https://files.pythonhosted.org/packages/40/a1/581a8c707b5e80efdbbe1dd94527404d33fe50bceb71f39d5a7e11bd57b7/llvmlite-0.47.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:92ec8a169a20b473c1c54d4695e371bde36489fc1efa3688e11e99beba0abf9c", size = 37232772, upload-time = "2026-03-31T18:29:37.952Z" },
+    { url = "https://files.pythonhosted.org/packages/11/03/16090dd6f74ba2b8b922276047f15962fbeea0a75d5601607edb301ba945/llvmlite-0.47.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa1cbd800edd3b20bc141521f7fd45a6185a5b84109aa6855134e81397ffe72b", size = 56275178, upload-time = "2026-03-31T18:29:42.58Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/cb/0abf1dd4c5286a95ffe0c1d8c67aec06b515894a0dd2ac97f5e27b82ab0b/llvmlite-0.47.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6725179b89f03b17dabe236ff3422cb8291b4c1bf40af152826dfd34e350ae8", size = 55128632, upload-time = "2026-03-31T18:29:46.939Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/79/d3bbab197e86e0ff4f9c07122895b66a3e0d024247fcff7f12c473cb36d9/llvmlite-0.47.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6842cf6f707ec4be3d985a385ad03f72b2d724439e118fcbe99b2929964f0453", size = 39153839, upload-time = "2026-03-31T18:29:51.004Z" },
+]
+
 [[package]]
 name = "markdown"
 version = "3.10.2"
@@ -647,6 +667,7 @@ version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
     { name = "huggingface-hub" },
+    { name = "numba" },
     { name = "numpy" },
     { name = "onnx" },
     { name = "plotly" },
@@ -665,6 +686,7 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "huggingface-hub", specifier = ">=1.14.0" },
+    { name = "numba", specifier = ">=0.65.1" },
     { name = "numpy", specifier = ">=2.4.4" },
     { name = "onnx", specifier = ">=1.21.0" },
     { name = "plotly", specifier = ">=6.7.0" },
@@ -689,6 +711,30 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" },
 ]
 
+[[package]]
+name = "numba"
+version = "0.65.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "llvmlite" },
+    { name = "numpy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f6/c5/db2ac3685833d626c0dcae6bd2330cd68433e1fd248d15f70998160d3ad7/numba-0.65.1.tar.gz", hash = "sha256:19357146c32fe9ed25059ab915e8465fb13951cf6b0aace3826b76886373ab23", size = 2765600, upload-time = "2026-04-24T02:02:56.551Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/79/37/14a4579049c1eb673afd0de0cb4842982acd55b9ce2643e763db858bcea0/numba-0.65.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:1735c15c1134a5108b4d6a5c77fc0947924ea066a738dc09a52008c13df9cad3", size = 2681344, upload-time = "2026-04-24T02:02:33.65Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/22/b8d873f6466b20aa563fc9b33acd48dec89a07803ddaa2f1c8ca1cd33126/numba-0.65.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c09f49117ef255e1f1c6dad0c7a1ed39868243862a73be5706793241a3755f1b", size = 3810619, upload-time = "2026-04-24T02:02:36.041Z" },
+    { url = "https://files.pythonhosted.org/packages/62/08/e16a8b5d9a018962ebb5c66be662317cde32b9f5dab08441f90bed5522fb/numba-0.65.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:594a8680b3fadac99e97e489b1fd89007177e5336713745c3b769528c635a464", size = 3509783, upload-time = "2026-04-24T02:02:38.245Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/a5/03c970d57f4c1741354837353ce39fb5206952ae1dba8922d29c86f64805/numba-0.65.1-cp313-cp313-win_amd64.whl", hash = "sha256:85be74c0d036842699a30058f82fb88fc5ffdc59f7615cab5792ea92914c9b62", size = 2750534, upload-time = "2026-04-24T02:02:39.903Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/2e/8aed9b726d9ba5f11ad287645fd479e88278db3060a25cb1225d730eb2b7/numba-0.65.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:33f5eb68eb1c843511615d14663ce60258525d6a4c65ab040e2c2b0c4cf17450", size = 2681554, upload-time = "2026-04-24T02:02:41.812Z" },
+    { url = "https://files.pythonhosted.org/packages/87/96/f3eb235fafa82a34e2ab5dd7dc9ffff998ebf5f0bbc23fa56a96aeb44da6/numba-0.65.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71e73029bf53a62cc6afcf96be4bd942290d8b4c55f0a454fb536158115790f7", size = 3779602, upload-time = "2026-04-24T02:02:43.726Z" },
+    { url = "https://files.pythonhosted.org/packages/09/90/b0f09b48752d23640b8284f22aa597737e8adaddc7fbfacc4708b7f73a4c/numba-0.65.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a07635e0be926b9bdbffb09137c230fb13f6ec0e564914ba937cee12ce3eb35", size = 3479532, upload-time = "2026-04-24T02:02:45.427Z" },
+    { url = "https://files.pythonhosted.org/packages/56/46/3f7fc04fb853559e74b210e0b62c19974ec844cefec611f9e535f4da3761/numba-0.65.1-cp314-cp314-win_amd64.whl", hash = "sha256:2a20fcdabdefbdacf88d85caf70c3b18c4bcb7ebb8f82e6a19486383dd26ab63", size = 2752637, upload-time = "2026-04-24T02:02:47.664Z" },
+    { url = "https://files.pythonhosted.org/packages/81/7b/c1a341a9067367778f4152a5f01061cf281fb09582c92c510ec4918cabf6/numba-0.65.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:548dd4b3a4508d5062768d1514b2cd7b015f9a25ec7af651c50dee243965e652", size = 2684600, upload-time = "2026-04-24T02:02:49.653Z" },
+    { url = "https://files.pythonhosted.org/packages/03/36/98ddbcf3e4f04a6dd07e1c67249955920579ba4af6bb6868e3088f4ed282/numba-0.65.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:78abc28feff2c2ff8307fff3975b6438352759c9acb797ecd6b1fb6e7e39e31d", size = 3817198, upload-time = "2026-04-24T02:02:51.266Z" },
+    { url = "https://files.pythonhosted.org/packages/a3/83/0dad21057ece5a835599f5d24099b091703995e23dbbf894f259e91c010b/numba-0.65.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee7676cb389555805f9b9a1840cbcd1ea6c8bd5376ab6918e3a29c5ea1dbda20", size = 3533862, upload-time = "2026-04-24T02:02:52.987Z" },
+    { url = "https://files.pythonhosted.org/packages/32/36/8be7118ffd4c8440881046eac3d0982cc5ab42909508cf5d67024d62a2e4/numba-0.65.1-cp314-cp314t-win_amd64.whl", hash = "sha256:20609346e3bd75204950dcbbfe383a8d7dbf4902f442aedbf00f97fef4aa8f38", size = 2758237, upload-time = "2026-04-24T02:02:54.612Z" },
+]
+
 [[package]]
 name = "numpy"
 version = "2.4.4"
diff --git a/viz_embeddings.py b/viz_embeddings.py
index 12dbb82..882dea8 100644
--- a/viz_embeddings.py
+++ b/viz_embeddings.py
@@ -13,6 +13,9 @@
 import torch
 
 from gpt import GPTLanguageModel, Hyperparameters, load_model_from_checkpoint
+import tokenizers as tok
+from tokenizers import CharTokenizer
+from tokenizers.base import Tokenizer
 
 CKPT_DIR = "checkpoints"
 
@@ -61,7 +64,18 @@ def char_display(c: str) -> str:
 @st.cache_data(show_spinner=False)
 def load_checkpoint(path: str) -> tuple[list[str], torch.Tensor, torch.Tensor]:
     ckpt = torch.load(path, map_location="cpu", weights_only=False)
-    chars: list[str] = ckpt["chars"]
+    # The embedding viz is char-tokenizer-only — per-character category
+    # coloring and similarity heatmaps don't translate to BPE subword tokens.
+    if "chars" in ckpt:
+        chars: list[str] = ckpt["chars"]
+    elif ckpt.get("tokenizer_type") == "char":
+        chars = ckpt["tokenizer_state"]["chars"]
+    else:
+        raise ValueError(
+            f"{os.path.basename(path)} is a non-char tokenizer "
+            f"({ckpt.get('tokenizer_type', 'unknown')}); the embedding viz "
+            f"only supports char-level checkpoints."
+        )
     sd = ckpt["model"]
     tok = sd["token_embedding_table.weight"].float()
     pos = sd["position_embedding_table.weight"].float()
@@ -71,8 +85,30 @@ def load_checkpoint(path: str) -> tuple[list[str], torch.Tensor, torch.Tensor]:
 @st.cache_resource(show_spinner="Loading model…")
 def load_model(path: str) -> tuple[GPTLanguageModel, list[str], Hyperparameters, str]:
     device = "cuda" if torch.cuda.is_available() else "cpu"
-    model, chars, hp = load_model_from_checkpoint(path, device=device)
-    return model, chars, hp, device
+    model, tokenizer, hp = load_model_from_checkpoint(path, device=device)
+    if not isinstance(tokenizer, CharTokenizer):
+        raise ValueError(
+            f"{os.path.basename(path)} uses {tokenizer.name} tokenizer; "
+            f"the embedding viz only supports char-level checkpoints."
+        )
+    return model, tokenizer.chars, hp, device
+
+
+@st.cache_data(show_spinner=False)
+def load_tokenizer(path: str) -> Tokenizer:
+    """Lightweight loader: pulls just the tokenizer state out of a
+    checkpoint without instantiating the model. Works for both char and
+    BPE checkpoints, plus legacy `chars`-only files."""
+    ckpt = torch.load(path, map_location="cpu", weights_only=False)
+    if "tokenizer_type" in ckpt and "tokenizer_state" in ckpt:
+        t = tok.get(ckpt["tokenizer_type"])
+        t.load_state_dict(ckpt["tokenizer_state"])
+    elif "chars" in ckpt:
+        t = tok.get("char")
+        t.load_state_dict({"chars": ckpt["chars"]})
+    else:
+        raise ValueError(f"{os.path.basename(path)} has no tokenizer info")
+    return t
 
 
 @torch.no_grad()
@@ -285,53 +321,118 @@ def position_scatter(coords: torch.Tensor, dim: int) -> go.Figure:
         default=list(CATEGORY_COLORS.keys()),
     )
 
-chars, tok_w, pos_w = load_checkpoint(ckpt_path)
+tokenizer = load_tokenizer(ckpt_path)
+
+# Embedding-based tabs need the model weight matrices and are
+# char-tokenizer-only. For BPE checkpoints we'll show the tokens tab
+# anyway and disable the rest.
+chars: list[str] | None = None
+tok_w: torch.Tensor | None = None
+pos_w: torch.Tensor | None = None
+try:
+    chars, tok_w, pos_w = load_checkpoint(ckpt_path)
+except ValueError as e:
+    st.warning(str(e))
 
 c1, c2, c3 = st.columns(3)
-c1.metric("vocab size", len(chars))
-c2.metric("token emb dim", tok_w.shape[1])
-c3.metric("block size", pos_w.shape[0])
+c1.metric("vocab size", tokenizer.vocab_size)
+c2.metric("tokenizer", tokenizer.name)
+c3.metric("block size", pos_w.shape[0] if pos_w is not None else "?")
 
-tab3d, tab2d, tab_sim, tab_pos, tab_resid = st.tabs(
-    ["3D tokens", "2D tokens", "Similarity heatmap", "Positions", "Residual stream"]
+tab_tokens, tab3d, tab2d, tab_sim, tab_pos, tab_resid = st.tabs(
+    ["Tokens", "3D tokens", "2D tokens", "Similarity heatmap", "Positions", "Residual stream"]
 )
 
-with tab3d:
-    coords = pca(tok_w, 3)
-    st.plotly_chart(
-        token_scatter(coords, chars, dim=3,
-                      selected_cats=selected_cats, show_labels=show_labels),
-        use_container_width=True,
+
+def _embedding_only(label: str) -> None:
+    st.info(
+        f"The **{label}** tab visualizes the learned token embedding "
+        f"matrix, which only has a meaningful per-character interpretation "
+        f"for `char` tokenizers. This checkpoint uses `{tokenizer.name}`."
     )
-    st.caption("Drag to rotate, scroll to zoom. Each point is one character "
-               "of the vocabulary projected into the top-3 principal components "
-               "of the learned token embedding matrix.")
 
-with tab2d:
-    coords = pca(tok_w, 2)
-    st.plotly_chart(
-        token_scatter(coords, chars, dim=2,
-                      selected_cats=selected_cats, show_labels=True),
-        use_container_width=True,
+
+with tab_tokens:
+    st.markdown(
+        f"Vocabulary learned by this checkpoint's `{tokenizer.name}` tokenizer. "
+        f"For BPE, each row is a byte sequence merged out of more frequent "
+        f"adjacent pairs during training; for char, each row is one character "
+        f"that appeared in the training corpus."
     )
-    st.caption("Same data as the 3D view, projected to the top-2 PCs.")
+    rows: list[dict[str, int | str]] = []
+    for token_id in range(tokenizer.vocab_size):
+        s = tokenizer.decode([token_id])
+        rows.append({
+            "id": token_id,
+            "token": repr(s),
+            "chars": len(s),
+            "bytes": len(s.encode("utf-8")),
+        })
+    st.dataframe(rows, use_container_width=True, height=600, hide_index=True)
+    if tokenizer.vocab_size > 256:
+        # BPE token-length histogram — char-level is uninteresting (all 1s).
+        from collections import Counter
+        length_counts: Counter[int] = Counter(int(r["chars"]) for r in rows)
+        xs: list[int] = sorted(length_counts)
+        ys: list[int] = [length_counts[x] for x in xs]
+        fig = go.Figure(data=go.Bar(x=xs, y=ys))
+        fig.update_layout(template="plotly_white",
+                          title="Token length distribution (chars)",
+                          xaxis_title="length", yaxis_title="count",
+                          height=300, margin=dict(l=0, r=0, t=40, b=0))
+        st.plotly_chart(fig, use_container_width=True)
+
+with tab3d:
+    if chars is None or tok_w is None:
+        _embedding_only("3D tokens")
+    else:
+        coords = pca(tok_w, 3)
+        st.plotly_chart(
+            token_scatter(coords, chars, dim=3,
+                          selected_cats=selected_cats, show_labels=show_labels),
+            use_container_width=True,
+        )
+        st.caption("Drag to rotate, scroll to zoom. Each point is one character "
+                   "of the vocabulary projected into the top-3 principal components "
+                   "of the learned token embedding matrix.")
+
+with tab2d:
+    if chars is None or tok_w is None:
+        _embedding_only("2D tokens")
+    else:
+        coords = pca(tok_w, 2)
+        st.plotly_chart(
+            token_scatter(coords, chars, dim=2,
+                          selected_cats=selected_cats, show_labels=True),
+            use_container_width=True,
+        )
+        st.caption("Same data as the 3D view, projected to the top-2 PCs.")
 
 with tab_sim:
-    st.plotly_chart(similarity_heatmap(tok_w, chars), use_container_width=True)
-    st.caption("Cosine similarity between every pair of token embedding rows. "
-               "Red = similar direction, blue = opposite. Hover for the pair "
-               "and exact value.")
+    if chars is None or tok_w is None:
+        _embedding_only("Similarity heatmap")
+    else:
+        st.plotly_chart(similarity_heatmap(tok_w, chars), use_container_width=True)
+        st.caption("Cosine similarity between every pair of token embedding rows. "
+                   "Red = similar direction, blue = opposite. Hover for the pair "
+                   "and exact value.")
 
 with tab_pos:
-    pos_dim = st.radio("dimensions", options=[3, 2], horizontal=True, index=0)
-    coords = pca(pos_w, pos_dim)
-    st.plotly_chart(position_scatter(coords, dim=pos_dim),
-                    use_container_width=True)
-    st.caption("Learned position embeddings, colored by position index and "
-               "connected in order. A smooth path means the model learned a "
-               "continuous notion of position.")
+    if pos_w is None:
+        _embedding_only("Positions")
+    else:
+        pos_dim = st.radio("dimensions", options=[3, 2], horizontal=True, index=0)
+        coords = pca(pos_w, pos_dim)
+        st.plotly_chart(position_scatter(coords, dim=pos_dim),
+                        use_container_width=True)
+        st.caption("Learned position embeddings, colored by position index and "
+                   "connected in order. A smooth path means the model learned a "
+                   "continuous notion of position.")
 
 with tab_resid:
+    if not isinstance(tokenizer, CharTokenizer):
+        _embedding_only("Residual stream")
+        st.stop()
     st.markdown(
         "Each prompt is encoded character-by-character and run through the "
         "model. We grab the residual stream at the chosen layer at the "