diff --git a/data/poetry-chinese-zhtw/convert_traditional_to_simplified.py b/data/poetry-chinese-zhtw/convert_traditional_to_simplified.py
new file mode 100644
index 0000000000..45b667314a
--- /dev/null
+++ b/data/poetry-chinese-zhtw/convert_traditional_to_simplified.py
@@ -0,0 +1,11 @@
+from opencc import OpenCC
+
+cc = OpenCC('t2s')  # Traditional -> Simplified
+
+with open("input.txt", "r", encoding="utf-8") as f:
+    text = f.read()
+
+converted = cc.convert(text)
+
+with open("output.txt", "w", encoding="utf-8") as f:
+    f.write(converted)
diff --git a/data/simplified_hanzi_mc/.gitignore b/data/simplified_hanzi_mc/.gitignore
new file mode 100644
index 0000000000..4bdf5a08d2
--- /dev/null
+++ b/data/simplified_hanzi_mc/.gitignore
@@ -0,0 +1,15 @@
+/char/
+/non_hanzi/
+/whole/
+/left/
+/right/
+/top/
+/bottom/
+/enclosure/
+/inside/
+/corner/
+/overlay/
+/other/
+/manifest.json
+__pycache__/
+*.pyc
diff --git a/data/simplified_hanzi_mc/README.md b/data/simplified_hanzi_mc/README.md
new file mode 100644
index 0000000000..0e59d53ec5
--- /dev/null
+++ b/data/simplified_hanzi_mc/README.md
@@ -0,0 +1,23 @@
+# Simplified Hanzi radical-location multicontext demo
+
+This folder demonstrates a reversible multicontext split for simplified Hanzi.
+`input.txt` may be either the bundled one-character-per-line corner-case fixture or an ordinary UTF-8 text corpus (for example `#Title:` / `#Poem:` records). `get_dataset.sh` treats the file as a character stream and creates one aligned multicontext timestep per Unicode code point:
+
+- `char`: the original simplified character, or `⧆` when the timestep is not simplified Hanzi.
+- `non_hanzi`: the original non-simplified-Hanzi code point, or `∅` when the timestep is simplified Hanzi. Together with `char`, this makes the representation a full-text 1:1 bijection.
+- `whole`, `left`, `right`, `top`, `bottom`, `enclosure`, `inside`, `corner`, `overlay`, `other`: radical/location signals.
+
+`∅` means “this simplified Hanzi has nothing in this category” (and is also the empty value in `non_hanzi` for simplified Hanzi). `⧆` means “this
+input code point is not treated as simplified Hanzi” in the `char`/radical lanes; the original code point is preserved in `non_hanzi` using line-safe escapes for control characters such as newlines.
+
+The decomposition table is intentionally small and transparent for tests. It can
+be replaced with a full Unihan/IDS-derived table without changing the lane
+contract or downstream training commands.
+
+Run:
+
+```bash
+bash data/simplified_hanzi_mc/get_dataset.sh
+```
+
+Each lane then contains `char_simplified_hanzi_mc/{train.bin,val.bin,meta.pkl}`.
diff --git a/data/simplified_hanzi_mc/build_simplified_hanzi_mc.py b/data/simplified_hanzi_mc/build_simplified_hanzi_mc.py
new file mode 100755
index 0000000000..3a5ad8e807
--- /dev/null
+++ b/data/simplified_hanzi_mc/build_simplified_hanzi_mc.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Build toy simplified-Hanzi radical-location multicontext lanes.
+
+This is intentionally small and transparent: it demonstrates a reversible
+(1:1) representation by carrying the original simplified Hanzi in a dedicated
+`char` lane and aligned radical-location lanes for model conditioning.
+"""
+from __future__ import annotations
+import argparse, json, re
+from pathlib import Path
+
+PLACEHOLDER = "∅"
+NON_HANZI = "⧆"
+LANES = ["char", "non_hanzi", "whole", "left", "right", "top", "bottom", "enclosure", "inside", "corner", "overlay", "other"]
+
+# Demonstration lookup table: enough cases to cover the location categories and
+# corner cases in input.txt. Values are radical/location signals, not full IDS.
+DECOMP = {
+    "一": {"whole":"一"}, "人": {"whole":"人"}, "口": {"whole":"口"},
+    "明": {"left":"日", "right":"月"}, "休": {"left":"亻", "right":"木"},
+    "林": {"left":"木", "right":"木"}, "好": {"left":"女", "right":"子"},
+    "苗": {"top":"艹", "bottom":"田"}, "尖": {"top":"小", "bottom":"大"},
+    "想": {"top":"相", "bottom":"心", "left":"木", "right":"目"},
+    "国": {"enclosure":"囗", "inside":"玉"}, "问": {"enclosure":"门", "inside":"口"},
+    "闪": {"enclosure":"门", "inside":"人"}, "医": {"enclosure":"匚", "inside":"矢"},
+    "区": {"enclosure":"匚", "inside":"乂"}, "同": {"enclosure":"冂", "inside":"一口"},
+    "这": {"enclosure":"辶", "inside":"文"}, "房": {"enclosure":"户", "inside":"方"},
+    "病": {"enclosure":"疒", "inside":"丙"}, "氧": {"enclosure":"气", "inside":"羊"},
+    "赢": {"corner":"亡口月贝凡"}, "器": {"corner":"口口口口", "inside":"犬"},
+    "乘": {"overlay":"禾北"}, "爽": {"overlay":"大乂乂乂乂"},
+    "坐": {"overlay":"人人土"}, "办": {"other":"力丶丶"}, "必": {"other":"心丿"},
+}
+# Tiny demo-only exclusions so the non-simplified-Hanzi vector is testable.
+TRADITIONAL_ONLY = set("體龍門馬愛學國風書樂車東長萬與興貓鳥魚")
+CJK_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]")
+
+def line_escape(value: str) -> str:
+    """Encode one lane value so each timestep stays on one physical line."""
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t")
+
+def is_simplified_hanzi(ch: str) -> bool:
+    return len(ch) == 1 and bool(CJK_RE.fullmatch(ch)) and ch not in TRADITIONAL_ONLY
+
+def encode_char(ch: str) -> dict[str, str]:
+    if not is_simplified_hanzi(ch):
+        row = {lane: NON_HANZI for lane in LANES}
+        row["non_hanzi"] = ch
+        return row
+    row = {lane: PLACEHOLDER for lane in LANES}
+    row["char"] = ch
+    row["non_hanzi"] = PLACEHOLDER
+    for lane, value in DECOMP.get(ch, {"other": ch}).items():
+        row[lane] = value
+    return row
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--input", default="input.txt")
+    ap.add_argument("--output_root", default=".")
+    ap.add_argument("--label", default="simplified_hanzi_mc")
+    args = ap.parse_args()
+    in_path = Path(args.input)
+    out_root = Path(args.output_root)
+    # Treat input.txt as an arbitrary UTF-8 text stream, not as one-character
+    # records. This lets regular corpora such as poem/title files flow through
+    # unchanged at the character timestep level: every code point gets one
+    # aligned multicontext vector; non-Hanzi code points become NON_HANZI.
+    chars = list(in_path.read_text(encoding="utf-8"))
+    if not chars:
+        raise ValueError(f"Input file is empty: {in_path}")
+    rows = [encode_char(ch) for ch in chars]
+    datasets = []
+    for lane in LANES:
+        lane_dir = out_root / lane
+        lane_dir.mkdir(parents=True, exist_ok=True)
+        lane_values = [line_escape(row[lane]) if lane == "non_hanzi" else row[lane] for row in rows]
+        (lane_dir / "input.txt").write_text("\n".join(lane_values) + "\n", encoding="utf-8")
+        datasets.append(f"simplified_hanzi_mc/{lane}/char_{args.label}")
+    manifest = {"tokenizer":"simplified_hanzi_radical_location_multicontext", "source":str(in_path), "lanes":LANES,
+                "multicontext_datasets":datasets, "placeholder":PLACEHOLDER, "non_hanzi":NON_HANZI,
+                "bijection":"The char lane stores simplified Hanzi; the non_hanzi lane stores original non-Hanzi/non-simplified code points; aligned radical lanes store location labels.",
+                "rows":len(rows)}
+    (out_root / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+"\n", encoding="utf-8")
+    print(json.dumps(manifest, ensure_ascii=False, indent=2))
+if __name__ == "__main__": main()
diff --git a/data/simplified_hanzi_mc/decode_multicontext_sample.py b/data/simplified_hanzi_mc/decode_multicontext_sample.py
new file mode 100755
index 0000000000..e35491c230
--- /dev/null
+++ b/data/simplified_hanzi_mc/decode_multicontext_sample.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Reconstruct simplified Hanzi from generated multicontext lane text.
+
+The `char` lane carries simplified Hanzi. For timesteps where `char` is `⧆`,
+the `non_hanzi` lane carries the original escaped code point, allowing full-text
+reconstruction instead of rendering `<NON_HANZI>` placeholders.
+"""
+from __future__ import annotations
+import argparse, json
+from pathlib import Path
+NON_HANZI="⧆"
+def line_unescape(value: str) -> str:
+    out=[]
+    i=0
+    while i < len(value):
+        if value[i] == "\\" and i + 1 < len(value):
+            nxt=value[i+1]
+            if nxt == "n": out.append("\n")
+            elif nxt == "r": out.append("\r")
+            elif nxt == "t": out.append("\t")
+            elif nxt == "\\": out.append("\\")
+            else:
+                out.append(nxt)
+            i += 2
+        else:
+            out.append(value[i])
+            i += 1
+    return "".join(out)
+
+def main():
+    ap=argparse.ArgumentParser()
+    ap.add_argument("--root", default="data/simplified_hanzi_mc")
+    ap.add_argument("--char_file", default=None, help="Optional generated char-lane text file; defaults to <root>/char/input.txt")
+    ap.add_argument("--non_hanzi_file", default=None, help="Optional generated non_hanzi-lane text file; defaults to <root>/non_hanzi/input.txt")
+    args=ap.parse_args()
+    root=Path(args.root)
+    manifest=json.loads((root/"manifest.json").read_text(encoding="utf-8"))
+    char_path=Path(args.char_file) if args.char_file else root/"char"/"input.txt"
+    non_hanzi_path=Path(args.non_hanzi_file) if args.non_hanzi_file else root/"non_hanzi"/"input.txt"
+    chars=[line.rstrip("\n") for line in char_path.read_text(encoding="utf-8").splitlines()]
+    non_hanzi=[line_unescape(line.rstrip("\n")) for line in non_hanzi_path.read_text(encoding="utf-8").splitlines()]
+    if len(chars) != len(non_hanzi):
+        raise ValueError(f"Lane length mismatch: char={len(chars)} non_hanzi={len(non_hanzi)}")
+    decoded=[nh if ch==NON_HANZI else ch for ch, nh in zip(chars, non_hanzi)]
+    print("".join(decoded))
+    print(f"decoded_steps={len(decoded)} lanes={','.join(manifest['lanes'])}")
+if __name__ == "__main__": main()
diff --git a/data/simplified_hanzi_mc/get_dataset.sh b/data/simplified_hanzi_mc/get_dataset.sh
new file mode 100755
index 0000000000..54034e465e
--- /dev/null
+++ b/data/simplified_hanzi_mc/get_dataset.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build simplified-Hanzi radical-location multicontext lanes, then tokenize each
+# lane into a labeled subfolder with prepare.py -s -S.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INPUT_TXT="${1:-${SCRIPT_DIR}/input.txt}"
+LABEL="${LABEL:-simplified_hanzi_mc}"
+METHOD="${METHOD:-char}"
+python3 "${SCRIPT_DIR}/build_simplified_hanzi_mc.py" --input "${INPUT_TXT}" --output_root "${SCRIPT_DIR}" --label "${LABEL}"
+LANES=(char non_hanzi whole left right top bottom enclosure inside corner overlay other)
+for lane in "${LANES[@]}"; do
+  echo "[prepare] ${lane}"
+  (cd "${SCRIPT_DIR}/${lane}" && python3 "${SCRIPT_DIR}/prepare.py" -t input.txt --method "${METHOD}" -s -S "${LABEL}")
+done
diff --git a/data/simplified_hanzi_mc/input.txt b/data/simplified_hanzi_mc/input.txt
new file mode 100644
index 0000000000..f044cd4149
--- /dev/null
+++ b/data/simplified_hanzi_mc/input.txt
@@ -0,0 +1,41 @@
+一
+人
+口
+明
+休
+林
+好
+苗
+尖
+想
+国
+问
+闪
+医
+区
+同
+这
+房
+病
+氧
+赢
+器
+乘
+爽
+坐
+办
+必
+體
+A
+。
+🙂
+门
+马
+爱
+学
+书
+鱼
+车
+龙
+万
+风
diff --git a/data/simplified_hanzi_mc/nanogpt_tokenizers.py b/data/simplified_hanzi_mc/nanogpt_tokenizers.py
new file mode 100644
index 0000000000..37677bd68a
--- /dev/null
+++ b/data/simplified_hanzi_mc/nanogpt_tokenizers.py
@@ -0,0 +1,1163 @@
+# nanogpt_tokenizers.py
+#
+# NOTE: this module is deliberately NOT named `tokenizers.py`. An earlier
+# revision was, which shadowed the third-party HuggingFace `tokenizers`
+# package on `sys.path` (because `data/*/prepare.py` are symlinks to
+# `data/template/prepare.py`, so Python's `sys.path[0]` resolves to this
+# directory) and broke `from transformers import AutoTokenizer`.
+import os
+import pickle
+import tempfile
+import sentencepiece as spm
+import tiktoken
+from tqdm import tqdm
+from collections import defaultdict, Counter
+import json
+import math
+import numpy as np
+import importlib.util
+from pathlib import Path
+try:
+    import torch
+    import torchaudio
+except ImportError:  # pragma: no cover - optional dependency
+    torch = None
+    torchaudio = None
+
+
+class Tokenizer:
+    def __init__(self, args):
+        self.args = args
+        self.token_counts = defaultdict(int) if getattr(args, "track_token_counts", False) else None
+
+    def tokenize(self, data):
+        raise NotImplementedError("Tokenize method must be implemented by subclasses.")
+
+    def detokenize(self, ids):
+        raise NotImplementedError("Detokenize method must be implemented by subclasses.")
+
+    def save_meta(self, meta):
+        meta_path = getattr(self.args, "meta_output_path", "meta.pkl")
+        with open(meta_path, "wb") as f:
+            pickle.dump(meta, f)
+
+    def record_token(self, token_id):
+        if self.token_counts is not None:
+            self.token_counts[token_id] += 1
+
+    def finalize_meta(self, meta):
+        if self.token_counts is not None:
+            meta["token_counts"] = dict(self.token_counts)
+        self.save_meta(meta)
+
+    @staticmethod
+    def get_key_from_meta(keyname, meta_path="meta.pkl"):
+        if os.path.exists(meta_path):
+            with open(meta_path, 'rb') as f:
+                meta = pickle.load(f)
+                return meta.get(keyname)
+        return None
+
+class SentencePieceTokenizer(Tokenizer):
+    def __init__(self, args, input_files=None):
+        super().__init__(args)
+        self.vocab_size = args.vocab_size
+        self.spm_model_file = args.spm_model_file
+        self.spm_vocab_file = args.spm_vocab_file
+        self.skip_tokenization = args.skip_tokenization
+        self.input_files = input_files
+        self.output_dir = os.path.dirname(getattr(args, "meta_output_path", ""))
+        self.sp = None
+
+        if self.spm_model_file:
+            self.sp = spm.SentencePieceProcessor()
+            self.sp.load(self.spm_model_file)
+        elif input_files:
+            self.sp = self.train_sentencepiece_model()
+
+    def train_sentencepiece_model(self):
+        spm_model_prefix = "trained_spm_model"
+        if self.output_dir:
+            os.makedirs(self.output_dir, exist_ok=True)
+            spm_model_prefix = os.path.join(self.output_dir, spm_model_prefix)
+        num_threads = os.cpu_count()
+        input_arg = ""
+        if isinstance(self.input_files, list):
+            with tempfile.NamedTemporaryFile(delete=False, mode="w") as tmpfile:
+                for input_file in self.input_files:
+                    with open(input_file, "r") as infile:
+                        tmpfile.write(infile.read())
+                input_arg = tmpfile.name
+        else:
+            input_arg = self.input_files
+
+        spm.SentencePieceTrainer.train(
+            num_threads=num_threads,
+            user_defined_symbols="\n, ",
+            input=input_arg,
+            model_prefix=spm_model_prefix,
+            split_digits=True,
+            vocab_size=self.vocab_size,
+            model_type="bpe",
+        )
+        print("SentencePiece model training complete.")
+
+        if isinstance(self.input_files, list):
+            os.remove(input_arg)
+
+        sp = spm.SentencePieceProcessor()
+        sp.load(f"{spm_model_prefix}.model")
+        return sp
+
+    def tokenize(self, data):
+        if not self.sp:
+            raise ValueError("SentencePiece model is not loaded.")
+        ids = self.sp.encode_as_ids(data)
+
+        # Record token counts
+        for token_id in ids:
+            self.record_token(token_id)
+
+        stoi = {self.sp.id_to_piece(i): i for i in range(self.sp.GetPieceSize())}
+        itos = {i: self.sp.id_to_piece(i) for i in range(self.sp.GetPieceSize())}
+
+        meta = {
+            "vocab_size": self.sp.GetPieceSize(),
+            "tokenizer": "sentencepiece",
+            "stoi": stoi,
+            "itos": itos,
+        }
+        self.finalize_meta(meta)
+        return ids
+
+    def detokenize(self, ids):
+        if not self.sp:
+            raise ValueError("SentencePiece model is not loaded.")
+        return self.sp.decode_ids(ids)
+
+class TiktokenTokenizer(Tokenizer):
+    def __init__(self, args):
+        super().__init__(args)
+        self.tiktoken_encoding = args.tiktoken_encoding
+        self.last_token_count = 0
+
+        # Load additional tokens if provided
+        self.additional_tokens = {}
+        if hasattr(args, 'additional_tokens_file') and args.additional_tokens_file:
+            with open(args.additional_tokens_file, 'r') as f:
+                self.additional_tokens = json.load(f)
+
+        # Get base encoding
+        base_enc = tiktoken.get_encoding(self.tiktoken_encoding)
+
+        if self.additional_tokens:
+            # Create custom encoding with additional tokens
+            self.enc = tiktoken.Encoding(
+                name=f"{self.tiktoken_encoding}_custom",
+                pat_str=base_enc._pat_str,
+                mergeable_ranks=base_enc._mergeable_ranks,
+                special_tokens={**base_enc._special_tokens,
+                                **self.additional_tokens},
+                disallowed_special=(),
+            )
+            self.special_tokens = self.additional_tokens
+        else:
+            self.enc = base_enc
+            self.special_tokens = {}
+
+    def tokenize(self, data):
+        """Tokenize the input data using tiktoken with support for special tokens."""
+        token_ids = []
+        current_pos = 0
+        data_len = len(data)
+
+        while current_pos < data_len:
+            # Try to match special tokens first
+            matched_special = False
+            for token, token_id in self.special_tokens.items():
+                if data.startswith(token, current_pos):
+                    token_ids.append(token_id)
+                    self.record_token(token_id)
+                    current_pos += len(token)
+                    matched_special = True
+                    break
+
+            if not matched_special:
+                # Find the next special token or end of text
+                next_special = data_len
+                for token in self.special_tokens:
+                    pos = data.find(token, current_pos)
+                    if pos != -1 and pos < next_special:
+                        next_special = pos
+
+                # Take the chunk up to the next special token and let tiktoken handle it
+                chunk = data[current_pos:next_special]
+                if chunk:
+                    # Use encode() for proper subword tokenization
+                    chunk_ids = self.enc.encode(
+                            chunk,
+                            allowed_special=set(),
+                            disallowed_special=(),
+                            )
+                    token_ids.extend(chunk_ids)
+                    for token_id in chunk_ids:
+                        self.record_token(token_id)
+                current_pos = next_special
+
+        # Save metadata
+        meta = {
+            "vocab_size": self.enc.n_vocab,
+            "tokenizer": "tiktoken",
+            "tiktoken_encoding": self.tiktoken_encoding,
+            "has_additional_tokens": bool(self.additional_tokens),
+            "special_tokens": self.special_tokens,
+            "itos": {i: self.enc.decode([i]) for i in set(token_ids)}
+        }
+        self.finalize_meta(meta)
+
+        self.last_token_count = len(token_ids)
+        return token_ids
+
+    def detokenize(self, token_ids):
+        """Detokenize the token IDs back to text."""
+        result = []
+        for token_id in token_ids:
+            # Check if it's a special token
+            found = False
+            for token, special_id in self.special_tokens.items():
+                if token_id == special_id:
+                    result.append(token)
+                    found = True
+                    break
+
+            if not found:
+                # Regular token
+                result.append(self.enc.decode([token_id]))
+
+        return ''.join(result)
+
+
+class HuggingFaceTokenizer(Tokenizer):
+    """Wrap any HuggingFace tokenizer via `transformers.AutoTokenizer`.
+
+    `AutoTokenizer` is a unified entry point that transparently loads both
+    fast (Rust `tokenizers`-backed) and slow (Python) variants, from either a
+    Hub name (e.g. ``"gpt2"``) or a local directory saved via
+    ``save_pretrained``.
+
+    This module is intentionally named ``nanogpt_tokenizers.py`` rather than
+    ``tokenizers.py`` so that it cannot shadow the third-party
+    ``tokenizers`` package that ``transformers`` imports internally (via
+    ``from tokenizers import decoders, normalizers, ...``).
+    """
+
+    def __init__(self, args):
+        super().__init__(args)
+        try:
+            from transformers import AutoTokenizer  # lazy import
+        except ImportError as exc:  # pragma: no cover - optional dependency
+            raise ImportError(
+                "HuggingFaceTokenizer requires the `transformers` package. "
+                "Install with `pip install transformers`."
+            ) from exc
+
+        self.hf_tokenizer_name = getattr(args, "hf_tokenizer_name", None)
+        if not self.hf_tokenizer_name:
+            raise ValueError(
+                "--hf_tokenizer_name must be provided for the huggingface method "
+                "(accepts a Hub id like 'gpt2' or 'google/gemma-3-270m', or a "
+                "local path to a directory written by save_pretrained)."
+            )
+
+        trust_remote_code = bool(getattr(args, "hf_trust_remote_code", False))
+        use_fast = getattr(args, "hf_use_fast", True)
+        if use_fast is None:
+            use_fast = True
+
+        # Optional Hub knobs. All of these mirror real `from_pretrained` args.
+        self.hf_revision = getattr(args, "hf_revision", None) or None
+        self.hf_subfolder = getattr(args, "hf_subfolder", None) or None
+        self.hf_cache_dir = getattr(args, "hf_cache_dir", None) or None
+        self.hf_token = getattr(args, "hf_token", None) or None
+
+        # Build a kwargs dict so we can omit None-valued knobs cleanly.
+        from_pretrained_kwargs = {
+            "trust_remote_code": trust_remote_code,
+            "use_fast": bool(use_fast),
+        }
+        if self.hf_revision is not None:
+            from_pretrained_kwargs["revision"] = self.hf_revision
+        if self.hf_subfolder is not None:
+            from_pretrained_kwargs["subfolder"] = self.hf_subfolder
+        if self.hf_cache_dir is not None:
+            from_pretrained_kwargs["cache_dir"] = self.hf_cache_dir
+        if self.hf_token is not None:
+            # `token=` is the modern (v4.32+) parameter; `use_auth_token=` is
+            # deprecated and emits a warning. Prefer `token=`.
+            from_pretrained_kwargs["token"] = self.hf_token
+
+        try:
+            self.tokenizer = AutoTokenizer.from_pretrained(
+                self.hf_tokenizer_name,
+                **from_pretrained_kwargs,
+            )
+        except Exception as exc:  # broad: HF raises a few specific subclasses
+            self._explain_load_error(exc)
+            raise
+
+        # Try to resolve the actual commit SHA the tokenizer was loaded from,
+        # for reproducibility. This is best-effort and never fatal.
+        self.hf_resolved_commit = self._resolve_commit_hash()
+
+        # Where to cache a local snapshot of the tokenizer so `sample.py` /
+        # `train.py` can reload it even without network access.
+        meta_output_path = getattr(args, "meta_output_path", "meta.pkl")
+        output_dir = os.path.dirname(meta_output_path) or "."
+        self.hf_local_dir = os.path.join(output_dir, "hf_tokenizer")
+        self.last_token_count = 0
+
+    def _explain_load_error(self, exc):
+        """Print a friendly hint when from_pretrained fails on a Hub repo."""
+        msg = str(exc)
+        repo = self.hf_tokenizer_name
+        cls_name = type(exc).__name__
+        # huggingface_hub.utils.GatedRepoError / 401 / 403 / "gated repo"
+        is_gated = (
+            "gated" in msg.lower()
+            or "GatedRepoError" in cls_name
+            or "401" in msg
+            or "403" in msg
+        )
+        is_missing = "RepositoryNotFoundError" in cls_name or "404" in msg
+        print()
+        if is_gated:
+            print(f"[huggingface] '{repo}' is a gated repository.")
+            print( "[huggingface] To use it you must:")
+            print(f"[huggingface]   1) visit https://huggingface.co/{repo} and accept the license")
+            print( "[huggingface]   2) authenticate locally via ONE of:")
+            print( "[huggingface]        - `huggingface-cli login`")
+            print( "[huggingface]        - environment: `export HF_TOKEN=hf_xxx`")
+            print( "[huggingface]        - this CLI flag: `--hf_token hf_xxx`")
+        elif is_missing:
+            print(f"[huggingface] Repository '{repo}' was not found on the Hub.")
+            print( "[huggingface] Check the spelling, or pass a local path "
+                  "(a directory previously written by save_pretrained).")
+        else:
+            print(f"[huggingface] Failed to load '{repo}': {cls_name}: {msg}")
+
+    def _resolve_commit_hash(self):
+        """Best-effort: return the actual commit SHA of the loaded tokenizer."""
+        # 1) transformers stamps `_commit_hash` into init_kwargs when it
+        #    downloaded from the Hub. This is the most reliable source.
+        try:
+            init_kwargs = getattr(self.tokenizer, "init_kwargs", {}) or {}
+            commit = init_kwargs.get("_commit_hash")
+            if commit:
+                return str(commit)
+        except Exception:
+            pass
+        # 2) Fall back to whatever the user pinned (could be a tag/branch/sha).
+        return self.hf_revision
+
+    def _build_vocab_maps(self):
+        try:
+            stoi = dict(self.tokenizer.get_vocab())
+        except Exception:
+            stoi = {}
+        itos = {int(v): k for k, v in stoi.items()}
+        return stoi, itos
+
+    def _effective_vocab_size(self):
+        # `len(tokenizer)` includes added/special tokens. Fall back gracefully.
+        try:
+            return len(self.tokenizer)
+        except Exception:
+            return int(getattr(self.tokenizer, "vocab_size", 0))
+
+    def tokenize(self, data):
+        # Encode without special tokens to match the behavior of our other
+        # subword tokenizers (tiktoken/sentencepiece) so that text resumes
+        # cleanly across chunks.
+        ids = self.tokenizer.encode(data, add_special_tokens=False)
+
+        for token_id in ids:
+            self.record_token(token_id)
+
+        stoi, itos = self._build_vocab_maps()
+
+        # Save a local snapshot of the tokenizer next to meta.pkl so it can
+        # be reloaded offline during sampling / training. This is what makes
+        # gated models like Gemma "just work" downstream: prepare-time is the
+        # only step that has to authenticate against the Hub.
+        hf_saved_path = None
+        try:
+            os.makedirs(self.hf_local_dir, exist_ok=True)
+            self.tokenizer.save_pretrained(self.hf_local_dir)
+            hf_saved_path = self.hf_local_dir
+        except Exception as exc:  # pragma: no cover - best effort
+            print(f"[huggingface] Warning: could not save tokenizer snapshot to "
+                  f"{self.hf_local_dir}: {exc}")
+
+        meta = {
+            "vocab_size": self._effective_vocab_size(),
+            "tokenizer": "huggingface",
+            "hf_tokenizer_name": self.hf_tokenizer_name,
+            "hf_tokenizer_path": hf_saved_path,
+            "hf_use_fast": bool(getattr(self.args, "hf_use_fast", True)),
+            "hf_trust_remote_code": bool(getattr(self.args, "hf_trust_remote_code", False)),
+            "hf_revision": self.hf_revision,
+            "hf_resolved_commit": self.hf_resolved_commit,
+            "hf_subfolder": self.hf_subfolder,
+            "stoi": stoi,
+            "itos": itos,
+        }
+        self.finalize_meta(meta)
+
+        self.last_token_count = len(ids)
+        return ids
+
+    def detokenize(self, ids):
+        return self.tokenizer.decode(list(ids), skip_special_tokens=False)
+
+
+class CustomTokenizer(Tokenizer):
+    def __init__(self, args):
+        super().__init__(args)
+        if args.tokens_file is None:
+            raise ValueError("Tokens file must be provided for custom tokenization method.")
+        with open(args.tokens_file, "r") as f:
+            self.tokens = [line.strip() for line in f.readlines() if line.strip()]
+            self.tokens = [token.replace("\\n", "\n").replace("\\t", "\t") for token in self.tokens]
+        self.stoi = {token: i for i, token in enumerate(self.tokens)}
+        self.itos = {i: token for i, token in enumerate(self.tokens)}
+
+    def tokenize(self, data):
+        encoded_data = []
+        i = 0
+        covered_chars = 0
+        data_len = len(data)
+        pbar = tqdm(total=data_len, desc="Tokenizing Custom Tokens")
+        while i < data_len:
+            matched = False
+            for token in self.tokens:
+                token_len = len(token)
+                if data.startswith(token, i):
+                    encoded_data.append(self.stoi[token])
+                    self.record_token(self.stoi[token])
+                    i += token_len
+                    covered_chars += token_len
+                    pbar.update(token_len)
+                    matched = True
+                    break
+            if not matched:
+                i += 1  # Skip character if no token matches
+                pbar.update(1)
+        pbar.close()
+        coverage = covered_chars / data_len
+        print(f"Data coverage by tokens: {coverage*100:.2f}%")
+        meta = {"vocab_size": len(self.tokens), "stoi": self.stoi, "itos": self.itos}
+        self.finalize_meta(meta)
+        return encoded_data
+
+    def detokenize(self, ids):
+        return ''.join([self.itos[id] for id in ids])
+
+class ByteTokenizer(Tokenizer):
+    def __init__(self, args):
+        super().__init__(args)
+
+    def tokenize(self, data):
+        data_bytes = data.encode('utf-8')
+        ids = list(data_bytes)
+        for token_id in ids:
+            self.record_token(token_id)
+        meta = {
+            "vocab_size": 256,
+            "tokenizer": "byte",
+            "itos": {i: bytes([i]) for i in range(256)},
+        }
+        self.finalize_meta(meta)
+        return ids
+
+    def detokenize(self, ids):
+        return bytes(ids).decode('utf-8', errors='replace')
+
+
+class CharTokenizer(Tokenizer):
+    def __init__(self, args, train_data, val_data):
+        super().__init__(args)
+        self.reuse_chars = args.reuse_chars
+        if self.reuse_chars:
+            self.chars = self.get_key_from_meta('chars', getattr(args, "meta_output_path", "meta.pkl"))
+            if self.chars is None:
+                raise ValueError("No chars found in meta.pkl. Cannot reuse chars.")
+        else:
+            self.chars = sorted(list(set(train_data + (val_data if val_data else ""))))
+            print(f"All unique characters: {''.join(self.chars)}")
+            print(f"Vocab size: {len(self.chars)}")
+        self.stoi = {ch: i for i, ch in enumerate(self.chars)}
+        self.itos = {i: ch for i, ch in enumerate(self.chars)}
+
+    def tokenize(self, data):
+        data_len = len(data)
+        ids = []
+        pbar = tqdm(total=data_len, desc="Tokenizing Characters")
+        for ch in data:
+            token_id = self.stoi[ch]
+            self.record_token(token_id)
+            ids.append(token_id)
+            pbar.update(1)
+
+        pbar.close()
+        meta = {"vocab_size": len(self.chars), "itos": self.itos, "stoi": self.stoi, "chars": self.chars}
+        self.finalize_meta(meta)
+        return ids
+
+    def detokenize(self, ids):
+        return ''.join([self.itos[id] for id in ids])
+
+
+class CharBPETokenizerWithByteFallback(Tokenizer):
+    def __init__(self, args, train_data, val_data=None):
+        super().__init__(args)
+        self.reuse_meta_path = getattr(args, "char_bpe_vocab_path", None)
+        if self.reuse_meta_path:
+            meta = self._load_char_bpe_meta(self.reuse_meta_path)
+            self.desired_vocab_size = meta["vocab_size"]
+            self.char_tokens = meta["char_tokens"]
+            self.sorted_char_tokens = meta.get(
+                "char_tokens_sorted",
+                sorted(self.char_tokens, key=lambda t: len(t), reverse=True),
+            )
+            self._build_vocab()
+            return
+
+        if getattr(args, "vocab_size", None) is None:
+            raise ValueError("vocab_size must be provided for char_bpe method.")
+        if args.vocab_size <= 256:
+            raise ValueError("vocab_size must be greater than 256 to allow space for byte fallback tokens.")
+
+        self.desired_vocab_size = args.vocab_size
+        corpus_text = train_data or ""
+        if val_data:
+            corpus_text += val_data
+
+        self.unique_chars = sorted(set(corpus_text))
+        if not self.unique_chars:
+            raise ValueError("Training data must contain at least one character for char_bpe tokenization.")
+
+        self.char_tokens = list(self.unique_chars)
+        self._train_merges(corpus_text)
+        self._build_vocab()
+
+    @staticmethod
+    def _load_char_bpe_meta(meta_path):
+        if not os.path.exists(meta_path):
+            raise FileNotFoundError(f"Char-BPE meta file not found: {meta_path}")
+        with open(meta_path, "rb") as f:
+            meta = pickle.load(f)
+        if meta.get("tokenizer") != "char_bpe":
+            raise ValueError("Provided meta file is not from a char_bpe tokenizer.")
+        if "char_tokens" not in meta or "vocab_size" not in meta:
+            raise ValueError("Meta file missing required char_bpe vocabulary fields.")
+        return meta
+
+    def _train_merges(self, text):
+        tokens = list(text)
+        # Nothing to merge if text empty or target vocab already satisfied
+        if len(tokens) < 2:
+            return
+
+        current_vocab_size = 256 + len(self.char_tokens)
+        merges_needed = self.desired_vocab_size - current_vocab_size
+
+        while merges_needed > 0:
+            pair_counts = Counter()
+            prev = None
+            for token in tokens:
+                if prev is not None:
+                    pair_counts[(prev, token)] += 1
+                prev = token
+
+            if not pair_counts:
+                break
+
+            best_pair, best_count = pair_counts.most_common(1)[0]
+            if best_count < 2:
+                break
+
+            new_token = ''.join(best_pair)
+            if new_token in self.char_tokens:
+                # Already present, skip to avoid duplicates
+                tokens = self._apply_merge(tokens, best_pair, new_token)
+            else:
+                self.char_tokens.append(new_token)
+                tokens = self._apply_merge(tokens, best_pair, new_token)
+                merges_needed -= 1
+
+            current_vocab_size = 256 + len(self.char_tokens)
+            merges_needed = self.desired_vocab_size - current_vocab_size
+            if merges_needed <= 0:
+                break
+
+        self.sorted_char_tokens = sorted(self.char_tokens, key=lambda t: len(t), reverse=True)
+
+    @staticmethod
+    def _apply_merge(tokens, pair, new_token):
+        merged = []
+        i = 0
+        max_index = len(tokens) - 1
+        while i <= max_index:
+            if i < max_index and tokens[i] == pair[0] and tokens[i + 1] == pair[1]:
+                merged.append(new_token)
+                i += 2
+            else:
+                merged.append(tokens[i])
+                i += 1
+        return merged
+
+    def _build_vocab(self):
+        self.stoi = {}
+        self.itos = {}
+
+        for b in range(256):
+            key = bytes([b])
+            self.stoi[key] = b
+            self.itos[b] = key
+
+        offset = 256
+        for idx, token in enumerate(self.char_tokens):
+            token_id = offset + idx
+            self.stoi[token] = token_id
+            self.itos[token_id] = token
+
+        self.vocab_size = len(self.itos)
+        self.sorted_char_tokens = sorted(self.char_tokens, key=lambda t: len(t), reverse=True)
+
+    def tokenize(self, data):
+        if not data:
+            return []
+
+        ids = []
+        i = 0
+        data_len = len(data)
+        pbar = tqdm(total=data_len, desc="Tokenizing Char BPE")
+
+        while i < data_len:
+            matched = False
+            for token in self.sorted_char_tokens:
+                if data.startswith(token, i):
+                    token_id = self.stoi[token]
+                    ids.append(token_id)
+                    self.record_token(token_id)
+                    i += len(token)
+                    pbar.update(len(token))
+                    matched = True
+                    break
+
+            if matched:
+                continue
+
+            ch = data[i]
+            if ch in self.stoi:
+                token_id = self.stoi[ch]
+                ids.append(token_id)
+                self.record_token(token_id)
+                i += len(ch)
+                pbar.update(len(ch))
+            else:
+                ch_bytes = ch.encode('utf-8')
+                for b in ch_bytes:
+                    token_id = self.stoi[bytes([b])]
+                    ids.append(token_id)
+                    self.record_token(token_id)
+                    pbar.update(1)
+                i += 1
+
+        pbar.close()
+
+        meta = {
+            "vocab_size": self.vocab_size,
+            "tokenizer": "char_bpe",
+            "stoi": self.stoi,
+            "itos": self.itos,
+            "char_tokens": self.char_tokens,
+            "char_tokens_sorted": self.sorted_char_tokens,
+            "byte_fallback": True,
+        }
+        self.finalize_meta(meta)
+        return ids
+
+    def detokenize(self, ids):
+        out_pieces = []
+        byte_buffer = []
+
+        for token_id in ids:
+            token = self.itos.get(token_id)
+            if token is None:
+                continue
+
+            if isinstance(token, bytes):
+                byte_buffer.append(token)
+            else:
+                if byte_buffer:
+                    combined = b''.join(byte_buffer)
+                    out_pieces.append(combined.decode('utf-8', errors='replace'))
+                    byte_buffer = []
+                out_pieces.append(token)
+
+        if byte_buffer:
+            combined = b''.join(byte_buffer)
+            out_pieces.append(combined.decode('utf-8', errors='replace'))
+
+        return ''.join(out_pieces)
+
+    def finalize_meta(self, meta):
+        super().finalize_meta(meta)
+        self._write_vocab_jsons(meta)
+
+    def _write_vocab_jsons(self, meta):
+        vocab_json = []
+        for idx in range(self.vocab_size):
+            token = self.itos[idx]
+            vocab_json.append(self._format_token_for_json(token))
+
+        with open("char_bpe_vocab.json", "w", encoding="utf-8") as f:
+            json.dump(vocab_json, f, ensure_ascii=False, indent=2)
+
+        if self.token_counts is not None:
+            counts_json = []
+            counts = meta.get("token_counts", {})
+            for idx in range(self.vocab_size):
+                token = self.itos[idx]
+                counts_json.append({
+                    "id": idx,
+                    "token": self._format_token_for_json(token),
+                    "count": counts.get(idx, 0)
+                })
+            with open("char_bpe_token_counts.json", "w", encoding="utf-8") as f:
+                json.dump(counts_json, f, ensure_ascii=False, indent=2)
+
+    @staticmethod
+    def _format_token_for_json(token):
+        if isinstance(token, bytes):
+            return f"<byte:{token[0]}>"
+        return token
+
+
+class CustomCharTokenizerWithByteFallback(Tokenizer):
+    """
+    In this version, we assign IDs 0..255 to raw bytes,
+    then custom tokens get IDs from 256 upwards.
+
+    During tokenization:
+      1) Convert text to UTF-8 bytes.
+      2) For each position in the byte sequence, attempt to match
+         a custom token's UTF-8 pattern. If we match, produce that token ID.
+         Otherwise, produce the ID for the single byte.
+
+    Detokenization:
+      - If ID < 256, it's a single raw byte.
+      - If ID >= 256, it's the custom token string.
+    """
+
+    def __init__(self, args):
+        super().__init__(args)
+        if args.custom_chars_file is None:
+            raise ValueError("Custom characters file must be provided for this tokenizer.")
+
+        # Load custom tokens from file
+        with open(args.custom_chars_file, "r", encoding="utf-8") as f:
+            self.custom_tokens = [line.strip() for line in f if line.strip()]
+
+        # Build vocab dictionaries (bytes first, then custom tokens)
+        self.build_vocab()
+
+    def build_vocab(self):
+        # Assign IDs 0..255 to individual bytes
+        self.stoi = {}
+        self.itos = {}
+
+        for b in range(256):
+            # Store key as the actual single byte
+            key = bytes([b])
+            self.stoi[key] = b  # ID = b
+            self.itos[b] = key
+
+        # Now assign IDs to the custom tokens from 256 onwards
+        offset = 256
+        self.custom_token_bytes = {}
+        for i, token_str in enumerate(self.custom_tokens):
+            token_id = offset + i
+            self.stoi[token_str] = token_id
+            self.itos[token_id] = token_str
+            self.custom_token_bytes[token_str] = token_str.encode('utf-8')
+
+        self.custom_char_count = len(self.custom_tokens)
+        self.vocab_size = 256 + self.custom_char_count
+
+    def tokenize(self, data):
+        # Convert entire string to UTF-8 bytes
+        data_bytes = data.encode('utf-8')
+        i = 0
+        n = len(data_bytes)
+        ids = []
+
+        # We'll try to match any custom token at the current position; otherwise single byte
+        pbar = tqdm(total=n, desc="Tokenizing Bytes First + Custom")
+        while i < n:
+            matched = False
+            # Check each custom token
+            for token_str, token_bytes in self.custom_token_bytes.items():
+                length = len(token_bytes)
+                # If next 'length' bytes match this custom token
+                if data_bytes[i:i+length] == token_bytes:
+                    token_id = self.stoi[token_str]  # e.g., 256+
+                    self.record_token(token_id)
+                    ids.append(token_id)
+                    i += length
+                    pbar.update(length)
+                    matched = True
+                    break
+
+            if not matched:
+                # No custom token matched, so we treat this as a single byte
+                single_byte = data_bytes[i:i+1]
+                token_id = self.stoi[single_byte]  # 0..255
+                self.record_token(token_id)
+                ids.append(token_id)
+                i += 1
+                pbar.update(1)
+
+        pbar.close()
+
+        # Finalize metadata with token_counts
+        meta = {
+            "vocab_size": self.vocab_size,
+            "tokenizer": "custom_char_with_byte_fallback",
+            "custom_chars": self.custom_tokens,  # i.e., custom tokens
+            "stoi": self.stoi,
+            "itos": self.itos,
+            "custom_char_count": self.custom_char_count,
+        }
+        self.finalize_meta(meta)
+        return ids
+
+    def detokenize(self, ids):
+        """
+        If ID < 256 => single byte
+        If ID >= 256 => custom token string
+        We'll accumulate bytes in a buffer, and whenever we see a custom token,
+        we flush the buffer as text, then append the custom token as is.
+        """
+        out_pieces = []
+        byte_buffer = []
+
+        for idx, token_id in enumerate(ids):
+            if token_id < 256:
+                # Single raw byte
+                byte_buffer.append(self.itos[token_id])  # e.g. b'\x61'
+            else:
+                # It's a custom token
+                # First flush any accumulated bytes
+                if byte_buffer:
+                    all_bytes = b''.join(byte_buffer)
+                    out_pieces.append(all_bytes.decode('utf-8', errors='replace'))
+                    byte_buffer = []
+                # Append the custom token string
+                custom_str = self.itos[token_id]
+                out_pieces.append(custom_str)
+
+        # Flush remaining bytes
+        if byte_buffer:
+            all_bytes = b''.join(byte_buffer)
+            out_pieces.append(all_bytes.decode('utf-8', errors='replace'))
+
+        return ''.join(out_pieces)
+
+class JsonByteTokenizerWithByteFallback(Tokenizer):
+    """
+    Similar to CustomCharTokenizerWithByteFallback, but loads tokens from a JSON array.
+    IDs 0..255 are reserved for raw bytes, then custom tokens get IDs from 256 upwards.
+
+    During tokenization:
+      1) Convert text to UTF-8 bytes.
+      2) For each position in the byte sequence, attempt to match
+         a custom token's UTF-8 pattern. If we match, produce that token ID.
+         Otherwise, produce the ID for the single byte.
+
+    Detokenization:
+      - If ID < 256, it's a single raw byte.
+      - If ID >= 256, it's the custom token string.
+    """
+
+    def __init__(self, args):
+        super().__init__(args)
+        if args.json_tokens_file is None:
+            raise ValueError("JSON tokens file must be provided for this tokenizer.")
+
+        # Load custom tokens from JSON file
+        with open(args.json_tokens_file, "r", encoding="utf-8") as f:
+            self.custom_tokens = json.load(f)
+            if not isinstance(self.custom_tokens, list):
+                raise ValueError("JSON file must contain an array of tokens")
+
+        # Build vocab dictionaries (bytes first, then custom tokens)
+        self.build_vocab()
+
+    def build_vocab(self):
+        # Assign IDs 0..255 to individual bytes
+        self.stoi = {}
+        self.itos = {}
+
+        for b in range(256):
+            # Store key as the actual single byte
+            key = bytes([b])
+            self.stoi[key] = b  # ID = b
+            self.itos[b] = key
+
+        # Now assign IDs to the custom tokens from 256 onwards
+        offset = 256
+        self.custom_token_bytes = {}
+        for i, token_str in enumerate(self.custom_tokens):
+            token_id = offset + i
+            self.stoi[token_str] = token_id
+            self.itos[token_id] = token_str
+            self.custom_token_bytes[token_str] = token_str.encode('utf-8')
+
+        self.custom_token_count = len(self.custom_tokens)
+        self.vocab_size = 256 + self.custom_token_count
+
+    def tokenize(self, data):
+        # Convert entire string to UTF-8 bytes
+        data_bytes = data.encode('utf-8')
+        i = 0
+        n = len(data_bytes)
+        ids = []
+
+        # We'll try to match any custom token at the current position; otherwise single byte
+        pbar = tqdm(total=n, desc="Tokenizing Bytes First + JSON Custom")
+        while i < n:
+            matched = False
+            # Check each custom token
+            for token_str, token_bytes in self.custom_token_bytes.items():
+                length = len(token_bytes)
+                # If next 'length' bytes match this custom token
+                if data_bytes[i:i+length] == token_bytes:
+                    token_id = self.stoi[token_str]  # e.g., 256+
+                    self.record_token(token_id)
+                    ids.append(token_id)
+                    i += length
+                    pbar.update(length)
+                    matched = True
+                    break
+
+            if not matched:
+                # No custom token matched, so we treat this as a single byte
+                single_byte = data_bytes[i:i+1]
+                token_id = self.stoi[single_byte]  # 0..255
+                self.record_token(token_id)
+                ids.append(token_id)
+                i += 1
+                pbar.update(1)
+
+        pbar.close()
+
+        # Finalize metadata with token_counts
+        meta = {
+            "vocab_size": self.vocab_size,
+            "tokenizer": "json_byte_fallback",
+            "custom_tokens": self.custom_tokens,  # i.e., custom tokens from JSON
+            "stoi": self.stoi,
+            "itos": self.itos,
+            "custom_token_count": self.custom_token_count,
+        }
+        self.finalize_meta(meta)
+        return ids
+
+    def detokenize(self, ids):
+        """
+        If ID < 256 => single byte
+        If ID >= 256 => custom token string
+        We'll accumulate bytes in a buffer, and whenever we see a custom token,
+        we flush the buffer as text, then append the custom token as is.
+        """
+        out_pieces = []
+        byte_buffer = []
+
+        for idx, token_id in enumerate(ids):
+            if token_id < 256:
+                # Single raw byte
+                byte_buffer.append(self.itos[token_id])  # e.g. b'\x61'
+            else:
+                # It's a custom token
+                # First flush any accumulated bytes
+                if byte_buffer:
+                    all_bytes = b''.join(byte_buffer)
+                    out_pieces.append(all_bytes.decode('utf-8', errors='replace'))
+                    byte_buffer = []
+                # Append the custom token string
+                custom_str = self.itos[token_id]
+                out_pieces.append(custom_str)
+
+        # Flush remaining bytes
+        if byte_buffer:
+            all_bytes = b''.join(byte_buffer)
+            out_pieces.append(all_bytes.decode('utf-8', errors='replace'))
+
+        return ''.join(out_pieces)
+
+
+def _load_python_token_processor():
+    """Load the PythonTokenProcessor helper without requiring a package install."""
+    tokenizer_path = Path(__file__).parent / "programming_tokenizers" / "python_tokenizer.py"
+    spec = importlib.util.spec_from_file_location("python_tokenizer", tokenizer_path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Unable to load python_tokenizer from {tokenizer_path}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module.PythonTokenProcessor
+
+
+class PythonProgrammingTokenizer(JsonByteTokenizerWithByteFallback):
+    """Tokenize Python code with reserved tokens and byte fallback for everything else."""
+
+    def __init__(self, args):
+        if args.json_tokens_file is None:
+            args.json_tokens_file = str(
+                Path(__file__).parent / "premade_vocab_sets" / "python_programming_tokens.json"
+            )
+        super().__init__(args)
+        python_processor_cls = _load_python_token_processor()
+        self.python_processor = python_processor_cls(self.custom_tokens)
+
+    def tokenize(self, data):
+        ids = []
+
+        def encode_bytes(segment: str) -> None:
+            for byte_val in segment.encode("utf-8"):
+                token_id = self.stoi[bytes([byte_val])]
+                self.record_token(token_id)
+                ids.append(token_id)
+
+        def emit_reserved(token_text: str) -> None:
+            token_id = self.stoi[token_text]
+            self.record_token(token_id)
+            ids.append(token_id)
+
+        self.python_processor.encode_with_reserved_tokens(data, encode_bytes, emit_reserved)
+
+        meta = {
+            "vocab_size": self.vocab_size,
+            "tokenizer": "python_json_byte_fallback",
+            "custom_tokens": self.custom_tokens,
+            "stoi": self.stoi,
+            "itos": self.itos,
+            "custom_token_count": self.custom_token_count,
+        }
+        self.finalize_meta(meta)
+        return ids
+
+
+class SineWaveTokenizer:
+    """Generate a deterministic sequence of sine wave samples."""
+
+    def __init__(self, args):
+        self.period = args.sine_period
+        self.points_per_period = args.sine_points_per_period
+        self.num_periods = args.sine_num_periods
+        self.amplitude = args.sine_amplitude
+        self.max_val = 255
+
+    def generate_wave(self):
+        total_points = self.num_periods * self.points_per_period
+        values = []
+        for i in range(total_points):
+            x = (i * 2 * math.pi) / self.points_per_period
+            y = 64 + self.amplitude * math.sin(x * self.period)
+            y_clamped = int(max(0, min(self.max_val, round(y))))
+            values.append(y_clamped)
+        return values
+
+    def tokenize(self, data=None):
+        # `data` is unused; generation is parameter driven.
+        return self.generate_wave()
+
+    def detokenize(self, ids):
+        array = np.asarray(ids, dtype=np.int64)
+        return ','.join(map(str, array.tolist()))
+
+      
+class WhisperMelCsvTokenizer(Tokenizer):
+    """Generate Whisper-style log-mel spectrogram frames suitable for CSV export."""
+
+    def __init__(self, args):
+        super().__init__(args)
+        if torch is None or torchaudio is None:
+            raise ImportError("WhisperMelCsvTokenizer requires torch and torchaudio.")
+        self.sample_rate = args.mel_sample_rate
+        self.n_fft = args.mel_n_fft
+        self.hop_length = args.mel_hop_length
+        self.win_length = args.mel_win_length
+        self.n_mels = args.mel_n_mels
+        self.f_min = args.mel_f_min
+        self.f_max = args.mel_f_max
+        self.center = args.mel_center
+        self.power = args.mel_power
+        self.normalize = args.mel_normalize
+        self.mel_transform = torchaudio.transforms.MelSpectrogram(
+            sample_rate=self.sample_rate,
+            n_fft=self.n_fft,
+            hop_length=self.hop_length,
+            win_length=self.win_length,
+            n_mels=self.n_mels,
+            f_min=self.f_min,
+            f_max=self.f_max,
+            center=self.center,
+            power=self.power,
+            mel_scale="slaney",
+            norm="slaney",
+        )
+
+    def _load_audio(self, path):
+        waveform, original_sample_rate = torchaudio.load(path)
+        if waveform.size(0) > 1:
+            waveform = waveform.mean(dim=0, keepdim=True)
+        if original_sample_rate != self.sample_rate:
+            waveform = torchaudio.functional.resample(
+                waveform, orig_freq=original_sample_rate, new_freq=self.sample_rate
+            )
+        return waveform
+
+    def _whisper_log_mel(self, mel_spec):
+        log_mel = torch.clamp(mel_spec, min=1e-10).log10()
+        log_mel = torch.maximum(log_mel, log_mel.max() - 8.0)
+        return (log_mel + 4.0) / 4.0
+
+    def tokenize(self, data):
+        waveform = self._load_audio(data)
+        mel_spec = self.mel_transform(waveform).squeeze(0)
+        if self.normalize:
+            mel_spec = self._whisper_log_mel(mel_spec)
+        mel_frames = mel_spec.transpose(0, 1).contiguous()
+        meta = {
+            "tokenizer": "whisper_mel_csv",
+            "sample_rate": self.sample_rate,
+            "n_fft": self.n_fft,
+            "hop_length": self.hop_length,
+            "win_length": self.win_length,
+            "n_mels": self.n_mels,
+            "f_min": self.f_min,
+            "f_max": self.f_max,
+            "center": self.center,
+            "power": self.power,
+            "normalize": self.normalize,
+        }
+        self.finalize_meta(meta)
+        return mel_frames.cpu().numpy()
+
+    def detokenize(self, ids):
+        array = np.asarray(ids, dtype=np.float32)
+        lines = [",".join(map(str, row)) for row in array.tolist()]
+        return "\n".join(lines)
+      
diff --git a/data/simplified_hanzi_mc/prepare.py b/data/simplified_hanzi_mc/prepare.py
new file mode 100644
index 0000000000..d0a260f4a7
--- /dev/null
+++ b/data/simplified_hanzi_mc/prepare.py
@@ -0,0 +1,322 @@
+# prepare.py
+import json
+import os
+import argparse
+import numpy as np
+from nanogpt_tokenizers import (
+    SentencePieceTokenizer,
+    TiktokenTokenizer,
+    HuggingFaceTokenizer,
+    CustomTokenizer,
+    ByteTokenizer,
+    CharTokenizer,
+    CharBPETokenizerWithByteFallback,
+    CustomCharTokenizerWithByteFallback,
+    JsonByteTokenizerWithByteFallback,
+    PythonProgrammingTokenizer,
+    SineWaveTokenizer,
+    WhisperMelCsvTokenizer,
+)
+from tqdm import tqdm
+import pickle
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Tokenize text data using different methods.")
+
+    # Input/output arguments
+    parser.add_argument("-t", "--train_input", type=str, required=True, help="Path to the input text file")
+    parser.add_argument("-v", "--val_input", type=str, help="Path to validation input file. If not provided, train_input will be split using percentage_train")
+    parser.add_argument("--train_output", type=str, default="train.bin", help="Path to save the training output file")
+    parser.add_argument("--val_output", type=str, default="val.bin", help="Path to save the validation output file")
+    parser.add_argument("-p", "--percentage_train", type=float, default=0.9, help="Percentage of data to use for training (between 0 and 1) when val_input is not provided")
+
+    # Tokenizer selection and configuration
+    parser.add_argument("--method", type=str,
+                       choices=["sentencepiece", "tiktoken", "huggingface", "char", "char_bpe", "custom", "byte", "custom_char_byte_fallback", "json_byte_fallback", "python_programming", "sinewave", "whisper_mel_csv"],
+                       default="tiktoken", help="Tokenization method")
+
+    # HuggingFace tokenizer arguments
+    parser.add_argument("--hf_tokenizer_name", type=str, default=None,
+                        help="HuggingFace tokenizer: a Hub repo id (e.g. 'gpt2', "
+                             "'google/gemma-3-270m', 'meta-llama/Llama-3.2-1B') or a "
+                             "local directory previously written by save_pretrained. "
+                             "Hub repos are downloaded and cached automatically.")
+    parser.add_argument("--hf_trust_remote_code", action="store_true",
+                        help="Trust remote code when loading a HuggingFace tokenizer "
+                             "(needed for some custom tokenizer classes shipped in repos)")
+    parser.add_argument("--hf_use_fast", action=argparse.BooleanOptionalAction, default=True,
+                        help="Use the fast (Rust-based) HuggingFace tokenizer variant if available")
+    parser.add_argument("--hf_revision", type=str, default=None,
+                        help="Pin the HuggingFace repo to a specific commit SHA, branch, or tag "
+                             "for reproducibility (forwarded to from_pretrained as `revision=`)")
+    parser.add_argument("--hf_subfolder", type=str, default=None,
+                        help="Subfolder inside the HuggingFace repo that holds the tokenizer files "
+                             "(forwarded to from_pretrained as `subfolder=`)")
+    parser.add_argument("--hf_cache_dir", type=str, default=None,
+                        help="Override the HuggingFace download cache directory "
+                             "(forwarded to from_pretrained as `cache_dir=`). Default honors "
+                             "HF_HOME / HF_HUB_CACHE / ~/.cache/huggingface/hub.")
+    parser.add_argument("--hf_token", type=str, default=None,
+                        help="HuggingFace auth token for gated repos (e.g. Gemma, Llama). "
+                             "Alternatively run `huggingface-cli login` once, or set the "
+                             "HF_TOKEN environment variable. You must also accept the model's "
+                             "license at https://huggingface.co/<repo> while logged in.")
+
+    # Sine wave tokenizer arguments
+    parser.add_argument("--sine_period", type=float, default=1.0,
+                        help="Period multiplier applied to the sine wave (in radians)")
+    parser.add_argument("--sine_points_per_period", type=int, default=64,
+                        help="Number of discrete points sampled per sine wave period")
+    parser.add_argument("--sine_num_periods", type=int, default=10,
+                        help="Total number of periods to generate")
+    parser.add_argument("--sine_amplitude", type=float, default=50.0,
+                        help="Amplitude of the generated sine wave prior to clamping")
+
+    # Whisper-style mel spectrogram tokenizer arguments
+    parser.add_argument("--mel_sample_rate", type=int, default=16000,
+                        help="Target sample rate for mel spectrogram computation")
+    parser.add_argument("--mel_n_fft", type=int, default=400,
+                        help="FFT size for mel spectrogram computation")
+    parser.add_argument("--mel_hop_length", type=int, default=160,
+                        help="Hop length between frames for mel spectrogram computation")
+    parser.add_argument("--mel_win_length", type=int, default=400,
+                        help="Window length for mel spectrogram computation")
+    parser.add_argument("--mel_n_mels", type=int, default=80,
+                        help="Number of mel filterbank channels")
+    parser.add_argument("--mel_f_min", type=float, default=0.0,
+                        help="Minimum frequency for mel filterbank")
+    parser.add_argument("--mel_f_max", type=float, default=8000.0,
+                        help="Maximum frequency for mel filterbank")
+    parser.add_argument("--mel_center", action=argparse.BooleanOptionalAction, default=True,
+                        help="Center frames during STFT computation")
+    parser.add_argument("--mel_power", type=float, default=2.0,
+                        help="Exponent for the magnitude spectrogram")
+    parser.add_argument("--mel_normalize", action=argparse.BooleanOptionalAction, default=True,
+                        help="Apply Whisper-style log-mel normalization")
+    parser.add_argument("--mel_csv_float_format", type=str, default="%.6f",
+                        help="Float format string used when writing mel CSV files")
+
+    # SentencePiece arguments
+    parser.add_argument("--vocab_size", type=int, default=500, help="Vocabulary size for SentencePiece model")
+    parser.add_argument("--spm_model_file", type=str, default=None, help="Path to the pre-trained SentencePiece model file")
+    parser.add_argument("--spm_vocab_file", type=str, default=None, help="Path to the SentencePiece vocabulary file")
+    parser.add_argument("--skip_tokenization", action="store_true", help="Skip creation of .bin files")
+
+    # Tiktoken arguments
+    parser.add_argument("-e", "--tiktoken_encoding",
+                       choices=["gpt2", "r50k_base", "p50k_base", "cl100k_base"],
+                       default="gpt2", help="Version of tiktoken encoding to utilize")
+    parser.add_argument("--additional_tokens_file", type=str, default=None,
+                       help="Path to JSON file containing additional special tokens for tiktoken (format: {'token': id})")
+
+    # Char tokenizer arguments
+    parser.add_argument("--reuse_chars", action="store_true", help="Reuse character list from meta.pkl")
+    parser.add_argument("--char_bpe_vocab_path", type=str, default=None,
+                        help="Path to a char_bpe meta.pkl to reuse its vocabulary/merges")
+
+    # Custom tokenizer arguments
+    parser.add_argument("--tokens_file", type=str, default=None, help="Path to the file containing newline-separated tokens for tokenization")
+    parser.add_argument("--custom_chars_file", type=str, default=None, help="Path to the file containing custom characters for the tokenizer")
+    parser.add_argument("--json_tokens_file", type=str, default=None, help="Path to JSON file containing tokens for json_byte_fallback tokenizer")
+
+    # Additional options
+    parser.add_argument("-T", "--track_token_counts", action="store_true", help="Track how often each token appears and store in meta.pkl")
+    parser.add_argument("-s", "--output_tokenization_subdir", action="store_true",
+                        help="Write meta.pkl/train.bin/val.bin into a subdirectory named after the selected tokenization method")
+    parser.add_argument("-S", "--output_subdir_suffix", type=str, default="",
+                        help="Optional suffix to append to the tokenization subdirectory name (e.g. sp_1000_suffix)")
+
+    return parser.parse_args()
+
+def save_tokens(ids, output_file, dtype):
+    """Save tokenized data to a binary file with progress bar."""
+    total = len(ids)
+    batch_size = 1024 * 1024  # 1 million tokens per batch
+    with open(output_file, 'wb') as f_out:
+        for i in tqdm(range(0, total, batch_size), desc=f"Saving {output_file}"):
+            batch = ids[i:i+batch_size]
+            np.array(batch, dtype=dtype).tofile(f_out)
+
+def save_mel_csv(frames, output_file, float_format):
+    with open(output_file, "w", encoding="utf-8") as f_out:
+        np.savetxt(f_out, frames, delimiter=",", fmt=float_format)
+
+def _read_input_data(path):
+    if os.path.isdir(path):
+        collected = []
+        for root, _, files in os.walk(path):
+            for name in sorted(files):
+                file_path = os.path.join(root, name)
+                with open(file_path, "r", encoding="utf-8", errors="replace") as f:
+                    collected.append(f.read())
+        return "\n".join(collected)
+    with open(path, 'r', encoding='utf-8', errors='replace') as f:
+        return f.read()
+
+
+def main():
+    args = parse_arguments()
+    output_dir = None
+    if args.output_tokenization_subdir:
+        if args.method == "json_byte_fallback" and args.json_tokens_file:
+            output_dir = os.path.splitext(os.path.basename(args.json_tokens_file))[0]
+        elif args.method == "sentencepiece":
+            output_dir = f"sp_{args.vocab_size}"
+        elif args.method == "huggingface" and args.hf_tokenizer_name:
+            sanitized = args.hf_tokenizer_name.replace("/", "_").replace(os.sep, "_")
+            output_dir = f"hf_{sanitized}"
+        else:
+            output_dir = args.method
+        if args.output_subdir_suffix:
+            output_dir = f"{output_dir}_{args.output_subdir_suffix}"
+    if output_dir:
+        args.meta_output_path = os.path.join(output_dir, "meta.pkl")
+        args.train_output = os.path.join(output_dir, os.path.basename(args.train_output))
+        if args.val_output:
+            args.val_output = os.path.join(output_dir, os.path.basename(args.val_output))
+    else:
+        args.meta_output_path = "meta.pkl"
+    if output_dir and not os.path.exists(output_dir):
+        os.makedirs(output_dir, exist_ok=True)
+
+    # Load training/validation data depending on tokenizer method
+    if args.method in {"sinewave", "whisper_mel_csv"}:
+        train_data = None
+        val_data = None
+    else:
+        train_data = _read_input_data(args.train_input)
+
+        if args.val_input:
+            val_data = _read_input_data(args.val_input)
+        else:
+            n = len(train_data)
+            train_data, val_data = train_data[:int(n * args.percentage_train)], train_data[int(n * args.percentage_train):]
+            if args.percentage_train == 1.0:
+                val_data = None
+
+    # Initialize tokenizer based on method
+    if args.method == "sentencepiece":
+        tokenizer = SentencePieceTokenizer(args, input_files=args.train_input)
+    elif args.method == "tiktoken":
+        tokenizer = TiktokenTokenizer(args)
+    elif args.method == "huggingface":
+        tokenizer = HuggingFaceTokenizer(args)
+    elif args.method == "custom":
+        tokenizer = CustomTokenizer(args)
+    elif args.method == "byte":
+        tokenizer = ByteTokenizer(args)
+    elif args.method == "char":
+        tokenizer = CharTokenizer(args, train_data, val_data)
+    elif args.method == "char_bpe":
+        tokenizer = CharBPETokenizerWithByteFallback(args, train_data, val_data)
+    elif args.method == "custom_char_byte_fallback":
+        tokenizer = CustomCharTokenizerWithByteFallback(args)
+    elif args.method == "json_byte_fallback":
+        tokenizer = JsonByteTokenizerWithByteFallback(args)
+    elif args.method == "python_programming":
+        tokenizer = PythonProgrammingTokenizer(args)
+    elif args.method == "sinewave":
+        tokenizer = SineWaveTokenizer(args)
+    elif args.method == "whisper_mel_csv":
+        tokenizer = WhisperMelCsvTokenizer(args)
+    else:
+        raise ValueError(f"Unknown tokenization method: {args.method}")
+
+    # Tokenize data
+    if args.method == "whisper_mel_csv":
+        train_ids = tokenizer.tokenize(args.train_input)
+    else:
+        train_ids = tokenizer.tokenize(train_data)
+    if args.method in ("tiktoken", "huggingface"):
+        print(f"[{args.method}] Total train tokens: {tokenizer.last_token_count:,}")
+    if args.method == "whisper_mel_csv" and args.val_input is None:
+        split_point = int(len(train_ids) * args.percentage_train)
+        val_ids = train_ids[split_point:]
+        train_ids = train_ids[:split_point]
+    elif args.method == "sinewave" and args.val_input is None:
+        split_point = int(len(train_ids) * args.percentage_train)
+        val_ids = train_ids[split_point:]
+        train_ids = train_ids[:split_point]
+    elif val_data is not None:
+        if args.method == "whisper_mel_csv":
+            val_ids = tokenizer.tokenize(args.val_input)
+        else:
+            val_ids = tokenizer.tokenize(val_data)
+        if args.method in ("tiktoken", "huggingface"):
+            print(f"[{args.method}] Total val tokens: {tokenizer.last_token_count:,}")
+    else:
+        val_ids = None
+
+    # Determine dtype based on vocabulary size from meta.pkl
+    if args.method == "whisper_mel_csv":
+        dtype = None
+    elif args.method == "sinewave":
+        dtype = np.uint16
+    else:
+        with open(args.meta_output_path, "rb") as f:
+            meta = pickle.load(f)
+        vocab_size = meta["vocab_size"]
+        dtype = np.uint32 if vocab_size > 65535 else np.uint16
+
+    # Ensure output directories exist if paths include folders
+    for output_path in [args.train_output, args.val_output, args.meta_output_path]:
+        if output_path:
+            out_dir = os.path.dirname(output_path)
+            if out_dir and not os.path.exists(out_dir):
+                os.makedirs(out_dir, exist_ok=True)
+
+    # Save tokenized data
+    if args.method == "whisper_mel_csv":
+        save_mel_csv(train_ids, args.train_output, args.mel_csv_float_format)
+        if val_ids is not None:
+            save_mel_csv(val_ids, args.val_output, args.mel_csv_float_format)
+    else:
+        save_tokens(train_ids, args.train_output, dtype)
+        if val_ids is not None:
+            save_tokens(val_ids, args.val_output, dtype)
+
+    if args.method == "sinewave":
+        meta = {
+            "tokenizer": "sinewave",
+            "vocab_size": 256,
+            "sine_period": args.sine_period,
+            "sine_points_per_period": args.sine_points_per_period,
+            "sine_num_periods": args.sine_num_periods,
+            "sine_amplitude": args.sine_amplitude,
+        }
+        with open(args.meta_output_path, "wb") as f:
+            pickle.dump(meta, f)
+    elif args.method == "whisper_mel_csv":
+        meta = {
+            "tokenizer": "whisper_mel_csv",
+            "sample_rate": args.mel_sample_rate,
+            "n_fft": args.mel_n_fft,
+            "hop_length": args.mel_hop_length,
+            "win_length": args.mel_win_length,
+            "n_mels": args.mel_n_mels,
+            "f_min": args.mel_f_min,
+            "f_max": args.mel_f_max,
+            "center": args.mel_center,
+            "power": args.mel_power,
+            "normalize": args.mel_normalize,
+        }
+        with open("meta.pkl", "wb") as f:
+            pickle.dump(meta, f)
+
+    # Save additional metadata for tiktoken if needed
+    if args.method == "tiktoken" and args.additional_tokens_file:
+        with open(args.additional_tokens_file, 'r') as f:
+            additional_tokens = json.load(f)
+        with open(args.meta_output_path, "rb") as f:
+            meta = pickle.load(f)
+        meta.update({
+            "has_additional_tokens": True,
+            "special_tokens": additional_tokens,
+            "tokenizer": "tiktoken",
+            "tiktoken_encoding": args.tiktoken_encoding
+        })
+        with open(args.meta_output_path, "wb") as f:
+            pickle.dump(meta, f)
+
+if __name__ == "__main__":
+    main()
diff --git a/demos/simplified_hanzi_mc_demo.sh b/demos/simplified_hanzi_mc_demo.sh
new file mode 100755
index 0000000000..2648c5fb1f
--- /dev/null
+++ b/demos/simplified_hanzi_mc_demo.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# End-to-end smoke demo for simplified Hanzi radical-location multicontext:
+# data prep -> tiny training run -> sampling -> bijective char-lane reconstruction.
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "${ROOT}"
+DATA_ROOT="data/simplified_hanzi_mc"
+OUT_DIR="out/simplified_hanzi_mc_demo"
+
+bash "${DATA_ROOT}/get_dataset.sh"
+
+mapfile -t DATASETS < <(python3 - <<'PY'
+import json
+m=json.load(open('data/simplified_hanzi_mc/manifest.json', encoding='utf-8'))
+print('\n'.join(m['multicontext_datasets']))
+PY
+)
+
+python3 train.py \
+  --training_mode multicontext \
+  --dataset "data/simplified_hanzi_mc/char/char_simplified_hanzi_mc" \
+  --multicontext \
+  --multicontext_datasets "${DATASETS[@]}" \
+  --out_dir "${OUT_DIR}" \
+  --eval_interval 250 \
+  --eval_iters 100 \
+  --log_interval 10 \
+  --always_save_checkpoint \
+  --max_iters "${MAX_ITERS:-10000}" \
+  --use_rotary_embeddings \
+  --no-use_abs_pos_embeddings \
+  --use_qk_norm \
+  --use_qk_norm_scale \
+  --batch_size 32 \
+  --block_size 256 \
+  --n_layer 10 \
+  --n_head 3 \
+  --n_embd 384 \
+  --dropout 0.0 \
+  --device "${DEVICE:-cuda:0}" \
+  --no-compile
+
+python3 sample.py \
+  --out_dir "${OUT_DIR}" \
+  --device "${DEVICE:-cuda:0}" \
+  --no-compile \
+  --multicontext \
+  --multicontext_datasets "${DATASETS[@]}" \
+  --multicontext_start "明" "∅" "∅" "日" "月" "∅" "∅" "∅" "∅" "∅" "∅" "∅" \
+  --max_new_tokens 16 \
+  --top_k 1 \
+  --num_samples 1 | tee "${OUT_DIR}/sample.txt"
+
+# Deterministic reconstruction smoke test from the prepared char lane. For model
+# output, save/generated the char-lane continuation and pass --char_file to this script.
+python3 "${DATA_ROOT}/decode_multicontext_sample.py" --root "${DATA_ROOT}" | tee "${OUT_DIR}/decoded_reference.txt"