Add hanzi mc experiments

data/simplified_hanzi_mc/.gitignore

@@ -0,0 +1,14 @@
+/char/
+/whole/
+/left/
+/right/
+/top/
+/bottom/
+/enclosure/
+/inside/
+/corner/
+/overlay/
+/other/
+/manifest.json
+__pycache__/
+*.pyc

data/simplified_hanzi_mc/README.md

@@ -0,0 +1,22 @@
+# Simplified Hanzi radical-location multicontext demo
+
+This folder demonstrates a reversible multicontext split for simplified Hanzi.
+`input.txt` may be either the bundled one-character-per-line corner-case fixture or an ordinary UTF-8 text corpus (for example `#Title:` / `#Poem:` records). `get_dataset.sh` treats the file as a character stream and creates one aligned multicontext timestep per Unicode code point:
+
+- `char`: the original simplified character, making the representation a 1:1 bijection.
+- `whole`, `left`, `right`, `top`, `bottom`, `enclosure`, `inside`, `corner`, `overlay`, `other`: radical/location signals.
+
+`∅` means “this simplified Hanzi has nothing in this category.” `⧆` means “this
+input code point is not treated as simplified Hanzi,” and is emitted in every lane.
+
+The decomposition table is intentionally small and transparent for tests. It can
+be replaced with a full Unihan/IDS-derived table without changing the lane
+contract or downstream training commands.
+
+Run:
+
+```bash
+bash data/simplified_hanzi_mc/get_dataset.sh
+```
+
+Each lane then contains `char_simplified_hanzi_mc/{train.bin,val.bin,meta.pkl}`.

data/simplified_hanzi_mc/build_simplified_hanzi_mc.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Build toy simplified-Hanzi radical-location multicontext lanes.
+
+This is intentionally small and transparent: it demonstrates a reversible
+(1:1) representation by carrying the original simplified Hanzi in a dedicated
+`char` lane and aligned radical-location lanes for model conditioning.
+"""
+from __future__ import annotations
+import argparse, json, re
+from pathlib import Path
+
+PLACEHOLDER = "∅"
+NON_HANZI = "⧆"
+LANES = ["char", "whole", "left", "right", "top", "bottom", "enclosure", "inside", "corner", "overlay", "other"]
+
+# Demonstration lookup table: enough cases to cover the location categories and
+# corner cases in input.txt. Values are radical/location signals, not full IDS.
+DECOMP = {
+    "一": {"whole":"一"}, "人": {"whole":"人"}, "口": {"whole":"口"},
+    "明": {"left":"日", "right":"月"}, "休": {"left":"亻", "right":"木"},
+    "林": {"left":"木", "right":"木"}, "好": {"left":"女", "right":"子"},
+    "苗": {"top":"艹", "bottom":"田"}, "尖": {"top":"小", "bottom":"大"},
+    "想": {"top":"相", "bottom":"心", "left":"木", "right":"目"},
+    "国": {"enclosure":"囗", "inside":"玉"}, "问": {"enclosure":"门", "inside":"口"},
+    "闪": {"enclosure":"门", "inside":"人"}, "医": {"enclosure":"匚", "inside":"矢"},
+    "区": {"enclosure":"匚", "inside":"乂"}, "同": {"enclosure":"冂", "inside":"一口"},
+    "这": {"enclosure":"辶", "inside":"文"}, "房": {"enclosure":"户", "inside":"方"},
+    "病": {"enclosure":"疒", "inside":"丙"}, "氧": {"enclosure":"气", "inside":"羊"},
+    "赢": {"corner":"亡口月贝凡"}, "器": {"corner":"口口口口", "inside":"犬"},
+    "乘": {"overlay":"禾北"}, "爽": {"overlay":"大乂乂乂乂"},
+    "坐": {"overlay":"人人土"}, "办": {"other":"力丶丶"}, "必": {"other":"心丿"},
+}
+# Tiny demo-only exclusions so the non-simplified-Hanzi vector is testable.
+TRADITIONAL_ONLY = set("體龍門馬愛學國風書樂車東長萬與興貓鳥魚")
+CJK_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]")
+
+def is_simplified_hanzi(ch: str) -> bool:
+    return len(ch) == 1 and bool(CJK_RE.fullmatch(ch)) and ch not in TRADITIONAL_ONLY
+
+def encode_char(ch: str) -> dict[str, str]:
+    if not is_simplified_hanzi(ch):
+        return {lane: NON_HANZI for lane in LANES}
+    row = {lane: PLACEHOLDER for lane in LANES}
+    row["char"] = ch
+    for lane, value in DECOMP.get(ch, {"other": ch}).items():
+        row[lane] = value
+    return row
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--input", default="input.txt")
+    ap.add_argument("--output_root", default=".")
+    ap.add_argument("--label", default="simplified_hanzi_mc")
+    args = ap.parse_args()
+    in_path = Path(args.input)
+    out_root = Path(args.output_root)
+    # Treat input.txt as an arbitrary UTF-8 text stream, not as one-character
+    # records. This lets regular corpora such as poem/title files flow through
+    # unchanged at the character timestep level: every code point gets one
+    # aligned multicontext vector; non-Hanzi code points become NON_HANZI.
+    chars = list(in_path.read_text(encoding="utf-8"))
+    if not chars:
+        raise ValueError(f"Input file is empty: {in_path}")
+    rows = [encode_char(ch) for ch in chars]
+    datasets = []
+    for lane in LANES:
+        lane_dir = out_root / lane
+        lane_dir.mkdir(parents=True, exist_ok=True)
+        (lane_dir / "input.txt").write_text("\n".join(row[lane] for row in rows) + "\n", encoding="utf-8")
+        datasets.append(f"simplified_hanzi_mc/{lane}/char_{args.label}")
+    manifest = {"tokenizer":"simplified_hanzi_radical_location_multicontext", "source":str(in_path), "lanes":LANES,
+                "multicontext_datasets":datasets, "placeholder":PLACEHOLDER, "non_hanzi":NON_HANZI,
+                "bijection":"The char lane stores the original simplified Hanzi, while aligned lanes store radical-location labels.",
+                "rows":len(rows)}
+    (out_root / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+"\n", encoding="utf-8")
+    print(json.dumps(manifest, ensure_ascii=False, indent=2))
+if __name__ == "__main__": main()

data/simplified_hanzi_mc/decode_multicontext_sample.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""Reconstruct simplified Hanzi from generated multicontext lane text.
+
+Because the `char` lane carries the original character, decoding is a direct
+bijection for simplified Hanzi. A timestep where every lane is `⧆` is rendered
+as `<NON_HANZI>`.
+"""
+from __future__ import annotations
+import argparse, json
+from pathlib import Path
+NON_HANZI="⧆"
+def main():
+    ap=argparse.ArgumentParser()
+    ap.add_argument("--root", default="data/simplified_hanzi_mc")
+    ap.add_argument("--char_file", default=None, help="Optional generated char-lane text file; defaults to <root>/char/input.txt")
+    args=ap.parse_args()
+    root=Path(args.root)
+    manifest=json.loads((root/"manifest.json").read_text(encoding="utf-8"))
+    char_path=Path(args.char_file) if args.char_file else root/"char"/"input.txt"
+    chars=[line.strip() for line in char_path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    decoded=["<NON_HANZI>" if ch==NON_HANZI else ch for ch in chars]
+    print("".join(decoded))
+    print(f"decoded_steps={len(decoded)} lanes={','.join(manifest['lanes'])}")
+if __name__ == "__main__": main()

data/simplified_hanzi_mc/get_dataset.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build simplified-Hanzi radical-location multicontext lanes, then tokenize each
+# lane into a labeled subfolder with prepare.py -s -S.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INPUT_TXT="${1:-${SCRIPT_DIR}/input.txt}"
+LABEL="${LABEL:-simplified_hanzi_mc}"
+METHOD="${METHOD:-char}"
+python3 "${SCRIPT_DIR}/build_simplified_hanzi_mc.py" --input "${INPUT_TXT}" --output_root "${SCRIPT_DIR}" --label "${LABEL}"
+LANES=(char whole left right top bottom enclosure inside corner overlay other)
+for lane in "${LANES[@]}"; do
+  echo "[prepare] ${lane}"
+  (cd "${SCRIPT_DIR}/${lane}" && python3 "${SCRIPT_DIR}/prepare.py" -t input.txt --method "${METHOD}" -s -S "${LABEL}")
+done

data/simplified_hanzi_mc/input.txt

@@ -0,0 +1,41 @@
+一
+人
+口
+明
+休
+林
+好
+苗
+尖
+想
+国
+问
+闪
+医
+区
+同
+这
+房
+病
+氧
+赢
+器
+乘
+爽
+坐
+办
+必
+體
+A
+。
+🙂
+门
+马
+爱
+学
+书
+鱼
+车
+龙
+万
+风