Add simplified hanzi support

data/poetry-chinese-zhtw/convert_traditional_to_simplified.py

@@ -0,0 +1,11 @@
+from opencc import OpenCC
+
+cc = OpenCC('t2s')  # Traditional -> Simplified
+
+with open("input.txt", "r", encoding="utf-8") as f:
+    text = f.read()
+
+converted = cc.convert(text)
+
+with open("output.txt", "w", encoding="utf-8") as f:
+    f.write(converted)

data/simplified_hanzi_mc/.gitignore

@@ -0,0 +1,15 @@
+/char/
+/non_hanzi/
+/whole/
+/left/
+/right/
+/top/
+/bottom/
+/enclosure/
+/inside/
+/corner/
+/overlay/
+/other/
+/manifest.json
+__pycache__/
+*.pyc

data/simplified_hanzi_mc/README.md

@@ -0,0 +1,23 @@
+# Simplified Hanzi radical-location multicontext demo
+
+This folder demonstrates a reversible multicontext split for simplified Hanzi.
+`input.txt` may be either the bundled one-character-per-line corner-case fixture or an ordinary UTF-8 text corpus (for example `#Title:` / `#Poem:` records). `get_dataset.sh` treats the file as a character stream and creates one aligned multicontext timestep per Unicode code point:
+
+- `char`: the original simplified character, or `⧆` when the timestep is not simplified Hanzi.
+- `non_hanzi`: the original non-simplified-Hanzi code point, or `∅` when the timestep is simplified Hanzi. Together with `char`, this makes the representation a full-text 1:1 bijection.
+- `whole`, `left`, `right`, `top`, `bottom`, `enclosure`, `inside`, `corner`, `overlay`, `other`: radical/location signals.
+
+`∅` means “this simplified Hanzi has nothing in this category” (and is also the empty value in `non_hanzi` for simplified Hanzi). `⧆` means “this
+input code point is not treated as simplified Hanzi” in the `char`/radical lanes; the original code point is preserved in `non_hanzi` using line-safe escapes for control characters such as newlines.
+
+The decomposition table is intentionally small and transparent for tests. It can
+be replaced with a full Unihan/IDS-derived table without changing the lane
+contract or downstream training commands.
+
+Run:
+
+```bash
+bash data/simplified_hanzi_mc/get_dataset.sh
+```
+
+Each lane then contains `char_simplified_hanzi_mc/{train.bin,val.bin,meta.pkl}`.

data/simplified_hanzi_mc/build_simplified_hanzi_mc.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Build toy simplified-Hanzi radical-location multicontext lanes.
+
+This is intentionally small and transparent: it demonstrates a reversible
+(1:1) representation by carrying the original simplified Hanzi in a dedicated
+`char` lane and aligned radical-location lanes for model conditioning.
+"""
+from __future__ import annotations
+import argparse, json, re
+from pathlib import Path
+
+PLACEHOLDER = "∅"
+NON_HANZI = "⧆"
+LANES = ["char", "non_hanzi", "whole", "left", "right", "top", "bottom", "enclosure", "inside", "corner", "overlay", "other"]
+
+# Demonstration lookup table: enough cases to cover the location categories and
+# corner cases in input.txt. Values are radical/location signals, not full IDS.
+DECOMP = {
+    "一": {"whole":"一"}, "人": {"whole":"人"}, "口": {"whole":"口"},
+    "明": {"left":"日", "right":"月"}, "休": {"left":"亻", "right":"木"},
+    "林": {"left":"木", "right":"木"}, "好": {"left":"女", "right":"子"},
+    "苗": {"top":"艹", "bottom":"田"}, "尖": {"top":"小", "bottom":"大"},
+    "想": {"top":"相", "bottom":"心", "left":"木", "right":"目"},
+    "国": {"enclosure":"囗", "inside":"玉"}, "问": {"enclosure":"门", "inside":"口"},
+    "闪": {"enclosure":"门", "inside":"人"}, "医": {"enclosure":"匚", "inside":"矢"},
+    "区": {"enclosure":"匚", "inside":"乂"}, "同": {"enclosure":"冂", "inside":"一口"},
+    "这": {"enclosure":"辶", "inside":"文"}, "房": {"enclosure":"户", "inside":"方"},
+    "病": {"enclosure":"疒", "inside":"丙"}, "氧": {"enclosure":"气", "inside":"羊"},
+    "赢": {"corner":"亡口月贝凡"}, "器": {"corner":"口口口口", "inside":"犬"},
+    "乘": {"overlay":"禾北"}, "爽": {"overlay":"大乂乂乂乂"},
+    "坐": {"overlay":"人人土"}, "办": {"other":"力丶丶"}, "必": {"other":"心丿"},
+}
+# Tiny demo-only exclusions so the non-simplified-Hanzi vector is testable.
+TRADITIONAL_ONLY = set("體龍門馬愛學國風書樂車東長萬與興貓鳥魚")
+CJK_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]")
+
+def line_escape(value: str) -> str:
+    """Encode one lane value so each timestep stays on one physical line."""
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t")
+
+def is_simplified_hanzi(ch: str) -> bool:
+    return len(ch) == 1 and bool(CJK_RE.fullmatch(ch)) and ch not in TRADITIONAL_ONLY
+
+def encode_char(ch: str) -> dict[str, str]:
+    if not is_simplified_hanzi(ch):
+        row = {lane: NON_HANZI for lane in LANES}
+        row["non_hanzi"] = ch
+        return row
+    row = {lane: PLACEHOLDER for lane in LANES}
+    row["char"] = ch
+    row["non_hanzi"] = PLACEHOLDER
+    for lane, value in DECOMP.get(ch, {"other": ch}).items():
+        row[lane] = value
+    return row
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--input", default="input.txt")
+    ap.add_argument("--output_root", default=".")
+    ap.add_argument("--label", default="simplified_hanzi_mc")
+    args = ap.parse_args()
+    in_path = Path(args.input)
+    out_root = Path(args.output_root)
+    # Treat input.txt as an arbitrary UTF-8 text stream, not as one-character
+    # records. This lets regular corpora such as poem/title files flow through
+    # unchanged at the character timestep level: every code point gets one
+    # aligned multicontext vector; non-Hanzi code points become NON_HANZI.
+    chars = list(in_path.read_text(encoding="utf-8"))
+    if not chars:
+        raise ValueError(f"Input file is empty: {in_path}")
+    rows = [encode_char(ch) for ch in chars]
+    datasets = []
+    for lane in LANES:
+        lane_dir = out_root / lane
+        lane_dir.mkdir(parents=True, exist_ok=True)
+        lane_values = [line_escape(row[lane]) if lane == "non_hanzi" else row[lane] for row in rows]
+        (lane_dir / "input.txt").write_text("\n".join(lane_values) + "\n", encoding="utf-8")
+        datasets.append(f"simplified_hanzi_mc/{lane}/char_{args.label}")
+    manifest = {"tokenizer":"simplified_hanzi_radical_location_multicontext", "source":str(in_path), "lanes":LANES,
+                "multicontext_datasets":datasets, "placeholder":PLACEHOLDER, "non_hanzi":NON_HANZI,
+                "bijection":"The char lane stores simplified Hanzi; the non_hanzi lane stores original non-Hanzi/non-simplified code points; aligned radical lanes store location labels.",
+                "rows":len(rows)}
+    (out_root / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+"\n", encoding="utf-8")
+    print(json.dumps(manifest, ensure_ascii=False, indent=2))
+if __name__ == "__main__": main()

data/simplified_hanzi_mc/decode_multicontext_sample.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Reconstruct simplified Hanzi from generated multicontext lane text.
+
+The `char` lane carries simplified Hanzi. For timesteps where `char` is `⧆`,
+the `non_hanzi` lane carries the original escaped code point, allowing full-text
+reconstruction instead of rendering `<NON_HANZI>` placeholders.
+"""
+from __future__ import annotations
+import argparse, json
+from pathlib import Path
+NON_HANZI="⧆"
+def line_unescape(value: str) -> str:
+    out=[]
+    i=0
+    while i < len(value):
+        if value[i] == "\\" and i + 1 < len(value):
+            nxt=value[i+1]
+            if nxt == "n": out.append("\n")
+            elif nxt == "r": out.append("\r")
+            elif nxt == "t": out.append("\t")
+            elif nxt == "\\": out.append("\\")
+            else:
+                out.append(nxt)
+            i += 2
+        else:
+            out.append(value[i])
+            i += 1
+    return "".join(out)
+
+def main():
+    ap=argparse.ArgumentParser()
+    ap.add_argument("--root", default="data/simplified_hanzi_mc")
+    ap.add_argument("--char_file", default=None, help="Optional generated char-lane text file; defaults to <root>/char/input.txt")
+    ap.add_argument("--non_hanzi_file", default=None, help="Optional generated non_hanzi-lane text file; defaults to <root>/non_hanzi/input.txt")
+    args=ap.parse_args()
+    root=Path(args.root)
+    manifest=json.loads((root/"manifest.json").read_text(encoding="utf-8"))
+    char_path=Path(args.char_file) if args.char_file else root/"char"/"input.txt"
+    non_hanzi_path=Path(args.non_hanzi_file) if args.non_hanzi_file else root/"non_hanzi"/"input.txt"
+    chars=[line.rstrip("\n") for line in char_path.read_text(encoding="utf-8").splitlines()]
+    non_hanzi=[line_unescape(line.rstrip("\n")) for line in non_hanzi_path.read_text(encoding="utf-8").splitlines()]
+    if len(chars) != len(non_hanzi):
+        raise ValueError(f"Lane length mismatch: char={len(chars)} non_hanzi={len(non_hanzi)}")
+    decoded=[nh if ch==NON_HANZI else ch for ch, nh in zip(chars, non_hanzi)]
+    print("".join(decoded))
+    print(f"decoded_steps={len(decoded)} lanes={','.join(manifest['lanes'])}")
+if __name__ == "__main__": main()

data/simplified_hanzi_mc/get_dataset.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build simplified-Hanzi radical-location multicontext lanes, then tokenize each
+# lane into a labeled subfolder with prepare.py -s -S.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INPUT_TXT="${1:-${SCRIPT_DIR}/input.txt}"
+LABEL="${LABEL:-simplified_hanzi_mc}"
+METHOD="${METHOD:-char}"
+python3 "${SCRIPT_DIR}/build_simplified_hanzi_mc.py" --input "${INPUT_TXT}" --output_root "${SCRIPT_DIR}" --label "${LABEL}"
+LANES=(char non_hanzi whole left right top bottom enclosure inside corner overlay other)
+for lane in "${LANES[@]}"; do
+  echo "[prepare] ${lane}"
+  (cd "${SCRIPT_DIR}/${lane}" && python3 "${SCRIPT_DIR}/prepare.py" -t input.txt --method "${METHOD}" -s -S "${LABEL}")
+done

data/simplified_hanzi_mc/input.txt

@@ -0,0 +1,41 @@
+一
+人
+口
+明
+休
+林
+好
+苗
+尖
+想
+国
+问
+闪
+医
+区
+同
+这
+房
+病
+氧
+赢
+器
+乘
+爽
+坐
+办
+必
+體
+A
+。
+🙂
+门
+马
+爱
+学
+书
+鱼
+车
+龙
+万
+风