Add nfc nfd comparison scripts for opus100

data/ko_commercial/get_dataset.py

@@ -62,14 +62,7 @@ def emit_json_contents(json_path, output_text_file):
 
 def main(output_text_file):
     parquet_files = {
-        "train-00000-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00000-of-00008-262a4650cba7cd42.parquet?download=true",
-        "train-00001-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00001-of-00008-911ee3e75f35d481.parquet?download=true",
-        "train-00002-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00002-of-00008-4f4791330d9553d5.parquet?download=true",
-        "train-00003-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00003-of-00008-9f822b7fe3799cd3.parquet?download=true",
-        "train-00004-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00004-of-00008-8daf63b596c127d7.parquet?download=true",
-        "train-00005-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00005-of-00008-68cf9afac57baa1c.parquet?download=true",
-        "train-00006-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00006-of-00008-275cbee982d3460c.parquet?download=true",
-        "train-00007-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00007-of-00008-e63b716fb34017d5.parquet?download=true",
+        "train-00000-of-00001": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00000-of-00001-1ae224438dce829b.parquet?download=true"
     }
     download_dir = "./downloaded_parquets"
     json_dir = "./json_output"

data/opus-100/compare_meta_vocab_tui.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""compare_meta_vocab_tui.py
+
+Textual TUI to compare vocabularies stored in two meta.pkl files.
+Supports sorting both vocab lists by byte length or frequency.
+"""
+
+from __future__ import annotations
+
+import argparse
+import pickle
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, List
+
+from textual.app import App, ComposeResult
+from textual.binding import Binding
+from textual.containers import Horizontal, Vertical
+from textual.widgets import DataTable, Footer, Header, Static
+
+
+@dataclass(frozen=True)
+class VocabEntry:
+    token_id: int
+    token: Any
+    byte_len: int
+    count: int
+
+
+def _load_meta(path: Path) -> dict:
+    with path.open("rb") as f:
+        return pickle.load(f)
+
+
+def _resolve_meta_path(path: Path) -> Path:
+    if path.is_dir():
+        return path / "meta.pkl"
+    return path
+
+
+def _token_byte_len(token: Any) -> int:
+    if isinstance(token, bytes):
+        return len(token)
+    if isinstance(token, str):
+        return len(token.encode("utf-8"))
+    return len(str(token).encode("utf-8"))
+
+
+def _display_token(token: Any) -> str:
+    if isinstance(token, bytes):
+        return repr(token)
+    return repr(token)
+
+
+def _coerce_token_id(raw_id: Any) -> int:
+    if isinstance(raw_id, int):
+        return raw_id
+    try:
+        return int(raw_id)
+    except (TypeError, ValueError):
+        return -1
+
+
+def _build_entries(meta: dict) -> List[VocabEntry]:
+    itos = meta.get("itos", {})
+    token_counts = meta.get("token_counts", {}) or {}
+    entries: List[VocabEntry] = []
+    for raw_id, token in itos.items():
+        token_id = _coerce_token_id(raw_id)
+        count = int(token_counts.get(raw_id, token_counts.get(token_id, 0)) or 0)
+        entries.append(
+            VocabEntry(
+                token_id=token_id,
+                token=token,
+                byte_len=_token_byte_len(token),
+                count=count,
+            )
+        )
+    return entries
+
+
+def _sorted_entries(entries: Iterable[VocabEntry], mode: str) -> List[VocabEntry]:
+    if mode == "bytes":
+        return sorted(entries, key=lambda e: (e.byte_len, e.token_id), reverse=True)
+    if mode == "freq":
+        return sorted(entries, key=lambda e: (e.count, e.token_id), reverse=True)
+    return sorted(entries, key=lambda e: e.token_id)
+
+
+class VocabCompareApp(App):
+    CSS = """
+    #main { height: 1fr; }
+    .panel { height: 1fr; width: 1fr; }
+    DataTable { height: 1fr; width: 1fr; }
+    """
+
+    BINDINGS = [
+        Binding("b", "sort_bytes", "Sort by bytes", show=True),
+        Binding("f", "sort_freq", "Sort by frequency", show=True),
+        Binding("i", "sort_id", "Sort by id", show=True),
+        Binding("q", "quit", "Quit", show=True),
+    ]
+
+    def __init__(self, left_meta: Path, right_meta: Path) -> None:
+        super().__init__()
+        self.left_meta_path = left_meta
+        self.right_meta_path = right_meta
+        self.left_meta = _load_meta(left_meta)
+        self.right_meta = _load_meta(right_meta)
+        self.left_entries = _build_entries(self.left_meta)
+        self.right_entries = _build_entries(self.right_meta)
+        self.sort_mode = "id"
+
+    def compose(self) -> ComposeResult:
+        yield Header()
+        with Horizontal(id="main"):
+            with Vertical(classes="panel"):
+                yield Static(self._panel_title(self.left_meta_path, self.left_meta), id="left_title")
+                yield DataTable(id="left_table", zebra_stripes=True)
+            with Vertical(classes="panel"):
+                yield Static(self._panel_title(self.right_meta_path, self.right_meta), id="right_title")
+                yield DataTable(id="right_table", zebra_stripes=True)
+        yield Footer()
+
+    def on_mount(self) -> None:
+        self._setup_table(self.query_one("#left_table", DataTable))
+        self._setup_table(self.query_one("#right_table", DataTable))
+        self._refresh_tables()
+
+    def _panel_title(self, path: Path, meta: dict) -> str:
+        tokenizer = meta.get("tokenizer", "unknown")
+        vocab_size = meta.get("vocab_size", "?")
+        return f"{path} | {tokenizer} | vocab={vocab_size}"
+
+    def _setup_table(self, table: DataTable) -> None:
+        self._ensure_columns(table)
+        table.clear()
+
+    def _ensure_columns(self, table: DataTable) -> None:
+        if table.columns:
+            return
+        table.add_column("id", key="id")
+        table.add_column("token", key="token")
+        table.add_column("bytes", key="bytes")
+        table.add_column("count", key="count")
+
+    def _refresh_tables(self) -> None:
+        left_table = self.query_one("#left_table", DataTable)
+        right_table = self.query_one("#right_table", DataTable)
+        self._fill_table(left_table, _sorted_entries(self.left_entries, self.sort_mode))
+        self._fill_table(right_table, _sorted_entries(self.right_entries, self.sort_mode))
+
+    def _fill_table(self, table: DataTable, entries: List[VocabEntry]) -> None:
+        self._ensure_columns(table)
+        table.clear()
+        for entry in entries:
+            table.add_row(
+                str(entry.token_id),
+                _display_token(entry.token),
+                str(entry.byte_len),
+                str(entry.count),
+            )
+
+    def action_sort_bytes(self) -> None:
+        self.sort_mode = "bytes"
+        self._refresh_tables()
+
+    def action_sort_freq(self) -> None:
+        self.sort_mode = "freq"
+        self._refresh_tables()
+
+    def action_sort_id(self) -> None:
+        self.sort_mode = "id"
+        self._refresh_tables()
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Compare vocabularies from two meta.pkl files.")
+    parser.add_argument("left", type=Path, help="Path to left meta.pkl or directory containing it")
+    parser.add_argument("right", type=Path, help="Path to right meta.pkl or directory containing it")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    left_meta = _resolve_meta_path(args.left)
+    right_meta = _resolve_meta_path(args.right)
+    if not left_meta.exists():
+        raise FileNotFoundError(f"Missing meta.pkl at {left_meta}")
+    if not right_meta.exists():
+        raise FileNotFoundError(f"Missing meta.pkl at {right_meta}")
+    app = VocabCompareApp(left_meta=left_meta, right_meta=right_meta)
+    app.run()
+
+
+if __name__ == "__main__":
+    main()
+

data/opus-100/compare_nfc_nfd.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# bash prepare_nfc_and_nfd_dataset_splits.sh input.txt
+
+for tokenization in "char_bpe"; do
+  for type in "nfc" "nfd"; do
+    for vocab_size in "2000" "3000" "4000" "5000"; do
+      python3 prepare.py -t p90_"$type".txt -v p10_"$type".txt --method "$tokenization" --vocab_size "$vocab_size" -s -S "$type"_"$vocab_size" -T
+      mv char_bpe_vocab.json "${tokenization}_${type}_${vocab_size}/"
+    done
+  done
+done

data/opus-100/hangul_nfc_to_nfd.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""
+Convert only precomposed Korean Hangul syllables from NFC to NFD.
+
+Everything outside the Hangul syllables block U+AC00..U+D7A3 is left unchanged.
+So Latin accents, emoji, punctuation, spaces, line endings, etc. are not normalized.
+
+Usage:
+  python3 hangul_nfc_to_nfd.py input.txt output.txt
+  python3 hangul_nfc_to_nfd.py --in-place input.txt
+  cat input.txt | python3 hangul_nfc_to_nfd.py > output.txt
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import tempfile
+import unicodedata
+from pathlib import Path
+
+
+HANGUL_SYLLABLES_START = 0xAC00
+HANGUL_SYLLABLES_END = 0xD7A3
+
+
+def is_precomposed_hangul_syllable(ch: str) -> bool:
+    codepoint = ord(ch)
+    return HANGUL_SYLLABLES_START <= codepoint <= HANGUL_SYLLABLES_END
+
+
+def hangul_only_nfd(text: str) -> str:
+    """
+    Apply Unicode NFD only to precomposed Hangul syllables.
+
+    Example:
+      한 U+D55C -> ᄒ U+1112 + ᅡ U+1161 + ᆫ U+11AB
+
+    Non-Hangul characters are returned exactly as they are.
+    """
+    return "".join(
+        unicodedata.normalize("NFD", ch)
+        if is_precomposed_hangul_syllable(ch)
+        else ch
+        for ch in text
+    )
+
+
+def convert_bytes(data: bytes, encoding: str) -> bytes:
+    # surrogateescape preserves invalid bytes when decoding/re-encoding.
+    text = data.decode(encoding, errors="surrogateescape")
+    converted = hangul_only_nfd(text)
+    return converted.encode(encoding, errors="surrogateescape")
+
+
+def convert_file(input_path: Path, output_path: Path, encoding: str) -> None:
+    data = input_path.read_bytes()
+    output_path.write_bytes(convert_bytes(data, encoding))
+
+
+def convert_file_in_place(path: Path, encoding: str) -> None:
+    converted = convert_bytes(path.read_bytes(), encoding)
+
+    fd, tmp_name = tempfile.mkstemp(
+        prefix=f".{path.name}.",
+        suffix=".tmp",
+        dir=str(path.parent),
+    )
+
+    try:
+        with os.fdopen(fd, "wb") as tmp:
+            tmp.write(converted)
+        os.replace(tmp_name, path)
+    except Exception:
+        try:
+            os.unlink(tmp_name)
+        except FileNotFoundError:
+            pass
+        raise
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Convert only Korean Hangul syllables from NFC to NFD."
+    )
+    parser.add_argument("input", nargs="?", help="Input file. If omitted, reads stdin.")
+    parser.add_argument("output", nargs="?", help="Output file. If omitted, writes stdout.")
+    parser.add_argument(
+        "-i",
+        "--in-place",
+        action="store_true",
+        help="Rewrite the input file in place.",
+    )
+    parser.add_argument(
+        "--encoding",
+        default="utf-8",
+        help="Text encoding to use. Default: utf-8.",
+    )
+
+    args = parser.parse_args()
+
+    if args.in_place:
+        if not args.input or args.output:
+            parser.error("--in-place requires exactly one input file and no output file.")
+        convert_file_in_place(Path(args.input), args.encoding)
+        return 0
+
+    if args.input:
+        input_path = Path(args.input)
+        if args.output:
+            convert_file(input_path, Path(args.output), args.encoding)
+        else:
+            sys.stdout.buffer.write(convert_bytes(input_path.read_bytes(), args.encoding))
+        return 0
+
+    if args.output:
+        parser.error("An output file requires an input file.")
+
+    sys.stdout.buffer.write(convert_bytes(sys.stdin.buffer.read(), args.encoding))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

data/opus-100/prepare_nfc_and_nfd_dataset_splits.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+FILE="$1"
+
+NFC_90="p90_nfc.txt"
+NFC_10="p10_nfc.txt"
+
+NFD_90="p90_nfd.txt"
+NFD_10="p10_nfd.txt"
+
+TOTAL=$(wc -l < "$FILE")
+PCT=$((TOTAL * 10 / 100))
+head -n -$PCT "$FILE" > "$NFC_90"
+tail -n $PCT "$FILE" > "$NFC_10"
+
+python3 hangul_nfc_to_nfd.py "$NFC_90" "$NFD_90"
+python3 hangul_nfc_to_nfd.py "$NFC_10" "$NFD_10"
+