From fe140d0f6f31cd38858775c18748890e110b1728 Mon Sep 17 00:00:00 2001 From: klei22 Date: Tue, 14 Jul 2026 00:12:48 -0700 Subject: [PATCH 1/2] Add means to compare nfc and nfd on opus-100 --- data/opus-100/compare_meta_vocab_tui.py | 198 ++++++++++++++++++ data/opus-100/compare_nfc_nfd.sh | 12 ++ data/opus-100/hangul_nfc_to_nfd.py | 125 +++++++++++ .../prepare_nfc_and_nfd_dataset_splits.sh | 18 ++ 4 files changed, 353 insertions(+) create mode 100644 data/opus-100/compare_meta_vocab_tui.py create mode 100644 data/opus-100/compare_nfc_nfd.sh create mode 100644 data/opus-100/hangul_nfc_to_nfd.py create mode 100644 data/opus-100/prepare_nfc_and_nfd_dataset_splits.sh diff --git a/data/opus-100/compare_meta_vocab_tui.py b/data/opus-100/compare_meta_vocab_tui.py new file mode 100644 index 0000000000..e05c413ce5 --- /dev/null +++ b/data/opus-100/compare_meta_vocab_tui.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""compare_meta_vocab_tui.py + +Textual TUI to compare vocabularies stored in two meta.pkl files. +Supports sorting both vocab lists by byte length or frequency. +""" + +from __future__ import annotations + +import argparse +import pickle +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, List + +from textual.app import App, ComposeResult +from textual.binding import Binding +from textual.containers import Horizontal, Vertical +from textual.widgets import DataTable, Footer, Header, Static + + +@dataclass(frozen=True) +class VocabEntry: + token_id: int + token: Any + byte_len: int + count: int + + +def _load_meta(path: Path) -> dict: + with path.open("rb") as f: + return pickle.load(f) + + +def _resolve_meta_path(path: Path) -> Path: + if path.is_dir(): + return path / "meta.pkl" + return path + + +def _token_byte_len(token: Any) -> int: + if isinstance(token, bytes): + return len(token) + if isinstance(token, str): + return len(token.encode("utf-8")) + return len(str(token).encode("utf-8")) + + +def _display_token(token: Any) -> str: + if isinstance(token, bytes): + return repr(token) + return repr(token) + + +def _coerce_token_id(raw_id: Any) -> int: + if isinstance(raw_id, int): + return raw_id + try: + return int(raw_id) + except (TypeError, ValueError): + return -1 + + +def _build_entries(meta: dict) -> List[VocabEntry]: + itos = meta.get("itos", {}) + token_counts = meta.get("token_counts", {}) or {} + entries: List[VocabEntry] = [] + for raw_id, token in itos.items(): + token_id = _coerce_token_id(raw_id) + count = int(token_counts.get(raw_id, token_counts.get(token_id, 0)) or 0) + entries.append( + VocabEntry( + token_id=token_id, + token=token, + byte_len=_token_byte_len(token), + count=count, + ) + ) + return entries + + +def _sorted_entries(entries: Iterable[VocabEntry], mode: str) -> List[VocabEntry]: + if mode == "bytes": + return sorted(entries, key=lambda e: (e.byte_len, e.token_id), reverse=True) + if mode == "freq": + return sorted(entries, key=lambda e: (e.count, e.token_id), reverse=True) + return sorted(entries, key=lambda e: e.token_id) + + +class VocabCompareApp(App): + CSS = """ + #main { height: 1fr; } + .panel { height: 1fr; width: 1fr; } + DataTable { height: 1fr; width: 1fr; } + """ + + BINDINGS = [ + Binding("b", "sort_bytes", "Sort by bytes", show=True), + Binding("f", "sort_freq", "Sort by frequency", show=True), + Binding("i", "sort_id", "Sort by id", show=True), + Binding("q", "quit", "Quit", show=True), + ] + + def __init__(self, left_meta: Path, right_meta: Path) -> None: + super().__init__() + self.left_meta_path = left_meta + self.right_meta_path = right_meta + self.left_meta = _load_meta(left_meta) + self.right_meta = _load_meta(right_meta) + self.left_entries = _build_entries(self.left_meta) + self.right_entries = _build_entries(self.right_meta) + self.sort_mode = "id" + + def compose(self) -> ComposeResult: + yield Header() + with Horizontal(id="main"): + with Vertical(classes="panel"): + yield Static(self._panel_title(self.left_meta_path, self.left_meta), id="left_title") + yield DataTable(id="left_table", zebra_stripes=True) + with Vertical(classes="panel"): + yield Static(self._panel_title(self.right_meta_path, self.right_meta), id="right_title") + yield DataTable(id="right_table", zebra_stripes=True) + yield Footer() + + def on_mount(self) -> None: + self._setup_table(self.query_one("#left_table", DataTable)) + self._setup_table(self.query_one("#right_table", DataTable)) + self._refresh_tables() + + def _panel_title(self, path: Path, meta: dict) -> str: + tokenizer = meta.get("tokenizer", "unknown") + vocab_size = meta.get("vocab_size", "?") + return f"{path} | {tokenizer} | vocab={vocab_size}" + + def _setup_table(self, table: DataTable) -> None: + self._ensure_columns(table) + table.clear() + + def _ensure_columns(self, table: DataTable) -> None: + if table.columns: + return + table.add_column("id", key="id") + table.add_column("token", key="token") + table.add_column("bytes", key="bytes") + table.add_column("count", key="count") + + def _refresh_tables(self) -> None: + left_table = self.query_one("#left_table", DataTable) + right_table = self.query_one("#right_table", DataTable) + self._fill_table(left_table, _sorted_entries(self.left_entries, self.sort_mode)) + self._fill_table(right_table, _sorted_entries(self.right_entries, self.sort_mode)) + + def _fill_table(self, table: DataTable, entries: List[VocabEntry]) -> None: + self._ensure_columns(table) + table.clear() + for entry in entries: + table.add_row( + str(entry.token_id), + _display_token(entry.token), + str(entry.byte_len), + str(entry.count), + ) + + def action_sort_bytes(self) -> None: + self.sort_mode = "bytes" + self._refresh_tables() + + def action_sort_freq(self) -> None: + self.sort_mode = "freq" + self._refresh_tables() + + def action_sort_id(self) -> None: + self.sort_mode = "id" + self._refresh_tables() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Compare vocabularies from two meta.pkl files.") + parser.add_argument("left", type=Path, help="Path to left meta.pkl or directory containing it") + parser.add_argument("right", type=Path, help="Path to right meta.pkl or directory containing it") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + left_meta = _resolve_meta_path(args.left) + right_meta = _resolve_meta_path(args.right) + if not left_meta.exists(): + raise FileNotFoundError(f"Missing meta.pkl at {left_meta}") + if not right_meta.exists(): + raise FileNotFoundError(f"Missing meta.pkl at {right_meta}") + app = VocabCompareApp(left_meta=left_meta, right_meta=right_meta) + app.run() + + +if __name__ == "__main__": + main() + diff --git a/data/opus-100/compare_nfc_nfd.sh b/data/opus-100/compare_nfc_nfd.sh new file mode 100644 index 0000000000..fbb5385740 --- /dev/null +++ b/data/opus-100/compare_nfc_nfd.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# bash prepare_nfc_and_nfd_dataset_splits.sh input.txt + +for tokenization in "char_bpe"; do + for type in "nfc" "nfd"; do + for vocab_size in "2000" "3000" "4000" "5000"; do + python3 prepare.py -t p90_"$type".txt -v p10_"$type".txt --method "$tokenization" --vocab_size "$vocab_size" -s -S "$type"_"$vocab_size" -T + mv char_bpe_vocab.json "${tokenization}_${type}_${vocab_size}/" + done + done +done diff --git a/data/opus-100/hangul_nfc_to_nfd.py b/data/opus-100/hangul_nfc_to_nfd.py new file mode 100644 index 0000000000..b5b88d4d1c --- /dev/null +++ b/data/opus-100/hangul_nfc_to_nfd.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +""" +Convert only precomposed Korean Hangul syllables from NFC to NFD. + +Everything outside the Hangul syllables block U+AC00..U+D7A3 is left unchanged. +So Latin accents, emoji, punctuation, spaces, line endings, etc. are not normalized. + +Usage: + python3 hangul_nfc_to_nfd.py input.txt output.txt + python3 hangul_nfc_to_nfd.py --in-place input.txt + cat input.txt | python3 hangul_nfc_to_nfd.py > output.txt +""" + +from __future__ import annotations + +import argparse +import os +import sys +import tempfile +import unicodedata +from pathlib import Path + + +HANGUL_SYLLABLES_START = 0xAC00 +HANGUL_SYLLABLES_END = 0xD7A3 + + +def is_precomposed_hangul_syllable(ch: str) -> bool: + codepoint = ord(ch) + return HANGUL_SYLLABLES_START <= codepoint <= HANGUL_SYLLABLES_END + + +def hangul_only_nfd(text: str) -> str: + """ + Apply Unicode NFD only to precomposed Hangul syllables. + + Example: + 한 U+D55C -> ᄒ U+1112 + ᅡ U+1161 + ᆫ U+11AB + + Non-Hangul characters are returned exactly as they are. + """ + return "".join( + unicodedata.normalize("NFD", ch) + if is_precomposed_hangul_syllable(ch) + else ch + for ch in text + ) + + +def convert_bytes(data: bytes, encoding: str) -> bytes: + # surrogateescape preserves invalid bytes when decoding/re-encoding. + text = data.decode(encoding, errors="surrogateescape") + converted = hangul_only_nfd(text) + return converted.encode(encoding, errors="surrogateescape") + + +def convert_file(input_path: Path, output_path: Path, encoding: str) -> None: + data = input_path.read_bytes() + output_path.write_bytes(convert_bytes(data, encoding)) + + +def convert_file_in_place(path: Path, encoding: str) -> None: + converted = convert_bytes(path.read_bytes(), encoding) + + fd, tmp_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=str(path.parent), + ) + + try: + with os.fdopen(fd, "wb") as tmp: + tmp.write(converted) + os.replace(tmp_name, path) + except Exception: + try: + os.unlink(tmp_name) + except FileNotFoundError: + pass + raise + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Convert only Korean Hangul syllables from NFC to NFD." + ) + parser.add_argument("input", nargs="?", help="Input file. If omitted, reads stdin.") + parser.add_argument("output", nargs="?", help="Output file. If omitted, writes stdout.") + parser.add_argument( + "-i", + "--in-place", + action="store_true", + help="Rewrite the input file in place.", + ) + parser.add_argument( + "--encoding", + default="utf-8", + help="Text encoding to use. Default: utf-8.", + ) + + args = parser.parse_args() + + if args.in_place: + if not args.input or args.output: + parser.error("--in-place requires exactly one input file and no output file.") + convert_file_in_place(Path(args.input), args.encoding) + return 0 + + if args.input: + input_path = Path(args.input) + if args.output: + convert_file(input_path, Path(args.output), args.encoding) + else: + sys.stdout.buffer.write(convert_bytes(input_path.read_bytes(), args.encoding)) + return 0 + + if args.output: + parser.error("An output file requires an input file.") + + sys.stdout.buffer.write(convert_bytes(sys.stdin.buffer.read(), args.encoding)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data/opus-100/prepare_nfc_and_nfd_dataset_splits.sh b/data/opus-100/prepare_nfc_and_nfd_dataset_splits.sh new file mode 100644 index 0000000000..b561b3b3dd --- /dev/null +++ b/data/opus-100/prepare_nfc_and_nfd_dataset_splits.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +FILE="$1" + +NFC_90="p90_nfc.txt" +NFC_10="p10_nfc.txt" + +NFD_90="p90_nfd.txt" +NFD_10="p10_nfd.txt" + +TOTAL=$(wc -l < "$FILE") +PCT=$((TOTAL * 10 / 100)) +head -n -$PCT "$FILE" > "$NFC_90" +tail -n $PCT "$FILE" > "$NFC_10" + +python3 hangul_nfc_to_nfd.py "$NFC_90" "$NFD_90" +python3 hangul_nfc_to_nfd.py "$NFC_10" "$NFD_10" + From da565e547fa59621c23d3346994dd9801a7c1f01 Mon Sep 17 00:00:00 2001 From: klei22 Date: Tue, 14 Jul 2026 00:13:04 -0700 Subject: [PATCH 2/2] Update ko-commercial dataset script --- data/ko_commercial/get_dataset.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/data/ko_commercial/get_dataset.py b/data/ko_commercial/get_dataset.py index 1db517a024..bd2bae4d4d 100644 --- a/data/ko_commercial/get_dataset.py +++ b/data/ko_commercial/get_dataset.py @@ -62,14 +62,7 @@ def emit_json_contents(json_path, output_text_file): def main(output_text_file): parquet_files = { - "train-00000-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00000-of-00008-262a4650cba7cd42.parquet?download=true", - "train-00001-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00001-of-00008-911ee3e75f35d481.parquet?download=true", - "train-00002-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00002-of-00008-4f4791330d9553d5.parquet?download=true", - "train-00003-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00003-of-00008-9f822b7fe3799cd3.parquet?download=true", - "train-00004-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00004-of-00008-8daf63b596c127d7.parquet?download=true", - "train-00005-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00005-of-00008-68cf9afac57baa1c.parquet?download=true", - "train-00006-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00006-of-00008-275cbee982d3460c.parquet?download=true", - "train-00007-of-00008": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00007-of-00008-e63b716fb34017d5.parquet?download=true", + "train-00000-of-00001": "https://huggingface.co/datasets/MarkrAI/KoCommercial-Dataset/resolve/main/data/train-00000-of-00001-1ae224438dce829b.parquet?download=true" } download_dir = "./downloaded_parquets" json_dir = "./json_output"